Using COMPA and COMPB correctly

The pulses look OK to me.

100-20=80 80/16=5 microseconds. at 0.5 milliseconds (500 microseconds) per division the pulse would be about 1/100th of a division.

10000-20 = 9980. 9980/16=623 microseconds or a little over 1 division.

I suspect the long delay before triggering is a wrap-around of the timer. Try resetting more of the registers:

#include <avr/io.h>
#include <avr/interrupt.h>
#define LEDPIN 13

void setup(){  
  pinMode(LEDPIN, OUTPUT);
  cli();                    // disable global interrupts
  TCCR1A = 0;               // Clear TCCR1A
  TCCR1B = 0;               // Clear TCCR1B
  TCCR1C = 0;               // Clear TCCR1C
  TCNT1 = 0;                // Clear the timer
  TIMSK1 = 0;               // Clear the interrupt mask
  
  TIMSK1 |= (1 << OCIE1A);  // enable timer compare interrupt channel A:
  TIMSK1 |= (1 << OCIE1B);  // enable timer compare interrupt channel B:
  TCCR1A = B11110000;      // Set OC1A/OC1B on Compare Match, clear OC1A/OC1B at BOTTOM
  OCR1A = 20;              // COMPA at 20/16ths microseconds
  OCR1B = 40;              // COMPB at 40/16th microseconds

  sei();                    // Enable global interrupts
  attachInterrupt(0, start, RISING);
}

void loop(){
}

void start(){
  //want to start/ restart the timer here
  TCNT1 = 0;                // Clear the timer
  TCCR1B |= (1 << CS10);    // prescaler=1 16 MHz clock
}

ISR(TIMER1_COMPA_vect){
  digitalWrite(LEDPIN, HIGH);
}

ISR(TIMER1_COMPB_vect){
  digitalWrite(LEDPIN, LOW);
  TCCR1B = 0;  //stop timer here
}