Timing on a attiny 85 - crashes after few actions

I built this oven timer thing - You have a pot that sets the time (0 - 15minutes, past 290 degrees is infinity) and then a button that you use to start / stop the timer.
There is an LED that blinks fast when it is running, blinks even faster when it is set to infinity, continuously shines when the timer is not running and flashes in sync with a buzzer when the timer finishes. And also there is a relay that is turned on when the timer is running (duh).

Schematic / Diagram is in the files and here:

Now when I run the code runs fine but after like 50 presses of the button (it needs less presses when its set to endless) it starts doing weird stuff - led stays lit and relay turns on, buzzer just continuously screams, led turns on and off when I turn the pot sometimes. It almost feels like the internal crystal is stopping but I built two pieces, and they do the same stuff.

Code is here:

#define PotPin A1
#define ButtonPin 0
#define LEDPin 1
#define BuzzerPin 4
#define RelayPin 3

bool TimerRunning = false;
bool TimerEndless = false;
uint32_t TimerDuration;
bool TimerFinished = false;

bool LedState;
uint8_t LEDSequence;
bool BuzzerState;
uint8_t BuzzerSequence;
bool ButtonPrev;

uint32_t LEDPMill; //Im making all of this only 32 bit because thats enough and I dont want the code to be big
uint32_t BuzzerPMill;
uint32_t TimerPMill;

void setup() {
  pinMode(LEDPin, OUTPUT);
  pinMode(BuzzerPin, OUTPUT);
  pinMode(RelayPin, OUTPUT);
  pinMode(ButtonPin, INPUT_PULLUP);
}

void LEDBlink(uint16_t timing, uint8_t sequence) { //just making the LED blink
  uint32_t CMill = millis();
  if (CMill - LEDPMill >= timing) {
    LEDPMill = CMill;
    digitalWrite(LEDPin, bitRead(sequence, LEDSequence));
    if (LEDSequence < 7) LEDSequence++;
    else LEDSequence = 0;
  }
}

void BuzzerControl(uint16_t timing, uint8_t sequence) { //just making the buzzer beep
  uint32_t CMill = millis();
  if (CMill - BuzzerPMill >= timing) {
    BuzzerPMill = CMill;
    digitalWrite(BuzzerPin, bitRead(sequence, BuzzerSequence));
    if (BuzzerSequence < 7) BuzzerSequence++;
    else BuzzerSequence = 0;
  }
}

void BuzzerAndLEDControl(uint16_t timing, uint8_t sequence) { //just making the LED and the buzzer blink / beep
  uint32_t CMill = millis();
  if (CMill - BuzzerPMill >= timing) {
    BuzzerPMill = CMill;
    digitalWrite(BuzzerPin, bitRead(sequence, BuzzerSequence));
    digitalWrite(LEDPin, bitRead(sequence, BuzzerSequence));
    if (BuzzerSequence < 7) BuzzerSequence++;
    else BuzzerSequence = 0;
  }
}

void loop() {
  uint32_t CMill = millis();

  bool ButtonState = !digitalRead(ButtonPin); //button change react
  if (ButtonState != ButtonPrev) {
    ButtonPrev = ButtonState;
    TimerPMill = CMill;
    if (ButtonState) {
      if (TimerFinished) TimerFinished = false;
      else TimerRunning = !TimerRunning;
      if (analogRead(PotPin) < 767) { //pot to milliseconds - I have to sample the time only when the timer starts
        TimerDuration = map(analogRead(PotPin), 0, 767, 0, 900000);
        TimerEndless = 0;
      }
      else if (analogRead(PotPin) < 990) { //past 3/4 of the pot is just 15 minutes
        TimerDuration = 900000;
        TimerEndless = 0;
      }
      else TimerEndless = 1; //or it just goes endless
    }
  }

  switch (TimerRunning) {
    case true:
      switch (TimerEndless) {
        case true:
          digitalWrite(RelayPin, 1);
          digitalWrite(BuzzerPin, 0);
          LEDBlink(50, 0b01010101);
          break;
        case false:
          if (CMill - TimerPMill >= TimerDuration) {
            TimerRunning = false;
            TimerFinished = true;
          }
          else {
            digitalWrite(RelayPin, 1);
            digitalWrite(BuzzerPin, 0);
            LEDBlink(100, 0b10001000);
          }
          break;
      }
      break;
    case false:
      digitalWrite(RelayPin, 0);
      if (!TimerFinished) {
        digitalWrite(BuzzerPin, 0);
        digitalWrite(LEDPin, 1);
      }
      break;
  }

  if (TimerFinished) {
    BuzzerAndLEDControl(300, 0b10110000);
  }
}

I hope I described everything well. I really want to get this working without having to rewrite the whole code.

Thanks for help and replies, Marek.