Using Serial imput to run a code

I suspect this line

void checkToggle() {
  buttonNewData = NO;   <---

You should only set it to NO once you have handled the new button data.

I think that (for starters) you have to rework your code a bit. There are too many places where you call your functions checkSerial and checkButton.

The idea is that you call those functions once (as you do in loop) and nowhere else. This will, as far as I can see, only involve a change in checkToggle.

Below are some snippets of changes I made to your code. I do not know if it works, but it does compile.

Move the new #defines to before the declaration of toggle; this will allow you to initialise toggle straight away with RESTART.

I have also added a new variable previousToggle so the code can do a detection of the change of toggle. This replaces the buttonNewData.

...
...
int mandmcount = 0;

#define ISPRESSED LOW
// sterretje: moved new #defines to above toggle definition so RESTART can be used there
#define YES 1
#define NO 0

#define START 0
#define PAUSE 1
#define RESTART 2
#define TERMINATE 3

const int buttonPin = 2;
// sterretje: initialise with correct value
int toggle = RESTART;
// sterretje: added so a state change in toggle can be detected in check in checkToggle
//            replaces buttonNewData
int previousToggle = RESTART;

unsigned long lastDebounceTime = 0;  // last time the output pin was toggled
unsigned long debounceDelay = 50;    // the debounce time
int buttonState = !ISPRESSED;       // the current reading from the input pin
int lastButtonState = !ISPRESSED;   // the previous reading from the input pin
// sterretje: removed, not needed
//int buttonNewData = 0;
...
...

In setup, you can get rid of the setting of toggle

void setup()
{
  ...
  ...
  bottomservo.write(0);
  // sterretje: removed as it's now initialised in the global section
  // toggle = RESTART;
  lcd.begin(16, 2);
  ...
  ...
}

Next you can modify checkToggle. It compares previousToggle and toggle to check if once-off things need to be done (preventing frequent updates of serial monitor and the lcd).

void checkToggle() {
  if (toggle == PAUSE) {
    if (toggle != previousToggle)
    {
      previousToggle = toggle;
      Serial.println ("-Paused-");
      lcd.clear();
      lcd.setCursor(0, 0);
      lcd.print ("-Paused-");
    }
  }
  if (toggle == START) {
    if (toggle != previousToggle)
    {
      previousToggle = toggle;
      mandmStart();
    }
    color();
    mandmColor();
  }
  if (toggle == RESTART) {
    if (toggle != previousToggle)
    {
      previousToggle = toggle;
      Serial.println("<Initializing...>");
      lcd.clear();
      lcd.setCursor(0, 0);
      lcd.print("<Initializing..>");
      lcd.setCursor(0, 1);
      lcd.print("M&M Sorter - DH");
      bottomservo.write(0);
      delay(1000);
      Serial.println("M&M Sorter - D.H.");
      lcd.clear();
      delay(50);
      Serial.println("- Awaiting Start Command Prompt -");
      lcd.setCursor(0, 0);
      lcd.print("Awaiting Start");
      lcd.setCursor(0, 1);
      lcd.print("Command Prompt");
      delay(2);
      mandmcount = 0;
    }
  }
  if (toggle == TERMINATE) {
    lcd.clear();
    lcd.setCursor(0, 0);
    lcd.print("PROGRAM");
    lcd.setCursor(0, 1);
    lcd.print("TERMINATED");
    for (;;) {
      delay(1000);
    }
  }
}

Lastly you can remove the button check from the end of mandmColor

void mandmColor() {
  ...
  ...
  Serial.print("  - #");
  Serial.println(mandmcount);
  // sterretje: removed
  //startMillis = millis();
  //currentMillis = millis();
  //while (currentMillis - startMillis <= period) {
  //  currentMillis = millis();
  //  checkButton();
  //}
}

Hope this gets you on track.