CSV file read stops after 9 lines?

Good morning all,

I am in the process of assembling a project using an Arduino Mega 2560. It consists, in part, of reading in a CSV data file that will have a sequence of a relay addresse followed by each relay's on-time and off-time. As you can surmise; the sketch causes relays connected to the Mega to turn on then off. The CSV is formatted with integers in the following manner: relay number, time on (in millis), time off (in millis) terminated by a new line. Here is a sequence I used to test all relays (for now 16, but this project will have 32 relays in the end):
1,250,250
2,250,250
3,250,250
4,250,250

...etc, you can see where this is going. I currently have the sketch designed so this information outputs to a serial monitor (on my Linux box, I use a serial port terminal which permits the ability to read in the CSV file directly from my PC and send it to the USB port);in order to see what the sketch is doing with the info.

The issue I have encountered is that the whole thing stops after the 9th relay. I've tried changing the variable types, played with the on-times and off-times making them longer and shorter, but nothing seems to change. I can only presume the Mega is running out of memory, but the array I use (data[3]) is recycled at every iteration, so the memory addresses used by the Mega have to be overwritten. I attempted using the Serial.flush() function right after sending the data array to the relay activation subroutine, but it does not seem to help.

Here is the entire sketch:

int data[3];
int counter;
int inbyte;
int relaypin;

void setup(){
  // initialize serial communication:
  Serial.begin(19200);
  counter = 0;
  // initialize the relay pins:
  for (relaypin = 22; relaypin < 38; relaypin++) {
    pinMode(relaypin, OUTPUT);
  }
}
void loop(){
  if (Serial.available()){
    inbyte = Serial.parseInt();// check for comma separated data
    data[counter] = inbyte;
    counter++;
    if(Serial.read() ==  '\n'){  // end of line test
      data[counter] = inbyte;
      counter = 0;
      //send data to actRelay func
      actRelay(data);
      Serial.print("Relay:");
      Serial.println(data[0]);
      Serial.print("Time ON:");
      Serial.println(data[1]);
      Serial.print("Time OFF:");
      Serial.println(data[2]);
    }
    
  }
}

void actRelay(int* array){
        switch (data[0]) {
        case 1:
          digitalWrite(22, HIGH);
          delay(data[1]);
          digitalWrite(22,LOW);
          delay(data[2]);
          break;
        case 2:
          digitalWrite(23, HIGH);
          delay(data[1]);
          digitalWrite(23,LOW);
          delay(data[2]);
          break;
        case 3:
          digitalWrite(24, HIGH);
          delay(data[1]);
          digitalWrite(24,LOW);
          delay(data[2]);
          break;
        case 4:
          digitalWrite(25, HIGH);
          delay(data[1]);
          digitalWrite(25,LOW);
          delay(data[2]);
          break;
        case 5:
          digitalWrite(26, HIGH);
          delay(data[1]);
          digitalWrite(26,LOW);
          delay(data[2]);
          break;
        case 6:
          digitalWrite(27, HIGH);
          delay(data[1]);
          digitalWrite(27,LOW);
          delay(data[2]);
          break;
        case 7:
          digitalWrite(28, HIGH);
          delay(data[1]);
          digitalWrite(28,LOW);
          delay(data[2]);
          break;
        case 8:
          digitalWrite(29, HIGH);
          delay(data[1]);
          digitalWrite(29,LOW);
          delay(data[2]);
          break;
        case 9:
          digitalWrite(30, HIGH);
          delay(data[1]);
          digitalWrite(30,LOW);
          delay(data[2]);
          break;
        case 10:
          digitalWrite(31, HIGH);
          delay(data[1]);
          digitalWrite(31,LOW);
          delay(data[2]);
          break;
        case 11:
          digitalWrite(32, HIGH);
          delay(data[1]);
          digitalWrite(32,LOW);
          delay(data[2]);
          break;
        case 12:
          digitalWrite(33, HIGH);
          delay(data[1]);
          digitalWrite(33,LOW);
          delay(data[2]);
          break;
        case 13:
          digitalWrite(34, HIGH);
          delay(data[1]);
          digitalWrite(34,LOW);
          delay(data[2]);
          break;
        case 14:
          digitalWrite(35, HIGH);
          delay(data[1]);
          digitalWrite(35,LOW);
          delay(data[2]);
          break;
        case 15:
          digitalWrite(36, HIGH);
          delay(data[1]);
          digitalWrite(36,LOW);
          delay(data[2]);
          break;
        case 16:
          digitalWrite(37, HIGH);
          delay(data[1]);
          digitalWrite(37,LOW);
          delay(data[2]);
          break;
        }
}

Thank you in advance for helping.

You are not checking Serial.available() before every Serial.read() I fear - you must check on every single read.

I attempted using the Serial.flush() function right after sending the data array to the relay activation subroutine, but it does not seem to help.

What good is blocking until all serial data has been sent, when you aren't sending any serial data?

MarkT; thanks for the reply. Would I then have to re-write the section that tests for the new line in the following manner?

void loop(){
  if (Serial.available()){
    inbyte = Serial.parseInt();// check for comma separated data
    data[counter] = inbyte;
    counter++;
    if (Serial.available() && Serial.read() ==  '\n'){  // end of line test
      data[counter] = inbyte;
      counter = 0;
    }

The original code seems to detect the \n (EOL). I may be going about parsing the data in the wrong way. This code has been edited from a few ideas I've found on this forum and others.

    inbyte = Serial.parseInt();// check for comma separated data

That's not what parseInt() does.

Hello PaulS

Can you explain what the Serial.parseInt() does? It appears to function in my sketch as to actually parse the comma separated values in the file. It returns the actual number in before each comma (which is what I need to pass on the the Switch subroutine in order to use the numbers to activate the relays and provide timing).

My sketch does not sanitize the data sent to it; ensuring it is numbers and not other characters: I am relying on the CSV to be correct to begin with. I had used a simple Serial.read() command to get the data, but it returned the ascii code value for each one. Made programming more complicated until I began experimenting with Serial.parseInt(). What I really need is the numbers, not their ascii equivalent.

Thanks.

Can you explain what the Serial.parseInt() does?

It reads from the serial stream until a non-digit character arrives, or until a time limit is exceeded, and converts the data that it read to an int. It does not "check for comma separated data".

I had used a simple Serial.read() command to get the data, but it returned the ascii code value for each one.

Guess what. That's what parseInt() does, only it's smart enough to know that the value it retrieves is really a char, not an int.

Very cool PaulS.

Using Serial.parseInt() is good, but I need to keep in mind that the comma is not what is actually causing the command to parse: anything not a number will suffice.

What about the response from MarT? Would you weigh in on why my sketch stops after the 9th line?

void loop(){
  if (Serial.available()){
    inbyte = Serial.parseInt();// check for comma separated data
    data[counter] = inbyte;
    counter++;
    if (Serial.available() && Serial.read() ==  '\n'){  // end of line test
      data[counter] = inbyte;
      counter = 0;
    }

You have a number of naming issues. inbyte is not a byte.

You use parseInt() to get an int value, that you store in an int called inbyte. You store inbyte in the array and increment the index.

Then, if there is serial data (and there likely will be because something caused parseInt() to stop reading), and if that character is a carriage return, you store the last int received in the array again.

Why is that?

The issue I have encountered is that the whole thing stops after the 9th relay.

You'll need to elaborate on this. Does the RX led continue to blink while you are diddling with the relays? Does the file contain data for 9 relays in consecutive order? Does changing the order matter?

Does the sending application keep sending data while you aren't paying attention, and the serial buffer simply overflows and discards all the extra data?

I suspect that you're overflowing the serial buffer; at that baud rate, you'll be sending ~1900 characters a second. Your half second delay on the first relay activation alone will cause overflow. I assume your program gets through the data in the buffer accounting for the successful nine and then the rest has been thrown away because there was no space.

Is this text CSV? I would think so. What happens after #9?
1
2
3
4
5
6
7
8
9
10

How many characters is that last number?
Just a thought...

edit: I did this with network settings for the w5100 using a microSD card file in this post:

I read the input into a buffer until the buffer size is reached or a cr/lf is encountered. Then it uses sscanf to retrieve the values.

In response to PaulS:

  1. Good point on the naming convention.
  2. Do I really need to use an array to store the three pieces of data I need to activate the relays? Maybe not, I will rewrite the code to use variables called: relay, t_on, t_off and I will retest.
  3. I will perform tests when I get back to the lab and provide feedback on the comms questions

In response to wildbill:
I will slow down the baud rate and see if this helps. On that note; what is the method to prevent such an overflow without having to play with baud rate? The issue may be more prominent when the project gets used in the field: I will be using 1-wire comms to transmit the CSV to the Mega; which will activate the relays. The length of the 2 wires used to communicate may vary and I don't want to have to adjust the baud rate to prevent overflow every time something changes on the wiring.

Thank you all for weighing in. More data to come after spending more time in the lab.

Slowing down the baud rate won't get you much - your delays are long enough that you'll suffer the same issue. You can test it using the IDE or other terminal program and type the relay commands by hand. If that's the problem, you should be able to go way beyond nine. Don't just cut & paste the dat & send it all though.

I will slow down the baud rate and see if this helps.

Only is you slow it down so far that the delay()s are shorter than the time needed to transmit a buffer full of data.

On that note; what is the method to prevent such an overflow without having to play with baud rate?

Handshaking. The receiver (the Arduino) sends a message to the sender (the PC) that says "Whew, done with that last batch of data; send me some more".

Of course, it doesn't have to be quite that verbose...

I will be using 1-wire comms to transmit the CSV to the Mega

From? You'll need at least two (ground). A third (to allow two way communication) would be even better.

The length of the 2 wires used to communicate may vary and I don't want to have to adjust the baud rate to prevent overflow every time something changes on the wiring.

I don't think you have accurate enough equipment to measure the transmission time of serial data down a short wire vs. a long wire.

Wire length has virtually nothing to do with your issue.

Hey Paul,

You'll most likely get to see more questions on the 1-wire comms stuff; that's the next phase of this project. I will be going from testing the code PC-to-Arduino, then going Arduino (Master Control Module)-to-Arduino (Field Module). The 1-Wire comms will be Dallas 1-wire. The protocol performs bi-directional comms using 1 wire plus ground.

Stay tuned...

What is this system intended to do? How many relay commands will there be? Right now your test example suggests you'll be turning each relay on only once - is that true for the final product? If you have a small (preferably known) number of actions, you can just read them into an array as fast as the serial port can deliver them. If there are too many then PaulS' suggestion of software flow control (e.g. XON/XOFF) may well help.

Finally, if you're prepared to assume that your relays are on adjacent pins (as you did setting their pinmode), you can simplify your actRelay routing to this:

void actRelay(int* array)
{
digitalWrite(array[0]+22, HIGH);
delay(array[1]);
digitalWrite(array[0]+22,LOW);
delay(array[2]);
}

Hello again everyone,

Here is my new code; I've tightened things up a bit. I also googled and searched this forum for flow control on the serial buffer; and found that according to those in the know: there is no flow control on the Arduino's serial interface. Please correct me if I'm wrong (and my wish is that someone please corrects me). I would love to see code that would perform this in order to incorporate it in my current proof of concept sketch. I've found a few things on Handshaking, but they all seem to relate to Arduino to some external device serial comms, rather than a constant stream from DTE to the Arduino.

The new code permits the execution up to relay 13 at 1200 baud. If I remove the lines that actually print to the serial port (for visual feedback of where the CSV file is at); the entire test CSV up to relay 16 does cycle through; I can see the relay indicator LEDs light up sequentially. If I bring the baud rate to 9600, the sequence stops at 9 as it did before; even without the serial printing.

int data[3];
int arr_ndx;
int relaypin;

void setup(){
  // initialize serial communication:
  Serial.begin(1200);
  arr_ndx = 0;
  // initialize the relay pins:
  for (relaypin = 22; relaypin < 38; relaypin++) {
    pinMode(relaypin, OUTPUT);
  }
}
void loop(){
  if (Serial.available()){
    data[arr_ndx] = Serial.parseInt();// check for comma separated data
    arr_ndx++;
    if(Serial.available() && Serial.read() ==  '\n'){  // end of line test
      arr_ndx = 0;
      actRelay(data);//send data to actRelay func
/*      Serial.print("Relay:");
      Serial.println(data[0]);
      Serial.print("Time ON:");
      Serial.println(data[1]);
      Serial.print("Time OFF:");
      Serial.println(data[2]);*/
    }

  }
}

void actRelay(int* array)
{
  digitalWrite(array[0]+22, HIGH);
  delay(data[1]);
  digitalWrite(array[0]+22,LOW);
  delay(data[2]);
}

Thanks Wildbill for the snippet that reduced the size and complexity of the actRelay function.

I appreciate all your help on this.

I don't believe you've ever shared the contents of this mysterious file you are reading from. We can't even try to duplicate your problem without that.

The file is just a text file containing lines as shown in my very first post. I run through a sequence of numbers corresponding to the relay numbers: 1 through 16 followed by relay on-time and off-time. It looks something like this:

1,50,50
2,50,50
3,50,50
4,50,50
5,50,50
6,50,50
7,50,50
8,50,50

...and so on and so on. The Serial.parseInt() helps separate the values within the sketch. I can confirm that changing the delays changes the number of lines the sketch can go through (compounded by whether I allow the on-screen printing of the results). I understand that the relay activation is not an amazing feat, but this project is a proof of concept. These relays will eventually perform tasks and the sequence and timing will be much different. Coming up against these challenges now permits me to understand the behavior of the Arduino and programming in C++.

The final proof will be using a Leonardo connected to an SD card reader containing the CSV file, it will transmit this file's data via Dallas 1-wire comms (to keep the wiring down to 2 physical wires only), and having a Mega 2560 receive the data from the Leonardo to activate the relays.

I haven't used XON/XOFF in a long time, let alone in any arduino related application, but it should be easy enough to test. Can you send control-s (character 19) before you call the relay function and control-q ( character 17) afterwards using Serial.write?