c# receive problem.

Hi, I have been trying to figure out my problem with sending serial data from arduino to c#.
My arduino sends out readings for Temp, RH, CO2, Light, HighTemp, LowTemp, HighRH, LowRH, HiCo2, LowCo2 every 10seconds, every 5 minutes it takes the average of these values and stores them in eprom.
I pass the values to

void addPacketData(uint32_t S, uint16_t T, uint16_t H, uint16_t C, uint32_t L, uint16_t fil)
{
      savePacket.Sec = S;
      savePacket.Temp = T;
      savePacket.RH = H;
      savePacket.CO2 = C;
      savePacket.Lights = L;
      savePacket.filler = fil;
      eeWrite(getAddress(),savePacket);
      delay(5);
}

getAddress() gets the address Im storing in address 0x0 again this address is stored using a struct, the starting address is 0x40, once the data is stored I increment the address by 16 and store that so the next reading goes into the new address, savepacket is the struct. All that seems to be working fine and in short runs the readings come through fine but when I leave the device for long periods and the data being sent is large it seems to fail and it doesnt even get to the parsing point in c# as I have lots of console.writelines so I can see whats happening.

This is my getHistoricData() which is called when i press a button on the application, Im a little unsure if this is the best way to send all that data, I was going to just send everything as a packet eg 25,85,1600,1,198762537 but if you have 5000 records it may take a while as your reading from eprom / put onto serial port / etc etc.

void getHistoricData()
{
  int numofrecords = (epromaddress.SA - 0x40) / 16;
  //Serial.println(numofrecords, DEC); 
  int TempArray[numofrecords];
  int RHArray[numofrecords];
  int COArray[numofrecords];
  int LightArray[numofrecords];
  long int TimeS[numofrecords];
  
  for(unsigned int i = 0x40, p = 0; i < epromaddress.SA; p++, i = i + 16)
  {
    Eprom.eeRead(i,histPacket);
    delay(10);
    TempArray[p] = histPacket.Temp;
    RHArray[p] = histPacket.RH;
    COArray[p] = histPacket.CO2;
    LightArray[p] = histPacket.Lights;
    TimeS[p] = histPacket.Sec;
  }
  Serial.print("*");
  for(int i = 0; i < numofrecords; i++)
  {
    Serial.print(TempArray[i], DEC);
    Serial.print(",");
  }
  Serial.println("*");
  delay(500);
  Serial.print("{");
  for(int i = 0; i < numofrecords; i++)
  {
    Serial.print(RHArray[i], DEC);
    Serial.print(",");
  }
  Serial.println("{");
  delay(500);
  Serial.print("+");
  for(int i = 0; i < numofrecords; i++)
  {
    Serial.print(COArray[i], DEC);
    Serial.print(",");
  }
  Serial.println("+");
  delay(500);
  Serial.print("-");
  for(int i = 0; i < numofrecords; i++)
  {
    Serial.print(LightArray[i], DEC);
    Serial.print(",");
  }
  Serial.println("-");
  delay(500);
  Serial.print(":");
  for(int i = 0; i < numofrecords; i++)
  {
    Serial.print(TimeS[i], DEC);
    Serial.print(",");
  }
  Serial.println(":");
}

Once sent to the c# application and it goes through the readserial function I seperate using this

if (text[0] == '[')
                {
                    ParseCurrentData(text);
                }
                if (text[0] == '*')
                {

                    t_globalstring = text.Substring(1, text.Length - 3);
                    System.Console.WriteLine(t_globalstring);
                }
                if (text[0] == '{')
                {
                    h_globalstring = text.Substring(1, text.Length - 3);
                    System.Console.WriteLine(h_globalstring);
                }
                if (text[0] == '+')
                {
                    c_globalstring = text.Substring(1, text.Length - 3);
                    System.Console.WriteLine(c_globalstring);
                }
                if (text[0] == '-')
                {
                    l_globalstring = text.Substring(1, text.Length - 3);
                    System.Console.WriteLine(l_globalstring);

                }
                if (text[0] == ':')
                {
                    ti_globalstring = text.Substring(1, text.Length - 3);
                    addData(t_globalstring, h_globalstring, c_globalstring, l_globalstring, ti_globalstring);
                    t_globalstring = "";
                    h_globalstring = "";
                    c_globalstring = "";
                    l_globalstring = "";
                    ti_globalstring = "";
                }

And then it goes into this function

private void addData(string a, string b, string c, string d, string e)
        {
            System.Console.WriteLine(a);
            System.Console.WriteLine(b);
            System.Console.WriteLine(c);
            System.Console.WriteLine(d);
            System.Console.WriteLine(e);
            System.Console.WriteLine("Add data to database here!!");
            string[] datT = a.Split(',');
            string[] datH = b.Split(',');
            string[] datC = c.Split(',');
            string[] datL = d.Split(',');
            string[] datTi = e.Split(',');
            histlistt.Clear();
            histlisth.Clear();
            histlistc.Clear();
            histlistl.Clear();
            if (datT.Length > 2)
            {
                for (int i = 0; i < datT.Length - 1; i++)
                {
                    this.histlistt.Add(histlistt.Count, int.Parse(datT[i]));
                    this.histlisth.Add(histlisth.Count, int.Parse(datH[i]));
                    this.histlistc.Add(histlistc.Count, int.Parse(datC[i]));

                    //DataRow dRow = ds1.Tables["THCdata"].NewRow();
                    //dRow[1] = datT[i];
                    //dRow[2] = datH[i];
                    //dRow[3] = datC[i];
                    //dRow[5] = datTi[i];
                    if (int.Parse(datL[i]) > 4)
                    {
                        this.histlistl.Add(histlistc.Count, 1);
                        //dRow[4] = 1;
                    }
                    else
                    {
                        this.histlistl.Add(histlistc.Count, 0);
                        //dRow[4] = 0;
                    }
                    //ds1.Tables["THCdata"].Rows.Add(dRow);

                }
                UpdateHistGraphs();
                //da.Update(ds1, "THCdata");
                //port.Write("3");
                //GetDataFromDatabase();
            }

        }

I dont think its a problem with c#, before switching to structing everything up I was just allocating long lines of eprom address for each variable and iterating through them, worked no problem for weeks on end. Ive tried changing the port timeout on c# but they have no effect on my problem, i was getting timeout exceptions though this problem because i need to reset the arduino after asking for historic data and it fails.

Regards

getAddress() gets the address Im storing in address 0x0 again this address is stored using a struct, the starting address is 0x40, once the data is stored I increment the address by 16 and store that so the next reading goes into the new address, savepacket is the struct.

What happens when you get to the end of EEPROM? EEPROM is fairly small. Saving 16 bytes every 5 minutes will exhaust EEPROM, on a UNO or other 328 based board, after 60 iterations (5 hours).

All that seems to be working fine and in short runs the readings come through fine but when I leave the device for long periods and the data being sent is large it seems to fail and it doesnt even get to the parsing point in c# as I have lots of console.writelines so I can see whats happening.

What is a "long period"? An hour? Five hours?

I was going to just send everything as a packet eg 25,85,1600,1,198762537 but if you have 5000 records it may take a while as your reading from eprom / put onto serial port / etc etc.

You'll never have more than 60 records on a UNO or other 328 based board.

  int TempArray[numofrecords];
  int RHArray[numofrecords];
  int COArray[numofrecords];
  int LightArray[numofrecords];
  long int TimeS[numofrecords];

12 bytes per record. Unless you have limited the number of records to something reasonable, and there is no evidence that you are, you'll quickly exhaust all memory. Especially with all those Serial.print()s.

Sorry I forgot to mention Im using a 512kb eprom. I was testing yesterday and all day the historic data was being sent when I asked for it, when left over night and I ask for data in the morning it sends but doesnt get as far as parsing, then the arduino stops functioning and I have to pull plug out and reset.

when left over night and I ask for data in the morning it sends but doesnt get as far as parsing, then the arduino stops functioning and I have to pull plug out and reset.

You haven't said how many records that amounts to. Even with external EEPROM, the SRAM on the Arduino is limited. It sounds like you are expecting to read all 512 KB of data into the Arduino's 2 KB (to 8 KB, depending on the particular model) of SRAM.

I'm sure that the Arduino starts sending, but I am almost certain that it never finishes because you have corrupted memory, and the Arduino resets.

That exactly right, it does start sending but fails through the process. Why had it been ok when I was just allocating 2016 addresses of eprom and sending the whole lot through the serial? since yesterday I have only stored 288 records so im sending 288 temp values 26,26,27 etc , then 288 rh values etc. is it because Im creating my arrays to put the values into? then iterating through the arrays to send the data from RAM.

since yesterday I have only stored 288 records so im sending 288 temp values 26,26,27 etc , then 288 rh values etc.

No, you are collecting 288 temp values AND 288 humidity values AND 288 CO values AND 288 light values, and storing them all in memory at once.

is it because Im creating my arrays to put the values into? then iterating through the arrays to send the data from RAM.

Exactly.

Since you are sending all the temp values first, then all the humidity values, etc., there is no reason to store them in arrays first.

Even though you write the data to the EEPROM in 16 byte chunks, you do not need to read it the same way. You can determine the offset from the base address where the temperature is stored, then read a temperature, send it, advance 16 bytes, read a temperature, send it, and repeat for each temperature.

Then, read and send all the other data. The only difference will be the offset from the start of the chunk in the EEPROM for the specific piece of data for a record.

No arrays needed.

Ok that sounds right, I thought there was something wrong in the way I was implementing it plus not thinking about how much ram the system actually had.

Thanks

Thats solution worked, although I ended up just sending a struct at a time between * *, that cut down the parsing code in c# and now only have one if statement if parse function, the sendhistoricdata function is down to a few lines of code, it looked very nice streaming in because I was updating the graphs every packet, now ill redirect them into the database then refresh.

Thats solution worked, although I ended up just sending a struct at a time between * *

Great. Thanks for the feedback.