Problems in TX/RX data between C# and Arduino

I'm trying to use arduino to read four analog inputs from some black box (all at the same freq), send the data to C# through serial and finally use C# to print it in a file. From C# I manage to tell arduino how many reads to get, and give the indication of transmission. However, i'm not receiving the correct data or sometimes i receive the correct data except that it is incomplete (if I indicate 100 readings, I only get 60). Can anybody point me into the right direction? I have tried everything that I have found and yet have not accomplish my goal. Below are my codes for Arduino and part of the C#. I will appreciate your help.

Thank you.

Arduino Code:

//Variable Declaration List
#define Input0 A0 //Assigning PINs for the operation
#define Input1 A1
#define Input2 A2
#define Input3 A3
float Input0_Data = 0; //Assigning Variables to hold the values read from the PINs
float Input1_Data = 0;
float Input2_Data = 0;
float Input3_Data = 0;
const float ToAnalogCoefficient = 5.0/1023.0; // Multiplication by ToAnalogCoefficient converts ASCII-encoded decimal back to Analog
int N = 0; //Variable for the N-Domain

boolean started = false;
boolean ended = false;
char buffer[30];//creating buffer large enough
int serialIn = 0;//counter for the incoming data
int atoiHolder = 50;//holds the value of atoi, default value 50

void setup()
{
      Serial.begin(19200); //opens serial port and sets data rate to 19200bpm
}

void loop()
{
      while(Serial.available() > 0)
      {
            char incomingData = Serial.read(); //reading from the serial ( data from C# )

            if(incomingData == '<')//check for started packet
            {
                  started = true;//the string started
                  ended = false;
            }
            else if(incomingData == '>')//check for ended packet
            {
                  ended = true;//indicated end of the reading
                  break; //break out of the loop
            }
            else
            {
                  buffer[serialIn] = incomingData;//begining to store data in the buffer
                  serialIn++;
                  buffer[serialIn] = '\0';// NULL terminate the array
            }
      }//end of while loop

      //-------------- Executing C# Commands --------------
      if(started && ended)
      {  
            if(buffer[0] == 'N')
            {
                  buffer[0] = '0';//replaces N by '0' and now buffer holds '0','X','X','X','X'
                  atoiHolder = atoi(buffer);//atoiHolder now holds XXXX (number of reads decided by user)
            } 
            if(buffer[0] == 'T' && buffer[1] == 'X')//indication from C# to TX data
            {
                  while(N <= atoiHolder)
                  {
                  Serial.print("======================");
                  Serial.print(atoiHolder);Serial.print("    ");Serial.print(N);
                  Serial.println("======================");

                        if(N == atoiHolder)
                        {
                                        Serial.print("IM AT THE END OF COM");
                              Serial.end();//when exciding the desired number of reads, end serial communication
                        }

                        //ADC OPERATION
                        Input0_Data  = analogRead(Input0);//Reading inputs from analog inputs
                        Input1_Data  = analogRead(Input1);
                        Input2_Data  = analogRead(Input2);
                        Input3_Data  = analogRead(Input3);

                        Input0_Data = Input0_Data * ToAnalogCoefficient; //converting ASCII-encoded decimal back to Analog
                        Input1_Data = Input1_Data * ToAnalogCoefficient;
                        Input2_Data = Input2_Data * ToAnalogCoefficient;
                        Input3_Data = Input3_Data * ToAnalogCoefficient;

                        Serial.print(N);Serial.print(';');//Packaging the data to be outputted to the computer
                        Serial.print(Input0_Data,8);Serial.print(';');//The selected format  N;Input0_Data;Input1_Data;Input2_Data;Input3_Data;
                        Serial.print(Input1_Data,8);Serial.print(';');
                        Serial.print(Input2_Data,8);Serial.print(';');
                        Serial.print(Input3_Data,8);Serial.println(';');
                        N = N + 1; //Incrementing the N-Domain
                        delay(10);//delay of 10ms to let ADC recover ( SEARCH IF NECESSARY )
                  }

            }//end of transmitting section 

            serialIn = 0;//resetting serialIn
            buffer[serialIn] = '\0';//Null the buffer array
            started = false; //reset to false
            ended = false; //reset to false

      }//end of Executing C# Commands
}//end of loop

C# Code:

//Upper variable declarations

DateTime CurrentTime = DateTime.Now; //Setting the clock to current system clock time
string desktopPath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop); //path to the desktop made into strings
string RxData;//String to hold the data from arduino serial port

       //Opening the selected Port
        private void OpenPortButtom_Click(object sender, EventArgs e)
        {

            if (PortSelectBox.SelectedItem == null)//check if not port selected
            {
                MessageBox.Show("No Serial Ports was selected, Try again.");//message box telling user that there is not ports open
            }

            else
            {
                //++++++++SETTINGS FOR THE SERIAL PORT+++++++++++
                serialPort1.PortName = PortSelectBox.SelectedItem.ToString();//setting serialPort1 to a selected port by user 
                serialPort1.BaudRate = 19200;
                serialPort1.DataBits = 8;
                serialPort1.ReadBufferSize = 10240;  //10 * 1024bit = 10KB
                serialPort1.WriteBufferSize = 2048; //2 * 1024bit = 2KB

                if (serialPort1.IsOpen == true)
                {
                    serialPort1.Close();//closing port if previously open
                    serialPort1.Open();//opening port  
                }
                else
                {
                    serialPort1.Open();//opening port when the program starts
                }
                
                OpenPortButtom.Enabled = false; //disabling open port button
                ClosePortButtom.Enabled = true;//enabling the closing port button
                SettupButton.Enabled = true;//enabling the settup button
                StartArduinoButton.Enabled = true;//enabling the flush serial button
            }
        }//end of OpenPortButtom

        //settings for the button that starts arduino serial transmission
        private void StartArduinoButton_Click(object sender, EventArgs e)
        {

            if (serialPort1.IsOpen == true)//checking if any port is open in order to close it
            {
                //creating output file
                string ArduinoLogName = @"" + desktopPath + "\\" + "ArduinoData_" + DateTime.Now.ToString("MMM-d-yyyy") + ".txt";//creates the log file
                FileStream ArduinoLogFile = new FileStream(ArduinoLogName, FileMode.Append);//makes the files writable if existent or creates a new file 
                StreamWriter writer = new StreamWriter(ArduinoLogFile);//variable to write 

                serialPort1.Write("<TX>");//Sending arduino the transmitting command
                MessageBox.Show("Arduino is transmitting data...");//message box telling user that arduino was flushed
                try
                {
                    RxData = serialPort1.ReadExisting();//reading data from the port
                    writer.WriteLine(RxData);//writing the transmitted data
                }
                catch(Exception RzData)
                {
                    RxData = RzData.ToString();
                }

                ArduinoLogFile.Close();//Closing the data file
            }
            else
            {
                MessageBox.Show("There is no Serial Ports open.");//message box telling user that there is not ports open
                StartArduinoButton.Enabled = false;//disabling the start arduino button
            }
        }//end of StartArduinoButton

        //settings for button that tells arduino how many readings to take
        private void SettupButton_Click(object sender, EventArgs e)
        {

            if (serialPort1.IsOpen == true)//checking if any port is open in order to close it
            {
                if (SelNumberOfReading.SelectedItem == null)
                {
                    MessageBox.Show("No number of reading was selected, using default (50 readings).");
                    serialPort1.Write("<N50>");//Sending the readings selected [default (50 readings)]
                }
                else
                {
                    serialPort1.Write("<N" + SelNumberOfReading.SelectedItem.ToString() + ">");//Sending arduino the readings selected, format <N#####>
                    MessageBox.Show("Arduino has been set to " + SelNumberOfReading.SelectedItem.ToString() + " number of readings.");//message box telling user the selected number of readings
                }

                OpenPortButtom.Enabled = false; //disabling open port button
                ClosePortButtom.Enabled = true;//enabling the closing port button
            }
            else
            {
                MessageBox.Show("There is no Serial Ports open.");//message box telling user that there is not ports open
                SettupButton.Enabled = false;//disabling the settup button
            }
        }//end of SettupButton
            Serial.print(Input0_Data,8);Serial.print(';');

8 decimal places, huh? Do you think you are really getting 8 decimal places of accuracy from whatever the sensor is?

However, i'm not receiving the correct data

How do you know you are not receiving the correct data?

                serialPort1.ReadBufferSize = 10240;  //10 * 1024bit = 10KB

Where did you allocate a buffer this big? I didn't see one.

Thanks for the reply PaulS. The black box give 10 decimal places, or so i as told. I confirmed that I'm receiving the correct data, however, its incomplete as indicated above. how can i allocate a buffer? string buffer? Is there any examples on how to do it?

It appears that you don't really need that big a buffer. It also appears that you are telling the SerialPort class to allocate that buffer.

Serial data transmission is relatively slow. You don't need a huge buffer to hold incoming data. You just need to read it often enough, which you appear to be doing.

The black box give 10 decimal places

I have no idea what this black box is, but expecting to read data accurate to 10 decimal places, or even 8, is unrealistic, in my opinion.

What are you doing with that data, after getting it back to the PC?

What evidence do you have (you haven't provided any) that you are not getting back complete and accurate data?

Do you see something in the file like "Measurement 8 of 100" that eventually increments to "Measurement 60 of 100" which is the last entry in the file?

Do you echo the data coming into the C# application, before writing it to the file?

After I get the data into the computer i'm planning in doing FFT, filtering, plotting and few more stuff regarding signal comparison. As for the accuracy, I do not mind reading up to 3 or 4 decimal places. When I get the resulting file from Arduino and C#, I have a header before each reading that indicates at what measurement i'm at and that is what indicates me that the data is incomplete. am I failing to write everything to the file?

Do you echo the data coming into the C# application, before writing it to the file?

how do I echo data coming into the C# application? what does echo really means?

how do I echo data coming into the C# application?

Run the application in debug mode, with plenty of calls to Console.WriteLine().

what does echo really means?

Just printing it out, before doing anything with it.

I changed the serial port buffer size to 4096 (4KB), and the precition to 4 decimal places. Yet, I still have not been able to allocate the buffer. By the way, Is there a way for me to receive data from arduino everytime I request it from the C# application without the need of opening the serial monitor and closing it again? After I upload the code into the board and run C# to extract the data, I can only get data one time. Then, I have attempted to request data a second time during the same session but nothing is TX :o. I close the port and then open it again to request more data from arduino. But nothing is being transmitted (TX light does not blinks). Has anyone experienced something like this?

From my C# application, I can talk to the Arduino all day.

without the need of opening the serial monitor and closing it again?

How does this help. Only one application on the PC can talk to the serial port at a time. If the C# application has opened the port, the Serial Monitor application should not be able to.

You are right about the ports, once C# opens it the serial monitor does not works. My problem refer to the fact that once arduino TX for the first time, it will not transmitt again for a second time without me 1) closing the port using C# 2) opening and closing the serial monitor 3) reopening the port with C#.

I would venture a guess, then, that the problem is that the Arduino code is the culprit.

Set up a button that sends a command to the Arduino to take one reading. Have the Arduino send that one reading back.

Can you press that button more than once. with results being returned each time?

Opening the Serial Monitor, after closing it in the C# application, causes the Arduino to be reset.

The procedure you mentioned is the exact same procedure i have, and the first time i press the button and request to TX I get data. But when i request data again by pressing the button, I do not receive nothing. And yes, I think that opening the Serial Monitor, after closing it in the C# application, causes the arduino to be reset. ;D How can I deal with that? Has that happend before?

I went back and looked at your Arduino code. At the end of the loop that responds to a request for data you have this:

                              Serial.end();//when exciding the desired number of reads, end serial communication

And you wonder why the Arduino won't listen to you anymore?

Oops... Is there any other way to tell arduino to stop sending data until indicated again to keep on sending data?

The Arduino sends data in response to requests. Quit asking, and it will quit sending.

Sounds good. Now i only need to work on the buffer allocation and properly writing to the file. Is it possible to program arduino's chip and take it out of the board and use it dirrectly from a bread board? Eventually, i will need to burn a few of them trying to get at least one correctly into PBC board. Will the ADC still work?

Now i only need to work on the buffer allocation

You are telling the SerialPort class to allocate a receive buffer. I don't see that you have anything more to do.

and properly writing to the file.

Is there a problem with this? Have you confirmed that you are getting all the data to the C# application?

Is it possible to program arduino's chip and take it out of the board and use it dirrectly from a bread board?

With a few extra pieces, yes. "standalone" is the search term you want.

Will the ADC still work?

But, of course.

By echoing the data I confirmed that Arduino is giving me all the values that I request, however, I have not manage to get all the data in the files created. Is there any better approach to writing the data into a file? I'm using the following C#

        private void StartArduinoButton_Click(object sender, EventArgs e)
        {
            if (serialPort1.IsOpen == true)//checking if any port is open in order to close it
            {
                //creating output file
                string ArduinoLogName = @"" + desktopPath + "\\" + "ArduinoData_" + DateTime.Now.ToString("MMM-d-yyyy") + ".txt";//creates the log file
                FileStream ArduinoLogFile = new FileStream(ArduinoLogName, FileMode.Create);//creates a new file or overwrites a previous file created
                StreamWriter writer = new StreamWriter(ArduinoLogFile);//variable to write 

                serialPort1.Write("<TX>");//Sending arduino the transmitting command
                int NumberOfReadings = int.Parse(SelNumberOfReading.SelectedItem.ToString());//converting the selected number of readings from string to ints 

                for (int count = 0; count <= NumberOfReadings; count++)
                {
                    try
                    {
                        RxData = serialPort1.ReadLine();//reading data from the port
                        writer.WriteLine(RxData);//writing the transmitted data
                        Console.WriteLine(RxData);
                    }
                    catch (Exception RzData)
                    {
                        RxData = RzData.ToString();
                        writer.WriteLine(RxData);//writing the transmitted data
                        Console.WriteLine(RxData);
                    }
                }

                MessageBox.Show("The Transmission from Arduino has ended.");
                ArduinoLogFile.Close();//Closing the data file
            }
            else
            {
                MessageBox.Show("There is no Serial Ports open.");//message box telling user that there is not ports open
                StartArduinoButton.Enabled = false;//disabling the start arduino button
            }

            serialPort1.Close();//closing the serial port
            this.Refresh();
            this.Close();//closing the application
        }//end of StartArduinoButton

Also, even though I close the port and the C# application, I still face the problem of needing to open the Serial Monitor for the Arduino to be reset. Is it manageable to have a function or something do that? would I experience the same problem if i use the ATmega328 as standalone?

Go back and look at your Arduino code. How many lines is it sending if you request one reading? 2, by my count.

How many lines does it send if you request 5 readings. 10, by my count.

How many lines are you reading in the C# application using serialPort1.read(), after requesting 1 reading? 2, by my count (count == 0 and count == 1).

How many lines are you reading in the C# application using serialPort1.read(), after requesting 5 readings? 6, by my count (count == 0, count == 1, count ==2, count ==3, count == 4, and count == 5).

Also, notice that you open a file, and get a stream writer object from the file. After writing some stuff (buffered), you close the file, without closing the streams into that file. The streams might automatically be flushed and closed, but I never rely on that happening. Flush the streams and close them, before closing the file.

I did not manage to understand the comment below. I took care of the stream writer ( flush it and close it ), yet I do not manage to write all the data into the file. When I echo it, I see every point requested by the C# command.

Go back and look at your Arduino code. How many lines is it sending if you request one reading? 2, by my count.

How many lines does it send if you request 5 readings. 10, by my count.

How many lines are you reading in the C# application using serialPort1.read(), after requesting 1 reading? 2, by my count (count == 0 and count == 1).

How many lines are you reading in the C# application using serialPort1.read(), after requesting 5 readings? 6, by my count (count == 0, count == 1, count ==2, count ==3, count == 4, and count == 5).

This is how the reading part of C# is looking like, can some comments be done?

        //settings for the button that starts arduino serial transmission
        private void StartArduinoButton_Click(object sender, EventArgs e)
        {
            if (serialPort1.IsOpen == true)//checking if any port is open in order to close it
            {
                //creating output file
                string ArduinoLogName = @"" + desktopPath + "\\" + "ArduinoData_" + DateTime.Now.ToString("MMM-d-yyyy") + ".txt";//creates the log file
                FileStream ArduinoLogFile = new FileStream(ArduinoLogName, FileMode.Create);//creates a new file or overwrites a previous file created
                StreamWriter writer = new StreamWriter(ArduinoLogFile);//variable to write 

                serialPort1.Write("<TX>");//Sending arduino the transmitting command
                int NumberOfReadings;

                if (SelNumberOfReading.SelectedItem == null)
                {
                    NumberOfReadings = 50;//default number of readings
                }
                else
                {
                    NumberOfReadings = int.Parse(SelNumberOfReading.SelectedItem.ToString());//converting the selected number of readings from string to ints 
                }

                for (int count = 0; count <= NumberOfReadings; count++)
                {
                    try
                    {
                        RxData = serialPort1.ReadLine();//reading data from the port
                        writer.WriteLine(RxData);//writing the transmitted data
                        Console.WriteLine(RxData);
                    }
                    catch (Exception RzData)
                    {
                       RxData = RzData.ToString();
                       writer.WriteLine(RxData);//writing the transmitted data
                       Console.WriteLine(RxData);
                    }
                }

                MessageBox.Show("The Transmission from Arduino has ended.");
                writer.Flush();//flush the streams
                writer.Close();//close the streams
                ArduinoLogFile.Close();//Closing the data file
            }
            else
            {
                MessageBox.Show("There is no Serial Ports open.");//message box telling user that there is not ports open
                StartArduinoButton.Enabled = false;//disabling the start arduino button
            }

            serialPort1.Close();//closing the serial port
            this.Refresh();
            this.Close();//closing the application
        }//end of StartArduinoButton

Your Arduino code contains:

      Serial.print("======================");
      Serial.print(atoiHolder);Serial.print("    ");Serial.print(N);
      Serial.println("======================");

      if(N == atoiHolder)
      {
                Serial.print("IM AT THE END OF COM");
            Serial.end(); // Presumably, this has been removed.
      }

<snip>

      Serial.print(N);Serial.print(';');
      Serial.print(Input0_Data,8);Serial.print(';');
      Serial.print(Input1_Data,8);Serial.print(';');
      Serial.print(Input2_Data,8);Serial.print(';');
      Serial.print(Input3_Data,8);Serial.println(';');

If one reading is requested, there will be 3 lines of output (not two as I suggested earlier).

If ten readings are requested, there will be 21 lines of output (not 20 as I suggested earlier.

Your C# code has this:

    for (int count = 0; count <= NumberOfReadings; count++)

If one reading is requested, NumberOfReadings will be 1, and the loop will execute when count = 0 and when count = 1, reading 2 of the three lines.

If ten readings are requested, NumberOfReadings will be 10, and the loop will execite when count = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, and 10, reading 11 of the 21 lines sent.