Text file transfer between PC and Arduino using Batch scripts

Hello,

I have a personal project based on file transfer between an Arduino Due and my PC. Theses files are in text format and can go over 10Mb.

I firstly bought an SD SPI module to connect an external SD card to my Arduino and a compatible micro-SD card.

SD card sided, everything works well, the card is detected and data storage is also working.

To transfer data from my PC, I am using Batch scripts that send text file content over Serial port used by Arduino.

The problem comes: I am unable to transfer over 65535 length file over Serial port at once.

If the file contains more than 65535 characters, the transfer stuck and I am unable to do anything until I reset the Arduino card. I tried with a bit more characters, but it only writes firsts characters in the file.

So, if my file does 65635 characters, only the first 100 characters will be written in the new file.

But as I said, my goal is to transfer huge files.

To illustrate my problem I give you some codes on Arduino and Batch scripts.

Arduino part

I launch different states with buttons.

  1. First one run a "send_file" batch script that will send data to serial port and write into a new file on SD card.
  2. A second one will run a "read_serial" batch script that will read file content sent over Serial port by Arduino.
  3. The third one is used to delete the file created before.

Note that I use RGB led for different states, and also Keyboard library to do actions on my computer.

Here is the complete code:

#include <Wire.h>
#include <Keyboard.h>
#include <SPI.h>
#include <SD.h>

File myFile;

const int chipSelect = 53;

String AltGrazerty = "~#{[|`\\^@#]}";
String shiftazerty = "QBCDEFGHIJKL?NOPARSTUVZXYW1234567890 Q+QQQQM%Q./Q>";
String azerty = "qbcdefghijkl,noparstuvzxyw&q\"'(-q_qq )=q$q*mqq;:!<";
const byte scancode[] = { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 100 };

int BP1 = 28, BP2 = 29, BP3 = 31, BP4 = 30, BP5 = 33, BP6 = 32;

int red_light_pin= 9;
int green_light_pin = 11;
int blue_light_pin = 10;

int ledstate = 0;

int val1 = 0, val2 = 0, val3 = 0, val4 = 0, val5 = 0, val6 = 0, flag = 0;

void setup() {
  Serial.begin(115200);
  Keyboard.begin();

  if (!SD.begin(chipSelect)) {
    while (1);
  }
  
  myFile = SD.open("test.txt");
  while (myFile.available()) {
          Serial.write(myFile.read());
        }
  myFile.close();
  Serial.flush();
      
  pinMode(BP1, INPUT_PULLUP);
  pinMode(BP2, INPUT_PULLUP);
  pinMode(BP3, INPUT_PULLUP);
  pinMode(BP4, INPUT_PULLUP);
  pinMode(BP5, INPUT_PULLUP);
  pinMode(BP6, INPUT_PULLUP);
  
  pinMode(red_light_pin, OUTPUT);
  pinMode(green_light_pin, OUTPUT);
  pinMode(blue_light_pin, OUTPUT);
}

void loop() {

  val1 = digitalRead(BP1);
  val2 = digitalRead(BP2);
  val3 = digitalRead(BP3);
  val4 = digitalRead(BP4);
  val5 = digitalRead(BP5);
  val6 = digitalRead(BP6);

  switch (ledstate){
    case 0 : RGB_color(255, 239, 0); 
    break;
    case 1 : RGB_color(255, 0, 0); 
    break;
    case 2 : RGB_color(0, 255, 0); 
    break;
    case 3 : RGB_color(0, 0, 255); 
    break;
  }

  if (val1 == 0) 
  {
    if (flag == 0) 
    {
      
      Keyboard.press(KEY_LEFT_GUI);
      Keyboard.press('r');
      Keyboard.releaseAll();
      delay(250);
      Keyfr("C:/Users/***/Desktop/send_data.bat");
      delay(500);
      Keyboard.press(KEY_RETURN);
      Keyboard.releaseAll();

      String text = Serial.readString();
      
      myFile = SD.open("test.txt",FILE_WRITE);

      if(myFile){
      myFile.println(text);
      myFile.close();
      } else {}

      ledstate = 1;
      flag = 1;
    }
  } else if (val2 == 0) {
    if (flag == 0) {

      Serial.flush();

      delay(500);

      Keyboard.press(KEY_LEFT_GUI);
      Keyboard.press('r');
      Keyboard.releaseAll();
      delay(250);
      Keyfr("C:/Users/***/Desktop/read_data.bat");
      delay(500);
      Keyboard.press(KEY_RETURN);
      Keyboard.releaseAll();
    
      ledstate = 2;
      flag = 1;
    }
  } else if (val3 == 0) {
    if (flag == 0) {

      Keyboard.press(KEY_LEFT_GUI);
      Keyboard.press('r');
      Keyboard.releaseAll();
      delay(250);
      Keyfr("C:/Users/***/Desktop/del.bat");
      delay(500);
      Keyboard.press(KEY_RETURN);
      Keyboard.releaseAll();
      
      myFile = SD.open("test.txt");
      if (myFile) {

         SD.remove("test.txt");

        myFile.close();
      } else {}

      ledstate = 3;
      flag = 1;
    }
  } else if (val4 == 0) {
    if (flag == 0) {
      
      flag = 1;
    }
  } else if (val5 == 0)
  {
    if (flag == 0) {
      
      flag = 1;
    }
  } else if (val6 == 0) {
    if (flag == 0) {

      flag = 1;
    }
  } else if (val1 == 1 && val2 == 1 && val3 == 1 && val4 == 1 && val5 == 1 && val6 == 1) 
  {
    flag = 0;
  }

  delay(50);
}

void Keyfr(const String &Texte) {  
  int j = -1;

  for (unsigned int i = 0; i < Texte.length(); i++) {
    char c = Texte.charAt(i);

    if (c == '\t') {
      Keyboard.write(KEY_TAB);
    }

    int index = azerty.indexOf(c);
    if (index > -1) {
      j = scancode[index] + 136;
      Keyboard.write(j);
    } else {
      index = shiftazerty.indexOf(c);
      if (index > -1) {
        j = scancode[index] + 136;
        Keyboard.press(KEY_LEFT_SHIFT);
        Keyboard.press(j);
        Keyboard.releaseAll();
      } else {
        index = AltGrazerty.indexOf(c);
        if (index > -1) {
          j = scancode[index + 27] + 136;
          Keyboard.press(KEY_LEFT_CTRL);
          Keyboard.press(KEY_LEFT_ALT);
          Keyboard.write(j);
          Keyboard.releaseAll();
          if (index == 0 || index == 7) {
            Keyboard.press(KEY_LEFT_CTRL);
            Keyboard.press(KEY_LEFT_ALT);
            Keyboard.write(j);
            Keyboard.releaseAll();
            Keyboard.write(KEY_BACKSPACE);
          }
        }
      }
    }
  }
}

void RGB_color(int red_light_value, int green_light_value, int blue_light_value)
 {
  analogWrite(red_light_pin, red_light_value);
  analogWrite(green_light_pin, green_light_value);
  analogWrite(blue_light_pin, blue_light_value);
}

Batch part

I give you in this part only send and read scripts, even if I think theses scripts are not a problem.

Send script :

@echo off
setlocal EnableDelayedExpansion

@echo mode COM5 BAUD=115200 PARITY=n DATA=8

set "cmd=findstr /R /N "^^" file.txt | find C/ ":""

@type file.txt > COM5

Read script :

@echo off
setlocal EnableDelayedExpansion

@mode COM5 BAUD=115200 PARITY=n DATA=8 DTR=ON

set "cmd=findstr /R /N "^^" file.txt | find /C ":""

@type COM5>file2.txt

I hope my problem is understandable and my English is correct (not my primary language).

Thanks in advance.

Romain.

My guess would be that the maximum length of a String is the limiting factor here.

You could read one byte at a time in a loop to circumvent this.

I would implementation of give XON/XOFF a try, preferably at a lower baudrate (I know it will take longer).

//edit: lower baudrate will probably have no effect on boards with native USB :frowning:

I would consider transfer of a binay file (XON/XOFF will not work with it) and write a dedicated application to transfer.

The below sketch demonstrates the XON/XOFF software handshake.

This was tested on a SparkFun ProMicro. I doubt that serial feedback is possible so I've implemented a blink to display error codes
Notes:

  1. The ProMicro does not have a pin 13 LED hence the use of the RX led.
  2. Use of SdFat library and CS on A0
// https://forum.arduino.cc/t/text-file-transfer-between-pc-and-arduino-using-batch-scripts/988565

#include <SPI.h>
#include "SdFat.h"
#include "sdios.h"
// SD card chip select pin. Adjusted for ProMicro setup with CS on A0
const uint8_t chipSelect = A0;

// File system object.
SdFat sd;
// Use for file creation in folders.
SdFile file;
char filename[13] = "988565.txt";

const uint8_t XON = 0x13;
const uint8_t XOFF = 0x11;
const uint32_t timeOut = 10000;

const uint8_t RXLED = 17;  // The RX LED has a defined Arduino pin

void setup()
{
  Serial.begin(115200);
  while (!Serial);

  pinMode(RXLED, OUTPUT);  // Set RX LED as an output
  digitalWrite(RXLED, LOW);


  if (!sd.begin(chipSelect, SD_SCK_MHZ(50)))
  {
    // blink forever
    for (;;)
    {
      blink(4);
    }
  }

  if (!file.open(filename, O_WRONLY | O_CREAT))
  {
    // blink forever
    for (;;)
    {
      blink(8);
    }
  }

}

void loop()
{
  static bool pause = false;
  static uint32_t counter;
  static uint32_t lastRcvTime;

  if (Serial.available() > 0)
  {
    lastRcvTime = millis();
    counter++;
    if (pause == false)
    {
      // send XOFF
      Serial.print(XOFF);
      pause = true;
    }

    // if file notopen yet
    if (file.isOpen() == false)
    {
      if (!file.open(filename, FILE_WRITE))
      {
        // blink forever
        for (;;)
        {
          blink(2);
        }
      }
    }

    char ch = Serial.read();
    file.print(ch);
  }
  else
  {
    if (pause == true)
    {
      // send XON
      Serial.print(XON);

      pause = false;
    }
  }
  // assume transfer complete if we haven't received for N seconds
  if (millis() - lastRcvTime > timeOut)
  {
    // close file if open
    if (file.isOpen())
    {
      file.close();
    }
    for (;;)
    {
      blink(1);
    }
  }
}

void blink(int numBlinks)
{
  for (int cnt = 0; cnt < numBlinks; cnt++)
  {
    digitalWrite(RXLED, HIGH);
    delay(1000 / numBlinks);
    digitalWrite(RXLED, LOW);
    delay(1000 / numBlinks);
  }
  delay(2000);
}

It's very crude. Every time data is received, the sketch sends XOFF to tell the sender to stop sending; so if you have 10 characters in the serial buffer, it will only send one XOFF. From experience with long-ago playing around, it seems that it can take the PC a while to react on the XOFF so you might have receive more than you can handle resulting in data loss; hence the XOFF is send immediately at the first byte.

Once the serial buffer is empty, it will send a XON to tell the sender to continue sending.

I've implemented a 10 second timeout after which the file will be closed (as said, it's crude).

I have transmitted a 1.7 Mbyte file without data loss; it took approx. 2.5 minutes. Results were verified with WinMerge.

bat file used (com9 is my ProMicro)

time
mode com9 baud=115200 parity=n data=8 XON=ON
type text.04.txt > COM9
time

Thanks for your answer, I will try to adapt your code to my needs and test it.

for (unsigned int i = 0; i < Texte.length(); i++) {

unsigned int can only count to 65,535. Try unsigned long

An Arduino Due (as used by OP) has a 32 bit processor; so an int should be 4 bytes.

Umm, no.

What do you mean?

@madmark2150 The Arduino reference says this for the int datatype:

On the Arduino Uno (and other ATMega based boards) an int stores a 16-bit (2-byte) value. This yields a range of -32,768 to 32,767 (minimum value of -2^15 and a maximum value of (2^15) - 1).

On the Arduino Due and SAMD based boards (like MKR1000 and Zero), an int stores a 32-bit (4-byte) value. This yields a range of -2,147,483,648 to 2,147,483,647 (minimum value of -2^31 and a maximum value of (2^31) - 1).

@sterretje
I tried your code with my specific CS PIN, I have my LED blinking only 1 time, so I may be in this part of the code :

if (millis() - lastRcvTime > timeOut)
  {
    // close file if open
    if (file.isOpen())
    {
      file.close();
    }
    for (;;)
    {
      blink(1);
    }
  }

One in this part I run the bat that you gave me with my port and my file but nothing happen. Do I have to change the file name at the begining or not ?

I'm not sure where it goes wrong. If the led only flashes once, that is not caused by my code as far as i can see. The part that you quoted flashes continously.

What does the RX led on your board do during the transfer? It should look like it's continously on till the transfer is completed.

I'm not familiar with the Due. On the ProMicro, I used a single reset to make sure that it was in a known state before I started the bat file.

You are aware that you have to press <enter> to get past the first time command.

Have you tried a manual approach using the two relevant commands (mode and type)? Observe the behaviour of the built-in led after the mode command to see if the board is stable (no flashing or fading) before issuing the type command.

Lastly, the Due has two ports; maybe it works on the other port.

The RX led is only blinking once when I run the batch script. But it's is not blinking continously, the port maybe is not the same on the DUE, so I changed it to another external LED.

My Batch script is 100% similar to yours so I dont understand why it is not working.

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.