read big string from SD and split in chunks

Projects Programming
1

I have a really big string (2kb) on txt in my SD card, i need to read that and transform in chunks of n chars, thats because i want to send it via LoRa and then i need to break it in small packages, the problem is that Arduino freezes due lot of data, if i reduce the amount of data in string to 1.4 or 1.5kb everything goes fine, the whole LoRa thing is O,. but i knock my head to resolve string problem, i know that string is not the best way, but witch way ?

/* Includes ---------------------- */
#include "./LoRaMESH.h"
#include "./Button.cpp"
#include <SoftwareSerial.h>
#include <stdint.h>
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#include <SD.h>
#include <SPI.h>

/* Globals ------------------------ */
#define CMD_ANALOG  50

uint8_t bufferPayload[MAX_PAYLOAD_SIZE] = {0};

uint8_t payloadSize = 0;

uint16_t localId;

uint16_t remoteId;

uint32_t localUniqueId;

uint16_t localNet;

uint8_t command;

bool isMaster;

String packagesContainer[] = {};

Button sendButton(2);

File myFile;

int pinCS = 53;

SoftwareSerial* hSerialCommands = NULL;

String fullString;

char fullStringBuffer[] = {};

/* Initialization routine */
void setup() {
  sendButton.begin();
  delay(1000);
  Serial.begin(9600); /* Initialize monitor serial */
  Serial.println("begin");

  /* Initialize SoftwareSerial depending on Board*/
#if defined(ARDUINO_AVR_UNO)
  hSerialCommands = SerialCommandsInit(6, 7, 9600);
#elif defined(ARDUINO_AVR_NANO)
  hSerialCommands = SerialCommandsInit(6, 7, 9600);
#elif defined(ARDUINO_AVR_MEGA2560)
  hSerialCommands = SerialCommandsInit(10, 11, 9600);
#endif

  /* Gets the local device ID */
  if (LocalRead(&localId, &localNet, &localUniqueId) != MESH_OK)
    Serial.print("Couldn't read local ID\n\n");
  else
  {
    Serial.print("Local ID: ");
    Serial.println(localId);
    Serial.print("Local NET: ");
    Serial.println(localNet);
    Serial.print("Local Unique ID: ");
    Serial.println(localUniqueId, HEX);
    Serial.print("\n");
  }

  pinMode(pinCS, OUTPUT);

  // SD Card Initialization
  if (SD.begin())
  {
    Serial.println("SD card is ready to use.");
  } else
  {
    Serial.println("SD card initialization failed");
    return;
  }

}

/* Main loop */
void loop() {
  if (sendButton.isPushed()) {

    myFile = SD.open("machine.txt");
    if (myFile) {
      while (myFile.available()) {

        myFile.read(fullStringBuffer,50);

      }
      myFile.close();
    }
    else {
      Serial.println("error opening txt");
    }


            char buffer[41] = {0}; // note space for terminator
            size_t len = strlen(fullStringBuffer);      // doesn't count terminator
            size_t blen = sizeof(buffer) - 1; // doesn't count terminator
            size_t i = 0;
            int packageCounter = 0;
    
            Serial.print("Package size: ");
            Serial.println(len);

    //      // the actual loop that enumerates your buffer
    //      for (i=0; i<len/blen; ++i)
    //      {
    //        memcpy(buffer, fullSring + (i*blen), blen);
    //        puts(buffer);
    //        packagesContainer[packageCounter] = buffer;
    //        packageCounter++;
    //      }
    //      packageCounter ++;
    //      // if there is anything left over
    //      if (len % blen)
    //      puts(fullSring + (len - len % blen));
    //      packagesContainer[packageCounter] = fullSring + (len - len % blen);
    //
    //      //Não ta chegando aqui
    //      Serial.println("Cheguei aqui 3");
    //
    //      for(int f=0;f<=packageCounter;f++){
    //        delay(2000);
    //         byte bufferPayload[packagesContainer[f].length()];
    //
    //      packagesContainer[f].getBytes(bufferPayload,packagesContainer[f].length()+1);
    //
    //      PrepareFrameCommand(localId, CMD_ANALOG, bufferPayload, packagesContainer[f].length());
    //      SendPacket();
    //      for (int k = 0; k<packagesContainer[f].length(); k++){
    //        Serial.print(bufferPayload[k]);
    //        Serial.print(" ");
    //
    //      }
    //      Serial.print("");
    //      }
    //
    //      Serial.println("Cheguei aqui 4");


  }
}

This is some atempt that i make but the string comes with 0 char to my array

2

Please post or attach an example of the 2K file
How should the file be split when sending it ?
Fixed length segments or segments delimited by start/end markers, perhaps ?

Over what period of time is the file created ?
Could each segment of it be sent as it is created rather than waiting until you have to deal with a 2K file ?

3

The text file is just a Lorem Ipsum that simulates the machine exports, i need to split in chunks of 40 bytes (40 characters) transform then in bytes and send via LoRa. BUT!!!! I just saw that i can use the read() method of SD instead readString() and it will give me bytes, now i just need to know how to create a array of chunks of bytes, iterate that array and send it

so i need to know how to read a file from SD in that:

ARRAY(
0: 42 154 123 456 78 125 45 255 123 21 20 45 02...(until 40 bytes)
1: 42 154 123 456 78 125 45 255 123 21 20 45 02...(until 40 bytes)
2: 42 154 123 456 78 45 135 78 198 211 20 45 02...(until 40 bytes)
)

machine.txt (1.96 KB)

4

Why do you need an array ? Can't you read 40 bytes, send them, and repeat this until end of file ?

5

Do you need put the whole file into an array before sending row by row it or would it be acceptable to read 40 bytes, send 40 bytes, read 40 bytes, send 40 bytes and so on until the whole file has been read ?

6

Hmm ok i agree with read, send, read send... i ll try that and post the results

7

Why make it so difficult?

#define BUFSIZE 40
byte buffer[BUFSIZE];

while (sd.available())
{
  size_t n = sd.readBytes(buffer, BUFSIZE);
  sendBytes(buffer, n);
}
8

Danois90, this will read 0-40 then 41-81 then 81-121...and so on ?

9

augusto3691:
Danois90, this will read 0-40 then 41-81 then 81-121...and so on ?

Yes, unless there is a timeout on the stream or EOF is less than 40 bytes away from the current position. "n" will hold the numbers of bytes read from the stream, so you can test "n" to see how much data was read.

10

Ok that WORKS !!!

i still need to work in how to send the number of package, and how many packages need to be sended to my broker know how to re-group that particular message, and if something fails to transmit try to send again, with some ACK, but this is for future.

thats my code now:

/* Includes ---------------------- */
#include "./LoRaMESH.h"
#include "./Button.cpp"
#include <SoftwareSerial.h>
#include <stdint.h>
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#include <SD.h>
#include <SPI.h>

/* Globals ------------------------ */
#define CMD_ANALOG  50

uint8_t bufferPayload[MAX_PAYLOAD_SIZE] = {0};

uint8_t payloadSize = 0;

uint16_t localId;

uint16_t remoteId;

uint32_t localUniqueId;

uint16_t localNet;

uint8_t command;

bool isMaster;

String packagesContainer[] = {};

Button sendButton(2);

File myFile;

int pinCS = 53;

SoftwareSerial* hSerialCommands = NULL;

byte package[] = {};

/* Initialization routine */
void setup() {
  sendButton.begin();
  delay(1000);
  Serial.begin(9600); /* Initialize monitor serial */
  Serial.println("begin");

  /* Initialize SoftwareSerial depending on Board*/
#if defined(ARDUINO_AVR_UNO)
  hSerialCommands = SerialCommandsInit(6, 7, 9600);
#elif defined(ARDUINO_AVR_NANO)
  hSerialCommands = SerialCommandsInit(6, 7, 9600);
#elif defined(ARDUINO_AVR_MEGA2560)
  hSerialCommands = SerialCommandsInit(10, 11, 9600);
#endif

  /* Gets the local device ID */
  if (LocalRead(&localId, &localNet, &localUniqueId) != MESH_OK)
    Serial.print("Couldn't read local ID\n\n");
  else
  {
    Serial.print("Local ID: ");
    Serial.println(localId);
    Serial.print("Local NET: ");
    Serial.println(localNet);
    Serial.print("Local Unique ID: ");
    Serial.println(localUniqueId, HEX);
    Serial.print("\n");
  }

  pinMode(pinCS, OUTPUT);

  // SD Card Initialization
  if (SD.begin())
  {
    Serial.println("SD card is ready to use.");
  } else
  {
    Serial.println("SD card initialization failed");
    return;
  }

}

/* Main loop */
void loop() {
  if (sendButton.isPushed()) {
    int i = 0;
    myFile = SD.open("machine.txt");
    if (myFile) {
      Serial.println("Start sending...");
      while (myFile.available()) {
        delay(2000);
        Serial.println("Read file");
        size_t n = myFile.readBytes(package, 40);
        PrepareFrameCommand(localId, CMD_ANALOG, package, n);
        Serial.println("Sending package...");
        SendPacket();
      }
      Serial.println("Finish sending");
      myFile.close();
    }
    else {
      Serial.println("error opening txt");
    }
  }
}
11

That's not gonna work well for you since "package" has no size..

12

how's that ? the code works well you mean in relation of future features ?

13

Unallocated memory usage:

byte package[] = {};