writing mqtt payload into a file (SPIFFS file system) and reading it later

Hello,

I'm trying to put the payload, which is a byte*, inside a file to later read it into a variable, for example.

So far, I've tried some things, but I don't know if they are correct. My mqtt sketch has a function that triggers when a message is received and checks the topic it belongs to. For one particular topic, I need to store the contents of the message payload into a file, because later, if certain conditions are met, I will need to do something with that payload. This is the code:

I have globally declared File crt;

This is inside the void receivedCallback(char* topic, byte* payload, unsigned int length):

if (strcmp(topic, FILE_TOPIC)==0){
     if(!SPIFFS.exists("/crt")){
        crt = SPIFFS.open("/crt", "r+");
        Serial.print("CREATE");
      }
     crt.println((char*)payload);
     crt.close();
}

and this is what I'm using to read the contents later:

crt = SPIFFS.open("/crt", "r");
size_t filesize = crt.size(); //the size of the file in bytes     
char buff[filesize + 1];   // + 1 for '\0' char at the end      

crt.read((uint8_t *)buff, sizeof(buff));  
crt.close(); 
buff[filesize] = '\0';

Could someone tell me what I'm doing wrong?

if the file exists, you don't open it before writing into it ?

what does the payload look like ? is that a null terminated cString?
You might be better off writing a byte buffer using write() rather than println()

For one particular topic, I need to store the contents of the message payload into a file, because later, if certain conditions are met, I will need to do something with that payload.

I am doing something similar but decided to save the payload as a string to make reading by humans easier

void callback(char* topic, byte * payload, unsigned int length)
{
  char received[50];
  Serial.println();
  Serial.print("message received from topic : ");
  Serial.println(topic);
  memcpy(received, payload, length);  //convert to string
  received[length] = '\0';
  Serial.println(received);
  if (strcmp(topic, "garage/outputs") == 0) //got an output
  {
    parseMessage(received);
    if (strcmp(data.command, "list") == 0)
    {
      listRecords();
    }
    else if (strcmp(data.command, "count") == 0)
    {
      countRecords();
    }
    else if (strcmp(data.command, "file") == 0)
    {
      saveToSPIFFS();
    }
    else if (strcmp(data.command, "*delete file*") == 0)
    {
      deleteFile();
    }
    else if (strcmp(data.command, "debug") == 0)
    {
      debug = !debug;
      Serial.println(debug == true ? "debug mode on" : "debug mode off");
    }
  }
}

The parseMessage() function splits the data into elements of a struct to be used later

Here is an example of listing the file to the Serial monitor

void listRecords()
{
  File logFile = SPIFFS.open("/garage2.log", "r");
  if (!logFile)
  {
    Serial.println("Could not open file to list");
    return;
  }
  Serial.println("*************************");
  Serial.println("start of file");
  unsigned long byteCount = 0;
  unsigned long recordCount = 0;
  while (logFile.available())
  {
    char inChar = logFile.read();
    Serial.write(inChar);
    if (inChar == '\r')
    {
      recordCount++;
    }
    byteCount++;
  }
  logFile.close();
  Serial.println("end of file");
  Serial.print("records read : ");
  Serial.println(recordCount);
  Serial.print("bytes read : ");
  Serial.println(byteCount);
  showTotals();
  Serial.println("*************************");
}

Obviously you could read a single record, parse and act on it
On my system a record looks like this

29-04-20,10:47:00,990,18.00,58.00

J-M-L:
if the file exists, you don't open it before writing into it ?

what does the payload look like ? is that a null terminated cString?
You might be better off writing a byte buffer using write() rather than println()

About not opening the file if it exists, yes that was wrong, thank you.
The payload is a byte*, I tried doing

file.write(payload, length)

, but it seems like the file is not written, because the way of reading doesn't look to me as the source of the problem.

UKHeliBob:
I am doing something similar but decided to save the payload as a string to make reading by humans easier

void callback(char* topic, byte * payload, unsigned int length)

{
  char received[50];
  Serial.println();
  Serial.print("message received from topic : ");
  Serial.println(topic);
  memcpy(received, payload, length);  //convert to string
  received[length] = '\0';
  Serial.println(received);
  if (strcmp(topic, "garage/outputs") == 0) //got an output
  {
    parseMessage(received);
    if (strcmp(data.command, "list") == 0)
    {
      listRecords();
    }
    else if (strcmp(data.command, "count") == 0)
    {
      countRecords();
    }
    else if (strcmp(data.command, "file") == 0)
    {
      saveToSPIFFS();
    }
    else if (strcmp(data.command, "delete file") == 0)
    {
      deleteFile();
    }
    else if (strcmp(data.command, "debug") == 0)
    {
      debug = !debug;
      Serial.println(debug == true ? "debug mode on" : "debug mode off");
    }
  }
}



The parseMessage() function splits the data into elements of a struct to be used later


Here is an example of listing the file to the Serial monitor


void listRecords()
{
  File logFile = SPIFFS.open("/garage2.log", "r");
  if (!logFile)
  {
    Serial.println("Could not open file to list");
    return;
  }
  Serial.println("");
  Serial.println("start of file");
  unsigned long byteCount = 0;
  unsigned long recordCount = 0;
  while (logFile.available())
  {
    char inChar = logFile.read();
    Serial.write(inChar);
    if (inChar == '\r')
    {
      recordCount++;
    }
    byteCount++;
  }
  logFile.close();
  Serial.println("end of file");
  Serial.print("records read : ");
  Serial.println(recordCount);
  Serial.print("bytes read : ");
  Serial.println(byteCount);
  showTotals();
  Serial.println("
");
}



Obviously you could read a single record, parse and act on it
On my system a record looks like this


29-04-20,10:47:00,990,18.00,58.00

I'm trying to find a way so I don't need to save anything in memory twice, by passing the payload directly to the file write or print function. I'll try the reading part of your code. Thanks!

I'm trying to find a way so I don't need to save anything in memory twice,

The payload is a pointer to an array of bytes. You could iterate through the array from 0 to length minus 1 and save each of them individually

In my case I wanted the payload as a string for further manipulation before filing, hence the copy and conversion

UKHeliBob:
The payload is a pointer to an array of bytes. You could iterate through the array from 0 to length minus 1 and save each of them individually

or just use crt.write(payload, length);

J-M-L:
or just use crt.write(payload, length);

I tried that with this code, but nothing is displayed on the serial monitor

if(!SPIFFS.exists("/crt")){
        crt = SPIFFS.open("/crt", "r+");
        Serial.print("CREATE");
      }
     crt = SPIFFS.open("/crt", "r+");
     crt.write(payload, length);
     
     while(crt.available()){
      Serial.write(crt.read());
     }
     crt.close();

For a start, it's binary data so you can't hope to get something readable if that was not ASCII in the first place --> you'll want to transform that in human readable text

Then in your specific code, you open the file in the r+ mode which means

r+ Open for reading and writing. The stream is positioned at the beginning of the file.

So you write data at the beginning of the crt file, overwriting whatever as there before with your buffer and once you are done writing, the pointer in the file points at the very end (unless there was more stuff in the file from a previous write).
crt.available() will tell you there is stuff to read AFTER Your buffer and you should not expect to read what you just wrote. For that you need to move the file cursor position back to the beginning using crt.seek(0, SeekSet);

J-M-L:
For a start, it's binary data so you can't hope to get something readable if that was not ASCII in the first place --> you'll want to transform that in human readable text

Then in your specific code, you open the file in the r+ mode which means
So you write data at the beginning of the crt file, overwriting whatever as there before with your buffer and once you are done writing, the pointer in the file points at the very end (unless there was more stuff in the file from a previous write).
crt.available() will tell you there is stuff to read AFTER Your buffer and you should not expect to read what you just wrote. For that you need to move the file cursor position back to the beginning using crt.seek(0, SeekSet);

Thank you. I've managed to check the written content by:

if(!SPIFFS.exists("/crt")){
        crt = SPIFFS.open("/crt", "r+");
        Serial.print("CREATE");
      }
     crt = SPIFFS.open("/crt", "r+");
     crt.write(payload, length);
     crt.seek(0, SeekSet);
     while(crt.available()){
      Serial.print((char)crt.read());
     }
     crt.close();

However, in terms of reading it and saving it into a char* variable is where I'm having trouble now. I'm using this code:

crt = SPIFFS.open("/key", "r");
      while (crt.available())
      {
        content = (char*)crt.read();
      }
      crt.close();

Before implementing the file system thing, what I did was to memcpy the payload inside a byte array and then content=(char*)myPayload. I don't know how to do the same with files here

You need to fill in a buffer

const size_t maxBufferSize = 200; // needs to be big enough...
char bigBuffer[maxBufferSize+1]; 
crt = SPIFFS.open("/key", "r");
size_t index = 0;
while (crt.available() && (index < maxBufferSize)) {
  bigBuffer[index++] = (char) crt.read();
}
bigBuffer[index] = '\0'; // mark the end of the buffer - terminates the cString so that you can use print()
crt.close();
Serial.print(F("Your buffer is: "));
Serial.println(bigBuffer);

(typed here so may be some typos, but you get the idea I suppose)

this assumes that your buffer was NOT holding NULL characters. otherwise you’ll still be building the right buffer but printing it then will be truncated at the first NULL.

You need to fill in a buffer

  char received[50];
  memcpy(received, payload, length);
  received[length] = '\0';

I assumed he meant reading that much later, directly from SPIFFS, when the payload is long gone and you don’t even know its size (well could be found by looking at the size of the SPIFFS file)

otherwise to ensure adequate space is allocated, that would be

  char received[length+1];
  memcpy(received, payload, length);
  received[length] = '\0';

J-M-L:
I assumed he meant reading that much later, directly from SPIFFS, when the payload is long gone and you don’t even know its size (well could be found by looking at the size of the SPIFFS file)

otherwise to ensure adequate space is allocated, that would be

  char received[length+1];

memcpy(received, payload, length);
  received[length] = ‘\0’;

My payload is a certificate. I’m trying to send it from the server via mqtt message (so it’s inside the payload) and then save it into a file. I have to read it from the file, but I cannot find a way to make it work. I have to do the same with the key. I know I can write the payload into the file and printing it shows the correct certificate, but for reading that doesn’t work. This is the code where I read the cert from the file:

crt = SPIFFS.open("/crt", "r");
      key = SPIFFS.open("/key", "r");
      const size_t maxBufferSize = 1024; // needs to be big enough...
      char cert[maxBufferSize+1];
      crt = SPIFFS.open("/key", "r");
      size_t index = 0;
      while (crt.available() & index < maxBufferSize) {
        cert[index++] = (char) crt.read();
      }
      cert[index] = '\0'; // mark the end of the buffer - terminates the cString so that you can use print()
      crt.close();
      Serial.print("***************************************\n");
      for(int i=0;i<sizeof(cert);i++){
        Serial.print((char*)cert[i]);
      }
      
            Serial.print("***************************************\n");

      test_client_cert = (char*)cert;

Nothing is being printed inbetween “******” adn sometimes I get a LoadProhibited exception. Again, sending the certificate inside the payload and not doing the file thing works, but if the esp32 loses its power source they obviously dissappear