ESP32 SIM800L File.Available() & File.Read() Issue - Inverted question mark ⸮

I am trying to read a file from SD card and store it char by char in a string buffer and once the string is of 1000 bytes, i send it to web server via ftp. I do get successful some times, but other time the program fails and gets into endless loop.

I am reading from a file and i also check the size of the file and it shows correct information. File1.available() decrements itself after reading every character via File1.Read(). But sometimes File1.available() resets its counter to the size of the file and never decrements and File1.read() returns “⸮” at every read.

while (File.available()>0) {                                    
  char_buffer = File.read();//Takes one by one character in char_buffer from File.read()
  string_buffer.concat(char_buffer);//Concats every single character from char_buffer and makes string_buffer
  i++;
  if (i == buffer_space) {
  sendATcommand("AT+FTPPUT=2," + String(buffer_space), "AT+FTPPUT=2,10", "ERROR", 1000);
  sendATcommand(string_buffer, "OK", "ERROR", 5000);
  string_buffer = "";
  i = 0;
  }
}

When the File1.available() counter resets and stops decrementing itself, my program gets into infinite loop and keeps on sending “⸮”. The FTP connection never ends and file on server keeps on increasing and in fact gets more than its original size.
Any help will be highly appreciated. I am putting down the complete code below as well.
“Please pardon my coding skills and i am still a newbie in electronics programming”

#include <SPI.h>
#include <Wire.h>
#include <SD.h>
#include <SoftwareSerial.h>
SoftwareSerial MySerial(26, 27); // RX, TX

// TTGO T-Call pins
#define MODEM_RST            5
#define MODEM_PWKEY          4
#define MODEM_POWER_ON       23
#define MODEM_TX             27
#define MODEM_RX             26
#define I2C_SDA              21
#define I2C_SCL              22


#include "FS.h"
#include "SD.h"

#define IP5306_ADDR          0x75
#define IP5306_REG_SYS_CTL0  0x00

char* file_name1 = "/data2.txt";
char char_buffer;
String string_buffer = "";
int buffer_space = 1000; //Bytes i can send together, Network dependent.
File File1;

SPIClass SPI1(HSPI);
#define MY_CS       33
#define MY_SCLK     18
#define MY_MISO     19
#define MY_MOSI     32

void setup() {
  Serial.begin(19200);
  MySerial.begin(19200);
  delay(10);

  // Keep power when running from battery
  bool isOk = setPowerBoostKeepOn(1);
  Serial.println(String("IP5306 KeepOn ") + (isOk ? "OK" : "FAIL"));
  
setupSDCard();

  File1 = SD.open(file_name1);
  if (File1) {
    Serial.println("Opening the file: " + String(file_name1) + " done.");
  }else {
    Serial.println("Error opening " + String(file_name1));
    while(true);
  }
 setupModem();

 Wire.begin(I2C_SDA, I2C_SCL, 400000);
 
  Serial.println("Starting...");
  int reply = gprs_modem_function ();
  Serial.println("The end... Response: " + String(reply));
}

void setupSDCard()
{
    SPI1.begin(MY_SCLK, MY_MISO, MY_MOSI, MY_CS);
    //Assuming use of SPI SD card
    if (!SD.begin(MY_CS, SPI1)) {
        Serial.println("Card Mount Failed");
    } else {
        Serial.println("SDCard Mount PASS");
        String size = String((uint32_t)(SD.cardSize() / 1024 / 1024)) + "MB";
        Serial.println(size);
    }
}

bool setPowerBoostKeepOn(int en){
  Wire.beginTransmission(IP5306_ADDR);
  Wire.write(IP5306_REG_SYS_CTL0);
  if (en) {
    Wire.write(0x37); // Set bit1: 1 enable 0 disable boost keep on
  } else {
    Wire.write(0x35); // 0x37 is default reg value
  }
  return Wire.endTransmission() == 0;
}

void setupModem()
{
    // Set modem reset, enable, power pins
  pinMode(MODEM_PWKEY, OUTPUT);
  pinMode(MODEM_RST, OUTPUT);
  pinMode(MODEM_POWER_ON, OUTPUT);
  digitalWrite(MODEM_PWKEY, LOW);
  digitalWrite(MODEM_RST, HIGH);
  digitalWrite(MODEM_POWER_ON, HIGH);
}

void loop() {
}

byte gprs_modem_function (){
  byte reply = 1;
  int i = 0;
  while (i < 10 && reply == 1){ //Try 10 times...
  reply = sendATcommand("AT+CREG?","+CREG: 0,1","ERROR", 1000);
  i++;
  delay(1000);
  }
  if (reply == 0){
  reply = sendATcommand("AT+SAPBR=3,1,\"Contype\",\"GPRS\"","OK","ERROR", 1000);
  if (reply == 0){
  reply = sendATcommand("AT+SAPBR=3,1,\"APN\",\"airtelgprs.com\"", "OK", "ERROR", 1000);
  if (reply == 0){
  //reply = sendATcommand("AT+SAPBR=3,1,\"USER\",\"entelpcs\"", "OK", "ERROR", 1000);
  if (reply == 0){
  //reply = sendATcommand("AT+SAPBR=3,1,\"PWD\",\"entelpcs\"", "OK", "ERROR", 1000);
  if (reply == 0){
  reply = 2;
  i = 0;
  while (i < 3 && reply == 2){ //Try 3 times...
  reply = sendATcommand("AT+SAPBR=1,1", "OK", "ERROR", 10000);
  if (reply == 2){
  sendATcommand("AT+SAPBR=0,1", "OK", "ERROR", 10000);
  }
  i++;
  }
  if (reply == 0){
  reply = sendATcommand("AT+SAPBR=2,1", "OK", "ERROR", 1000);
  if (reply == 0){
  reply = sendATcommand("AT+FTPCID=1", "OK", "ERROR", 1000);
  if (reply == 0){
  reply = sendATcommand("AT+FTPSERV=\"ftp.example.com\"", "OK", "ERROR", 1000);
  if (reply == 0){
  reply = sendATcommand("AT+FTPPORT=21", "OK", "ERROR", 1000);
  if (reply == 0){
  reply = sendATcommand("AT+FTPUN=\"username@example.com\"", "OK", "ERROR", 1000);
  if (reply == 0){
  reply = sendATcommand("AT+FTPPW=\"example123\"", "OK", "ERROR", 1000);
  if (reply == 0){
  reply = sendATcommand("AT+FTPPUTNAME=\"" + String(file_name1) + "\"", "OK", "ERROR", 1000);
  if (reply == 0){
  reply = sendATcommand("AT+FTPPUTPATH=\"/\"", "OK", "ERROR", 1000);
  if (reply == 0){
  unsigned int ptime = millis();
  reply = sendATcommand("AT+FTPPUT=1", "+FTPPUT: 1,1", "+FTPPUT: 1,6", 60000);
  Serial.println("Time: " + String(millis() - ptime));
  if (reply == 0){
  if (File1) {
  int i = 0;
  while (File1.available()>0) {                                    
  char_buffer = File1.read();//Takes one by one character in char_buffer from File.read()
  string_buffer.concat(char_buffer);//Concats every single character from char_buffer and makes string_buffer
  i++;
  if (i == buffer_space) {
  sendATcommand("AT+FTPPUT=2," + String(buffer_space), "AT+FTPPUT=2,10", "ERROR", 1000);
  sendATcommand(string_buffer, "OK", "ERROR", 5000);
  Serial.println("string_buffer: " + String(string_buffer));//Printing the data once it gets of size 1000 bytes
  Serial.println("File.available: " + String(File.available()));//Checking the size of after sending every 1000 bytes
  string_buffer = "";
  i = 0;
  }
  }
  if (string_buffer != ""){
  sendATcommand("AT+FTPPUT=2," + String(i), "AT+FTPPUT=2,10", "ERROR", 1000);
  sendATcommand(string_buffer, "OK", "ERROR", 5000);
  sendATcommand("AT+FTPPUT=2,0", "OK", "ERROR", 1000);
  Serial.println("File.available: " + String(File.available()));//Checking the size of after sending every 1000 bytes
  }
  File1.close();
  }
  }
  }
  }
  }
  }
  }
  }
  }
  }
  }
  }
  }
  }
  }
  }
  return reply;
}

byte sendATcommand(String ATcommand, String answer1, String answer2, unsigned int timeout){
  byte reply = 1;
  String content = "";
  char character;
 
  //Clean the modem input buffer
  while(MySerial.available()>0) MySerial.read();

  //Send the atcommand to the modem
  MySerial.println(ATcommand);
  delay(100);
  unsigned int timeprevious = millis();
  while((reply == 1) && ((millis() - timeprevious) < timeout)){
    while(MySerial.available()>0) {
      character = MySerial.read();
      content.concat(character);
      Serial.print(character);
      delay(10);
    }
    //Stop reading conditions
    if (content.indexOf(answer1) != -1){
      reply = 0;
    }else if(content.indexOf(answer2) != -1){
      reply = 2;
    }else{
      //Nothing to do...
    }
  }
  return reply;
}

Output:
In the attached picture.

File.read() returns an int so that you can check for -1 if a character is not available. Change the char_buffer variable to an int and check for -1 to see if any data is available.

ToddL1962:
File.read() returns an int so that you can check for -1 if a character is not available. Change the char_buffer variable to an int and check for -1 to see if any data is available.

Thanks for the reply, but just curious to know that how does i work some times and fails some times?

ToddL1962:
File.read() returns an int so that you can check for -1 if a character is not available. Change the char_buffer variable to an int and check for -1 to see if any data is available.

I get this weird output when using int for char_buffer variable. Data output which i should get is not the same as shown in attached picture of my original post.

21:40:08.478 -> File1.available: 137457
21:40:08.572 -> 
21:40:08.572 -> +FTPPUT: 2,500
21:40:10.190 -> 
21:40:10.190 -> OK
21:40:10.239 -> 
21:40:10.284 -> +FTPPUT: 1,1,1360
21:40:10.733 -> string_buffer: 5848514448464852444548465248444846574813104952544450485048454856454948444953585052584851444846485244454846524844484657481310495255445048504845485645494844495358505258485144484648524445484652484448465748131049525644504850484548564549484449535850525848514448464852444548465248444846574813104952574450485048454856454948444953585052584851444846485244454846524844484657481310495348445048504845485645494844495358505258485144484648524445484652484448465748131049534944504850484548564549484449535850525848514448464852444548465248444846574813104953504450485048454856454948444953585052584851444846485244454846524844484657481310495351445048504845485645494844495358505258485244484648524445484652484448465748131049535244504850484548564549484449535850525848524448464954444548464955444946485013104953534450485048454856454948444953585052584852444846495444454846495544494648501310495354445048504845485645494844495358505258485244484649544445484649554449464850131049535544504850484548564549484449535850525848524448464954
21:40:11.005 -> File1.available: 136957
21:40:11.052 -> 
21:40:11.098 -> +FTPPUT: 2,500
21:40:12.677 -> 
21:40:12.723 -> OK
21:40:12.769 -> 
21:40:12.769 -> +FTPPUT: 1,1,136

Guys, any help on this? Really need help of you guys.

I get this weird output when using int for char_buffer variable.

You are seeing the ascii values. I would use char so that you can read/interpret what you see.

When I print some of the ascii you report I can see output similar to your first image.

146,2020-08-10,15:24:03,0.04,-0.40,0.90
147,2020-08-10,15:24:03,0.04,-0.40,0.90
148,2020-08-10,15:24:03,0.04,-0.40,0.90
149,2020-08-10,15:24:03,0.04,-0.40,0.90
150,2020-08-10,15:24:03,0.04,-0.40,0.90

My thinking is that when you get the backwards question marks you have unprintable characters do to some corruption.

How do you know the data on the SD card is clean? Can you read it in a computer and see what you have?

Can you write a minimal example program which demonstrates your issue. For example if you input 100 characters from serial, perform the concatenated read and then print an output, can you produce a failure? Or does the failure involve the SD card reading?

cattledog:
How do you know the data on the SD card is clean? Can you read it in a computer and see what you have?

Can you write a minimal example program which demonstrates your issue. For example if you input 100 characters from serial, perform the concatenated read and then print an output, can you produce a failure? Or does the failure involve the SD card reading?

I have made a minimal example as well, where i just read data from sd card and print it on serial monitor. This runs without any error.

#include <SPI.h>
#include <Wire.h>
#include <SD.h>

#include "FS.h"
#include "SD.h"

//char* file_name1 = "/basic.png";
//char* file_name1 = "/xl.xlsx";
char* file_name1 = "/data2.txt";
char char_buffer;
String string_buffer = "";
int buffer_space = 1000;
File File1;

SPIClass SPI1(HSPI);
#define MY_CS       33
#define MY_SCLK     18
#define MY_MISO     19
#define MY_MOSI     32

int answer2 = 0;
void setupSDCard()
{
    SPI1.begin(MY_SCLK, MY_MISO, MY_MOSI, MY_CS);
    //Assuming use of SPI SD card
    if (!SD.begin(MY_CS, SPI1)) {
        Serial.println("Card Mount Failed");
    } else {
        Serial.println("SDCard Mount PASS");
        String size = String((uint32_t)(SD.cardSize() / 1024 / 1024)) + "MB";
        Serial.println(size);
    }
}

void setup() {
  Serial.begin(19200);
  delay(10);
  setupSDCard();


 
//  Serial.println("Starting...");
//  read_char_function();
//  Serial.println("The end...");
}
void loop() {

    File1 = SD.open(file_name1);
    if (File1) {
    Serial.println("Opening the file: " + String(file_name1) + " done.");
    }else {
    Serial.println("Error opening " + String(file_name1));
    while(true);
    }
  
  // put your main code here, to run repeatedly:
  Serial.println("Starting...");
  if(answer2 == 0){
  answer2 = read_char_function();
  }
  if(answer2 == 1){
  Serial.println("The end...");
  answer2 = 2;
  }
}

byte read_char_function(){
  int answer = 0;
    if (File1) {
      int i = 0;
      while (File1.available()>0) {                                    
        char_buffer = File1.read();//Takes one by one character in char_buffer from File.read()
        string_buffer.concat(char_buffer);//Concats every single character from char_buffer and makes string_buffer
        i++;
        if (i == buffer_space) {
          Serial.println("string_buffer: " + String(string_buffer));
          string_buffer = "";
          i = 0;
        }
      }
      if (string_buffer != ""){
        Serial.println("string_buffer: " + String(string_buffer));
      }
      File1.close();
    }
    return answer;
}

Problem only occurs when i use two serial communications, one is interacting with SIM800L module and other is with the serial monitor. Does new line and carriage return has something to do with it? Is it a problem of buffer? or is SIM800L writing that unwanted character on my serial monitor?

Problem occurs in the loop of checking File.available() and File.read(). It starts getting unwanted characters and File.available counter resets the file size and never decrements then.

I observed one more thing. I made one separate program in which i just initialize the modem and do not send anything and just check the operation of file read and print. It still gives an error. But in the same program if i skip modem initialization, it reads and prints file data flawlessly.

I observed one more thing. I made one separate program in which i just initialize the modem and do not send anything and just check the operation of file read and print. It still gives an error. But in the same program if i skip modem initialization, it reads and prints file data flawlessly.

You seem to have made good progress in isolating some interaction between the sim800l and the sd reading or print out.

Problem only occurs when i use two serial communications, one is interacting with SIM800L module and other is with the serial monitor. Does new line and carriage return has something to do with it? Is it a problem of buffer? or is SIM800L writing that unwanted character on my serial monitor?

I would think that the issue is with corrupted readings from the sd card, and not with the printing of the read data back out to the serial monitor.

You might be able to test this by printing the data back to a different file on the sd card and see what it looks like there. Alternatively you can go back to printing the data as bytes instead of chars, and see what the backwards question marks actually are.

You are in a pretty unique hardware environment with the LilyGo-T-Call-SIM800. The modem pins look fixed, but you might want try to move the sd/spi aspects of the program to different pins in case there is some sort of subtle pin conflict.

cattledog: You might be able to test this by printing the data back to a different file on the sd card and see what it looks like there. Alternatively you can go back to printing the data as bytes instead of chars, and see what the backwards question marks actually are.

I'll definitely give this a try, do you think the issue can be because of using String variable?

while (File1.available()) {                                    
    char_buffer = File1.read();//Takes one by one character in char_buffer from File.read()
    string_buffer.concat(char_buffer);//Concats every single character from char_buffer and makes string_buffer
    i++;    
    if (i == buffer_space) {
      sendATcommand("AT+FTPPUT=2," + String(buffer_space), "AT+FTPPUT=2,10", "ERROR", 1000);
      sendATcommand(string_buffer, "OK", "ERROR", 5000);
      Serial.println("File1.available: " + String(File1.available()));
      string_buffer = "";
      i = 0;
    }
  }

string_buffer is a String with capital S and i read somewhere yesterday that using this may cause buffer overflow. Does that problem still persist?

I have avoided commenting on your method of reading/concatenating 1000 bytes from the SD using Strings because you said that everything worked correctly without the modem.

I have made a minimal example as well, where i just read data from sd card and print it on serial monitor. This runs without any error.

That said, I have many issues with your methodology, but I'm not certain that modifying the reading gets you to a solution if you can read your way through a file without issues when you just print to the Serial monitor.

I personally would be reading line by line given the cr/lf termination.

Why do you need to send the data in the 1000 byte chunks instead of smaller pieces?

Perhaps there are output buffer limitations with the modem. Can you send a large String message which you haven't read from the card, but just code into the sketch?

Still, you said the issue starts with a .begin() call and not the actual sending so this concern may not be relevant.

cattledog: I have avoided commenting on your method of reading/concatenating 1000 bytes from the SD using Strings because you said that everything worked correctly without the modem.

So the problem was with Usage of String. Instead of concatenating it in a string, i directly made a Serial.write operation byte by byte on modem and it fixed the issue.

Instead of concatenating it in a string, i directly made a Serial.write operation byte by byte on modem and it fixed the issue.

Well done. :)

I don't understand why the modem was involved in the Strings failing, but they certainly can cause problems when not properly managed. The esp's are less memory limited than most Arduinos and String usage is common in the examples. I always avoid Strings when possible.

I was never clear on why you were putting the message into the 1000 byte chunks. I assumed it was some serious issue with transmission time.

Perhaps there is some sweet spot between the one byte and the 1000.

Like I said, the "obvious" chunk to me is each line of data beginning with an index and ending with a cr/lf.

Reading and sending in larger blocks can certainly be done without Strings, but rather using null terminated character arrays instead.

cattledog: Well done. :)

I don't understand why the modem was involved in the Strings failing, but they certainly can cause problems when not properly managed. The esp's are less memory limited than most Arduinos and String usage is common in the examples. I always avoid Strings when possible.

I was never clear on why you were putting the message into the 1000 byte chunks. I assumed it was some serious issue with transmission time.

Perhaps there is some sweet spot between the one byte and the 1000.

Like I said, the "obvious" chunk to me is each line of data beginning with an index and ending with a cr/lf.

Reading and sending in larger blocks can certainly be done without Strings, but rather using null terminated character arrays instead.

Maximum allowable limit was 1360 by network and i was being at safe side by limiting it to 1000. Now the example i had from internet was using string with concatenation and new byte was added to that sting in a loop till it reaches 1000 bytes. However, i read somewhere over the internet that String is a very critical thing to use, each time you concatenate the string and if there is not enough memory to place new bytes in the memory, the entire string is moved to the end of the memory. Hence it causes issues, now i am not sure if that was the issue, but avoiding it certainly worked for me.

Huge thanks to you for trying out to help me in this matter.