I was implementing the OTA functionality on my own esp32 for fun and faced a lot of problems. looking at the online community I feel many people are having problems with this functionality. so I thought wouldn't it be awesome if all of us could collaborate in developing this functionality and gift it to the open-source community? this thread could serve as all in one package for anyone looking for help.
I am setting up a Github repository for this so that the development can begin. Will be actively accepting pull requests. I understand that not everyone has access to a gsm so I have already completed that part and updated the repository with it.
I have set up the repository in such a way that if you clone it and open it in vscode it should work right out of the box.
development requirements:
(board) ESP32 dev kit, vscode with platform io extension, tinygsm++ library installed.
Tasks to be performed by this function
-
Make an HTTP get request from gsm (in this case sim800l ) to a remote server (for developing purposes I'm using an amazon s3 bucket to store my firmware in it). (done)
-
check/verify if the update is intact after downloading (done)
-
saving the payload or only the update part to an sd card and increment its version to keep track of the latest version (more on this later) (firmware1.bin) (firmware2.bin) point of focus
-
once writing to the sd card is complete verify again that the write was successful, if not repeat process TODO
-
let the actual updating of the firmware begin. (means reading from the sd card and overwriting flash with it) DONE
-
once the update is done, make sure that the update is in fact working. if not, roll back to the factory default firmware stored in the sd card or in a flash partition itself. once this happens it should remember that an update did occur but wasn't successful. and should try to update again whenever the situation allows it (gsm internet connection is stable) TODO
-
store in the EEPROM version of firmware it is running for keeping track of it and debugging purposes. TODO
I will be updating the post with the latest updates, Would appreciate all the help I can get from you guys. Regards
UPDATE #1------------- 2022-08-12T23:50:00Z
Successfully able to get data with gsm from amazon s3 server and write it to spiffs.. once written updates the firmware from spiff.. this method is a bit closer to what we want in that data maintains its integrity.
I chose to mark this as an update because i'm one step closer to what we want. since spiffs and sd card act similarly except a few minor changes our next taks would be to convert this into sd card (write to sd card and upgrade firmware from sd card) shouldn't be that much of a problem. following is the code.
#include <Update.h>
#define TINY_GSM_MODEM_SIM800
// Increase RX buffer
#define TINY_GSM_RX_BUFFER 1030
//change according to your needs
const char apn[] = "internet";
const char user[] = "";
const char pass[] = "";
#define SerialMon Serial
#define sim800 Serial2
#include <TinyGsmClient.h>
#include <CRC32.h>
#include "FS.h"
#include "SPIFFS.h"
TinyGsm modem(sim800);
TinyGsmClient client(modem);
const char server[] = ""; //will have to replace with your own server. im using amazon s3 bucket
const int port = 80;
const char resource[] = "/ota-test.bin"; //path to bin file inside server
uint32_t knownCRC32 = 0x6f50d767;
uint32_t knownFileSize = 1024; // In case server does not send it
void setup()
{
sim800.begin(9600);
// Set console baud rate
Serial.begin(115200);
delay(6000);
if (!SPIFFS.begin(true))
{
Serial.println("SPIFFS Mount Failed");
return;
}
SPIFFS.format();
listDir(SPIFFS, "/", 0);
// Restart takes quite some time
// To skip it, call init() instead of restart()
Serial.println("Initializing modem...");
modem.restart();
String modemInfo = modem.getModemInfo();
Serial.print("Modem: ");
Serial.println(modemInfo);
// Unlock your SIM card with a PIN
//modem.simUnlock("1234");
}
void loop()
{
Serial.print("Waiting for network...");
if (!modem.waitForNetwork())
{
Serial.println(" fail");
delay(10000);
return;
}
Serial.println(" OK");
Serial.print("Connecting to ");
Serial.print(apn);
if (!modem.gprsConnect(apn, user, pass))
{
Serial.println(" fail");
delay(10000);
return;
}
Serial.println(" OK");
Serial.print("Connecting to ");
Serial.print(server);
// if you get a connection, report back via serial:
if (!client.connect(server, port))
{
Serial.println(" fail");
delay(10000);
return;
}
Serial.println(" OK");
// Make a HTTP request:
client.print(String("GET ") + resource + " HTTP/1.0\r\n");
client.print(String("Host: ") + server + "\r\n");
client.print("Connection: close\r\n\r\n");
long timeout = millis();
while (client.available() == 0)
{
if (millis() - timeout > 5000L)
{
Serial.println(">>> Client Timeout !");
client.stop();
delay(10000L);
return;
}
}
Serial.println("Reading header");
uint32_t contentLength = knownFileSize;
File file = SPIFFS.open("/update.bin", FILE_APPEND);
while (client.available())
{
String line = client.readStringUntil('\n');
line.trim();
//Serial.println(line); // Uncomment this to show response header
line.toLowerCase();
if (line.startsWith("content-length:"))
{
contentLength = line.substring(line.lastIndexOf(':') + 1).toInt();
}
else if (line.length() == 0)
{
break;
}
}
timeout = millis();
uint32_t readLength = 0;
CRC32 crc;
unsigned long timeElapsed = millis();
printPercent(readLength, contentLength);
while (readLength < contentLength && client.connected() && millis() - timeout < 10000L)
{
int i = 0;
while (client.available())
{
// read file data to spiffs
if (!file.print(char(client.read())))
{
Serial.println("Appending file");
}
//Serial.print((char)c); // Uncomment this to show data
//crc.update(c);
readLength++;
if (readLength % (contentLength / 13) == 0)
{
printPercent(readLength, contentLength);
}
timeout = millis();
}
}
file.close();
printPercent(readLength, contentLength);
timeElapsed = millis() - timeElapsed;
Serial.println();
client.stop();
Serial.println("stop client");
modem.gprsDisconnect();
Serial.println("gprs disconnect");
Serial.println();
float duration = float(timeElapsed) / 1000;
/*
Serial.print("Tamaño de Archivo: ");
Serial.println(contentLength);
Serial.print("Leido: ");
Serial.println(readLength);
Serial.print("Calculado. CRC32: 0x");
Serial.println(crc.finalize(), HEX);
Serial.print("Conocido CRC32: 0x");
Serial.println(knownCRC32, HEX);
Serial.print("Bajado en: ");
Serial.print(duration);
Serial.println("s");
Serial.println("Se genera una espera de 3 segundos");
for (int i = 0; i < 3; i++)
{
Serial.print(String(i) + "...");
delay(1000);
}
*/
//readFile(SPIFFS, "/update.bin");
updateFromFS();
// Do nothing forevermore
while (true)
{
delay(1000);
}
}
void appendFile(fs::FS &fs, const char *path, const char *message)
{
Serial.printf("Appending to file: %s\n", path);
File file = fs.open(path, FILE_APPEND);
if (!file)
{
Serial.println("Failed to open file for appending");
return;
}
if (file.print(message))
{
Serial.println("APOK");
}
else
{
Serial.println("APX");
}
}
void readFile(fs::FS &fs, const char *path)
{
Serial.printf("Reading file: %s\n", path);
File file = fs.open(path);
if (!file || file.isDirectory())
{
Serial.println("Failed to open file for reading");
return;
}
Serial.print("Read from file: ");
while (file.available())
{
Serial.write(file.read());
delayMicroseconds(100);
}
}
void writeFile(fs::FS &fs, const char *path, const char *message)
{
Serial.printf("Writing file: %s\n", path);
File file = fs.open(path, FILE_WRITE);
if (!file)
{
Serial.println("Failed to open file for writing");
return;
}
if (file.print(message))
{
Serial.println("File written");
}
else
{
Serial.println("Write failed");
}
}
void listDir(fs::FS &fs, const char *dirname, uint8_t levels)
{
Serial.printf("Listing directory: %s\n", dirname);
File root = fs.open(dirname);
if (!root)
{
Serial.println("Failed to open directory");
return;
}
if (!root.isDirectory())
{
Serial.println("Not a directory");
return;
}
File file = root.openNextFile();
while (file)
{
if (file.isDirectory())
{
Serial.print(" DIR : ");
Serial.println(file.name());
if (levels)
{
listDir(fs, file.name(), levels - 1);
}
}
else
{
Serial.print(" FILE: ");
Serial.print(file.name());
Serial.print(" SIZE: ");
Serial.println(file.size());
}
file = root.openNextFile();
}
}
void deleteFile(fs::FS &fs, const char *path)
{
Serial.printf("Deleting file: %s\n", path);
if (fs.remove(path))
{
Serial.println("File deleted");
}
else
{
Serial.println("Delete failed");
}
}
void updateFromFS()
{
File updateBin = SPIFFS.open("/update.bin");
if (updateBin)
{
if (updateBin.isDirectory())
{
Serial.println("Directory error");
updateBin.close();
return;
}
size_t updateSize = updateBin.size();
if (updateSize > 0)
{
Serial.println("Starting update");
performUpdate(updateBin, updateSize);
}
else
{
Serial.println("Error, archivo vacío");
}
updateBin.close();
// whe finished remove the binary from sd card to indicate end of the process
//fs.remove("/update.bin");
}
else
{
Serial.println("no such binary");
}
}
void performUpdate(Stream &updateSource, size_t updateSize)
{
if (Update.begin(updateSize))
{
size_t written = Update.writeStream(updateSource);
if (written == updateSize)
{
Serial.println("Writes : " + String(written) + " successfully");
}
else
{
Serial.println("Written only : " + String(written) + "/" + String(updateSize) + ". Retry?");
}
if (Update.end())
{
Serial.println("OTA finished!");
if (Update.isFinished())
{
Serial.println("Restart ESP device!");
ESP.restart();
}
else
{
Serial.println("OTA not fiished");
}
}
else
{
Serial.println("Error occured #: " + String(Update.getError()));
}
}
else
{
Serial.println("Cannot beggin update");
}
}
void printPercent(uint32_t readLength, uint32_t contentLength)
{
// If we know the total length
if (contentLength != -1)
{
Serial.print("\r ");
Serial.print((100.0 * readLength) / contentLength);
Serial.print('%');
}
else
{
Serial.println(readLength);
}
}
void setupModem()
{
sim800.begin(9600);
delay(6000);
SerialMon.println("Initializing modem...");
modem.restart();
String modemInfo = modem.getModemInfo();
SerialMon.print("Modem Info: ");
SerialMon.println(modemInfo);
}