ESP8266 / EPS32 more special tool to store the sourcecode in the flash-rom

Hi everybody,

as I'm writing code for a device that is used by a friend I had the idea / the question

There is an IDE-tool for storing data in the ESP8266's /ESP32's flash-ROM.
This could be any kind of a textfile or html-code or whatever.

Then I was asking myself is there more special tool to store the sourcecode of the compiled binary that is running on the ESP into flash?

This would mean that the sourcecode is always "at hand" where the device is.
Of course I can do this with the standard LittleFS-data-upload tool
by
copying the source-code-file(s) into a subfolder "data" and then start the standard-LittleFS-upload-tool.

But as computers are there to take over schematic work that follows an always the same pattern this uploading could be done by a tool similar to the standard LittleFS-data-uploadtool.

best regards Stefan

Check how you program the part, you will probably lose your source code if not done properly as it will erase and then reprogram it. Why not put it on a thumb drive.

Pretty easy, IMO. Several links to SPIFFS and how it is managed:

https://www.google.com/search?q=esp32+spiffs

I was able to store the entire OUI (MAC --> vendor) lookup table which I downloaded as an ASCII text file.

Stored data looks like this, in part:


|00-00-06|XEROX CORPORATION|
|00-00-07|XEROX CORPORATION|
|00-00-08|XEROX CORPORATION|
|00-00-09|XEROX CORPORATION|
|00-00-FF|CAMTEC ELECTRONICS LTD.|
|00-01-00|EQUIP'TRANS|
|00-01-01|Private|
|00-01-FF|Data Direct Networks, Inc.|
|00-02-00|Net & Sys Co., Ltd.|
|00-02-01|IFM Electronic gmbh|
|00-02-FD|Cisco Systems, Inc|
|00-02-FE|Viditec, Inc.|
|00-02-FF|Handan BroadInfoCom|
|00-03-00|Barracuda Networks, Inc.|
|00-03-01|EXFO|

One way to extract the file(s):
https://techexplorations.com/blog/esp32/blog-esp32-how-to-retrieve-a-file-from-the-spiffs/

Or, just write a sketch to open and send the file context to the PC where it can be displayed or redirected to a local file.

The OUI sketch:

/*
   Arduino 1.6.13 on Linux Mint 17.3 (1.8.1 tested)
   Open Source demo source by M. Ray Burnette (ray.burne) 
*/


#define maxBlks     7680                                                // determined by direct analysis

#include "FS.h"
#include <ESP8266WiFi.h>

const char* ssid     = ;
const char* password = ;

unsigned char ASCII2HEX( char c)                                        // http://stackoverflow.com/questions/2499000/how-to-convert-an-ascii-hex-character-to-its-value-0-15
{
  return ( c <= '9' ) ? ( c - '0' ) : ( (c | '\x60') - 'a' + 10 ) ;
}

struct        OUI {                                                     // Custom data structure to hold linked list of O1::02 file pointers
  unsigned char c_Octet1 ;
  unsigned char c_Octet2 ;
  uint16_t      f_offset ;                                              // linked list: beginning of current record TO beginning of next line /n included
}  ;

OUI oui_array[maxBlks] ;                                                // define the array type and size

String        OUI_matchup ;
char          cArrayOUI[7] ;
char          currentCharacter ;
char          * pEnd ;

unsigned char cOctet1 ;
unsigned char cOctet2 ;
unsigned char cOctet3 ;

unsigned long fileByteOffset    = 0 ;
unsigned long iSum              = 0 ;
unsigned long previousPosition  = 0 ;
unsigned long currentPosition   = 0 ;
unsigned long timeMarker ;

uint16_t      iRecIdx = 0 ;
uint8_t       OUI_match_Index = 0 ;
uint8_t       io1 ;
uint8_t       io2 ;
uint8_t       io3 ;

bool          ok ;
bool          lFlag = true ;
bool          lFound = true ;;

File          FileRead ;
WiFiServer    server(80);


void setup() {
  Serial.begin(115200) ;
  delay(100) ;
  Serial.print("\n\r ***** Starting Up *****\r\n") ;
  Serial.print("Connecting to ");
  Serial.println(ssid);
  
  WiFi.begin(ssid, password);
  
  while (WiFi.status() != WL_CONNECTED) {
    delay(1000);
    Serial.print(".");
  }
  Serial.println("");
  Serial.println("WiFi connected");
  
  server.begin();                                                       // Start the server
  Serial.println("Server started");
  Serial.println(WiFi.localIP());                                       // Print the IP address
  
  OUI_matchup.reserve(8) ;                                              // xx-xx-xx
  ok = SPIFFS.begin() ;                                                 // https://github.com/esp8266/Arduino/blob/master/doc/filesystem.md

  oui_array[0].c_Octet1 = 0x0000 ;                                      // initialize array as a placeholder
  oui_array[0].f_offset = 0 ;
  previousPosition      = 0 ;                                           // Walking var for varying length descriptions field
  iRecIdx               = 1 ;                                           // Beginning array indexpreviousPosition

  Serial.print("\r\nBegin reading OUI.txt located in SPIFFS and building array... about 50 seconds\r\n") ;
  timeMarker = millis() + 5000;
  FileRead = SPIFFS.open("/oui.txt", "r") ;
  FileRead.seek(fileByteOffset, SeekSet) ;                              // Seek to record

    while (ok && FileRead.available() > 0)                              // While not EOF
    {
      currentCharacter = FileRead.read() ;

      if (OUI_match_Index < 8 )  {                                      // gather the first 8 characters of a line as: xx-xx-xx
        cArrayOUI[OUI_match_Index] = currentCharacter ;
        ++OUI_match_Index ;
      }

      if (currentCharacter == 9) {                                      // TAB ?
          cOctet1 = (ASCII2HEX(cArrayOUI[1])) + (ASCII2HEX(cArrayOUI[0]) * 16) ;
          cOctet2 = (ASCII2HEX(cArrayOUI[4])) + (ASCII2HEX(cArrayOUI[3]) * 16) ;
          // the magic in the test below is only possible because the imported OUI list is in order by 1st::2nd octet
          if ((cOctet2 > oui_array[iRecIdx - 1].c_Octet2) || ( cOctet1 > oui_array[iRecIdx - 1].c_Octet1)) 
          {
            oui_array[iRecIdx].c_Octet1 = cOctet1 ;
            oui_array[iRecIdx].c_Octet2 = cOctet2 ;                     // iRecIdx points to last-built array elements
            oui_array[iRecIdx].f_offset = (currentPosition - previousPosition) ;
            previousPosition = currentPosition ;


            if( millis() > timeMarker ) {
              Serial.print(".") ;                                       // Let user know something is happening
              timeMarker = millis() + 5000 ;
            }
            OUI_match_Index = 0 ;                                       // prepare for next OUI parsing
            OUI_matchup = "" ;                                          // null out the capture String;
            ++iRecIdx ;                                                 // Block count
        }
      }

      if (currentCharacter == 10)                                       // End-of-line ?
      {
          // previousPosition  = currentPosition ;                         // Delta
          currentPosition   = FileRead.position() ;                     // DO NOT nest this function
          OUI_match_Index = 0 ;                                         // prepare for next OUI parsing
          OUI_matchup = "" ;                                            // null out the capture String;
      }
    }                                                                   // EOF

    Serial.print( "\n\rReached End Of File\n\r") ;
}


void loop() 
{
  if (lFound == !true)  {
  Serial.print("\r\nFailure to locate requested OUI record...\n\r") ;
  lFound = true ;
  }

  if (lFlag) {
    Serial.print("\n\rEnter 6 characters to represent 3 HEX octet... followed by ENTER...\n\r ") ;
    lFlag = !(lFlag) ;
  }

  WiFiClient client = server.available();                               // Check if a client has connected
  if (!client) {
    return;
  }
  
  Serial.println("new client");                                         // Wait until the client sends some data
  while(!client.available()){
    delay(1);
  }
  
  String req = client.readStringUntil('\r');                            // Read the first line of the request
  Serial.print("Request == ") ; Serial.println(req);
  client.flush();

  req.toCharArray(cArrayOUI, 8) ;
  // io1 = io2 = io3 = 0 ;

  io1 = (ASCII2HEX(cArrayOUI[1])) + (ASCII2HEX(cArrayOUI[0]) * 16) ;
  io2 = (ASCII2HEX(cArrayOUI[4])) + (ASCII2HEX(cArrayOUI[3]) * 16) ;
  io3 = (ASCII2HEX(cArrayOUI[7])) + (ASCII2HEX(cArrayOUI[6]) * 16) ;

  //if (Serial.available() > 0)
  if (true) ;
  {
    /*
    cArrayOUI[7] = cArrayOUI[6] = cArrayOUI[4] = cArrayOUI[3] = cArrayOUI[1] = cArrayOUI[0] = '00' ;
    cArrayOUI[0] = Serial.read() ;                                      // reuse cArrayOUI
    cArrayOUI[1] = Serial.read() ;
    cArrayOUI[3] = Serial.read() ;
    cArrayOUI[4] = Serial.read() ;
    cArrayOUI[6] = Serial.read() ;
    cArrayOUI[7] = Serial.read() ;
    char junk = Serial.read() ; //CR or LF                              // ArduinoIDE Monitor must be set for "Both NL & CR"
    junk = Serial.read() ; //LF or CR                                   // remark this line if you wish to only send one EOL character

    io1 = (ASCII2HEX(cArrayOUI[1])) + (ASCII2HEX(cArrayOUI[0]) * 16) ;
    io2 = (ASCII2HEX(cArrayOUI[4])) + (ASCII2HEX(cArrayOUI[3]) * 16) ;
    io3 = (ASCII2HEX(cArrayOUI[7])) + (ASCII2HEX(cArrayOUI[6]) * 16) ;

    Serial.print("HEX ") ;
    Serial.print(io1, HEX) ; Serial.print("-") ; Serial.print(io2, HEX) ; Serial.print("-") ; Serial.print(io3, HEX) ;
    Serial.print(" == DEC ") ; Serial.print(io1, DEC) ; Serial.print(":") ; Serial.print(io2, DEC) ;
    Serial.print(":") ; Serial.print(io3, DEC) ; Serial.print("  ") ;
    */
    
    iSum = 0 ;
    lFlag = true ;
    uint16_t x ;
    lFound = false ;

    for (x = 0; x <= maxBlks; x++)
    {
      iSum += oui_array[x].f_offset ;

      if ((io1 == oui_array[x].c_Octet1) && (io2 == oui_array[x].c_Octet2))
      {
        // here because we found the first and second matching octets in the array
        // We begin our search further for the 3rd octet which is in SPIFFS
        lFound = true ;
      }
      if( lFound) break ;
    }

    // here I should have the SPIFFS pointer set to the record with match to .x and .y octets
    // rest is just brute force sequential searching

    if (lFound) {
      lFound = false ;                                                  // reset for SPIFFS lookup - part 2 of search
      OUI_match_Index = 0 ;
      OUI_matchup = "" ;
      FileRead.seek(iSum, SeekSet) ;                                    // Seek to record
      cOctet2 = io2 ;                                                   // User octet2 input value

      while (FileRead.available() > 0 && !(lFound))                     // While not EOF and not-found
      {
        currentCharacter = FileRead.read() ;

        if (OUI_match_Index < 8 )                                       // gather the first 8 characters of a line as: xx-xx-xx
        {
          cArrayOUI[OUI_match_Index] = currentCharacter ;
          ++OUI_match_Index ;
        }

        if (currentCharacter  == 9)                                     // TAB
        {
            cOctet1 = (ASCII2HEX(cArrayOUI[1])) + (ASCII2HEX(cArrayOUI[0]) * 16) ;
            cOctet2 = (ASCII2HEX(cArrayOUI[4])) + (ASCII2HEX(cArrayOUI[3]) * 16) ;
            cOctet3 = (ASCII2HEX(cArrayOUI[7])) + (ASCII2HEX(cArrayOUI[6]) * 16) ;

            if ((cOctet3 == io3) && (cOctet2 == io2) && (cOctet1 == io1)) {
              lFound = true ;
            } else if( (cOctet1 > io1) || (cOctet2 > io2) ) {
              lFound = false ;
              break ;
            }
        }

        if (currentCharacter == 10)                                   // newLine
        {
              OUI_match_Index = 0 ;                                   // setup variables for next OUI discovery
              OUI_matchup = "" ;                                      // null out the String
        }

        if (lFound) {
          // Prepare the response
          String s = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<!DOCTYPE HTML>\r\n<html>\r\nGPIO is now ";
          lFlag == true ;
          currentCharacter = 0 ;                                        // send description to the printer
          do {
            currentCharacter = FileRead.read() ;
            // Serial.print(currentCharacter) ;
            s += currentCharacter ;
          } while ( currentCharacter != 10 ) ;

          s += "</html>\n" ;
          client.flush();
          // Send the response to the client
          client.print(s);
          delay(1);
          Serial.println("Client disonnected");
        }
      }                                                                 // while not EOF
    }                                                                   // found a record ?
  }                                                                     // Serial test
  delay(100);
}


Why should it get lost if there is a partition-scheme?
Why would somebody do the effort of using a tool like LittleFS-Data-Upload-Tool if it can happen that the content gets erased?
Or
Do you mean the existing LittleFS-Data-Upload-Tool is working this way:
earse all in the FS-partition and write everything new from the subfolder data?
hm - this is an idea too:
not a second upload-to-flash-tool but a tool that copies the actual sourcecode to a subfolder of subfolder "data" and then using the existing LittleFS-Data-Upload-Tool.
Will to have to test if the existing LittleFS-Data-Upload-Tool manages sub-sub-folders.

best regards Stefan

Espressif's tool is just a flat-file system. It is partition aware:
https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/spiffs.html

1 Like

Let us know how it works. I trashed my FS-Data section when programing the code, now I know why. I do not have the LittleFS-Data-Upload-Tool and did not know about it. Thanks I learned something.