Tracking the cause of crashes

[StefanL38]

@arneko

but what should I install on my PC to receive and write all these messages?

Sending UPD

WiFiUDP Udp;

    Udp.beginPacket(remoteIP, remotePort);
    Udp.write((const uint8_t*)UDP_Msg_SS.c_str(), UDP_Msg_SS.length() );  
    Udp.endPacket();

Here is a demo-code that create messages with a timestamp.
At the end of the code is as a comment a python-script that receives these messages
and store them into a textfile

The message has a header and a body.
The header is used for the filename
This means the python-script is able to store messages in different files if you use different headers

// I wrote some basic documentation about receiving the UDP-messages with python at the end of the file
#include <WiFi.h>
#include <SafeString.h>

#define MaxMsgLength 1024
createSafeString(UDP_Msg_SS,MaxMsgLength); 
uint8_t UDP_Msg_uint8_Buffer[MaxMsgLength + 1]; // for some strange reasons on ESP32 the udp.write-function needs an uint8_t-array

#define MaxHeaderLength 32 
createSafeString(Header_SS,MaxHeaderLength);

#define MaxTimeStampLength 64 
createSafeString(TimeStamp_SS,MaxTimeStampLength);


char HeaderDelimiter = '$'; // must match the delimiter defined in the python-code 


void InitSensor() {
  //Init Chip
}


void ReadPressureSensor(){

}


//const   char *ssid     = "FRITZ!Box 7490";
const char *ssid     = "WLANBuero_EXT"; 

const char *password = "80841232631090916208";

IPAddress    remoteIP     (192, 168, 178, 160); // receiver-IP
unsigned int remotePort = 4210;                 // receiver port to listen on must match the portnumber the receiver is listening to

WiFiUDP Udp;

const char* ntpServer = "fritz.box";
const long  gmtOffset_sec = 0;
const int   daylightOffset_sec = 7200;

#include <time.h>                   // time() ctime()
time_t now;                         // this is the epoch
tm myTimeInfo;                      // the structure tm holds time information in a more convient way


boolean TimePeriodIsOver (unsigned long &expireTime, unsigned long TimePeriod) {
  unsigned long currentMillis  = millis();
  if ( currentMillis - expireTime >= TimePeriod )
  {
    expireTime = currentMillis; // set new expireTime
    return true;                // more time than TimePeriod) has elapsed since last time if-condition was true
  }
  else return false;            // not expired
}

const byte OnBoard_LED = 2;
int BlinkTime = 500;

void BlinkHeartBeatLED(int IO_Pin, int BlinkPeriod) {
  static unsigned long MyBlinkTimer;
  pinMode(IO_Pin, OUTPUT);

  if ( TimePeriodIsOver(MyBlinkTimer, BlinkPeriod) ) {
    digitalWrite(IO_Pin, !digitalRead(IO_Pin) );
  }
}

unsigned long TestTimer;
unsigned long UDP_SendTimer;

int myCounter = 0;
int HeaderNr  = 0;


void PrintFileNameDateTime()
{
  Serial.print("Code running comes from file ");
  Serial.println(__FILE__);
  Serial.print(" compiled ");
  Serial.print(__DATE__);
  Serial.println(__TIME__);
}

void showTime() {
  time(&now);                       // read the current time
  localtime_r(&now, &myTimeInfo);           // update the structure tm with the current time
  Serial.print("year:");
  Serial.print(myTimeInfo.tm_year + 1900);  // years since 1900
  Serial.print("\tmonth:");
  Serial.print(myTimeInfo.tm_mon + 1);      // January = 0 (!)
  Serial.print("\tday:");
  Serial.print(myTimeInfo.tm_mday);         // day of month
  Serial.print("\thour:");
  Serial.print(myTimeInfo.tm_hour);         // hours since midnight  0-23
  Serial.print("\tmin:");
  Serial.print(myTimeInfo.tm_min);          // minutes after the hour  0-59
  Serial.print("\tsec:");
  Serial.print(myTimeInfo.tm_sec);          // seconds after the minute  0-61*
  Serial.print("\twday");
  Serial.print(myTimeInfo.tm_wday);         // days since Sunday 0-6
  if (myTimeInfo.tm_isdst == 1)             // Daylight Saving Time flag
    Serial.print("\tDST");
  else
    Serial.print("\tstandard");
    
  Serial.println();
}


void StoreTimeStampIntoSS(SafeString& p_RefToSS, tm p_myTimeInfo) {

  time(&now);                               // read the current time
  localtime_r(&now, &myTimeInfo);           // update the structure tm with the current time

  //p_RefToSS = " ";
  p_RefToSS  = myTimeInfo.tm_year + 1900;
  p_RefToSS += ".";

  // month
  if (p_myTimeInfo.tm_mon + 1 < 10) {
    p_RefToSS += "0";
  }  
  p_RefToSS += myTimeInfo.tm_mon + 1;

  p_RefToSS += ".";

  // day
  if (p_myTimeInfo.tm_mday + 1 < 10) {
    p_RefToSS += "0";
  }    
  p_RefToSS += myTimeInfo.tm_mday;

  p_RefToSS += "; ";

  // hour
  if (p_myTimeInfo.tm_hour < 10) {
    p_RefToSS += "0";
  }    
  p_RefToSS += myTimeInfo.tm_hour;
  p_RefToSS += ":";

  // minute
  if (p_myTimeInfo.tm_min < 10) {
    p_RefToSS += "0";
  }    
  p_RefToSS += myTimeInfo.tm_min;
  
  p_RefToSS += ":";

  // second
  if (p_myTimeInfo.tm_sec < 10) {
    p_RefToSS += "0";
  }    
  p_RefToSS += myTimeInfo.tm_sec;
  //p_RefToSS += ",";  
}


void connectToWifi() {
  Serial.print("Connecting to "); 
  Serial.println(ssid);

  WiFi.persistent(false);
  WiFi.mode(WIFI_STA);

  WiFi.begin(ssid, password);

  while (WiFi.status() != WL_CONNECTED) {
    BlinkHeartBeatLED(OnBoard_LED, 333);
    delay(332);
    Serial.print(".");
  }
  Serial.print("\n connected.");
  Serial.println(WiFi.localIP() );

}

void synchroniseWith_NTP_Time() {
  Serial.print("configTime uses ntpServer ");
  Serial.println(ntpServer);
  configTime(gmtOffset_sec, daylightOffset_sec, ntpServer);
  Serial.print("synchronising time");
  
  while (myTimeInfo.tm_year + 1900 < 2000 ) {
    time(&now);                       // read the current time
    localtime_r(&now, &myTimeInfo);
    BlinkHeartBeatLED(OnBoard_LED, 100);
    delay(100);
    Serial.print(".");
  }
  Serial.print("\n time synchronsized \n");
  showTime();    
}

void setup() {
  Serial.begin(115200);
  Serial.println("\n Setup-Start \n");
  PrintFileNameDateTime();
  InitSensor();
  Serial.print("InitSensor() done \n");
  
  connectToWifi();
  synchroniseWith_NTP_Time();
  Header_SS = "Header"; 
}

void PrintMsg() {
  Serial.print("UDP_Msg_SS #");
  Serial.print(UDP_Msg_SS);
  Serial.println("#");
}

void loop() {
  BlinkHeartBeatLED(OnBoard_LED, BlinkTime);

  if (TimePeriodIsOver(UDP_SendTimer, 2000) ) {
    ReadPressureSensor();
    Serial.print("Send Message to #");
    Serial.print(remoteIP);
    Serial.print(":");
    Serial.println(remotePort);

    UDP_Msg_SS = "";

    UDP_Msg_SS = Header_SS;
    UDP_Msg_SS += HeaderDelimiter;

    StoreTimeStampIntoSS(TimeStamp_SS,myTimeInfo);
    UDP_Msg_SS += TimeStamp_SS;

    UDP_Msg_SS += ",my Testdata1,";
    UDP_Msg_SS += 123;
    UDP_Msg_SS += ",my Testdata2,";

    UDP_Msg_SS += 789;
    UDP_Msg_SS += ",";
    
    UDP_Msg_SS += myCounter++; 

    Serial.print("Send UDP_Msg #");
    Serial.print(UDP_Msg_SS);
    Serial.println("#");
    dbg("length:",UDP_Msg_SS.length());
    Udp.beginPacket(remoteIP, remotePort);
    Udp.write((const uint8_t*)UDP_Msg_SS.c_str(), UDP_Msg_SS.length() );  
    Udp.endPacket();
  }    
}

/*

This is a democode that demonstrates how to send TCP/UDP-messages with a timestamp 
The time is synchronized using a NTP-server. Most local routers like Fritz!Box can be used as the NTP-timer-server

The message has a userdefinable header which could be used for identifying the sender on the recieverside
There is a user-definable Header-delimiter that can be used to identify which characters of the
UDP-message belong to the header and which to the userdata

The code makes use of the PString-library. PStrings don't cause memory-problems like datatype "Strings"
and are easier to use than arrays of char. Example adding an integer to a PString-variable is as easy as
MyPString = myInteger;

The userdata has commas between each data so you can import the textfile 
into table-calculation-software or databases as CSV-file comma separated values

the code has some additional useful functions as there are
- PrintFileNameDateTime() printing the path and filename of sourcecode file this program was generated with

- boolean TimePeriodIsOver  a non-blocking timing-function based on millis which is suitable for 
  timed execution in a regular manner (repeat every n milliseconds)

- BlinkHeartBeatLED() blinks the onboard-LED of ESP32 nodeMCU-boards. Gives visual feedback if the code is running  

The lines of code are grouped by functionality into several functions
The functions name says what the function does

I use this code for easy datalogging on a computer with a python-code that acts as the UDP-listener for 
incoming messages. Inside your ESP32-code you have to adjust the IP-adress to the receiver 
and the portnumber must be the same on sender (ESP32) and receiver-side 

In the python-code The header is used to create a file with the header as filename and extension ".txt" or if file 
is already existant to append the actual received UDP-message at the end of the file.

here is the python-code that does this. I tested it with python 3.9.2 for windows
#Start of python-code
# very simple and short upd-receiver based on python-code I found here
# https://www.studytonight.com/network-programming-in-python/working-with-udp-sockets#

import socket

sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)      # For UDP

udp_host = socket.gethostname()            # Host IP
udp_port = 4210                     # specified port to connect

sock.bind((udp_host,udp_port))
print ("Waiting for client...")

HeaderDelimitChar = "$"

while True:
  data,addr = sock.recvfrom(1024)         #receive data from client
  print("data #",data,"#")
  Msg = data.decode('utf-8')
  print ("Received Message: #",Msg,"# from",addr)
  EndOfHeader = Msg.find(HeaderDelimitChar)
  HeaderBytes = Msg[0:EndOfHeader]
  FileName = HeaderBytes + ".txt"
  print("Filename #",FileName,"#")
  myFile = open(FileName, "a+")
  EndOfStr = data.find(0)
  MsgToWrite = Msg[EndOfHeader + 1 :1024] + '\r'
  myFile.write(MsgToWrite);
  myFile.close()
  print ("Data #",MsgToWrite,"#")

#End of python-code

For learning more about python just google with keyword "python" "your keyword of interest"  
 */

[arneko]

I reset the memory by using

#include "mbed.h"
#include "LittleFileSystem.h"
#include "BlockDevice.h"

mbed::LittleFileSystem fs("kv");
mbed::BlockDevice *bd = mbed::BlockDevice::get_default_instance();

void setup() {
    int err = fs.mount(bd);
    if (err != 0) {
        fs.reformat(bd);
    }
}

After I had to reload the wifi firmware using the firmware updater, but still if I try my code with KVStore or yours that should give an error code it crashes the OS before setup loads

[cattledog]

@arneko

but still if I try my code with KVStore or yours that should give an error code it crashes the OS before setup loads

I have no idea what changed from when KV store was running, but I would think that KVStore is now attempting to use a memory region reserved for something else.

I am not familiar with the Giga and don't know how/where the memory is mapped or what memory regions KV store uses.

[arneko]

I appreciate the effort you put into this! It's going to take a while to understand everything though, I've never used Python before.

Am I correct in understanding this will write continuous statusupdates to the receiver, potentially creating a massive log file?

[StefanL38]

@arneko

Am I correct in understanding this will write continuous statusupdates to the receiver, potentially creating a massive log file?

Whenever the python-script receives an UDP-message this message is written into the textfile.

If your ESP32 sends a message each 100 milliseconds the textfile will grow very fast
As explained before the message
has to have a header then header-delimiter (in this case "$" then the data
example a message

"myHeader$myData"

will be stored in a file with filename "myHeader.txt" and the content will be "myData"

"LogESP32$temp=28-blablabla"

will be stored in a file with filename "LogESP32.txt" and the content will be "temp=28-blablabla"

[farmer_giles]

@arneko Did you solve the problem of kvstore crashing the Giga? I have the same issue.
It works fine when I upload a simple kvstore demo sketch. As soon as I upload a much larger one (that uses 80% of memory) kv_reset crashes the board. I suspect what @cattledog said is right - kvstore might be trampling on the running code of the board.

[arneko]

Never got it working again, not even a simple 10 line sketch.

[farmer_giles]

ah.... sorry to hear that.

[StefanL38]

I don't use any of these expensive Arduinos.

If I need to store a lot of logdata I use an ESP32 that sends the data to a Raspberry Pi wireless by UDP
The raspberry Pi can store hundreds of Megabytes of logdata.

Never had any problem with storing data this way

Same thing for storing data collected by an ESP32 and storing it on a SD-Card.

Third option on ESP32's is: litteFS (little file-system)
The flash is partioned so data and program storage is strictly in several places.

If I write ESP32 I mean a "REAL" and standalone ESP32

NOT an Arduino Nano ESP32
NOT an Arduino Uno R4
In my opinion the Arduino team came too late to the ESP32-party
and created somehow strange "double-microcontroller"-boards

[JoeHuber]

I would agree with Stefan. Try to get the log data off the GIGA and to a safe place as quickly as possible. The data will be safer and most importantly your logging will be less disruptive to the normal flow of your code that you're trying to investigate. Pausing to write stuff to the flash is likely a major disruption to the timing of your normal code and may hide your actual issue and yet cause other apparent issues that aren't in your normal code.

Even just using the serial console to log your messages is a good first step.