Storing Data in nano 33 BLE Flash

Hi,

I am trying to store a fair amount of data in flash for my project, about 28.8Kbytes. I am using the nano 33 BLE based on the nRF52840 processor from Nordic. I started out using the NanoBLEFlashPrefs Library. And the example NanoBLEFlashPrefsTest worked just fine. I then ported it over to my project and started to run into size limitations. I understand that the Physical Page size is 1024 (4-byte) words and by default, the Virtual Page size is the same. Furthermore, I understand that the 1st 5 words are used for the record header leaving 1019 words. I have read the Nordic documentation and it leads me to believe I can increase the Virtual Page Size and the number of Virtual Pages. I found how to change this down in the core files. But no matter the combinations I use, I can only seem to write two records with two different RECORD_IDs. The size of my data into each of the two records is 1010 words. So first off, if I try to write any more than that I get a FDS_ERR_RECORD_TOO_LARGE error. and when I go to write a 3 record I get a FDS_ERR_NO_SPACE_IN_FLASH error - even with 10 virtual pages. Has anybody had any experience with this? I can work around only being able to write a PHYSICAL PAGE size, but I really need to be able to write more data than two PHYSICAL PAGE sizes. Thats only 8152 bytes, my less than I need.

Perhaps the defines I am changing do not make a difference. The Nordic documentation keeps referencing NRF_CONFIG macro, but I don't know where to start with that.

Any feedback is much appreciated.

Thanks,

Greg Hauck

Often not welcome advice, but are you sure the data is packed as much as it can be?

Hi,

I appreciate the suggestion. Unfortunately, all the data is high precision measurements and need all the bytes. Thanks for the thoughts.

-Greg

Can you share the details? Not to put too fine a point on it, but many people hold mistaken views on this matter...

Because, work arounds often have a very short shelf life. It's up to you, but you might find that tinkering at a low level with the Flash library could give you issues in the future when updates happen for example.

Point well taken about the work arounds down in the inner libraries. I went through the same thing within the mbed core related to a ring buffer issue that was fixed in the Arduino Mbed OS Boards 2.3.1. So I have documented what I changed in the core files to make sure future updates applied to my problem.

#define WAYPOINT_ROWS 84 //needs to be less that 1019 words or 4076 bytes. So 8468(double word bytes)= 4032bytes
#define WAYPOINT_COLUMNS 6
#define MAX_ROUTES 3
//Flash Related
#include <NanoBLEFlashPrefs.h>
NanoBLEFlashPrefs myFlashPrefs;
// Preferences structure. Arbitrary, but must not exeed 1019 words (4076 byte)

typedef struct flashStruct {
int bufferNumber;
double waypoint[WAYPOINT_ROWS][WAYPOINT_COLUMNS] ={NULL};
} flashPrefs;
flashPrefs prefs;
flashPrefs prefsOut;

The data I want to eventually store in flash is declared in this 2D array:

double waypoint[WAYPOINT_ROWS][WAYPOINT_COLUMNS] = {NULL};

My actual read, write and delete functions are:

void readFromFlashBuffer(void)
{
int tempRC;
// See if we already have a preference record
if(serialDebugPrintEnable)
{
Serial.println("Read record...");
Serial.println(sizeof(prefsOut));
Serial.println(activeRouteNumber);
}
tempRC = myFlashPrefs.readPrefs(&prefsOut, sizeof(prefsOut));
if (tempRC == FDS_SUCCESS)
{
if(serialDebugPrintEnable)
{
Serial.print("bufferNumber is: ");
Serial.println(prefsOut.bufferNumber);
}
for( int16_t i=0;i<WAYPOINT_ROWS;i++)
{
for( int16_t j=0;j<WAYPOINT_COLUMNS;j++)
{
if(serialDebugPrintEnable)
{
Serial.print(prefsOut.waypoint[i][j]);
Serial.print(",");
}
waypoint[i][j] = prefsOut.waypoint[i][j];
}
if(serialDebugPrintEnable)
{
Serial.println();
}
}
}
else
{
if(serialDebugPrintEnable)
{
Serial.print("No preferences found. Return code: ");
Serial.print(tempRC);
Serial.print(", ");
Serial.println(myFlashPrefs.errorString(tempRC));
Serial.println("");
}
}
}
void writeToFlashBuffer(void)
{
int tempRC;
// Prepare preference record for writing
prefs.bufferNumber = activeRouteNumber;
for( int16_t i=0;i<WAYPOINT_ROWS;i++)
{
for( int16_t j=0;j<WAYPOINT_COLUMNS;j++)
{
prefs.waypoint[i][j] = waypoint[i][j];
}
}
//lets double check what we have to write
if(serialDebugPrintEnable)
{
Serial.print("bufferNumber is:");
Serial.println(prefs.bufferNumber);
for( int16_t i=0;i<WAYPOINT_ROWS;i++)
{
for( int16_t j=0;j<WAYPOINT_COLUMNS;j++)
{
Serial.print(prefs.waypoint[i][j]);
Serial.print(",");
}
Serial.println();
}
// Write preference record
Serial.println("Write preferences...");
Serial.println(sizeof(prefs));
}
for(int k=0;k<2;k++)
{
tempRC = myFlashPrefs.writePrefs(&prefs, sizeof(prefs));
if (tempRC == FDS_SUCCESS)
{
// Wait until completion
while (!myFlashPrefs.operationCompleted()) {
}
if(serialDebugPrintEnable)
{
Serial.print("successful write");
Serial.println("");
}
k=2; //break out for loop
}
else
{
if(serialDebugPrintEnable)
{
Serial.print("No preferences found. Return code: ");
Serial.print(tempRC);
Serial.print(", ");
Serial.println(myFlashPrefs.errorString(tempRC));
Serial.println("");
}
//most likely out of flash
deleteFlashBuffer();
}
}
}
void deleteFlashBuffer(void)
{
int tempRC;
tempRC = myFlashPrefs.deletePrefs();
if (tempRC == FDS_SUCCESS)
{
if(serialDebugPrintEnable)
{
Serial.print("successful delete");
Serial.println("");
}

  // Wait until completion
  myFlashPrefs.garbageCollection();
  if(serialDebugPrintEnable)
  {
    Serial.print("go do garbage collection");
    Serial.println();
  }

  while (!myFlashPrefs.operationCompleted()) {
  }

  }
  else 
  {
    if(serialDebugPrintEnable)
    {
      Serial.print("No preferences found. Return code: ");
      Serial.print(tempRC);
      Serial.print(", ");
      Serial.println(myFlashPrefs.errorString(tempRC));
      Serial.println("");
    }
  }

}

I made no changes in the library NanoBLEFlashPrefs.cpp except for:

#define FILE_ID 0x0001 /* The ID of the file to write the records into. /
#define RECORD_KEY 0x1111 /
A key for the preferences record. */

See lines I tried changing marked with "gfh" below. They are in sdk_config.h.

#ifndef FDS_VIRTUAL_PAGES
//gfh
#define FDS_VIRTUAL_PAGES 10
#endif

// FDS_VIRTUAL_PAGE_SIZE - The size of a virtual flash page.

// Expressed in number of 4-byte words.
// By default, a virtual page is the same size as a physical page.
// The size of a virtual page must be a multiple of the size of a physical page.
// <1024=> 1024
// <2048=> 2048

#ifndef FDS_VIRTUAL_PAGE_SIZE
//gfh
#define FDS_VIRTUAL_PAGE_SIZE 1024
#endif

I very much appreciate your consideration.

Thanks,

Greg Hauck

Can you please edit your post and put the code in code tags? Please, first use ctrl-T to perform auto formatting which provides proper indenting to make the code blocks easy to identify, unless those were stripped away by the cut and paste process you used. Thanks.

Please let me know if this helps:

#define WAYPOINT_ROWS 84  //needs to be less that 1019 words or 4076 bytes.  So 84*6*8(double word bytes)= 4032bytes
#define WAYPOINT_COLUMNS 6
#define MAX_ROUTES 3
//Flash Related
#include <NanoBLEFlashPrefs.h>
NanoBLEFlashPrefs myFlashPrefs;
// Preferences structure. Arbitrary, but must not exeed 1019 words (4076 byte)

typedef struct flashStruct {
    int  bufferNumber;
    double waypoint[WAYPOINT_ROWS][WAYPOINT_COLUMNS] ={NULL};
} flashPrefs;
flashPrefs prefs;
flashPrefs prefsOut; 
double waypoint[WAYPOINT_ROWS][WAYPOINT_COLUMNS] = {NULL};
 void readFromFlashBuffer(void)
 {
    int tempRC;
      // See if we already have a preference record
     if(serialDebugPrintEnable)
     {
       Serial.println("Read record...");
       Serial.println(sizeof(prefsOut));
       Serial.println(activeRouteNumber);
     }
    tempRC = myFlashPrefs.readPrefs(&prefsOut, sizeof(prefsOut));
    if (tempRC == FDS_SUCCESS) 
    {
      if(serialDebugPrintEnable)
      {
        Serial.print("bufferNumber is: ");
        Serial.println(prefsOut.bufferNumber);
      }
      for( int16_t i=0;i<WAYPOINT_ROWS;i++)
      {
        for( int16_t j=0;j<WAYPOINT_COLUMNS;j++)
        {
          if(serialDebugPrintEnable)
          {
            Serial.print(prefsOut.waypoint[i][j]);
            Serial.print(",");
          }
          waypoint[i][j] = prefsOut.waypoint[i][j];
        }
        if(serialDebugPrintEnable)
        {
            Serial.println();
         }
      }
    } 
    else 
    {
      if(serialDebugPrintEnable)
      {
        Serial.print("No preferences found. Return code: ");
        Serial.print(tempRC);
        Serial.print(", ");
        Serial.println(myFlashPrefs.errorString(tempRC));
        Serial.println("");
      }
    }
 }
void writeToFlashBuffer(void)
 {  
   int tempRC;      
   // Prepare preference record for writing
   prefs.bufferNumber = activeRouteNumber;
   for( int16_t i=0;i<WAYPOINT_ROWS;i++)
   {
     for( int16_t j=0;j<WAYPOINT_COLUMNS;j++)
     {
       prefs.waypoint[i][j] = waypoint[i][j];
     }
   }
   //lets double check what we have to write
   if(serialDebugPrintEnable)
   {
     Serial.print("bufferNumber is:");
     Serial.println(prefs.bufferNumber);
     for( int16_t i=0;i<WAYPOINT_ROWS;i++)
       {
       for( int16_t j=0;j<WAYPOINT_COLUMNS;j++)
       {
         Serial.print(prefs.waypoint[i][j]);
         Serial.print(",");
       }
       Serial.println();
       }
     // Write preference record
     Serial.println("Write preferences...");
     Serial.println(sizeof(prefs));
    }
    for(int k=0;k<2;k++)
    {
      tempRC = myFlashPrefs.writePrefs(&prefs, sizeof(prefs));
      if (tempRC == FDS_SUCCESS) 
      {
        // Wait until completion
        while (!myFlashPrefs.operationCompleted()) {
        } 
        if(serialDebugPrintEnable)
        {
        Serial.print("successful write");
        Serial.println("");
        }
        k=2; //break out for loop
      }
      else 
      {
        if(serialDebugPrintEnable)
        {
          Serial.print("No preferences found. Return code: ");
          Serial.print(tempRC);
          Serial.print(", ");
          Serial.println(myFlashPrefs.errorString(tempRC));
          Serial.println("");
        }
        //most likely out of flash
        deleteFlashBuffer(); 
      }
    }  
 }
 void deleteFlashBuffer(void)
 {
     int tempRC;
      tempRC = myFlashPrefs.deletePrefs();
    if (tempRC == FDS_SUCCESS) 
    {
      if(serialDebugPrintEnable)
      {
        Serial.print("successful delete");
        Serial.println("");
      }

      // Wait until completion
      myFlashPrefs.garbageCollection();
      if(serialDebugPrintEnable)
      {
        Serial.print("go do garbage collection");
        Serial.println();
      }

      while (!myFlashPrefs.operationCompleted()) {
      }

      }
      else 
      {
        if(serialDebugPrintEnable)
        {
          Serial.print("No preferences found. Return code: ");
          Serial.print(tempRC);
          Serial.print(", ");
          Serial.println(myFlashPrefs.errorString(tempRC));
          Serial.println("");
        }
      }
 }

Well, I think you can see that it's not quite right... perhaps it won't help much with the issue at hand, so maybe it doesn't matter at this point.