How to increase the Record Limit of Arduino Extended Database Library?

Dear all,

I am using the Arduino Extended Database Library in my arduino sketch. In the website of
Arduino Playground - ExtendedDatabaseLibrary, it states that it can support a theoretical maximum of 4,294,967,295 records (unsigned long).

However, in my sketch, I write the following code to define the table size to be 30000.

#define TABLE_SIZE 30000

Then, I define the database structure as follows.

struct LogEvent {
int id;
int temperature;
}

logEvent;

And use the following procedure to look at the record limit of the database.

void recordLimit()
{
Serial.print("Record Limit: ");
Serial.println(db.limit());
}

The sketch also output Record Limit: 2045, no matter how large I set the TABLE_SIZE to. How to increase the Record Limit of the database because 2045 must not enough for my project.

Thanks!

What's your memory? How do you initialize the DB?

I use MicroSD card as the memory. Actually I use the example EDB_SDCARD in the Arduino Extended Database Library. The codes to initialize DB is as follows. Thanks!

#include <SPI.h>
#include <SD.h>
#include "max6675.h"
#include <SoftwareSerial.h>
#include <Wire.h>//import necessary libraries: SoftwareSerial, Wire
#include <DS3231.h>
#include "Arduino.h" 
#include <EDB.h> // for database and SD card
#define SD_PIN 4  // SD Card CS pin
//#define TABLE_SIZE 8192 // for database and SD card
#define TABLE_SIZE 10000 // for database and SD card

// Init the DS3231 using the hardware interface
DS3231  rtc(SDA, SCL);

SoftwareSerial I2CBT(9,10);//define PIN9 and PIN10 as RX and TX
int ktcSO = 2;
int ktcCS = 3;
int ktcCLK = 8;
MAX6675 ktc(ktcCLK, ktcCS, ktcSO);

const int soilMoist = 0;
// const long int recordPeriod = 43200000; // 12 hours
const long int recordPeriod = 1000;
char val; // variable to receive data from the serial port
char* db_name = "/db/edb_test.db"; // path of db files in SD card
File dbFile;  
int logid; // variable to store id of log

// Arbitrary record definition for this table.
// This should be modified to reflect your record needs.
struct LogEvent {
    int id;
    String logdate;
    String logtime;
    int temp;
    int soilmoist;
}
logEvent;

// The read and write handlers for using the SD Library
// Also blinks the led while writing/reading
void writer (unsigned long address, byte data) {
    dbFile.seek(address);
    dbFile.write(data);
    dbFile.flush();
}

byte reader (unsigned long address) {
    dbFile.seek(address);
    byte b = dbFile.read();
    return b;
}

// Create an EDB object with the appropriate write and read handlers
EDB db(&writer, &reader);

void setup() {
  
  // Open serial communications and wait for port to open:
  Serial.begin(115200);
  // Initialize the rtc object
  rtc.begin();
  // The following lines can be uncommented to set the date and time
  //rtc.setDOW(SUNDAY);     // Set Day-of-Week to SUNDAY
  //rtc.setTime(9, 1, 0);     // Set the time to 12:00:00 (24hr format)
  //rtc.setDate(31, 7, 2016);   // Set the date to January 1st, 2014
  
  I2CBT.begin(9600); //藍牙 BAUD RATE 9600
  while (!Serial) {
    ; // wait for serial port to connect. Needed for native USB port only
  }
  
  if (!SD.begin(SD_PIN)) {
      Serial.println("No SD-card.");
      return;
  }

  // Check dir for db files
  if (!SD.exists("/db")) {
      Serial.println("Dir for Db files does not exist, creating...");
      SD.mkdir("/db");
  }
  if (SD.exists(db_name)) {
      dbFile = SD.open(db_name, FILE_WRITE);
      // Sometimes it wont open at first attempt, espessialy after cold start
      // Let's try one more time
      if (!dbFile) {
          dbFile = SD.open(db_name, FILE_WRITE);
      }

      if (dbFile) {
          Serial.print(F("Openning current table... "));
          EDB_Status result = db.open(0);
          if (result == EDB_OK) {
              Serial.println(F("DONE"));
          } else {
              Serial.println(F("ERROR"));
              Serial.println("Did not find database in the file " + String(db_name));
              Serial.print(F("Creating new table... "));
              db.create(0, TABLE_SIZE, (unsigned int)sizeof(logEvent));
              Serial.println(F("DONE"));
              return;
          }
      } else {
            Serial.println("Could not open file " + String(db_name));
            return;
      }
      } else {
        Serial.print("Creating table... ");
        // create table at with starting address 0
        dbFile = SD.open(db_name, FILE_WRITE);
        db.create(0, TABLE_SIZE, (unsigned int)sizeof(logEvent));
        Serial.println("DONE");
     }    
  recordLimit();    
  deleteAll();
  
  // read sensors and append to the string:
  int sensor;
  String ltime;
  sensor = analogRead(soilMoist);
  logEvent.id = logid;
  logid = db.count()+1;
  //logEvent.logdate  = rtc.getDateStr();
  logEvent.logtime = rtc.getTimeStr();
  Serial.println("TIME"+logEvent.logtime);
  logEvent.soilmoist  = sensor;
  logEvent.temp =  int(ktc.readCelsius()); // temperature sensor data
  EDB_Status result = db.appendRec(EDB_REC logEvent);
  if (result != EDB_OK) printError(result);
  Serial.println(F("Append 1 record"));    
  delay(5000); 
  // read sensors and append to the string:
  sensor = analogRead(soilMoist);
  logEvent.id = logid;
  logid = db.count()+1;
  //logEvent.logdate  = rtc.getDateStr();
  logEvent.logtime = rtc.getTimeStr();
  Serial.println("TIME"+logEvent.logtime);
  logEvent.soilmoist  = sensor;
  logEvent.temp =  int(ktc.readCelsius()); // temperature sensor data
  result = db.appendRec(EDB_REC logEvent);
  if (result != EDB_OK) printError(result);
  Serial.println(F("Append 1 record"));    
  delay(5000); 
  selectAll();
  dbFile.close();
}

void loop() {
  // make a string for assembling the data to log:
  String dataString = "";
 
  byte cmmd[20]; 
  int insize; 
    
  if ((insize=(I2CBT.available()))>0)
  {  //判斷有沒有訊息接收 
    dataString += String(insize);
    dataString += ",";
     for (int i=0; i<insize; i++) {
       cmmd[i]=char(I2CBT.read());//將接收的訊息顯示出來
       dataString += cmmd[i];
       dataString += ",";
     }
  } 
  delay(1000); 
   
}

void recordLimit()
{
    Serial.print(F("Record Limit: "));
    Serial.println(db.limit());
}

void countRecords()
{
    Serial.print(F("Record Count: "));
    Serial.println(db.count());
}

void printError(EDB_Status err)
{
    Serial.print(F("ERROR: "));
    switch (err)
    {
        case EDB_OUT_OF_RANGE:
            Serial.println(F("Recno out of range"));
            break;
        case EDB_TABLE_FULL:
            Serial.println(F("Table full"));
            break;
        case EDB_OK:
        default:
            Serial.println(F("OK"));
            break;
    }
}

void selectAll()
{
    for (int recno = 1; recno <= db.count(); recno++)
    {
        EDB_Status result = db.readRec(recno, EDB_REC logEvent);
        if (result == EDB_OK)
        {
            Serial.print(F("Recno: "));
            Serial.print(recno);
            Serial.print(F(" ID: "));
            Serial.print(logEvent.id);
            //Serial.print(F(" Date: "));
            //Serial.print(logEvent.logdate);
            Serial.print(F(" Time: "));
            Serial.print(logEvent.logtime);
            Serial.print(F(" Temp: "));
            Serial.println(logEvent.temp);
            Serial.print(F(" Soil Moisture: "));
            Serial.println(logEvent.soilmoist);
        }
        else printError(result);
    }
}

void deleteAll()
{
    Serial.print("Truncating table... ");
    db.clear();
    Serial.println("DONE");
}

You should post complete sketches and not snippets; there is no relationship between the codes that you posted so we don't know if you first print and next initialize or the other way around.

And please use code tags
type
** **[code]** **

paste your code after that
type
** **[/code]** **

How about reading the forum "stickies"?

Please use code tags.

Read this before posting a programming question

How to use this forum

I don't suppose this will help the OP, but given the code:

struct LogEvent {
    int id;
    String logdate;
    String logtime;
    int temp;
    int soilmoist;
}
logEvent;

Can you really put a String inside of a struct like that? I wouldn't think that would work correctly (and that a character array would be required).

Just curious.

Sorry I can't help,

Brad.

If the compiler lets you do it, then it's valid. The String object can be considered just a complex pointer, with the real contents of the String allocated on the heap.

You can do it, but if you try writing that struct to a database you won't get the String contents, just the internal pointers (as MorganS said). It's doubtful it will do what you want.

When I tested that library and ran the EDB_Simple sketch, changing TABLE_SIZE to 30000, and adding these lines after the create:

  Serial.print("Record Limit: ");
  Serial.println(db.limit());
  return;

I got this printed:

Extended Database Library + Arduino Internal EEPROM Demo

Record Count: 0
Record Limit: 7497

That would be 300000 minus the EDB header size, divided by the record size (4). So I definitely got more than 2045.

Just delete db file on SPIFFS using

SPIFFS.begin();
SPIFFS.remove("/db/edb_test.db");

or

SPIFFS.begin();
SPIFFS.format();

to recreate using a new file using your new TABLE_SIZE value.

If you don't delete, db file doesn't change db.limit().