Database Library

Here's a little database library I created that has been very useful for me, so I thought I would share. It makes use of the Arduino's EEPROM memory to store records in a table. I kept it simple to keep it small. You will have to spin your own locate and other functions like validation.

How to use in a nutshell:

  • include Db.h and EEPROM.h
  • declare an instance of DB, db
  • define a structure for your records
  • pick an address in EEPROM for the table to start
  • make a little sketch that creates the table by calling db.create(address,sizeof(myRecStruct))
  • in your application's sketch open the table with db.open(address)
  • use the define DB_REC to cast your structure as a byte pointer when passing it as a parameter for write and read operations, this helps make the code more readable.

example snippet:

#include <EEPROM.h>
#include <DB.h>
DB db;
#define MY_TABLE 128
struct MyRec {
  byte id;
  char name[9];
  int    val;
} myrec;

  db.create(MY_TBL,sizeof(myrec));
  myrec.id = 1;
  myrec.name[0]='O';
  myrec.name[1]='N';
  myrec.name[2]='E';
  myrec.name[30=0;
  db.append(DB_REC myrec);
   .....

  db.open(MY_TBL);
  for (int i=1;i<db.nRecs();i++)
  {
    db.read(i, DB_REC myrec);
    Serial.println(myrec.name);
  }
\

DB.h

/*
  DB.h
  Database library for Arduino 
  Written by Madhusudana das
  
  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

#ifndef DB_PROM
#define DB_PROM

#include "EEPROM.h"

struct DB_Header
{
  byte n_recs;
  byte rec_size;
};

// slightly padded for the time being
#define DB_HEAD_SIZE 4

// DB_error values
#define DB_OK 0
#define DB_RECNO_OUT_OF_RANGE 1

#define DB_REC (byte*)(void*)&

typedef byte* DB_Rec;

class DB {
  public:
    void    create(int head_ptr, byte recsize);
    void    open(int head_ptr);
    boolean write(byte recno, const DB_Rec rec);
    boolean read(byte recno, DB_Rec rec);
    boolean deleteRec(byte recno);                      // delete is a reserved word
    boolean insert(byte recno, const DB_Rec rec);
    void    append(DB_Rec rec);
      byte   nRecs();
    DB_Header DB_head;
    byte DB_error;
  private:
    int writeHead();
    int readHead();
    int EEPROM_dbWrite(int ee, const byte* p);
    int EEPROM_dbRead(int ee, byte* p);
    int DB_head_ptr;
    int DB_tbl_ptr;
};

extern DB db;

#endif

DB.cpp

// DB.cpp
/*
  Database library for Arduino 
  Written by Madhusudana das
  
  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

#include "WProgram.h"
#include "DB.h"

/**************************************************/
// private functions
int DB::writeHead()
{
    byte * p = (byte*)(void*)&DB_head;
      int ee = DB_head_ptr;
    int i;
    for (i = 0; i < (int)sizeof(DB_head); i++)
      EEPROM.write(ee++, *p++);
    return i;
}

int DB::readHead()
{
    byte* p = (byte*)(void*)&DB_head;
      int ee = DB_head_ptr;
    int i;
    for (i = 0; i < (int)sizeof(DB_head); i++)
      *p++ = EEPROM.read(ee++);
    return i;
}

int DB::EEPROM_dbWrite(int ee, const byte* p)
{
    int i;
    for (i = 0; i < DB_head.rec_size; i++)
      EEPROM.write(ee++, *p++);
    return i;
}

int DB::EEPROM_dbRead(int ee, byte* p)
{  
    int i;
    for (i = 0; i < DB_head.rec_size; i++)
      *p++ = EEPROM.read(ee++);
    return i;
}

/**************************************************/
// public functions

void DB::create(int head_ptr, byte recsize)
{
  DB_head_ptr = head_ptr;
  DB_head.n_recs   = 0;
  DB_head.rec_size = recsize;
  writeHead();
}

void DB::open(int head_ptr)
{
  DB_head_ptr = head_ptr;
  DB_tbl_ptr  = head_ptr + DB_HEAD_SIZE;
  readHead();
}
//other operations commit DB_head edits to EEPROM so no need for a DB_close


boolean DB::write(byte recno, const DB_Rec rec)
{
  DB_error = DB_OK;
  if (recno>0 && recno<=DB_head.n_recs+1)
    EEPROM_dbWrite(DB_tbl_ptr+((recno-1)*DB_head.rec_size), rec);
  else
    DB_error = DB_RECNO_OUT_OF_RANGE;
  return DB_error==DB_OK;
}


boolean DB::read(byte recno, DB_Rec rec)
{
  DB_error = DB_OK;
  if (recno>0 && recno<=DB_head.n_recs)
    EEPROM_dbRead(DB_tbl_ptr+((recno-1)*DB_head.rec_size), rec);
  else
    DB_error = DB_RECNO_OUT_OF_RANGE;
  return DB_error==DB_OK;
}


boolean DB::deleteRec(byte recno)
{
  DB_error = DB_OK;
  if (recno<0 || recno>DB_head.n_recs)
  {  Serial.println("recno out of range");
    DB_error = DB_RECNO_OUT_OF_RANGE;
    return false;
  }
  DB_Rec rec = (byte*)malloc(DB_head.rec_size);
  for (int i=recno+1; i<=DB_head.n_recs; i++)
  {
    read(i,rec);
    write(i-1,rec);
  }  
  free(rec);
  DB_head.n_recs--;
  EEPROM.write(DB_head_ptr,DB_head.n_recs);
  return true;
}


boolean DB::insert(byte recno, DB_Rec rec)
{
  DB_error = DB_OK;
  if (recno<0 || recno>DB_head.n_recs)
  {  Serial.println("recno out of range");
    DB_error = DB_RECNO_OUT_OF_RANGE;
    return false;
  }
  DB_Rec buf = (byte*)malloc(DB_head.rec_size);
  for (int i=DB_head.n_recs; i>=recno; i--)
  {
    read(i,buf);
    write(i+1,buf);
  }
  free(buf);
  write(recno,rec);  
  DB_head.n_recs++;
  EEPROM.write(DB_head_ptr,DB_head.n_recs);
  return true;
}

void DB::append(DB_Rec rec)
{
  DB_error = DB_OK;
  DB_head.n_recs++;
  write(DB_head.n_recs,rec);
  EEPROM.write(DB_head_ptr,DB_head.n_recs);
}

byte DB::nRecs()
{
  return DB_head.n_recs;
}

Very nice, thanks.

Thanks, this will be useful!

Looks useful, is it well enough polished to go into the playground?

It seems bug free enough. I'm not sure if I mentioned earlier that I used a BYTE type for the record number which limits tables to 255 records, but that seems like plenty for Arduino memory. An integer based library would be nice for SD memory card applications, but I imagine that's getting off the beaten path for most Arduino users.

I took a peek at the playground but I was a little intimidated by the whole procedure as I'm totally unfamiliar with this "wiki" thing, even though I've been writing HTML for years. A little more encouragement and I might take it on, unless someone else wants to volunteer :slight_smile:

Pick an entry that looks nice, click edit, and copy the source :slight_smile: Dump it into your favorite text editor and then go through and replace the content with your own.

Cool library, I'm looking forward to using it.

Well I can't say I had a good experience with the @!%&* wiki stuff. Give me a simple html text edit anyday over all these layers of smoke and mirrors!

Anyway, I've created a basic page in the playground for this now:
http://www.arduino.cc/playground/Code/DatabaseLibrary

Enjoy

Cool, looks good. I'm working on some C# code to marshal structs from the AVR onto the computer, it looks like it will work nicely with this database library.

I'm trying to adapt this library to write to an AT24C1024B eeprom providing for about 128KB (up to 512KB) of storage for use as a logging queue. I saw you mentioned you used the BYTE type for the record number, so I could quickly outgrow the max record limitation.

In short, I changed n_recs to an unsigned long which seems to work (able to write and retrieve >256 records). However, as soon as the DB Header is re-read, the value from db.nRecs() switches from 300 to 44. So I guess something is overflowing when the header is read but I can't figure out what.

Before I describe what I tried to do, the standard disclaimers apply: sorry for a long post, not a programmer, don't know C++, etc. :wink:

First, I had to make a new EEPROM library that accommodates the larger 17 bit addresses used my my EEPROM and functions the same as the existing Arduino EEPROM library. This piece seems to be working fine as I've verified I'm able to write and read values from the entire EEPROM address space.

E24C1024.h:

//This is based on the Arduino EEPROM library.  
#ifndef E24C1024_h
#define E24C1024_h

#include <WConstants.h>
#include <Wire.h>

#define WORD_MASK 0xFFFF

class E24C1024
{
  public:
    E24C1024();
    void write(unsigned long, uint8_t);
    uint8_t read(unsigned long);
};

extern E24C1024 EEPROM1024;

#endif

E24C1024.cpp:

#include <Wire.h>
#include "E24C1024.h"

E24C1024::E24C1024(void)
{
   Wire.begin();
}

void E24C1024::write(unsigned long dataAddress, uint8_t data)
{
   Wire.beginTransmission((uint8_t)((0x500000 | dataAddress) >> 16)); // B1010xxx
   Wire.send((uint8_t)((dataAddress & WORD_MASK) >> 8)); // MSB
   Wire.send((uint8_t)(dataAddress & 0xFF)); // LSB
   Wire.send(data);
   Wire.endTransmission();
   delay(5);
}

uint8_t E24C1024::read(unsigned long dataAddress)
{
   uint8_t data = 0x00;
   Wire.beginTransmission((uint8_t)((0x500000 | dataAddress) >> 16)); // B1010xxx
   Wire.send((uint8_t)((dataAddress & WORD_MASK) >> 8)); // MSB
   Wire.send((uint8_t)(dataAddress & 0xFF)); // LSB
   Wire.endTransmission();
   Wire.requestFrom(0x50,1);
   if (Wire.available()) data = Wire.receive();
   return data;
}

E24C1024 EEPROM1024;

Now I'm getting stuck trying to decipher the pointer magic that's happening in the DB library. I get that n_recs needs to be bigger than a BYTE, so I tried changing it to unsigned long which seemed to work... unti I add more than 256 items.

Current code (renamed everything to EDB to avoid conflicts with DB):

EDB.h:

/*
  EDB.h
  Database library for Arduino 
  Written by Madhusudana das
  License: LGPL
*/

#ifndef EDB_PROM
#define EDB_PROM
#include "E24C1024.h"

struct EDB_Header
{
  unsigned long n_recs;
  byte rec_size;
};

// slightly padded for the time being
#define EDB_HEAD_SIZE 10

// EDB_error values
#define EDB_OK 0
#define EDB_RECNO_OUT_OF_RANGE 1

#define EDB_REC (byte*)(void*)&

typedef byte* EDB_Rec;

class EDB {
  public:
    void    create(int head_ptr, byte recsize);
    void    open(int head_ptr);
    boolean write(unsigned long recno, const EDB_Rec rec);
    boolean read(unsigned long recno, EDB_Rec rec);
    boolean deleteRec(unsigned long recno);
    boolean insert(unsigned long recno, const EDB_Rec rec);
    void    append(EDB_Rec rec);
        unsigned long nRecs();
    EDB_Header EDB_head;
    byte EDB_error;
  private:
    int writeHead();
    int readHead();
    int EEPROM_dbWrite(int ee, const byte* p);
    int EEPROM_dbRead(int ee, byte* p);
    int EDB_head_ptr;
    int EDB_tbl_ptr;
};
extern EDB edb;
#endif

EDB.cpp:

/*
EDB.cpp
Database library for Arduino 
Written by Madhusudana das
License: LGPL 
*/

#include "WProgram.h"
#include "EDB.h"

/**************************************************/
// private functions
int EDB::writeHead()
{
  byte * p = (byte*)(void*)&EDB_head;
      int ee = EDB_head_ptr;
  int i;
  for (i = 0; i < (int)sizeof(EDB_head); i++)
    EEPROM1024.write(ee++, *p++);
  return i;
}

int EDB::readHead()
{
  byte* p = (byte*)(void*)&EDB_head;
      int ee = EDB_head_ptr;
  int i;
  for (i = 0; i < (int)sizeof(EDB_head); i++)
    *p++ = EEPROM1024.read(ee++);
  return i;
}

int EDB::EEPROM_dbWrite(int ee, const byte* p)
{
  int i;
  for (i = 0; i < EDB_head.rec_size; i++)
    EEPROM1024.write(ee++, *p++);
  return i;
}

int EDB::EEPROM_dbRead(int ee, byte* p)
{  
  int i;
  for (i = 0; i < EDB_head.rec_size; i++)
    *p++ = EEPROM1024.read(ee++);
  return i;
}

/**************************************************/
// public functions

void EDB::create(int head_ptr, byte recsize)
{
  EDB_head_ptr = head_ptr;
  EDB_head.n_recs   = 0;
  EDB_head.rec_size = recsize;
  writeHead();
}

void EDB::open(int head_ptr)
{
  EDB_head_ptr = head_ptr;
  EDB_tbl_ptr  = head_ptr + EDB_HEAD_SIZE;
  readHead();
}
//other operations commit EDB_head edits to EEPROM1024 so no need for a EDB_close

boolean EDB::write(unsigned long recno, const EDB_Rec rec)
{
  EDB_error = EDB_OK;
  if (recno>0 && recno<=EDB_head.n_recs+1)
    EEPROM_dbWrite(EDB_tbl_ptr+((recno-1)*EDB_head.rec_size), rec);
  else
    EDB_error = EDB_RECNO_OUT_OF_RANGE;
  return EDB_error==EDB_OK;
}


boolean EDB::read(unsigned long recno, EDB_Rec rec)
{
  EDB_error = EDB_OK;
  if (recno>0 && recno<=EDB_head.n_recs)
    EEPROM_dbRead(EDB_tbl_ptr+((recno-1)*EDB_head.rec_size), rec);
  else
    EDB_error = EDB_RECNO_OUT_OF_RANGE;
  return EDB_error==EDB_OK;
}


boolean EDB::deleteRec(unsigned long recno)
{
  EDB_error = EDB_OK;
  if (recno<0 || recno>EDB_head.n_recs)
  {  Serial.println("recno out of range");
    EDB_error = EDB_RECNO_OUT_OF_RANGE;
    return false;
  }
  EDB_Rec rec = (byte*)malloc(EDB_head.rec_size);
  for (int i=recno+1; i<=EDB_head.n_recs; i++)
  {
    read(i,rec);
    write(i-1,rec);
  }  
  free(rec);
  EDB_head.n_recs--;
  EEPROM1024.write(EDB_head_ptr,EDB_head.n_recs);
  return true;
}

boolean EDB::insert(unsigned long recno, EDB_Rec rec)
{
  EDB_error = EDB_OK;
  if (recno<0 || recno>EDB_head.n_recs)
  {  Serial.println("recno out of range");
    EDB_error = EDB_RECNO_OUT_OF_RANGE;
    return false;
  }
  EDB_Rec buf = (byte*)malloc(EDB_head.rec_size);
  for (int i=EDB_head.n_recs; i>=recno; i--)
  {
    read(i,buf);
    write(i+1,buf);
  }
  free(buf);
  write(recno,rec);  
  EDB_head.n_recs++;
  EEPROM1024.write(EDB_head_ptr,EDB_head.n_recs);
  return true;
}

void EDB::append(EDB_Rec rec)
{
  EDB_error = EDB_OK;
  EDB_head.n_recs++;
  write(EDB_head.n_recs,rec);
  EEPROM1024.write(EDB_head_ptr,EDB_head.n_recs);
}

unsigned long EDB::nRecs()
{
  return EDB_head.n_recs;
}

I used this sketch to create a database and populate it with 300 records:

//sample code snippet
#include "WProgram.h"
#include <Wire.h>
#include <E24C1024.h>
#include <EDB.h>
#include "string.h"

EDB db;

#define LOG_TABLE 0
#define CREATE_DB true

struct LogEvent {
  int id;
  char date[11];
  char time[9];
  int temperature;
} logEvent;

void setup()
{
  Serial.begin(9600);
  randomSeed(analogRead(0));
  if (CREATE_DB) db.create(LOG_TABLE, sizeof(logEvent));
  db.open(LOG_TABLE);
  
  if (CREATE_DB)
  {
    for (int j = 1; j <= 300; j++)
    {
      logEvent.id = j; 
      strncpy(logEvent.date, "2009-11-13", 11);
      strncpy(logEvent.time, "12:00:00", 9);
      logEvent.temperature = random(1, 125);
      db.append(EDB_REC logEvent);
      Serial.println("DONE");  
    }
  }

  Serial.print("Record Size: "); Serial.println(sizeof(logEvent));
  Serial.print("Record Count: "); Serial.println(db.nRecs());
  selectAll();
}

void loop()
{
}

void selectAll()
{
   for (int i = 1; i <= db.nRecs(); i++)
   {
     db.read(i, EDB_REC logEvent);
     Serial.print("RecNum: "); Serial.println(i);
     Serial.print("ID: "); Serial.println(logEvent.id);
     Serial.print("Date: "); Serial.println(logEvent.date); 
     Serial.print("Time: "); Serial.println(logEvent.time);
     Serial.print("Temp: "); Serial.println(logEvent.temperature);
     Serial.println();  
  }
}

This worked fine. The select all looped through all 300 records.

Finally the problem:

When I change CREATE_DB to false, and re-upload, nRecs() returns 44 records. So something is overflowing but I can't figure out what. I'm confused why it seemingly works until I re-read the database header.

Is anyone interested in helping make this work on larger amounts of memory? Hopefully what I did isn't too far off the mark and one of you programmer types can help connect the dots.

An integer based library would be nice for SD memory card applications, but I imagine that's getting off the beaten path for most Arduino users.

Actually thats exactly what I'm after if you can get it to go .
So I can use it as a logger direct

Also how many tables can I create. Could make one called today, one today+1, today+2 etc?
I would be working toward say 250 readings a day stored in each table and say a table a day for 365 days?

Good stuff

didn't get it all. I'm no programmer and came to c++ with arduino.
But there's a typo in the playgroundsite:

when filling the table, the 30th arrayelement has a missing "]".

I tried giving it 250 records under the example program in playground...I changed temperature to a float

I got a lot of temperature=0 . No time at all on many.Dates messed with . The write operation seemed correct.

80-120 OK
208-250 Ok The rest are screwed up
Is this normal running the example , does it make any mistakes in practical use?

Recnum: 1

Date: 28

Time: 

Temperature: 0.00

-----

Recnum: 2

Date: 05

Time: 

Temperature: 0.00

-----

Recnum: 3

Date: 13

Time: 

Temperature: 0.00

-----

Recnum: 4

Date: 24

Time: 

Temperature: 0.00

-----

Recnum: 5

Date: 24

Time: 

Temperature: 0.00

-----

Recnum: 6

Date: 01

Time: 

Temperature: 0.00

-----

Recnum: 7

Date: 19

Time: 

Temperature: 0.00

-----

Recnum: 8

I changed temperature to a float

I'm not that experianced on the software side, and haven't used this library yet (but plan to sometime) so I could be wrong on the following guess.

Changing the variable to a float from a int increases the memory requirement for each record (2 bytes Vs 4 bytes per variable) so you may be exceeding the size of the EEPROM memory, wrapping around and clobbering records?

Lefty

Mmm. I had to reduce it to 40 records before they looked correct . I think there is something else going on as well

With the example in the playground, each record is 22 bytes long, so in 1K of EEPROM you could get at most 46 records.
Changing the last field to a float makes that 24 bytes per record, so at most 42 records.
(double those for a Mega)
[edit]Groove RTFM's - wow! Only 512 bytes for a 168 (23 / 21 records) 1K for a 328 (46 / 42 records) but a whopping 4K bytes for a Mega (still only 186 / 170 records)[/edit]

If your address pointer ever got larger than 256, you'd experience issues. I made some additional changes to the library and posted them here:

http://www.arduino.cc/playground/Code/ExtendedDatabaseLibrary

I've successfully added 1300+, 45 byte records and retrieved them with no issues using the Extended Database Library and a 512 kilobyte AT24C1024 eeprom.

Anyone have a source and price for the AT24C1024 in DIP package? A quick check on goggle only showed datasheets.

Lefty

I got mine at digikey for about $4 each.

I have just got EDB_24XX512 Example working with my 24AA1025.

The problem i'm having is storing Text in the database field.

example:
Id = 1
Date = 24/03/2010
Time = 16:00
Value = 1023

I ether get the first letter of the string or a "0".

I have tried changing

struct LogEvent {
  int id;
  int Date;
  int Time;
  int Value;
}

to

struct LogEvent {
  int id;
  char Date;
  char Time;
  char Value;
}

What am I doing wrong?

Thanks in Advance

Post the entire Sketch.