EEPROM Management

As I've had occasion to note elsewhere on this forum elsewhere, that handy EEPROM on most or all AVR chips is seldom used.

I've been playing with LCD's lately (and thanks to everyone who has provided code!) and noticed that the "character generator ROM", which either is or could be declared constant is included in sketch after sketch. Sometimes it's accessed from PGM space, sometimes not. How about storing it in that otherwise unused EEPROM?

Okay, so here are some needs: how does the sketch know if the data is there? how does it find it? how do I remember what I burned where?

After going over a few options, I came up with a (very) simple directory structure, just a two-letter "file name" and a pointer to the base address of the data block. The "directory" starts at the top (high addresses) of the EEPROM and grows downwards, the data blocks are stored at the beginning of the EEPROM (low address) and grow upwards. To be consistent, I stored the names and addresses in little endian order.

Here's my code (the EMULATE def and ifdef's let me develop it in gcc before debugging on Arduino):

/* burn and/or verify eeprom
 * using #include file provided.
 *
 * DdelV 20140320
 *
 * This version implements directory management
 * 
 * 
 */
 
//#define EMULATE

#ifndef EMULATE
#include <EEPROM.h>
#include <progmem.h>
#include <StreamPrint.h>
#else
#include <stdio.h>
#include <stdint.h>
#include "emulate.h"
#include "EEPROM.h"
#endif

/* the user-supplied include (eeprom data and parameters)
 * must define
 * uint8_t fileName[2] = { 'X','X' };	// use unique character pair for directory entry

 * static const unsigned char Buff[] = { <data> };
 */
#include "ASCII5x7font.h"
//#include "EEPROM_spec.h"

uint16_t fileSize = sizeof(Buff);

#define E2PROMSIZE E2END+1

#ifdef EMULATE
#define readEEPROM(a) EEPROMread(a)
#define writeEEPROM(a,d) EEPROMwrite(a,d)
extern unsigned char EEPROMread(int );
extern void EEPROMwrite(int, unsigned char );
#else 
#define readEEPROM(a) EEPROM.read(a)
#define writeEEPROM(a,d) EEPROM.write(a,d)
#endif

#ifndef EMULATE
char myGetch() {
  while(Serial.available() == 0) {
    ;                    // wait here (busy)
  }
  return Serial.read();
}
#else
char myGetch() {
  char c;
  
  while(!isprint(c=getchar())) { ; }
  return c;
}
#endif

void gobble() {
#ifndef EMULATE
  while (Serial.available())Serial.read();
#endif  
}

char Menu() {
  
  gobble();

  Serialprint("\nB)urn EEPROM\n");
  Serialprint("D)ump EEPROM\n");
  Serialprint("S)how Directory\n");
  Serialprint("V)erify EEPROM\n");
  Serialprint("W)ipe EEPROM\n");
  Serialprint("Q)uit\n");
  Serialprint("\nChoose: ");
  
  return myGetch();
  
}
int yesno() {
// return true if user types 'y' or 'Y', false otherwise.

  switch (myGetch()) {
  case 'Y': 
  case 'y': 
    return 1;
  default: 
    return 0;
  }
}
void showParams(uint16_t address, uint16_t size) {
  
  Serialprint("%d bytes to be written to EEPROM at address: %04x\n",size,address);
   
}
void verifyError(uint16_t address, uint8_t sb, uint8_t is) {
  
  Serialprint("Verify error! Address: %04x should be %02x, is %02x\n",address, sb, is);
    
}

typedef struct {
  uint16_t ep;
  uint8_t nm[2];
  uint16_t fp;
} eepdir ;

eepdir dir;

uint16_t isNM(uint16_t f) {
  if (isupper(f >> 8) && isupper(f & 127))
    return 1;
  else
    return 0;
}
uint16_t isEEaddr(uint16_t f) {
  if (f <= E2END)
    return 1;
  else
    return 0;
}

uint16_t fetch16(uint16_t eepp) {
  uint16_t field;
// Little endian  
  
  field = (readEEPROM(eepp--)) << 8;
  field += readEEPROM(eepp);
  return field;
}
// for debug:
void dumpDir(uint16_t ep) {
  printf("ep: %04x\nnm: %c%c (%02x%02x)\nfp: %04x\n\neep=%04x\n",dir.ep,dir.nm[0],dir.nm[1],dir.nm[0],dir.nm[1],dir.fp,ep);
}

uint16_t burnFile(uint16_t S, uint8_t *Name) {
    
  Serialprint("Burning included file to EEPROM\nPlease wait...\n");

  uint16_t eep = E2END;
  uint16_t i,w;
  dir.ep = 0;
  
  // find the freepointer,
  do {
    dir.nm[0] = fetch16(eep) & 0xff;
    dir.nm[1] = fetch16(eep) >> 8;
    if (dir.nm[0] & dir.nm[1] == 0xff) break;
    eep -= 2;
    if ((dir.fp = fetch16(eep--)) == 0xffff)
      break;
    eep -= 2;
    if ((dir.ep = fetch16(eep)) == 0xffff)
      break;
    eep -= 2;
  } while (eep);	// search entire EEPROM if needed.
  
  // now sort out conditions
//  dumpDir(eep);
  if (E2END == eep) {	// new directory, add name and freepointer
    Serialprint("New Directory\n");
    dir.nm[0] = Name[0];
    dir.nm[1] = Name[1];
    dir.fp = S + 1;
  } else if (dir.ep == 0xffff) {	// un-named block has freepointer
    Serialprint("New Entry\n");
    dir.ep = dir.fp;
    dir.nm[0] = Name[0];
    dir.nm[1] = Name[1];
    dir.fp = dir.ep + S + 1;
  }
//  dumpDir(eep);
  if (dir.fp < (eep-4)) {
    Serialprint("Okay to burn\n");
    Serialprint("Burning Directory entry\n");
    if(dir.ep)eep++;
    writeEEPROM(eep,dir.nm[1]);
    writeEEPROM(--eep,dir.nm[0]);
    writeEEPROM(--eep,(dir.fp >> 8));
    writeEEPROM(--eep,(dir.fp & 0xff));
// ensure sentinal is present.    
    writeEEPROM(--eep,0xff);
    writeEEPROM(--eep,0xff);
    Serialprint("Burning block\n");
    w = dir.ep;
    for (i=0; i<S; i++) {
      writeEEPROM(w++,Buff[i]);
    }
    Serialprint("Done!\n");
  }
  else Serialprint("Sorry! Not enough room.\n");
  
  return dir.ep;

}
void verifyFile(uint16_t A, uint16_t S) {
  int i, E=0;
  uint8_t D;
  
  Serialprint("Verifying include file versus EEPROM\n");
  for (i=0 ; i<S ; i++) {
    if (Buff[i] != (D = readEEPROM(A))) {
      verifyError(A,Buff[i],D);
      E++;    // count errors
    }
    ++A;
  }
  char c = (E==1)?'.':'s';
  Serialprint("%d verify error%c.\n",E,c);
}
void dumpEEPROM() {
#define NCOLS 32
int i, addr;
uint8_t value;

// mostly pretty-printing...
  Serialprint("\n     ");
  for (i=0; i<NCOLS; i++) {
    Serialprint("%02x ",i);
  }
  Serialprint("\n---+");
  for (i=0; i<NCOLS; i++) {
    Serialprint("---");
  }
  
  for (addr=0; addr<=E2END; addr++) {
    if((addr % NCOLS) == 0) {
      Serialprint("\n%03x: ",addr);
    } 
    else {
      Serialprint(" ");
    }
    value = readEEPROM(addr);    // there! did something.
    Serialprint("%02x",value);  
  }
}
void wipeEEPROM() {
  int i;
  
  Serialprint("This will erase all %d bytes of EEPROM!\n",E2PROMSIZE);
  Serialprint("ARE YOU SURE?");
  if (yesno()) {
    for (i=0; i<=E2END; i++) {
      writeEEPROM(i,255);
    }
    Serialprint("Done.\n");
  }
  else Serialprint("Wipe aborted.\n");
}
void showDirectory() {
  uint16_t eep;
  uint16_t addr = 0;
  char Name[3];
  
  Name[2] = '\0';
  for (eep = E2END ; eep ; eep -= 4) {
    Name[1] = readEEPROM(eep);
    Name[0] = readEEPROM(eep-1);
    if (!isalpha(Name[0]) || !isalpha(Name[1])) break;
    Serialprint("Block named %s at 0x%04x\n",Name,addr);
    addr = fetch16(eep-2);
  }
  Serialprint("Free space @ 0x%04x\n",addr);
}
  
void setup() {
#ifndef EMULATE
  Serial.begin(115200);
  while(!Serial) ;
#endif  
  Serialprint("%d bytes to be written to EEPROM.\n",fileSize);
  Serialprint("Total EEPROM size is %d\n",E2PROMSIZE);
}
void loop() {
  char c;
  uint16_t start;
  
  c = Menu();
  switch(c) {
    case 'b':
    case 'B': start = burnFile(fileSize, fileName);
              break;
    case 'v':
    case 'V': verifyFile(start, fileSize);
              break;
    case 'd':
    case 'D': dumpEEPROM();
              break;
    case 'w':
    case 'W': wipeEEPROM();
	      break;
    case 's':
    case 'S': showDirectory();
	      break;
    case 'q':
    case 'Q': 
#ifndef EMULATE
      while(1) { ; }
#else
      exit(0);
#endif
    default: Serialprint("Bad Command! (%02x) Try again.\n",c);
  }
}

I'm not claiming it's great (kinda flat-footed, which is often best), but it seems to work. Could use some polish.

The "directory" format is a bit wasteful of EEPROM space in that it costs 4 bytes for each block. With a bit more complexity 3 (or even 4) character names could be packed into the 32 bits along with the 10-bit (for 1K EEPROM) address pointers.

Now to get to work on LCD routines that find and use this data (and some benchmarks!).

Any questions or suggestions are welcome.

One of the problems is that Arduinos are configured to erase the EEPROM every time you upload a program.

That makes it difficult to develop this sort of code. How exactly do you put data in EEPROM instead of flash? You can only do it by changing fuse bits, something that most people can't do.

fungus:
One of the problems is that Arduinos are configured to erase the EEPROM every time you upload a program.

I had to go check, because I would have thought this wasn't the case, but indeed it is!

That makes it difficult to develop this sort of code. How exactly do you put data in EEPROM instead of flash? You can only do it by changing fuse bits, something that most people can't do.

EEPROM will only be erased if code is uploaded via ICSP, most people don't see EEPROM get erased because they use the bootloader to upload.

I agree using EEPROM for small sets of static data is a good idea.
I'm using EEPROM to store data based on class instances. As I know the class instances ( and as such the data and data types)and I "programmed" a sequence of class instances I can just start at the bottom and go up.
This method has some drawbacks to. Changing classes (on behalf of how data is stored) makes the data corrupt.
There is some progmem lost due to the knowledge in the code what needs to be stored and what the sequence is.
I guess you always lose something when trying to use something extra.

As you are asking input for how you handle it.
I think 2 chars (=bytes) for a filename is to much. Lets face it with 1K bytes you have 2*255 or 510 file names or 4 bytes per file name.
For space considerations I would go for blocks of 4 bytes (1024/256) that makes that with 1 byte you can point to the start of the data.
If you start your filenames at the bottom of the EEPROM and your data at the top 256=FF is not possible as data pointer. so you can use FF as "end of file name indicator"
The file itself should start with the number of blocks used (1byte) and if a nice filename is wanted number of chars in the file name.

Note that only the last file has possibility to grow and shrink so this can only be used for "a relative static file system"
I hope this helps

Best regards
Jantje

fungus:
One of the problems is that Arduinos are configured to erase the EEPROM every time you upload a program.

That makes it difficult to develop this sort of code. How exactly do you put data in EEPROM instead of flash? You can only do it by changing fuse bits, something that most people can't do.

oh no they don't!
just use the standard EEPROM.h library

I've been using <avr/eeprom.h> some and I'm surprised that more people don't use it. No need to bother with addresses unnecessarily, just define variables with the EEMEM attribute. Plus it supports 16- and 32-bit integers, floats, and blocks of bytes in addition to single bytes.

Get a more recent version of AVR Libc than comes with the Arduino IDE and the eeprom_update_* functions are available. These write values to EEPROM only if they are different from what is already stored there.

Hah! You got me there....

Interesting suggestions, Jantje.
Your 4-byte block ("sector/cluster") idea is interesting, although I believe (without working out the proofs) that "fragmentation" due to blocksize % 4 != 0 looses you more bytes than my simple scheme. If it were really crucial to save space, and you were willing to limit filenames to upper or lower case + digits only, you could pack them (like DEC's Radix-50, remember that?). A ten-bit address field would leave room for four character names packed into the four bytes with address (and a couple of bits left over). Honestly, the number of blocks is specious in a mere 1024 bytes. You might have three or so. So the names can be quite short without collision. For example, my LCD character generator is named 'CG'.
My scheme uses name = 0xffff as the marker for the free pointer. No need for file sizes and files are stored contiguously, so size is difference between adjacent directory entries, or to the free pointer at the end.
I have made a few improvements to the code. I'll post if anyone cares. I do only write if the value already there is different, but the EEPROM can be re-written 10 times as often as the Flash!
And, even if you use ISP to program, use Nick Gammon's programmer and you can ask it not to erase the EEPROM. Flashing with the bootloader leaves it intact, at least on the ATmega328(p)'s I use.