Ethercard success, but duplicate buffers sent to callback - SOLVED

I am using an ethernet module and the ethercard library.

In my project, I want to read data from a web page ( a php script that echos a page of data ) and then save that data to a file on the SD card.

I have the ethernet module working and communicating with the online web page, and receiving the data.

I also have the writing and reading of the data to / from the SD card.

Am using a Mega 2560 board, with common pins for ethercard and SDcard on pins 50, 51, 52. Pin 53 is set as Output, and ethercard uses CS pin 46, and SD card CS pin 42.

No problems so far, except that some of the packets received in the Ethernet buffer sent to the callback are duplicated, and I can't see any reason why this would be so, or how to overcome this ( except for having to read all the data in the SD card file and check that the buffer data received in the callback has not already been saved to the SD card.

In the Serial Monitor data below, the following are duplicated :

my_callback initiated. Block : 3

is a duplicate of :

my_callback initiated. Block : 1
my_callback initiated. Block : 5

is a duplicate of :

my_callback initiated. Block : 4

The working code that I have so far is :

#include <EtherCard.h>
#include <SPI.h>
#include <SD.h>

// ethernet interface mac address, must be unique on the LAN
static byte mymac[] = { 0x74,0x69,0x70,0x2D,0x30,0x31 };
static byte static_ip[] = { 192,168,1,219 }; //local IP of the ethercard module
static byte static_gw[] = { 192,168,1,254 }; // router IP address
static byte static_dns[] = { 192,168,1,254 }; // dns via router

byte Ethernet::buffer[700];  // buffer for ethercard to pass received data to callback function
byte ebuf2[700];  // array used to copy Etehrnet::buffer for use in the callback
int CallBackBlockCount = 0;

static uint32_t timer;
const char website[] PROGMEM = "www.galatime.co.za";

boolean SDavailable;
int LoopCntr = 0;
char TXbuf[20];

// SD Card variables
char SDvar1[5];
char SDvar2[5];
char SDvar3[5];
char SDvar4[5];
char SDvar5[5];


void ClearSDvars(){                 // Clears all the temporary holding variables used when reading data from the SD card
 memset(SDvar1, 0, sizeof(SDvar1)); SDvar1[0] = '\0';
 memset(SDvar2, 0, sizeof(SDvar2)); SDvar2[0] = '\0';
 memset(SDvar3, 0, sizeof(SDvar3)); SDvar3[0] = '\0';
 memset(SDvar4, 0, sizeof(SDvar4)); SDvar4[0] = '\0';
 memset(SDvar5, 0, sizeof(SDvar5)); SDvar5[0] = '\0';
}


void WriteSDfile(int WriteFileNo = 0, int RemFile = 0, char* wrTxt = ""){

 Serial.println("......................................");
 Serial.println("WriteSDfile got data : ...............");
 Serial.println("......................................");
 Serial.println(wrTxt);
 Serial.println("......................................");
 Serial.println("End of Data ..........................");
 Serial.println("......................................");

        File fh;
 if(RemFile == 1){
 if (WriteFileNo == 1) SD.remove("file1.txt");
 if (WriteFileNo == 2) SD.remove("file2.txt");
 if (WriteFileNo == 3) SD.remove("file3.txt");
 } 

 if (WriteFileNo >= 1 && WriteFileNo <= 3){

 // open the file. note that only one file can be open at a time, so you have to close this one before opening another.
 if (WriteFileNo == 1) fh = SD.open("file1.txt", FILE_WRITE);
 if (WriteFileNo == 2) fh = SD.open("file2.txt", FILE_WRITE);
 if (WriteFileNo == 3) fh = SD.open("file3.txt", FILE_WRITE);

 if (fh) { // if the file opened okay, write to it:
 Serial.print("Writing to file ...");
 fh.write(wrTxt);
 fh.close(); // close the file:
 Serial.println("done.");
       Serial.println("......................................");
       Serial.println("......................................");
 } else {
 Serial.println("error opening SD file for writing");  // if the file didn't open, print an error:
 }
 }
}


void ReadSDfile(int ReadFileNo = 0){

 if (ReadFileNo >= 1 && ReadFileNo <= 3){
 File fh;
 Serial.println("......................................");
 Serial.println("......................................");
 Serial.print("SD Read File No : ");
 Serial.println(ReadFileNo);
 Serial.print("SD Available : ");
 Serial.println(SDavailable);
 Serial.println("......................................");
 Serial.println("......................................");
 if(SDavailable != false) {
 if (ReadFileNo == 1) fh = SD.open("file1.txt", FILE_READ);
 if (ReadFileNo == 2) fh = SD.open("file2.txt", FILE_READ);
 if (ReadFileNo == 3) fh = SD.open("file3.txt", FILE_READ);
 int useSDvar = 1;
 int chPos = 0;
 int lineNo = 0;
 ClearSDvars();
 if(!fh) {
 Serial.println("Error : SD open fail");
 } else {
 while(fh.available()) {
 char ch = fh.read();                // read 1 char at a time from the SD file
 if (ReadFileNo != 2){
 if(ch == '\n') {                    //  \n = LF  ( used for all OS ) - End of Line - do something with the populated SDvars 
 // move the holding variables data to the data arrays
 // echo the vars to the Serial monitor
 Serial.print("var 1 : "); Serial.println(SDvar1);
 Serial.print("var 2 : "); Serial.println(SDvar2);
 Serial.print("var 3 : "); Serial.println(SDvar3);
 Serial.print("var 4 : "); Serial.println(SDvar4);
 Serial.print("var 5 : "); Serial.println(SDvar5);
 Serial.println("----------------------");
 ClearSDvars();                  // clear the holding variables
 useSDvar = 1;                   // move to the first holding variable
 chPos = 0;                      // move back to the first char position in the new holding variable
 lineNo++;
 } else if(ch == '\r') {           //  \r = CR  ( used by Windows only )
 // do nothing - ignore the \r because we reacted to the \n
 } else if(ch == ',') {        // this is a comma to delimit fields
 useSDvar++;                 // move to the next holding variable
 chPos = 0;                  // move back to the first char position in the new holding variable
 } else if(chPos < 59) {   // not \n or \r so deal with read char
 if(useSDvar == 1) SDvar1[chPos] = ch;  // add the char to the holding variable at position chPos
 if(useSDvar == 2) SDvar2[chPos] = ch;
 if(useSDvar == 3) SDvar3[chPos] = ch;
 if(useSDvar == 4) SDvar4[chPos] = ch;
 if(useSDvar == 5) SDvar5[chPos] = ch;
 chPos++;                // move to the next char position in the holding variable
 }
 }

 if (ReadFileNo == 2){
 Serial.print(ch);  // just dump the file content to the serial monitor
 }

 } // end of : while(fh.available)
 fh.close();
 }                         // end of : if SD opened OK
 }else{
 Serial.println("Error : SD connect failed");
 }
 ClearSDvars();                // clear the holding variables as the data has been moved to the data arrays
 }
}



}
// code continued ...

// called when the client request is complete
static void my_callback (byte status, word off, word len) {

        CallBackBlockCount++;
 Serial.print("my_callback initiated. Block : ");
 Serial.println(CallBackBlockCount);

 memcpy( ebuf2, Ethernet::buffer, sizeof(Ethernet::buffer) );  // immediately copy the ethernet buffer to ebuf2, in case the ethernet buffer is modified while my_callback is running
 ebuf2[off+len] = 0;  // after the offset and the length, add an end marker
 Ethernet::buffer[0] = 0;  // clear the ethernet incoming buffer array
 Ethernet::buffer[off] = 0;  // clear the ethernet incoming buffer array

        unsigned int bLen = len;  
        Serial.print("bLen = ");
 Serial.println(bLen);
        if(bLen < 512){
    Serial.println("Disabling Persistant Connection");
          ether.persistTcpConnection(false); // stop receiving more buffers
        }  

 
 Serial.print("Status : "); Serial.println(status);
 Serial.print("off : "); Serial.println(off);
 Serial.print("len : "); Serial.println(len);

 Serial.println((const char*) ebuf2 + off);
 Serial.println("Writing data to File 2");
 WriteSDfile(2,0,((char*) ebuf2 + off));

        if(bLen < 512){   // if this is the end of the page, then read and echo the SD file to the Serial Monitor
 Serial.println("......................................");
 Serial.println("......................................");
         Serial.println("Reading Contents of File 2");
 Serial.println("......................................");
 Serial.println("......................................");
         ReadSDfile(2);
        } 
}


void SendDataToServer(char* SendTxt) {
 Serial.println("SendDataToServer initiated");
 Serial.println("Browse Started");
 ether.persistTcpConnection(true);
 ether.browseUrl(PSTR("/inc/ArdEvents.php?"), SendTxt, website, my_callback);
 Serial.println("Browse Completed");
}


void setup () {

 Serial.begin(57600);
 Serial.println("Setup Started");

 pinMode(53, OUTPUT);
 digitalWrite(53, HIGH);  // set pin 53 as Output and High otherwise SPI will not work

 SDavailable = SD.begin(42);  // set the CS pin number for the SD card here
 Serial.println("SD Card Started");
 Serial.print("SD Available = ");
 Serial.println(SDavailable);

 if (ether.begin(sizeof Ethernet::buffer, mymac, 46) == 0){ 
 Serial.println(F("Failed to access Ethernet controller"));
 }else{
 ether.staticSetup(static_ip,static_gw,static_dns);
 Serial.println("Ethercard Started");
 }
 
 ether.printIp("IP:  ", ether.myip);
 ether.printIp("GW:  ", ether.gwip);  
 ether.printIp("DNS: ", ether.dnsip);  

 if (!ether.dnsLookup(website)) Serial.println("DNS failed");
 ether.printIp("SRV: ", ether.hisip);

}

void loop () {

 ether.packetLoop(ether.packetReceive());  // this line must be at the start of the loop

 if(LoopCntr == 0){
 LoopCntr = 1;
 SD.remove("file2.txt");  // remove the file from the SD card
 Serial.println();
 Serial.print("<<< REQ ");
 sprintf(&(TXbuf[0]), "editgala=%d", 5);
 SendDataToServer(TXbuf);  // get the data from the php page
 }

The Serial Monitor that I am getting back :

Setup Started
SD Card Started
SD Available = 1
Ethercard Started
IP:  192.168.1.219
GW:  192.168.1.254
DNS: 192.168.1.254
SRV: 196.22.142.58

<<< REQ SendDataToServer initiated
Browse Started
Browse Completed
my_callback initiated. Block : 1
bLen = 512
Status : 0
off : 54
len : 512
HTTP/1.1 200 OK
Date: Fri, 19 Dec 2014 21:52:30 GMT
Server: Apache
X-Powered-By: PHP/5.4.35-0+deb7u2
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Pragma: no-cache
Set-Cookie: PHPSESSID=1u6usf7u7u4n60rc85bmklu927; path=/
Vary: Accept-Encoding
Connection: close
Content-Type: text/html



<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="htt
Writing data to File 2
......................................
WriteSDfile got data : ...............
......................................
HTTP/1.1 200 OK
Date: Fri, 19 Dec 2014 21:52:30 GMT
Server: Apache
X-Powered-By: PHP/5.4.35-0+deb7u2
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Pragma: no-cache
Set-Cookie: PHPSESSID=1u6usf7u7u4n60rc85bmklu927; path=/
Vary: Accept-Encoding
Connection: close
Content-Type: text/html



<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="htt
......................................
End of Data ..........................
......................................
Writing to file ...done.
......................................
......................................
my_callback initiated. Block : 2
bLen = 512
Status : 1
off : 54
len : 512
p://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
<title>ArdEvents List</title>
<link rel="stylesheet" href="./style.css" type="text/css">
<script src="./jquery-1.11.1.min.js">
</script>
</head>
<body>

E86--P--H1--L101011
E86--P--H2--L011110
E86--P--H3--L011110
E87--P--H1--L111110
E87--P--H2--L111110
E8--P--H1--L001100
E10--P--H1--L001100
E11--P--H1--L011100
E13--P--H1--L011100
E14--P--H1--L011110
E14--P--H2
Writing data to File 2
......................................
WriteSDfile got data : ...............
......................................
p://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
<title>ArdEvents List</title>
<link rel="stylesheet" href="./style.css" type="text/css">
<script src="./jquery-1.11.1.min.js">
</script>
</head>
<body>

E86--P--H1--L101011
E86--P--H2--L011110
E86--P--H3--L011110
E87--P--H1--L111110
E87--P--H2--L111110
E8--P--H1--L001100
E10--P--H1--L001100
E11--P--H1--L011100
E13--P--H1--L011100
E14--P--H1--L011110
E14--P--H2
......................................
End of Data ..........................
......................................
Writing to file ...done.
......................................
......................................
my_callback initiated. Block : 3
bLen = 512
Status : 0
off : 54
len : 512
HTTP/1.1 200 OK
Date: Fri, 19 Dec 2014 21:52:30 GMT
Server: Apache
X-Powered-By: PHP/5.4.35-0+deb7u2
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Pragma: no-cache
Set-Cookie: PHPSESSID=1u6usf7u7u4n60rc85bmklu927; path=/
Vary: Accept-Encoding
Connection: close
Content-Type: text/html



<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="htt
Writing data to File 2
......................................
WriteSDfile got data : ...............
......................................
HTTP/1.1 200 OK
Date: Fri, 19 Dec 2014 21:52:30 GMT
Server: Apache
X-Powered-By: PHP/5.4.35-0+deb7u2
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Pragma: no-cache
Set-Cookie: PHPSESSID=1u6usf7u7u4n60rc85bmklu927; path=/
Vary: Accept-Encoding
Connection: close
Content-Type: text/html



<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="htt
......................................
End of Data ..........................
......................................
Writing to file ...done.
......................................
......................................
my_callback initiated. Block : 4
bLen = 512
Status : 1
off : 54
len : 512
--L011110
E15--P--H1--L001100
E16--P--H1--L011110
E17--P--H1--L011110
E17--P--H2--L011100
E18--P--H1--L111111
E19--P--H1--L111110
E19--P--H2--L111110
E19--P--H3--L011110
E20--P--H1--L011110
E20--P--H2--L011110
E21--P--H1--L111110
E21--P--H2--L011110
E22--P--H1--L111111
E22--P--H2--L111110
E23--P--H1--L011110
E23--P--H2--L011110
E25--P--H1--L001100
E27--P--H1--L001100
E28--P--H1--L011100
E30--P--H1--L011100
E31--P--H1--L011110
E31--P--H2--L011
Writing data to File 2
......................................
WriteSDfile got data : ...............
......................................
--L011110
E15--P--H1--L001100
E16--P--H1--L011110
E17--P--H1--L011110
E17--P--H2--L011100
E18--P--H1--L111111
E19--P--H1--L111110
E19--P--H2--L111110
E19--P--H3--L011110
E20--P--H1--L011110
E20--P--H2--L011110
E21--P--H1--L111110
E21--P--H2--L011110
E22--P--H1--L111111
E22--P--H2--L111110
E23--P--H1--L011110
E23--P--H2--L011110
E25--P--H1--L001100
E27--P--H1--L001100
E28--P--H1--L011100
E30--P--H1--L011100
E31--P--H1--L011110
E31--P--H2--L011
......................................
End of Data ..........................
......................................
Writing to file ...done.
......................................
......................................
my_callback initiated. Block : 5
bLen = 512
Status : 1
off : 54
len : 512
--L011110
E15--P--H1--L001100
E16--P--H1--L011110
E17--P--H1--L011110
E17--P--H2--L011100
E18--P--H1--L111111
E19--P--H1--L111110
E19--P--H2--L111110
E19--P--H3--L011110
E20--P--H1--L011110
E20--P--H2--L011110
E21--P--H1--L111110
E21--P--H2--L011110
E22--P--H1--L111111
E22--P--H2--L111110
E23--P--H1--L011110
E23--P--H2--L011110
E25--P--H1--L001100
E27--P--H1--L001100
E28--P--H1--L011100
E30--P--H1--L011100
E31--P--H1--L011110
E31--P--H2--L011
Writing data to File 2
......................................
WriteSDfile got data : ...............
......................................
--L011110
E15--P--H1--L001100
E16--P--H1--L011110
E17--P--H1--L011110
E17--P--H2--L011100
E18--P--H1--L111111
E19--P--H1--L111110
E19--P--H2--L111110
E19--P--H3--L011110
E20--P--H1--L011110
E20--P--H2--L011110
E21--P--H1--L111110
E21--P--H2--L011110
E22--P--H1--L111111
E22--P--H2--L111110
E23--P--H1--L011110
E23--P--H2--L011110
E25--P--H1--L001100
E27--P--H1--L001100
E28--P--H1--L011100
E30--P--H1--L011100
E31--P--H1--L011110
E31--P--H2--L011
......................................
End of Data ..........................
......................................
Writing to file ...done.
......................................
......................................
my_callback initiated. Block : 6  .. these 3 blocks are not duplicated
my_callback initiated. Block : 7
my_callback initiated. Block : 8
bLen = 178
Disabling Persistant Connection
Status : 1
off : 54
len : 178
P--H1--L011100
E41--P--H1-- ............. 
......................................
......................................
......................................
......................................
Reading Contents of File 2   ( the contents of the SD card are identical to the data that was written to the file above, including the duplicated blocks of data.

OK. New morning and a fresh brain installed.

My gut feeling is that these duplicate packets are not an error caused by the code ( well, not directly ) but quite probably by the ethernet module resending the packet for some reason.

I suspected that the sharing of the SPI may be causing it, so I commented out the line that writes the packet to the SD card file ( so it does not keep switching the SPI devices ) :

WriteSDfile(2,0,((char*) ebuf2 + off));

BINGO !!

The serial monitor now shows the expected packets with NO duplications. We're on the right track.

Next I tried adding a delay before the WriteSDfile line, in case the ethernet module needed time to confirm that the callback had the buffer. This made the problem worse, and duplications increased from 2 to 6 cases.

So again commented out the WriteSDfile and increased the delay to 500ms.

Now it went crazy and LOTS of duplications.

So my deduction is that the ethernet module waits a specific time ( lerts call this the ' callback timeout ' )to get a response that the callback has completed, otherwise it resends the packet to the callback.

Anything in the callback function that takes too long ( WriteSDfile or a 500ms delay ) is too long for that ' callback timeout ', and the packet is resent.

So now I need to find a way to increase that ' callback timeout '.

Anyone with experience with this ?

Am very glad to say I seem to have finally managed to solve this.

I am using an ethernet module ( ENC28J60 ) with the EtherCard library and SD Card both on SPI on a Mega 2560.

The CS pins are different for each, and pin 53 as an OUTPUT.

The project I want needs a web page ( output from a php script ) to be downloaded and the data saved to an SD card file.

The root of the problem is the async browseurl and the callback function.

From my understanding, the browseurl sends the web page request to the ethernet module, and then continues with it's code.

During the main loop, the ethercard library then populates the ethernet buffer with the received data, and calls the callback function.

From my experiences, I can say it appears that this works great, UNLESS you actually do something with the data in the buffer in the callback function, like try and write it to an SD card file.

When this happens, the callback receives duplicates of the data packet. I can only assume that this is caused by the library ( or the ethernet module ) not getting the message that the callback function has completed within a certain timeout period. It then sends a duplicate buffer.

I was able to confirm this theory by adding a delay (400ms) after writing the data to the SD Card file. This caused almost every single buffer to be duplicated.

My solution ( probably not the best, but works for me ) was to do the following :

I have 4 char arrays ( xBuf1, xBuf2, xBuf3, xBuf4 ) each 33 bytes.

When a buffer arrives ( 512 bytes ) make a MD2 hash if the data ( 33 bytes ).

Compare the 33 byte MD2 hash to the 4 xBuf arrays. ( 33 byte arrays use less memory to compare than would 4 x 700 byte arrays to store the complete buffer )

If there is a match, then this buffer was one of the 4 buffers previously received, so do nothing with it. This will skip the SD Card File write, and the callback function completes faster, and the buffer is never repeated.

If there is no match, then delete the oldest MD2 from the xBuf arrays, and add the MD2 of the new data to the xBuf array.

I hope this helps someone in the future.

i have tried the code several times but as my card, no is (e334444443)eg i have created a php file in the website but respect to the card no I supposed to get ok response but I always get false response.please to help... I have use em 18 readers and hr911105a ethernet...

#include <EtherCard.h>
int count = 0; // count = 0
char input[12]; // character array of size 12
boolean flag = 0; // flag =0
// ethernet interface mac address, must be unique on the LAN
static byte mymac[] = { 0x74,0x69,0x69,0x2D,0x30,0x31 };

byte Ethernet::buffer[700];
unsigned long timer;
//const char state = 0;
const char website[] PROGMEM = "www.hdddddd.info";
char line_buf[50];
int ethernet_requests = 0;
int ethernet_error = 0;
void setup ()
{
Serial.begin(9600);
Serial.println("03 - Basic Web Client");

if (ether.begin(sizeof Ethernet::buffer, mymac) == 0)
Serial.println( "Failed to access Ethernet controller");

// DHCP Setup
if (!ether.dhcpSetup())
Serial.println("DHCP failed");

ether.printIp("IP: ", ether.myip);
ether.printIp("GW: ", ether.gwip);
ether.printIp("DNS: ", ether.dnsip);

// DNS Setup
if (!ether.dnsLookup(website))
Serial.println("DNS failed");

ether.printIp("SRV: ", ether.hisip);
}

void loop () {
ether.packetLoop(ether.packetReceive());
if(Serial.available())
{
count = 0;
while(Serial.available() && count < 12) // Read 12 characters and store them in input array
{
input[count] = Serial.read();
count++;
delay(5);
}
Serial.print(input);
char myInt = input; // or whatever
char myIntAsString[7]; // 7 bytes is enough to contain any int, including minus sign and terminating zero
itoa(myInt, myIntAsString, 12);
// Print RFID tag number
//#define FEED = "input";
//String input1= "idd="+input;
// input1+=input;
if((input[0] ^ input[2] ^ input[4] ^ input[6] ^ input[8] == input[10]) &&
(input[1] ^ input[3] ^ input[5] ^ input[7] ^ input[9] == input[11]))
{Serial.println("No Error");

Serial.println("Request sent");
ether.browseUrl(PSTR("/dn/rf_id.php?id="), (const char *)myIntAsString, website, ethernet_callback);
// Stash::prepare(PSTR("GET /demo/saveTemp.php?temp=$H&pwd=$F HTTP/1.0" "\r\n" "Host: $F" "\r\n" "\r\n"),sd, password, website);
// SendDataToServer(input);
// Send some test data to the server:

}
else{
Serial.println("Error");
}
}
}
void SendDataToServer(char* SendTxt) {
//Serial.println(PSTR("/dn/rf_id.php?id="),SendTxt, website, ethernet_callback);
// ether.browseUrl(PSTR("/dn/rf_id.php?id="),SendTxt, website, ethernet_callback);
// ether.packetLoop(ether.packetReceive());
}
static void ethernet_callback(byte status, word off, word len) {
Serial.println(">>>");
Ethernet::buffer[off+300] = 0;
Serial.println((const char*) Ethernet::buffer + off);
}