Robust Ethernet Code

Hi

I am using an Ethernet shield (that utilises the Wisnet WS5100 hardware) and have used the Ethernet example sketches and standard Ethernet library from the IDE 1.6.6 but I have struggled with having a stable connection. I get a lot of hang ups when running the sketch but on an intermittent basis.

I am not looking for anybody to fix my code here but wonder If anybody has a robust implementation of an Ethernet sketch .

I have spent some time on the forum and there seems to be a lot of trouble with the Ethernet sketches being robust but I could not find an example of a sketch that has been stress tested and considered reliable.

Any assistance appreciated.

I am not looking for anybody to fix my code

What code? Please read #7 below:

http://forum.arduino.cc/index.php/topic,148850.0.html

There are examples in the playground. Here are a couple that I have tested extensively and found to be reliable.
Web client
Web server

arduino_woody:
I get a lot of hang ups when running the sketch but on an intermittent basis.

What do you mean by "hang ups"? Does the connection just stop working for a short time and then starts working again or does it stop working completely until it is reset?

Thanks Surfer Tim, I shall take a look at your links.

@pert, the code just completely stops working until reset. I shall do a little more digging and research until I post any code on here for commenting.

Normally when the ethernet sketch stops working until reset, you have no sockets available. Are you using a client or server sketch?

This will display the status of the sockets. Add it to your sketch and call it either on command with the serial monitor or regularly in your code before the point where you think it stops. If you do not have a socket with a status of 0x0 or 0x14, you have run out of sockets. Mine shows a status of 0x17 on all sockets when it stops.

#include <utility/w5100.h>

byte socketStat[MAX_SOCK_NUM];

void ShowSockStatus()
{
  for (int i = 0; i < MAX_SOCK_NUM; i++) {
    Serial.print(F("Socket#"));
    Serial.print(i);
    uint8_t s = W5100.readSnSR(i);
    socketStat[i] = s;
    Serial.print(F(":0x"));
    Serial.print(s,16);
    Serial.print(F(" "));
    Serial.print(W5100.readSnPORT(i));
    Serial.print(F(" D:"));
    uint8_t dip[4];
    W5100.readSnDIPR(i, dip);
    for (int j=0; j<4; j++) {
      Serial.print(dip[j],10);
      if (j<3) Serial.print(".");
    }
    Serial.print(F("("));
    Serial.print(W5100.readSnDPORT(i));
    Serial.println(F(")"));
  }
}

A status list:
0x0 = available
0x14 = waiting for a connection
0x17 = connected
0x1C = connected waiting for close
0x22 = UDP

After nearly 24hours of seamless operation the sketch fell over with no apparent cause (no predetermined errors printed in the serial monitor). Leads me to think its something inside the actual Ethernet library (unless there is anything obvious in my sketch).

Its not pretty - I am not a natural coder put it that way - but below in full is a copy of my sketch.

@Surfer Tim, I can slot the socket monitor into the sketch but what is the fix if this diagnosis is correct?

#include <SPI.h>
#include <Ethernet.h>

// Local Network Settings
byte mac[] = { 0x00, 0xAA, 0xBB, 0xCC, 0xDE, 0x01 }; // Must be unique on local network

// ThingSpeak Settings
char thingSpeakAddress[] = "api.thingspeak.com";
String writeAPIKey = "XXXXXXXXXXXXXX";
const int updateThingSpeakInterval = 16 * 1000;      // Time interval in milliseconds to update ThingSpeak (number of seconds * 1000 = interval)

// Variable Setup
long lastConnectionTime = 0; 
boolean lastConnected = false;
int failedCounter = 0;

//Heat Meter Variable Setup
float Volt_low = 184;
float Volt_high = 787;
float kW_low = 0;
float kW_high = 16.299;
float Power;
int analogue1; //Analogue 1 is A1



// Initialize Arduino Ethernet Client
EthernetClient client;

void setup()
{
  // Start Serial for debugging on the Serial Monitor
  Serial.begin(9600);

  //Disable SD Card
    digitalWrite(4,HIGH);
    
  // Start Ethernet on Arduino
  while (startEthernet()== 0) {delay(1000);};
}

void loop()
{
  // Read value from Analog Input Pin 0
 
  analogue1 = analogRead(1);
  Power = (analogue1 - Volt_low) * ((kW_high - kW_low) / (Volt_high-Volt_low));
  String analogValue0 = String(Power, DEC);
  delay(5);  // Pause for 5 ms.


  while (client.available())
  {
    char c = client.read();
    Serial.print(c);
  }

  // Disconnect from ThingSpeak
  if (!client.connected() && lastConnected)
  {
    Serial.println("...disconnected");
    Serial.println();
    
    client.stop();
  }
  
  // Update ThingSpeak
  if(!client.connected() && (millis() - lastConnectionTime > updateThingSpeakInterval))
  {
    Serial.println("Updating ThingSpeak"); 
    Serial.println("field1="+analogValue0); 
    updateThingSpeak("field1="+analogValue0);
  }
  
  // Check if Arduino Ethernet needs to be restarted
  if (failedCounter > 3 ) {
      failedCounter = 0;
      while (startEthernet()==0 ) {delay(1000);};
      }
  
  lastConnected = client.connected();
}




void updateThingSpeak(String tsData)
{
  if (client.connect(thingSpeakAddress, 80))
  {         
    client.print("POST /update HTTP/1.1\n");
    client.print("Host: api.thingspeak.com\n");
    client.print("Connection: close\n");
    client.print("X-THINGSPEAKAPIKEY: "+writeAPIKey+"\n");
    client.print("Content-Type: application/x-www-form-urlencoded\n");
    client.print("Content-Length: ");
    client.print(tsData.length());
    client.print("\n\n");

    client.print(tsData);
    
    lastConnectionTime = millis();
    
    if (client.connected())
    {
      Serial.println("Connecting to ThingSpeak...");
      Serial.println();
      
      failedCounter = 0;
    }
    else
    {
      failedCounter++;
  
      Serial.println("Connection to ThingSpeak failed ("+String(failedCounter, DEC)+")");   
      Serial.println();
    }
    
  }
  else
  {
    failedCounter++;
    
    Serial.println("Connection to ThingSpeak Failed ("+String(failedCounter, DEC)+")");   
    Serial.println();
    
    lastConnectionTime = millis(); 
  }
}




int startEthernet()
{
  
  client.stop();

  Serial.println("Connecting Arduino to network...");
  Serial.println();  

  delay(1);
  
  // Connect to network amd obtain an IP address using DHCP
  if (Ethernet.begin(mac) == 0)
  {
    Serial.println("DHCP Failed");
    Serial.println();
    return 0;
  }
  else
  {
    Serial.print("Arduino connected to network using DHCP IP:");
    Serial.println(Ethernet.localIP());
    Serial.println();
    return 1;
  }
  
  delay(1);
}

What was the last thing in the serial monitor?

If the server stalls or the connection breaks, the server will not send a disconnect message. That could possibly stop your code. The client.connected() function will always return true.

String. Don't use String.

@SurferTim I am letting it run at the moment connected to the serial monitor and will report back if/when it falls over, but it has been running ok for a couple of days now bizarrely!

@/dev please explain what you mean don't use string? Are you referring to the use of a string for the API key (which has been xxx out for posting on the forum) or something else?

Many thanks

what you mean don't use string?

Never, not ever, shalt thou use String for anything. Not for arguments, not for variables, not for API key, not a thing. You will save at least 1600 bytes of flash and untold hours of debugging when it stops working "bizarrely".

Why do this:

String writeAPIKey = "XXXXXXXXXXXXXX";
  .
  .
  .
    client.print("X-THINGSPEAKAPIKEY: "+writeAPIKey+"\n");

...instead of this:

    client.print("X-THINGSPEAKAPIKEY: XXXXXXXXXXXXXX\n");

A total waste. Not only that, you are using RAM for all these character string constants (not Strings). You should be using the F macro:

    client.print( F("X-THINGSPEAKAPIKEY: XXXXXXXXXXXXXX\n") );

The problem with String and the way you are using it is that it will eventually fragment the dynamic memory "heap", and further String operations will fail, causing weird behavior wherever the sketch needs characters from those String. But the failure will happen at different times if you have different string values and operations. Bad juju. Determinism is your friend.

Use char arrays for things that are really textual and that change (there aren't any in your program, as thingSpeakAddress must remain a RAM char *).

Or just use the F macro in calls that can take PROGMEM strings, like Serial.print or client.print. If a constant text snippet is used in multiple places, you can declare a truly constant char array with

      const char varname[] PROGMEM = "for text that never changes.";

Using it is a little different if you're not just passing it to print. The field name for Power is an example of that.

In general, keep "variable" values in their integer or floating-point form until the last possible part of the sketch, then "convert" it to text with print or itoa or whatever. The Power variable is a good example of this. The numeric forms are almost always smaller than their character representation, so you will end up saving even more RAM.

Here is your sketch with those suggested modifications:

#include <SPI.h>
#include <Ethernet.h>

// Local Network Settings
byte mac[] = { 0x00, 0xAA, 0xBB, 0xCC, 0xDE, 0x01 }; // Must be unique on local network

// ThingSpeak Settings
char thingSpeakAddress[] = "api.thingspeak.com";
const int updateThingSpeakInterval = 16 * 1000;      // Time interval in milliseconds to update ThingSpeak (number of seconds * 1000 = interval)

// Variable Setup
long lastConnectionTime = 0; 
boolean lastConnected = false;
int failedCounter = 0;

//Heat Meter Variable Setup
float Volt_low = 184;
float Volt_high = 787;
float kW_low = 0;
float kW_high = 16.299;
float Power;
int analogue1; //Analogue 1 is A1

const char powerFieldName[] PROGMEM = "field1";



// Initialize Arduino Ethernet Client
EthernetClient client;

void setup()
{
  // Start Serial for debugging on the Serial Monitor
  Serial.begin(9600);

  //Disable SD Card
  digitalWrite(4,HIGH);
    
  // Start Ethernet on Arduino
  while (startEthernet()== 0) {
    delay(1000);
  }
}

void loop()
{
  // Read value from Analog Input Pin 0
 
  analogue1 = analogRead(1);
  Power = (analogue1 - Volt_low) * ((kW_high - kW_low) / (Volt_high-Volt_low));
  delay(5);  // Pause for 5 ms.


  while (client.available())
  {
    char c = client.read();
    Serial.print(c);
  }

  // Disconnect from ThingSpeak
  if (!client.connected() && lastConnected)
  {
    Serial.println( F("...disconnected\n") );
    
    client.stop();
  }
  
  // Update ThingSpeak
  if (!client.connected() && 
      (millis() - lastConnectionTime > updateThingSpeakInterval))
  {
    char powerChars[16];
    dtostrf( Power, 5, 2, powerChars );
    Serial.println( F("Updating ThingSpeak") ); 
    Serial.print( powerFieldName );
    Serial.print( '=' );
    Serial.println( powerChars );
    updateThingSpeak( powerFieldName, powerChars );
  }
  
  // Check if Arduino Ethernet needs to be restarted
  if (failedCounter > 3 ) {
    failedCounter = 0;
    while (startEthernet()==0 ) {
      delay(1000);
    }
  }
  
  lastConnected = client.connected();
}




void updateThingSpeak( PGM_P fieldName, char *value )
{
  if (client.connect(thingSpeakAddress, 80))
  {         
    client.println( F("POST /update HTTP/1.1") );
    client.println( F("Host: api.thingspeak.com") );
    client.println( F("Connection: close") );
    client.println( F("X-THINGSPEAKAPIKEY: XXXXXXXXXXXXX")  );
    client.println( F("Content-Type: application/x-www-form-urlencoded")  );
    client.println( F("Content-Length: ") );
    client.println( strlen_P(fieldName) + 1 + strlen(value) );
    client.println();

    client.print( fieldName );
    client.print( '=' );
    client.print( value );
    
    lastConnectionTime = millis();
    
    if (client.connected())
    {
      Serial.println( F("Connecting to ThingSpeak...\n") );
      
      failedCounter = 0;
    }
    else
    {
      failedCounter++;
  
      Serial.print( F("Connection to ThingSpeak failed (") );
      Serial.print( failedCounter );
      Serial.println( ')' );
      Serial.println();
    }
    
  }
  else
  {
    failedCounter++;
    
    Serial.print( F("Connection to ThingSpeak failed (") );
    Serial.print( failedCounter );
    Serial.println( ')' );
    Serial.println();

    lastConnectionTime = millis(); 
  }
}




int startEthernet()
{
  
  client.stop();

  Serial.println( F("Connecting Arduino to network...\n") );

  delay(1);
  
  // Connect to network and obtain an IP address using DHCP
  if (Ethernet.begin(mac) == 0)
  {
    Serial.println( F("DHCP Failed\n") );
    return 0;
  }
  else
  {
    Serial.print( F("Arduino connected to network using DHCP IP:") );
    Serial.println(Ethernet.localIP());
    Serial.println();
    return 1;
  }
  
  delay(1);
}

Be sure to compare the program space and RAM numbers before and after these changes. (It compiles, but I can't test it.)

Cheers,
/dev

/dev

Awesome reply. Thanks for taking the time to run through this and I shall definitely take on board and implement in this and future coding.

Have really enjoyed starting from zero knowledge of coding a year ago to pulling together (albeit it in a cut and past fashion!) working sketches and support like this I find encourages me further!

The original code has been working for about a week now without falling over but will implement your improvements anyway as a point of interest.

If you have not changed this, your sketch may fail if the server stalls or the connection fails. I can easily create this fail by breaking the connection. You will not see the "...disconnected" message.

  if (!client.connected() && lastConnected)
  {
    Serial.println("...disconnected");
    Serial.println();
   
    client.stop();
  }

If you post your current code, I can test it for you and let you know what makes it fail.

Thanks Surfer Tim

The current code that I am running is below. I have not implemented any of the changes yet suggested by you and /Dev as I was leaving the original to run with serial monitor so I could get a handle on where it was crashing; I am somewhat surprised it has run for so long!

With reference to the last post is there an alternative that I can drop into to provide the "intended" function or should I just omit it?

/*

 Arduino --> ThingSpeak Channel via Ethernet
 
 The ThingSpeak Client sketch is designed for the Arduino and Ethernet.
 This sketch updates a channel feed with an analog input reading via the
 ThingSpeak API (https://thingspeak.com/docs)
 using HTTP POST. The Arduino uses DHCP and DNS for a simpler network setup.
 The sketch also includes a Watchdog / Reset function to make sure the
 Arduino stays connected and/or regains connectivity after a network outage.
 Use the Serial Monitor on the Arduino IDE to see verbose network feedback
 and ThingSpeak connectivity status.
 
 Getting Started with ThingSpeak:
 
   * Sign Up for New User Account - https://thingspeak.com/users/new
   * Create a new Channel by selecting Channels and then Create New Channel
   * Enter the Write API Key in this sketch under "ThingSpeak Settings"
 
 Arduino Requirements:
 
   * Arduino with Ethernet Shield or Arduino Ethernet
   * Arduino 1.0+ IDE
   
  Network Requirements:

   * Ethernet port on Router    
   * DHCP enabled on Router
   * Unique MAC Address for Arduino
 
 Created: October 17, 2011 by Hans Scharler (http://www.nothans.com)
 
 Additional Credits:
 Example sketches from Arduino team, Ethernet by Adrian McEwen
 
*/

#include <SPI.h>
#include <Ethernet.h>
#include <utility/w5100.h>

// Socket monitor variables
byte socketStat[MAX_SOCK_NUM];

// Local Network Settings
byte mac[] = { 0x00, 0xAA, 0xBB, 0xCC, 0xDE, 0x01 }; // Must be unique on local network

// ThingSpeak Settings
char thingSpeakAddress[] = "api.thingspeak.com";
String writeAPIKey = "xxxxxx";
const int updateThingSpeakInterval = 16 * 1000;      // Time interval in milliseconds to update ThingSpeak (number of seconds * 1000 = interval)

// Variable Setup
long lastConnectionTime = 0; 
boolean lastConnected = false;
int failedCounter = 0;

//Heat Meter Variable Setup
float Volt_low = 184;
float Volt_high = 787;
float kW_low = 0;
float kW_high = 16.299;
float Power;
int analogue1; //Analogue 1 is A1



// Initialize Arduino Ethernet Client
EthernetClient client;

void setup()
{
  // Start Serial for debugging on the Serial Monitor
  Serial.begin(9600);

  //Disable SD Card
    digitalWrite(4,HIGH);
    
  // Start Ethernet on Arduino
  while (startEthernet()== 0) {delay(1000);};
}

void loop()
{
  // Read value from Analog Input Pin 0
 
  analogue1 = analogRead(1);
  Power = (analogue1 - Volt_low) * ((kW_high - kW_low) / (Volt_high-Volt_low));
  String analogValue0 = String(Power, DEC);
  delay(5);  // Pause for 5 ms.


  while (client.available())
  {
    char c = client.read();
    Serial.print(c);
  }

  // Disconnect from ThingSpeak
  if (!client.connected() && lastConnected)
  {
    Serial.println("...disconnected");
    Serial.println();
    
    client.stop();
  }
  // check sockets
   ShowSockStatus()

  
  // Update ThingSpeak
  if(!client.connected() && (millis() - lastConnectionTime > updateThingSpeakInterval))
  {
    Serial.println("Updating ThingSpeak"); 
    Serial.println("field1="+analogValue0); 
    updateThingSpeak("field1="+analogValue0);
  }
  
  // Check if Arduino Ethernet needs to be restarted
  if (failedCounter > 3 ) {
      failedCounter = 0;
      while (startEthernet()==0 ) {delay(1000);};
      }
  
  lastConnected = client.connected();
}




void updateThingSpeak(String tsData)
{
  if (client.connect(thingSpeakAddress, 80))
  {         
    client.print("POST /update HTTP/1.1\n");
    client.print("Host: api.thingspeak.com\n");
    client.print("Connection: close\n");
    client.print("X-THINGSPEAKAPIKEY: "+writeAPIKey+"\n");
    client.print("Content-Type: application/x-www-form-urlencoded\n");
    client.print("Content-Length: ");
    client.print(tsData.length());
    client.print("\n\n");

    client.print(tsData);
    
    lastConnectionTime = millis();
    
    if (client.connected())
    {
      Serial.println("Connecting to ThingSpeak...");
      Serial.println();
      
      failedCounter = 0;
    }
    else
    {
      failedCounter++;
  
      Serial.println("Connection to ThingSpeak failed ("+String(failedCounter, DEC)+")");   
      Serial.println();
    }
    
  }
  else
  {
    failedCounter++;
    
    Serial.println("Connection to ThingSpeak Failed ("+String(failedCounter, DEC)+")");   
    Serial.println();
    
    lastConnectionTime = millis(); 
  }
}




int startEthernet()
{
  
  client.stop();

  Serial.println("Connecting Arduino to network...");
  Serial.println();  

  delay(1);
  
  // Connect to network amd obtain an IP address using DHCP
  if (Ethernet.begin(mac) == 0)
  {
    Serial.println("DHCP Failed");
    Serial.println();
    return 0;
  }
  else
  {
    Serial.print("Arduino connected to network using DHCP IP:");
    Serial.println(Ethernet.localIP());
    Serial.println();
    return 1;
  }
  
  delay(1);
}


void ShowSockStatus()
{
  for (int i = 0; i < MAX_SOCK_NUM; i++) {
    Serial.print(F("Socket#"));
    Serial.print(i);
    uint8_t s = W5100.readSnSR(i);
    socketStat[i] = s;
    Serial.print(F(":0x"));
    Serial.print(s,16);
    Serial.print(F(" "));
    Serial.print(W5100.readSnPORT(i));
    Serial.print(F(" D:"));
    uint8_t dip[4];
    W5100.readSnDIPR(i, dip);
    for (int j=0; j<4; j++) {
      Serial.print(dip[j],10);
      if (j<3) Serial.print(".");
    }
    Serial.print(F("("));
    Serial.print(W5100.readSnDPORT(i));
    Serial.println(F(")"));
  }
}