So I’m having trouble with a home monitoring project I’m working on. The project uses a Mega 2560 with Ethernet Shield 2 as a base station and three MKR WiFi 1010 as wireless peripheral devices. The base station’s job is to monitor six different aspects of my home and send me an email if it discovers a problem with any of them. Three of the aspects are monitored over hard wire connections to the Mega analog pins or interrupt pins. The other three are monitored wirelessly over my LAN.
Before I introduced the peripheral code, so just monitoring the three native pins, the system was stable and worked flawlessly for years.
The WiFi units, however, have never been stable. It works but crashes in different ways. When the base station (Mega 2560) boots up it successfully finds each of the three WiFi devices, connects to them and verifies that the device is a device it wants to be connected to. The base station then polls each device sequentially every ten seconds to see what their states are. By poll I mean it connects to the device, reads its status and disconnects.
It seems to work for a while, but seemingly at random one or more of the connections fails (usually only one). 10 seconds later that connection usually works again. I don’t know why these connections fail. I keep track of how many fails occur for each device and if any of them repeat enough times I de-register that device and send an email saying the device is lost. When I get the email I go check the WiFi device, reset it, and then prompt the base station to go look for it again. It does and it finds it, registers it and begins to poll it every 10 seconds.
Eventually all three connections fail every poll. Then the problem becomes the the Ethernet connection to my SMTP email server also fails and I will never know about the failure. The only option then is a board reset of the Mega 2560.
reboot, function, fail randomly, fail completely, repeat…
Obviously, something isn’t right, but I haven’t got a clue what. I must be missing something(s) annoying and/or fundamental. I find the whole Client/Server thing very confusing; how do you choose which device operates in which class?… Should I just connect once and then leave the connection as is, three simultaneous open socket connections?… how do you do that?…
Here’s the relevant code for the base station, Mega 2560:
// set Ethernet Retransmission count and timeout
Ethernet.setRetransmissionTimeout(100);
Ethernet.setRetransmissionCount(15); // 100 ms x 15 tries = 1500 ms timeout (default: 200ms x 8 1600ms)
// set connection Client timeout
Sentinel.setConnectionTimeout(900);
sometimes the connection fails with the Client.ConnectionTimeout…
sometimes the connection fails with the Ethernet.RetransmissionTimeout…
sometimes the connection fails in less than 3 ms…
void Poll()
{
// poll native sensors, done every loop
pollTemp();
//
pollGrid();
//
pollSmoke();
// poll WiFi sensors, done every 10 seconds (polling interval)
if(millis() - pollingIntervalStart > pollingInterval && emailSent/* && !sendMessage*/) // don't poll lan if sending email or phone mssg, ie emailSent = true and sendMessage = false
{
// unsigned long pollSpan = 0UL;
static int numFails[numDevices] = {0,0,0,0,0,0,0,0}; //sets all values to 0? numDevices assigned in lanConfig.h file
// turn both leds on
digitalWrite(redLed, HIGH);
digitalWrite(greenLed, HIGH);
Serial.println(F("heartbeat, 2"));
//
int ms = 0;
int dvcPrint = 0;
bool msPrint = false;
for(int dvc = dvr_Water; dvc < numDevices; dvc++)
{
// pollStart = millis(); // start polling timer for this device
// assemble full IP addr
// lanIPx = Ethernet.localIP();
// create current device's full IP address
lanIPx[3] = lanIPreg[dvc];
//
if(bitRead(guiReg, gbr_verbose))
{
Serial.print(F("Polling "));
Serial.print(deviceNames[dvc]);
}
// check if device is NO poll, derigistered, native or lost
if(!bitRead(guiReg, dvc + pollOffset) || !bitRead(deviceReg, dvc) || lanIPreg[dvc] == -2 || bitRead(guiReg, dvc + lostOffset))
{
if(bitRead(guiReg, gbr_verbose))
{
if(lanIPreg[dvc] == -2) // native sensor
{
Serial.print(F(" (native sensor)... "));
Serial.println(mssgDone);
}
else if(!bitRead(deviceReg, dvc)) // not registered
{
Serial.println(F(": no Device registered."));
}
else if(!bitRead(guiReg, dvc + pollOffset))
{
Serial.println(F(": not included in Poll."));
}
else if(bitRead(guiReg, dvc + lostOffset)) // device lost
{
Serial.println(F(": Device has been lost!"));
}
// int ms = (int)(millis() - pollSpan);
// Serial.print(ms);
// Serial.println(F(" ms"));
// if(ms > peak_ms)
// {
// peak_ms = ms;
// //
// Serial.print(F("peak_ms = "));
// Serial.print(peak_ms);
// Serial.print(F(" ms: "));
// Serial.println(deviceNames[dvc]);
// //
// DailyLogHeader(__LINE__);
// DailyLog.print(F("peak_ms = "));
// DailyLog.print(peak_ms);
// DailyLog.println(F(" ms."));
// DailyLog.flush();
// }
}
//
continue; // skip polling the affected device, move on to next device
}
if(bitRead(guiReg, gbr_verbose))
{
Serial.print(F(" at "));
Serial.print(lanIPx);
Serial.print(F("... "));
}
// poll the device
int result = pollWiFi(dvc);
// measure the time it took to poll the device
ms = (int)(millis() - pollStart);
if(ms > peak_ms)
{
peak_ms = ms;
msPrint = true;
dvcPrint = dvc;
}
//
if(result > 0) // poll failed, update gui and log fail
{
numFails[dvc]++;
//
if(!bitRead(guiReg, gbr_verbose)) // print fail anyway
{
Serial.print(F("Polling "));
Serial.print(deviceNames[dvc]);
// Serial.print(F("... "));
Serial.print(F(" at "));
Serial.print(lanIPx);
Serial.print(F("... "));
}
Serial.print(mssgFailed);
Serial.print(F(" ("));
Serial.print(numFails[dvc]);
Serial.print(F(" at "));
Serial.print(result);
Serial.print(F(") / "));
Serial.print(ms);
Serial.println(F(" ms"));
//
DailyLogHeader(__LINE__);
DailyLog.print(F("Polling "));
DailyLog.print(deviceNames[dvc]);
DailyLog.print(F(" "));
DailyLog.print(mssgFailed);
DailyLog.print(F(" ("));
DailyLog.print(numFails[dvc]);
DailyLog.print(F(" at "));
DailyLog.print(result);
DailyLog.print(F(") / "));
DailyLog.print(ms);
DailyLog.println(F(" ms"));
DailyLog.flush();
if(numFails[dvc] > 15)
{
// gridAlarm = false;
if(!gridAlarm) // this means power is still ON, so we've LOST the device
{
// device lost
lostDevice = dvc;
//
bitClear(deviceReg, dvc); // not registered
bitClear(guiReg, dvc + pollOffset); // skip
bitSet(guiReg, dvc + lostOffset); // lost
//
bitSet(guiReg, gbr_verbose);
//
deviceCount = countDevices();
//
lanIPreg[dvc] = -3;
saveLanIPreg();
// trigger email send
adminEmailSent = false;
//
Serial.print(F("Device Lost! -> "));
Serial.println(deviceNames[dvc]);
Serial.println(F("Deregistering this device."));
//
DailyLogHeader(__LINE__);
DailyLog.print(F("Device LOST! -> "));
DailyLog.println(deviceNames[dvc]);
DailyLog.println(F("Deregistering this device."));
// break out of for() loop to immediately send email
break;
}
else // power if OFF to the sensor, just reset numFails until power comes back on
{
Serial.println(F("Power grid offline. numFails reset to zero."));
DailyLogHeader(__LINE__);
DailyLog.println(F("Power grid offline. numFails reset to zero."));
}
//
numFails[dvc] = 0;
}
}
else // successful poll
{
// Serial.println(mssgDone);
numFails[dvc] = 0;
if(bitRead(guiReg, gbr_verbose))
{
Serial.print(mssgDone);
Serial.print(F(" "));
Serial.print(ms);
Serial.println(F(" ms"));
}
}
} // end for() loop
// print the new peak ms
if(msPrint)
{
Serial.print(F("peak_ms = "));
Serial.print(peak_ms);
Serial.print(F(" ms: "));
Serial.println(deviceNames[dvcPrint]);
//
DailyLogHeader(__LINE__);
DailyLog.print(F("peak_ms = "));
DailyLog.print(peak_ms);
DailyLog.println(F(" ms."));
DailyLog.println(deviceNames[dvcPrint]);
DailyLog.flush();
}
if(bitRead(guiReg, gbr_verbose)) Serial.println(F("<-->"));
//
pollingIntervalStart = millis();
// turn both leds off
digitalWrite(redLed, LOW);
digitalWrite(greenLed, LOW);
Serial.println(F("heartbeat, 0"));
}
}
int pollWiFi(int device)
{
int result = 0;
// poll device
hungComs = true; //watchdog boolean
pollStart = millis();
// Connect to device
if(!Sentinel.connect(lanIPx, 502))// lanIPx preset before call to here
{
Sentinel.stop(); // necessary to allow polling of other devices!
//
return __LINE__;
}
// get data from device
result = getRemoteData(device);
// close the connection
Sentinel.stop();
hungComs = false;
//
return result;
}
int getRemoteData(int device) // returns fail line number on fail or 0 (zero) on success
{
byte connectedDevice= 0;
// static byte lastState = 0; // bit Register for remembering previous alarm states
int serNum = 0; //
// connection still good, ask remote device for update
Sentinel.write(funcIDs[FUNC_INDEX]); // send Sentinel's serial num 0x2A
//
int idx = 0;
// listen for reply
while(idx < 4)
{
if(millis() - pollStart > 3999) break; // timeout safety
//
if (Sentinel.available())
{
while(Sentinel.available()) // this loop reads all the data in inBuffer...
{
if(millis() - pollStart > 3999) break;
//
byte b;
//
b = Sentinel.read();
if(idx > 3)continue; // but only saves the first four bytes.
inBuffer[idx] = b;
idx++;
}
}
}
//
Sentinel.stop();
// clear previous data from buffers
connectedDevice = 0;
serNum = 0;
stateData = 0;
// load new data into buffers
connectedDevice = inBuffer[0]; // device IP addrerss (Name)
serNum = inBuffer[1] << 8; // highbyte
serNum += inBuffer[2]; // lowbyte
stateData = inBuffer[3]; // state
// verify the data
bool fail = false;
if(connectedDevice < 2 && connectedDevice > 7)
{
Serial.println(F("Invalid Func_Index!"));
DailyLog.println(F("Invalid Func_Index!"));
DailyLog.flush();
//
fail = true;
}
else // connected device is valid
{
if(device != NULL)
{
if(device != connectedDevice)
{
Serial.println(F("Mismatched devices!"));
Serial.print(deviceNames[device]);
Serial.print(F(" / "));
Serial.println(deviceNames[connectedDevice]);
//
DailyLog.println(F("Mismatched devices!"));
DailyLog.print(deviceNames[device]);
DailyLog.print(F(" / "));
DailyLog.println(deviceNames[connectedDevice]);
DailyLog.flush();
fail = true;
}
}
else
{
device = connectedDevice; // device == NULL so copy valid connected device to it
// register the device
bitSet(deviceReg, device);
// loop() will detect change in devReg and save the new value
}
}
//
if(serNum != funcIDs[connectedDevice])
{
Serial.println(F("Invalid serial number for given device!"));
DailyLog.println(F("Invalid serial number for given device!"));
DailyLog.flush();
//
fail = true;
}
// valid states?
if(stateData == 0xFF)
{
Serial.println(F("Communication error!"));
fail = true;
}
else // verify state data
{
switch(connectedDevice)
{
case dvr_Water:
{
if(stateData > 1) fail = true; // only 1 valid bit
else waterAlarm = (bool)stateData;
//
if(bitRead(waterAlarm, 0) != bitRead(mainAlarmReg, dvr_Water))
{
// printBinary(stateData);
if(bitRead(waterAlarm, 0))
{
bitSet(mainAlarmReg, dvr_Water);
acknowledged = false;
bitClear(guiReg, gbr_Ack);
}
else bitClear(mainAlarmReg, dvr_Water);
//
emailSent = false;
}
}
break;
//
case dvr_Security:
{
if(stateData > 7) fail = true; // only three valid bits
else
{
securityAlarm = stateData; // 8 bit register, 3 are valid
//
if(bitRead(securityAlarm, 0) != bitRead(mainAlarmReg, dvr_Security))
{
// printBinary(stateData);
if(bitRead(securityAlarm, 0))
{
bitSet(mainAlarmReg, dvr_Security);
//
acknowledged = false;
bitClear(guiReg, gbr_Ack);
}
else bitClear(mainAlarmReg, dvr_Security);
//
emailSent = false;
}
}
}
break;
//
case dvr_Grid:
case dvr_Temp:
case dvr_Smoke:
{
// no code here
}
break;
//
case dvr_Boiler:
{
if(stateData > 31) fail = true; // 5 valid bits (4 boiler reads + alarm state), so 2^5 - 1 = 31
else boilerAlarm = stateData;
//
if(bitRead(boilerAlarm, 0) != bitRead(mainAlarmReg, dvr_Boiler))
{
// alarm state has changed, send alert
if(bitRead(boilerAlarm, 0)) // alarm is ON
{
bitSet(mainAlarmReg, dvr_Boiler);
acknowledged = false;
bitClear(guiReg, gbr_Ack);
}
else bitClear(mainAlarmReg, dvr_Boiler); // alarm is OFF
//
emailSent = false;
}
}
break;
}
}
// clear out inBuffer
for(byte i = 0; i < 4;i++) inBuffer[i] = 0;
//
if(fail)
{
//
int ms = millis() - pollStart;
if(ms > peak_ms)
{
peak_ms = ms;
Serial.print(F("peak_ms: "));
Serial.print(peak_ms);
Serial.println(F(" ms"));
//
DailyLog.print(F("peak_ms: "));
DailyLog.print(peak_ms);
DailyLog.println(F(" ms"));
DailyLog.flush();
}
if(connectedDevice >= 2 && connectedDevice <= 7) Serial.println(deviceNames[connectedDevice]); // name
else Serial.println(connectedDevice);
Serial.print(F("0x"));
Serial.println(serNum, HEX);
Serial.println(stateData);
Serial.print(F("ms: "));
Serial.println(ms); // elapsed time
//
if(connectedDevice >= 2 && connectedDevice <= 7) DailyLog.println(deviceNames[connectedDevice]); // name
else DailyLog.println(connectedDevice);
DailyLog.print(F("0x"));
DailyLog.println(serNum, HEX);
DailyLog.println(stateData);
DailyLog.print(F("ms: "));
DailyLog.println(ms);
DailyLog.flush();
}
// clear data from buffers
connectedDevice = 0;
serNum = 0;
stateData = 0;
//
if(fail) return __LINE__;
return 0;
}
and the relevant remote device code:
WiFiServer thisDevice(502);
WiFiClient ClientList[8];
// listen for incoming call from Sentinel controller
WiFiClient Incoming = thisDevice.accept();
unsigned long whileTimeout = 0UL;
if(Incoming)
{
// indicated connection to sentinel is made
WiFiDrv::analogWrite(blueLED, 30);
digitalWrite(commLED, HIGH);
whileTimeout = millis();
//
for(byte i = 0; i < 8; i++)
{
if(!ClientList[i])
{
// Incoming.print("Hello, client number: ");
// Incoming.println(i);
// send state data (4 bytes)
Incoming.write(FUNC_INDEX); // byte 0
Incoming.write(highByte(funcIDs[FUNC_INDEX])); // byte 1
Incoming.write(lowByte(funcIDs[FUNC_INDEX])); // byte 2
Incoming.write(boilerAlarmReg); // byte 3
// Once we "accept", the client is no longer tracked by EthernetServer
// so we must store it into our list of clients
ClientList[i] = Incoming;
//
break;
}
}
}
else updateState();
// check for incoming data from all clients
for (byte i = 0; i < 8; i++)
{
while (ClientList[i] && ClientList[i].available() > 0)
{
// read incoming data from the client
//
byte c = 0;
byte b = Incoming.read();
//
if(b != funcIDs[1] && b != funcIDs[FUNC_INDEX])
{
// send state data (4 bytes)
thisDevice.write(FUNC_INDEX); // byte 0
thisDevice.write(highByte(funcIDs[FUNC_INDEX])); // byte 1
thisDevice.write(lowByte(funcIDs[FUNC_INDEX])); // byte 2
thisDevice.write(0xFF); // byte 3
Serial.print(F("error: byte = "));
Serial.println(b);
//
break;
}
// parse byte
if(b == funcIDs[1]) // 0x2A
{
// send state data (4 bytes)
thisDevice.write(FUNC_INDEX); // byte 0
thisDevice.write(highByte(funcIDs[FUNC_INDEX])); // byte 1
thisDevice.write(lowByte(funcIDs[FUNC_INDEX])); // byte 2
thisDevice.write(boilerAlarmReg); // byte 3
//
hungComms = false;
}
else if(b == funcIDs[FUNC_INDEX])
{
// secondary action (eg. pause monitoring poll call timeout while Sentinel is busy with DTMF phone calling)
//
hungComms = false;
}
else if(b == 0)
{
c++;
if(c > 100)
{
// send state data (4 bytes)
thisDevice.write(FUNC_INDEX); // byte 0
thisDevice.write(highByte(funcIDs[FUNC_INDEX])); // byte 1
thisDevice.write(lowByte(funcIDs[FUNC_INDEX])); // byte 2
thisDevice.write(0xFF); // byte 3
Serial.print(F("error: counter = "));
Serial.println(c);
break;
}
}
else
{
Serial.print(F("error: b = 0x"));
if(b < 16) Serial.print(F("0"));
Serial.println(b, HEX);
break;
}
//
if(whileTimeout != 0UL && millis() - whileTimeout > 2999)
{
Serial.print(F("Data read timeout on line "));
Serial.println(__LINE__);
//
break;
}
}
}
// stop any clients which disconnect
for (byte i = 0; i < 8; i++)
{
if (ClientList[i] && !ClientList[i].connected())
{
ClientList[i].stop();
// connection closed
WiFiDrv::analogWrite(blueLED, LOW);
digitalWrite(commLED, LOW);
// reset loop() timer
pollCallStart = millis();
}
}
//
if(bitRead(boilerAlarmReg, 0)) non_blockingBlinkLED(alarmLED, 500, 500);
} // end of loop()
I hope someone can spot the bugs that destabilize the project.
Thanks.
Stephen
“Blood from a stone” analogy works too….
as I preferred the ESP32 architecture and underlying RTOS rather than the bare-metal architecture when in needs to do more complex stuff (essentially, ESP32 has the TCP/IP stack integrated in the same MCU with multitasking, whereas MKR Wi-Fi relies on a coprocessor and a SPI based library interface. That’s why ESP32 is more flexible for complex networking).