WiFiClient running slowly on ESP8266 NodeMCU

Hi all,

Apologies for yet another topic on this but I have tried all sorts of things and cant reliably get this working quickly so any tips or pointers would be much appreciated. The most obvious delays are:

  1. After the last byte has been loaded it takes a while for the page spinner to stop spinning - maybe 2-3seconds.
  2. It takes a bit of time to get the first request but presumably this is just the time to connect to the various servers
  3. You can see the individual chunks of JSON data appear one by one, with a delay of maybe 200ms between them.
    The overall page load is 6-7seconds with a 5kb JSON, though based on the actual page loading I would say it only takes 500ms for the data to appear out of the 6-7s total time.

Summary: I have a server device in my house with a TCP API on a custom port, and if you send it JSON queries then it will respond with the answers in JSON format. I have written an ESP8266 bridge so that you can send the JSON through a normal webpage (with an argument) to a page, it will take the argument and make the request from the API before passing back the response to the requester (it will also include the requester and the requested JSON).

This seems to work ok on small JSON responses from the API server (~5kB), but if this JSON response goes up to 20kB then it selectively doesn’t finish the request, even though going from 5-10kb seems to only take a few hundred ms longer.

I have tried setting various noDelay flags, changing the timeout and also changing the packet size but have so far failed. I even tried creating a 20kB char array and downloading the whole response from server before passing it back to the requester, which worked ok sometimes, but then obviously fails if you reach 20.1kB and you quickly run out of memory aswell!

I am using about 30% of storage and 45% of dynamic memory so should be plenty of space, hence I’d love to hear your thoughts on speeding it up. Is there a stream implementation that would be better, or can I do things in a different order to speed things up?

Thanks very much in advance

#include <ESP8266WiFi.h>
#include <WiFiClient.h>
#include <WiFiUdp.h>
#include <ESP8266WebServer.h>
#include <ESP8266HTTPUpdateServer.h>
#include <WiFiManager.h>
#include <ArduinoOTA.h>
#include <EEPROM.h>
//No longer needed as reading JsonChunkSize bytes at a time
//#include <PrintEx.h>

#define JsonChunkSize 2920

char jsontext[JsonChunkSize];

#define JsonHeaderText "HTTP/1.1 200 OK\r\nContent-Type: application/json;charset=utf-8\r\nServer: Hub Bridge\r\nConnection: close\r\n\r\n"


//////////////////////////////////////////////////////////////////////////////////

ESP8266WebServer server(80);
ESP8266HTTPUpdateServer httpUpdater;


void handleRoot() {
  //This returns the 'homepage' with links to each other main page
  
  char szTmp[1024];
  sprintf(szTmp, "<html>Web Commands
<a href='/relay'>Hub Relay</a></html>");
  server.send(200, "text/html", szTmp);
}

void handleRelay() {
    //Start building the response with the original request and then add in the hub response
    char commandchararray[500];
    char devicechararray[50];
    int commandlength;
    bool has_device_arg = false;
  
    if (server.hasArg("command")){
        //If we have a command then lets log it for manipulation
        strncpy(commandchararray, server.arg("command").c_str(), 499);
        //Ensure the ending is null char
        commandchararray[499] = '\0';
        Serial.println(commandchararray);
        commandlength = strlen(commandchararray);
        if (server.hasArg("device")) {
            has_device_arg = true;
            strncpy(devicechararray, server.arg("device").c_str(), 49);
            devicechararray[49] = '\0';
            }
    }
    else {
        //No command so send a response to say so
        Serial.println("No command received");
        WiFiClient client;
        client = server.client();
        client.print(JsonHeaderText);
        client.print("{\"No command received\":\"Add /?command=blahblahblah\"}");
        client.stop();
        return;
    }

    strcpy(jsontext, "{\"relaycommand\": ");
    strcat(jsontext, commandchararray);
    strcat(jsontext, ",\r\n");
    if (has_device_arg) {
        strcat(jsontext, "\"relaydevice\": \"");
        strcat(jsontext, devicechararray);
        strcat(jsontext, "\",\r\n");
    }
    strcat(jsontext, "\"relayresult\":");
    

    WiFiClient client; //This is main for the requester
    WiFiClient hubsrv; //This is for the Hub

   //Tried all of these with different varieties with no luck
    //client.setNoDelay(true);
    //hubsrv.setNoDelay(true);
    //client.setTimeout(200);
    //hubsrv.setTimeout(200);
    
    if (hubsrv.connect("192.168.1.100", 424)) {
        Serial.println("Connected to hub");
    }
    else {
        Serial.println("Connection to hub failed");
        return;
    }
  
    //Send command with the \0 otherwise the server rejects the command
    hubsrv.write((const uint8_t *)commandchararray, commandlength+1);
    
    //Now set up the client for printing the return
    client = server.client();
    client.print(JsonHeaderText);
    client.print(jsontext);

    //Just have this wait in case the response is not back from hub yet
    unsigned long timeout = millis();
    while (hubsrv.available() == 0) {
        if (millis() - timeout > 8000) {
            Serial.println(">>> Client Timeout !");
            hubsrv.stop();
            client.stop();
            return;
        }
    }

    while(hubsrv.available()){
        hubsrv.readBytes(jsontext, JsonChunkSize-1);
        jsontext[JsonChunkSize-1] = '\0';
        client.print(jsontext);
    }
    
    client.print("}\r\n");
    
    //Now we should close the connection to the client and hub
    hubsrv.stop();
    client.stop();
}


void handleNotFound(){
  String message = "File Not Found\n\n";
  message += "URI: ";
  message += server.uri();
  message += "\nMethod: ";
  message += (server.method() == HTTP_GET)?"GET":"POST";
  message += "\nArguments: ";
  message += server.args();
  message += "\n";
  for (uint8_t i=0; i<server.args(); i++){
    message += " " + server.argName(i) + ": " + server.arg(i) + "\n";
  }
  server.send(404, "text/plain", message);
}

void setup(void){

    Serial.begin(9600); //enable Serial if we need it for debugging

    WiFiManager wifiManager;
    //reset settings - for testing
    //wifiManager.resetSettings();
    wifiManager.setTimeout(180);
    
    //fetches ssid and pass and tries to connect
    //if it does not connect it starts an access point with the specified name
    //and goes into a blocking loop awaiting configuration
    if(!wifiManager.autoConnect("HubRelayBridge")) {
        Serial.println("failed to connect and hit timeout");
        delay(3000);
        //reset and try again, or maybe put it to deep sleep
        ESP.reset();
        delay(5000);
    }


    Serial.println("WiFi Connected - Local IP:");
    Serial.println(WiFi.localIP());
  
    server.on("/", handleRoot);
    server.on("/relay", handleRelay);
    //server.on("/config", HTTP_POST, handleConfig);
   

    //Link the HTTP Updater with the web sesrver
    httpUpdater.setup(&server);
    //Start listening for OTA update requests
    ArduinoOTA.begin();
    //Use default password
    ArduinoOTA.setPassword((const char *)"123");
  
    server.onNotFound(handleNotFound);
    server.begin();

    Serial.println("WiFi Server Started");
}


void loop(void){
    ArduinoOTA.handle();
    server.handleClient();
}

Here is the Chrome breakdown of loading time:
Queueing 4.00 ms

Connection Start TIME
Stalled 247.00 ms
DNS Lookup 0
Initial connection 1.61 s

Request/Response TIME
Request sent 0
Waiting (TTFB) 10.00 ms
Content Download 4.44 s

Total 6.32s

  char szTmp[1024];
  sprintf(szTmp, "<html>Web Commands
<a href='/relay'>Hub Relay</a></html>");

Are you sure that array is big enough to hold all that data?

        strncpy(commandchararray, server.arg("command").c_str(), 499);
        //Ensure the ending is null char
        commandchararray[499] = '\0';

The NULL belongs after the last byte, not at the end of the array.

:slight_smile: Haha, you're right. It was sized to be 1kb as I originally had other things in it, but then strimmed everything down to try and understand if there were any other factors affecting the loading speed.

As for the null at the end, I know it should be after the last byte, but I figured if the command array was less than 499 bytes it wouldnt matter since it would copy the null from the server.arg. I put that in to be 100% sure that the array would always have a valid ending in case the 'command' arg was 500+bytes long, though actually on investigation it is not needed due to the way strncpy works. Im far from a good coder(!!!), but just wanted to try and put in some safety code.

Any ideas anybody? A few things to investigate would be ace!
Thanks in advance!

Two things you should be looking at. From your code I notice you have quite some Serial.println() commands so you have a Serial console available.

    while(hubsrv.available()){

In this routine: add before it a line like

long int starTime = millis();

and then before the end of the loop add

Serial.println(millis()-startTime);

so you can see how long it take for the chunks to be sent out, and with it how long it takes for the server to send each chunk. Of course the slowness can be on both sides. Add more such lines for more ideas on timing.

It doesn't look like you're buffering much in memory, so that should be OK. Running out of memory would cause a crash rather than a slowdown.

This is not very nice, although it may not be your primary problem:

    while(hubsrv.available()){
        hubsrv.readBytes(jsontext, JsonChunkSize-1);
        jsontext[JsonChunkSize-1] = '\0';
        client.print(jsontext);
    }

You are requesting a datastream of a specific size (JsonChunkSize-1), which may force a wait until either the full data is collected or there is a timeout. The last ‘chunk’ could cause a long wait especially if your size specification of chunksize-1 does not exactly match what is available.
Better would be to use the value of hubsrv.available() to determine the length of the data stream that is currently available, then request only the smaller of (a) that available data stream size or (b) your buffer size (which incidentally may be much smaller).

There is some example code here in a slightly different context: TX problems over wifi · Issue #385 · esp8266/Arduino · GitHub (see around June 5-6 2015)

Thank you both @6v6gt and @wvmarle - good suggestions and have finally been able to test them out.

Your point about the ""hubsrv.readBytes(jsontext, JsonChunkSize-1);"" line is indeed a good shout and that does speed things up a bit - thank you. For future reference I changed it to:

 while (hubsrv.available()) {
   int bytestoread = _min(hubsrv.available(), JsonChunkSize - 1);
   hubsrv.readBytes(jsontext, bytestoread);
   jsontext[JsonChunkSize - 1] = '\0';
   client.print(jsontext);
}

However whilst this sometimes works perfectly, sometimes it cuts out halfway through the response and I get invalid JSON data. Hence I tried adding in the timing code and this is what I see:

gotcommand:0
processedresult:0
connectedhub:4
setupresponse:17
waitedfortimeout:398
availableloop:398
availableloop:5406
availableloop:10421
availableloop:10451

Clearly all of the time is taken up in the loop mentioned above! (hence I wont bother explaining each of the timing points - but I can if you want) Sometimes when running that code it will take 200ms to run through the entire block of JSON as a loop as below:

gotcommand:1
processedresult:1
connectedhub:11
setupresponse:23
waitedfortimeout:376
availableloop:376
availableloop:389
availableloop:418
availableloop:440
availableloop:464
availableloop:494
availableloop:515

I can reliably produce this working result on the first run after updating firmware. However on the second request (and subsequent requests) it always gives a result with the ~5s timeouts happening in the loop and broken JSON data.

I think this happens because data is being received through multiple connections, since sometimes I see JSON data in a weird order, but that may just be a hunch/bug.... Also when checking the size of the chunks, the first one is always 1452 (which is fairly standard TCP I guess), and all subsequent chunks tend to max out the size possible (2919 bytes on the 2920 array). Not sure if that is relevant since it doesnt change much between the first and subsequent requests, and if I drop this to 1452 every time it doesnt seem to make any difference.

Has anybody seen something like this before? Feel like this is great progress (so thank you again), but very keen to solve it now!!

EDIT: Ive just noticed that the hub sends a NULL at the end of its response, could that be messing something up?
EDIT2: Actually Im more convinced it is something to do with the connection not closing properly, perhaps I need to keep it as a global definition which can then stay open...?

What I encountered myself (browser reading a page, refreshing every 10 seconds, page created by the ESP-12F), is that sometimes the connection breaks. The browser connects, but no data arrives and the sketch gets stuck. It times out after 1 second.

I never have problems sending data, though. That seems to always come through in good order (the html page).

My working code, handling the incoming client response. This is called every 500 ms to handle incoming requests.

/**
 * Check for incoming WiFi requests, and handle them.
 */
HydroMonitorNetwork::Request HydroMonitorNetwork::WiFiConnection(SensorData sensorData) {

  // Check if a client has connected
  WiFiClient client = server.available();
  if (!client) return {client, ""};

  // Wait until the client sends some data - or timeout.
  long startTime = millis();
  while(!client.available()){
    if (millis() > startTime + 1000) return {client, ""};
    yield();
  }
 
  // Read the first line of the request
  String request = client.readStringUntil('\r');
  client.flush();
  if (request.indexOf("/JSON") != -1) {
    String jsondata = 
      String("{\"hydrodata\": {"
              "\"time\": {"
                "\"year\": " + String(year()) + ","
                "\"month\": " + String(month()) + ","
                "\"day\": " + String(day()) + ","
                "\"hour\": " + String(hour()) + ","
                "\"minute\": " + String(minute()) + ","
                "\"second\": " + String(second()) + "}");
    if (sensorData.useECSensor) jsondata += "\", EC\": " + String(sensorData.EC);
    if (sensorData.useBrightnessSensor) jsondata += "\", brightness\": " + String(sensorData.brightness);
    if (sensorData.useWaterTempSensor) jsondata += "\", waterTemp\": " + String(sensorData.waterTemp);
    if (sensorData.useWaterLevelSensor) jsondata += "\", waterLevel\": " + String(sensorData.waterLevel);
    if (sensorData.usePressureSensor) jsondata += "\", pressure\": " + String(sensorData.pressure);
    if (sensorData.useGrowlight) jsondata += "\", growlight\": " + String(sensorData.growlight);
    jsondata += "}";
    client.println(jsondata);
    return {client, ""};
  }
  
  if (request.indexOf("/settings") != -1) return {client, "settings"};
  if (request.indexOf("/calibrate_ec1") != -1) return {client, "calibrate_ec1"};
  if (request.indexOf("/calibrate_ec2") != -1) return {client, "calibrate_ec2"};
  if (request.indexOf("/calibrate_ec3") != -1) return {client, "calibrate_ec3"};
  if (request.indexOf("/calibrate_ec") != -1) return {client, "calibrate_ec"};

  // If no known URL given, just return the standard page.
  htmlResponse(client, createHtmlPage(createSensorHtml(sensorData), true));
  yield();
  return {client, ""};
}

Hmm, Im definitely getting a weird 'stuck' effect! Sometimes I can send 4-5 large commands and they will all work perfectly, and then suddenly it won't process another until I re-flash the firmware... So weird!

I think that the solution might be something to do with forcefully flushing all connections or something, as perhaps that it what is causing it to get stuck. Though total guesswork! As far as I can tell the only difference with your code is that it keeps the connection and just checks it every time it is used, rather than opening it every time it is needed?

It also seems to be related to the fact that client.print can sometimes become very slow, hence perhaps data is timing out and it all falls over. Though I cant work out why the function would work fine in lots of cases and then break irreparably until re-flashed.

It has a WiFiServer object that's instantiated at the start of the program, and then kept alive. This is the part that receives incoming connections.

The WiFiClient is re-instantiated every time this routine is called. The client object is indeed returned, but only if available, and that client handles only that one connection. This allows other parts of my code to send other pages to that same client.

Do note that this is part a library, which is called from another library which is called from my main sketch (which is down to a couple dozen lines of code and still needs some more stripping).

Hi all,

Im pulling my hair out at this, I have now put the server connection in the setup steps and hence there is only ever one instance of hubsrv " WiFiClient hubsrv; ", but I am still having the same problem…

The first time the page is requested it works fine, 100% of the time, but on every subsequent request the chance of failure increases, such that by the third request it would almost always fail.

Updated code below:

#include <ESP8266WiFi.h>
#include <WiFiClient.h>
#include <WiFiUdp.h>
#include <ESP8266WebServer.h>
#include <ESP8266HTTPUpdateServer.h>
#include <WiFiManager.h>
//#include <ESP8266mDNS.h>
//#include <WebSocketClient.h>
#include <ArduinoOTA.h>
#include <EEPROM.h>

//This enables flashing of ESP via OTA (WiFI)
#define PM_ENABLE_OTA_UPDATES

//Default values for SmartThings, these are only used when first booting and stored in EEPROM from first boot onwards, no need to change
#define JsonChunkSize 2920


char jsontext[JsonChunkSize];
#define JsonHeaderText "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n"

ESP8266WebServer server(80);
ESP8266HTTPUpdateServer httpUpdater;

WiFiClient hubsrv; //This is for the Hub


void handleRoot() {
  //This returns the 'homepage' with links to each other main page

  char szTmp[1024];
  sprintf(szTmp, "<html>Web Commands
<a href='/relay'>Hub Relay</a></html>");
  server.send(200, "text/html", szTmp);
}

void handleRelay() {
  //Start building the response with the original request and then add in the hub response
  char commandchararray[500];
  char devicechararray[50];
  int commandlength;
  bool has_device_arg = false;

  char timingstring[2000];
  char timestr[8];
  long int startTime = millis();
  timingstring[0] = '\0';

  strcat(timingstring, ", \"timingstr\":\"");
  strcat(timingstring, "start:");
  sprintf(timestr, "%d", millis()-startTime);
  strcat(timingstring, timestr);
  Serial.print("start:");
  Serial.println(millis()-startTime);

  if (server.hasArg("command")) {
    //If we have a command then lets log it for manipulation
    strncpy(commandchararray, server.arg("command").c_str(), 499);
    //Ensure the ending is null char
    commandchararray[499] = '\0';
    Serial.println(commandchararray);
    commandlength = strlen(commandchararray);
    if (server.hasArg("device")) {
      has_device_arg = true;
      strncpy(devicechararray, server.arg("device").c_str(), 49);
      devicechararray[49] = '\0';
    }
  }
  else {
    //No command so send a response to say so
    Serial.println("No command received");
    WiFiClient client;
    client = server.client();
    client.print(JsonHeaderText);
    client.print("{\"No command received\":\"Add /?command=blahblahblah\"}");
    client.stop();
    return;
  }

  strcat(timingstring, " gotcommand:");
  sprintf(timestr, "%d", millis()-startTime);
  strcat(timingstring, timestr);
  Serial.print("gotcommand:");
  Serial.println(millis()-startTime);

  strcpy(jsontext, "{\"relaycommand\": ");
  strcat(jsontext, commandchararray);
  strcat(jsontext, ",\r\n");
  if (has_device_arg) {
    strcat(jsontext, "\"relaydevice\": \"");
    strcat(jsontext, devicechararray);
    strcat(jsontext, "\",\r\n");
  }
  strcat(jsontext, "\"relayresult\":");

  strcat(timingstring, " processedresult:");
  sprintf(timestr, "%d", millis()-startTime);
  strcat(timingstring, timestr);
  Serial.print("processedresult:");
  Serial.println(millis()-startTime);

  WiFiClient client; //This is main for the requester

  if (hubsrv.connect("192.168.1.198", 424)) {
    //Serial.println("Connected to Hub");
  }
  else {
    Serial.println("Connection to Hub failed");
    return;
  }

  strcat(timingstring, " connectedhub:");
  sprintf(timestr, "%d", millis()-startTime);
  strcat(timingstring, timestr);
  Serial.print("connectedhub:");
  Serial.println(millis()-startTime);


  strcat(timingstring, " postwrite:");
  sprintf(timestr, "%d", millis()-startTime);
  strcat(timingstring, timestr);
  Serial.print("postwrite:");
  Serial.println(millis()-startTime);

  //Create client for response
  client = server.client();

  strcat(timingstring, " postclientconnect:");
  sprintf(timestr, "%d", millis()-startTime);
  strcat(timingstring, timestr);
  Serial.print("postclientconnect:");
  Serial.println(millis()-startTime);
  
  client.write(JsonHeaderText, strlen(JsonHeaderText));

  strcat(timingstring, " postheader:");
  sprintf(timestr, "%d", millis()-startTime);
  strcat(timingstring, timestr);
  Serial.print("postheader:");
  Serial.println(millis()-startTime);

  client.write((const uint8_t *)jsontext, strlen(jsontext));


  strcat(timingstring, " postjsoncommand:");
  sprintf(timestr, "%d", millis()-startTime);
  strcat(timingstring, timestr);
  Serial.print("postjsoncommand:");
  Serial.println(millis()-startTime);

  hubsrv.write((const uint8_t *)commandchararray, commandlength + 1);

  //Just have this wait in case the response is not back from hub yet
  unsigned long timeout = millis();
  while (hubsrv.available() == 0) {
    if (millis() - timeout > 8000) {
      Serial.println(">>> Client Timeout !");
      hubsrv.stop();
      client.stop();
      return;
    }
  }

  strcat(timingstring, " waitedfortimeout:");
  sprintf(timestr, "%d", millis()-startTime);
  strcat(timingstring, timestr);
  Serial.print("waitedfortimeout:");
  Serial.println(millis()-startTime);

  while (hubsrv.available()) {
    int bytestoread = _min(1452, _min(hubsrv.available(), JsonChunkSize - 2));
    
    strcat(timingstring, " availableloop:");
    sprintf(timestr, "%d", millis()-startTime);
    strcat(timingstring, timestr);
    strcat(timingstring, "-bytes:");
    sprintf(timestr, "%d", bytestoread);
    strcat(timingstring, timestr);
    Serial.print("availableloop:");
    Serial.println(millis()-startTime);
    Serial.print("-bytes:");
    Serial.println(bytestoread);

    hubsrv.readBytes(jsontext, bytestoread); //readBytes
    jsontext[bytestoread] = '\0';
    //jsontext[JsonChunkSize - 1] = '\0';
    //client.print(jsontext);
    client.write((const uint8_t *)jsontext, strlen(jsontext));
  }

  client.write((const uint8_t *)timingstring, strlen(timingstring));
  client.println("\"}\r\n");


  hubsrv.flush();
  client.stop();
    
  Serial.print("finished:");
  Serial.println(millis()-startTime);
}


void handleNotFound() {
  String message = "File Not Found\n\n";
  message += "URI: ";
  message += server.uri();
  message += "\nMethod: ";
  message += (server.method() == HTTP_GET) ? "GET" : "POST";
  message += "\nArguments: ";
  message += server.args();
  message += "\n";
  for (uint8_t i = 0; i < server.args(); i++) {
    message += " " + server.argName(i) + ": " + server.arg(i) + "\n";
  }
  server.send(404, "text/plain", message);
}

void setup(void) {

  Serial.begin(9600); //enable Serial if we need it for debugging


  //WiFiManager
  //Local intialization. Once its business is done, there is no need to keep it around
  WiFiManager wifiManager;
  //reset settings - for testing
  //wifiManager.resetSettings();

  //sets timeout until configuration portal gets turned off
  //useful to make it all retry or go to sleep
  //in seconds
  wifiManager.setTimeout(180);

  //fetches ssid and pass and tries to connect
  //if it does not connect it starts an access point with the specified name
  //and goes into a blocking loop awaiting configuration
  if (!wifiManager.autoConnect("HubRelayBridge")) {
    Serial.println("failed to connect and hit timeout");
    delay(3000);
    //reset and try again, or maybe put it to deep sleep
    ESP.reset();
    delay(5000);
  }

  Serial.println("WiFi Connected - Local IP:");
  Serial.println(WiFi.localIP());

  server.on("/", handleRoot);
  server.on("/relay", handleRelay);


#ifdef PM_ENABLE_OTA_UPDATES
  //Link the HTTP Updater with the web sesrver
  httpUpdater.setup(&server);
  //Start listening for OTA update requests
  ArduinoOTA.begin();
  //Use default password
  ArduinoOTA.setPassword((const char *)"123");
#endif

  server.onNotFound(handleNotFound);
  server.begin();

  Serial.println("WiFi Server Started");
}


void loop(void) {
#ifdef PM_ENABLE_OTA_UPDATES
  ArduinoOTA.handle();
#endif
  server.handleClient();
}

What happens exactly when the request fails?

There are a few problems:

  1. The small problem is that after sending data it then takes about 3+seconds to close the connection (i.e. spinning Chrome logo), but all data has already been sent and received. I am mentioning this only in case it is helpful.

  2. The main problem is two fold, and I cant work out how they are connected:
    2a) Each of the availableloop sections (line = while (hubsrv.available()) {) goes from taking <100ms to taking 5000ms, this seems to make data transmission time out and hence an incomplete response from the hub is received and an incomplete response is received when on the webpage on the ESP.
    2b) The packets of data which are sent can quite often arrive in the wrong order, i.e. if the page is 6kb, then it might be made of up a small ~40byte packets, then several 1452byte packets (the max based on my setting), and finally a remainder. This doesnt always seem to be the case, but they can be out of order and incomplete.

I hope this makes sense… As soon as problem 2 has happened once, it will always happen until the firmware is re-flashed.

Thank you again for your help

One thing that may help debugging is have your html output dumped on our Serial console. Then you see what the board is trying to send out to your browser, and in what order.

cjcharles:

  1. The small problem is that after sending data it then takes about 3+seconds to close the connection (i.e. spinning Chrome logo), but all data has already been sent and received. I am mentioning this only in case it is helpful.

Sounds like Chrome doesn’t realise the page is completed. Some end character missing.

For the second part: really not sure what’s happening there, I do notice that you have lots of html control code missing. This is why I use to send out a JSON string, or an html page, to the browser:

void htmlResponse(WiFiClient client, String response) {
  client.println(F("HTTP/1.1 200 OK"));
  client.println(F("Content-Type: text/html"));
  client.println(""); //  do not forget this one
  client.println(F("<!DOCTYPE HTML>"));
  
  // The maximum amount of bytes we can send out in one go is 2,920. The string
  // response may be longer than that, so we have to deal with it accordingly
  // and send the data in chunks of 2,900 bytes.
  Serial.println("Size of response: " + String(response.length()));
  
  int size = response.length();
  int i = 0;
  while (i < size) {
    if (size < i + 2900) client.print(response.substring(i));
    else client.print(response.substring(i, i+2900));
    i += 2900;
  }
  client.println();
  yield();
  return;
}

Response is a String that can contain html code or plain json data and it displays fine in the browser (Firefox in my case). I just realise I never close the client or anything, seems to work fine. Got one running now for weeks receiving requests every 10 seconds and sending out data every 10 minutes without a hitch.

The code to send out the data is this (here the client is stopped properly :))

sendData(SensorData sensorData) {

  // Try to make the connection with the server.
  if (!client.connect(settings.MysqlHostname, HTTPSPORT)) return;

  // Prepre the data string to be sent to the server.
  String getdata = "user=" + String(settings.MysqlUsername) + "&password=" + String(settings.MysqlPassword);
  if (sensorData.useECSensor) getdata += "&ec=" + String(sensorData.EC);
  if (sensorData.useBrightnessSensor) getdata += "&brightness=" + String(sensorData.brightness);
  if (sensorData.useWaterTempSensor) getdata += "&watertemp=" + String(sensorData.waterTemp);
  if (sensorData.useWaterLevelSensor) getdata += "&waterlevel=" + String(sensorData.waterLevel);
  if (sensorData.usePressureSensor) getdata += "&pressure=" + String(sensorData.pressure);
  if (sensorData.useGrowlight) getdata += "&growlight=" + String(sensorData.growlight);
  
  String url = settings.MysqlUrlBase + getdata;
  client.print(String("GET ") + settings.MysqlUrlBase + " HTTP/1.1\r\n" +
               "Host: " + settings.MysqlHostname + "\r\n" +
               "User-Agent: HydroMonitor\r\n" +
               "Connection: close\r\n\r\n");
  while (client.connected()) {
    String line = client.readStringUntil('\n');
    if (line == "\r") {

      // Headers all received.
      break;
    }
  }
  client.stop();
  return;
}