Extract Json-body from GET-request payload

I am trying to get the Json-Body of a GET request payload extracted.

Normally, I would use, for example, the ArduinoHTTPClient-library's client.responseBody() method. But in my case, I am using the EthernetSSLClient library and in there, the ArduinoHTTPClient functions do not work.

It is critical that I use SSL since our clients request this level of security. But now I am here with a library that does not seem to offer an easy GET-request json-body extraction.

My code (see below) returns the following response upon my GET-request:

HTTP/1.1 200 OK
Connection: close
Content-Type: application/json; charset=utf-8
Content-Length: 72
Date: Tue, 09 Nov 2021 17:27:49 GMT
Server: DelphiMVCFramework
X-Powered-By: DMVCFramework 3.2.1 (carbon)

{"Version":"1.4.724.2267","Application":"MyApplication Commander"}

As you can see, there is a header upfront and the needed payload is at very end (Json).

Can you tell me how to nicely extract the json-body from this ?

My workarround solution (far from robust as I think) is as follows:

  String payload = "";
  while (sslClient.available()) {
    char c = sslClient.read();
    payload += c;
  }
  int index = payload.indexOf("{");
  String body = payload.substring(index);

But I think simply finding the first "{" is not a good solution.

Can you suggest a library that works in combination with the EthernetSSLClient library that consists of a more robust solution to the json-body extraction problem ?

Or do you have any code example that improves my current solution ?

Thank you.

Here is my entire code:

#include "defines.h"
#include "trust_anchors.h"   // You must have SSL Certificates here
#include <ArduinoJson.h>

#define USE_THIS_SS_PIN   5   // default CS/SS pin for MKRWiFi1010 board or MKRZERO board

const uint16_t  server_port = 6060;
byte mac[] = { 0xA8, 0x61, 0x0A, 0xAE, 0x28, 0x9C }; // MAC Adress of Arduino MKR ETH Shield
IPAddress server_host(192, 168, 1, 43);
// IPAddress myDns(192, 168, 1, 1);
// IPAddress gateway(192, 168, 1, 1);
// IPAddress subnet(255, 255, 255, 0);

// Initialize the SSL client library
EthernetClient    client;
EthernetSSLClient sslClient(client, TAs, (size_t)TAs_NUM);

// Variables to measure the speed
unsigned long beginMicros, endMicros;
unsigned long byteCount = 0;
unsigned long previousMillis = 0;
unsigned long interval = 1000;
unsigned long connectionTime = 0;
unsigned long startTime = 0;

void makeRequest(String cmd) {
  sslClient.println(cmd);        
  sslClient.print("Host: ");
  sslClient.println(server_host);
  sslClient.println("User-Agent: TriMini Buttons");        
  sslClient.println();
}

void showReceivedBytes() {
  Serial.print("Received ");
  Serial.print(byteCount);
  Serial.print(" bytes in ");
  float seconds = (float)(endMicros - beginMicros) / 1000000.0;
  Serial.print(seconds, 4);
  float rate = (float)byteCount / seconds / 1000.0;
  Serial.print(" s, rate = ");
  Serial.print(rate);
  Serial.print(" kbytes/second");
  Serial.println();
}

void checkVersion() {

  // if there are incoming bytes available
  // from the server, read them
  int len = sslClient.available();
  
  if (len > 0) {

    const size_t capacity = 80; // JSON_OBJECT_SIZE(3) + JSON_ARRAY_SIZE(2) + 60;
    byte buffer[capacity];
    if (len > capacity) { len = capacity; }
    
    sslClient.read(buffer, len);
    byteCount = byteCount + len;
  }
}

void setup()
{
  // Open serial communications and wait for port to open:
  Serial.begin(115200);
  delay(2000);
  
  Serial.print("\nStart WebClient_SSL on " + String(BOARD_NAME));
  Serial.println(" with " + String(SHIELD_TYPE));
  Serial.println(ETHERNET_WEBSERVER_SSL_VERSION);
  
  ET_LOGWARN(F("=========== USE_ETHERNET ==========="));
  
  ET_LOGWARN(F("Default SPI pinout:"));
  ET_LOGWARN1(F("MOSI:"), MOSI);
  ET_LOGWARN1(F("MISO:"), MISO);
  ET_LOGWARN1(F("SCK:"),  SCK);
  ET_LOGWARN1(F("SS:"),   SS);
  ET_LOGWARN(F("========================="));
  
  ET_LOGWARN3(F("Board :"), BOARD_NAME, F(", setCsPin:"), USE_THIS_SS_PIN);

  Serial.print("MAC Adress: ");
  Serial.print(mac[0], HEX); Serial.print(" ");
  Serial.print(mac[1], HEX); Serial.print(" ");
  Serial.print(mac[2], HEX); Serial.print(" ");
  Serial.print(mac[3], HEX); Serial.print(" ");
  Serial.print(mac[4], HEX); Serial.print(" ");
  Serial.println(mac[5], HEX);
  
  Ethernet.init (USE_THIS_SS_PIN);
  
  // start the ethernet connection and the server:
  // Use DHCP dynamic IP and random mac
  // Ethernet.begin(mac[index], server_host, myDns, gateway, subnet);
  Ethernet.begin(mac);
         
  Serial.print(F("Connected! IP address: "));
  Serial.println(Ethernet.localIP());
  
  // give the Ethernet shield a second to initialize:
  delay(2000);
  
  Serial.print("Connecting to : ");
  Serial.print(server_host);
  Serial.print(", port : ");
  Serial.println(server_port);
  
  // if you get a connection, report back via serial:
  startTime = millis();
  
  // specify the server and port, 443 is the standard port for HTTPS
  if (sslClient.connect(server_host, server_port)) {
    connectionTime = millis() - startTime;
    
    Serial.print("Connected to ");
    Serial.println(client.remoteIP());  
    
    Serial.print("Connect took: ");
    Serial.println(connectionTime);
    
    // Make a HTTP request:
    makeRequest("GET /Cmd/GetVersion HTTP/1.1");
  } else {
    // if you didn't get a connection to the server:
    Serial.println("Connection failed");
  }
  
  beginMicros = micros();
}

void loop()
{
  unsigned long currentMillis = millis();

  String payload = "";
  while (sslClient.available()) {
    char c = sslClient.read();
    payload += c;
  }
  Serial.println(payload);
  // ?????????? HOW DO I EXTRACT JSON-BODY OF THIS payload ????????????

  // My workarround looks like this:
  int index = payload.indexOf("{");
  String body = payload.substring(index);

  // if the server's disconnected, stop the sslClient:
    if (!sslClient.connected()) {
    
        endMicros = micros();
        
        Serial.println();
        Serial.println("Disconnecting.");
        sslClient.stop();
        
        Serial.print("Received ");
        Serial.print(byteCount);
        Serial.print(" bytes in ");
        float seconds = (float)(endMicros - beginMicros) / 1000000.0;
        Serial.print(seconds, 4);
        float rate = (float)byteCount / seconds / 1000.0;
        Serial.print(" s, rate = ");
        Serial.print(rate);
        Serial.print(" kbytes/second");
        Serial.println();
        
        // do nothing forevermore:
        while (true) {
            delay(1);
        }
    }
}

When you are reading the payload, don't add anything to your string until you've seen an opening brace.

Thank you wildbill, do you have a code-example showing exactly what your solution consists of ?

  String payload = "";
  bool seenBrace = false;
  while (sslClient.available()) {
    char c = sslClient.read();
    if (c == '{') {
      seenBrace = true;
    }
    if (seenBrace) {
      payload += c;
    }
  }

The convention is that an empty line marks the end of the headers. If you use an HTTPClient object instead of a raw TCP client it will read the headers for you. Look for a "WebClient" example for your Arduino board.

The ArduinoJson library can do the reading for you so you don't need to read the whole body of the HTML page before you deserialize. See the V6 "Assistant" to write the JSON code for you:

@johnwasser, thank you for the HTTPClient hint. But as mentioned above, I'd love to use such a HTTPClient-object. How can I use it in combination with the given EthernetSSLClient sslClient ? The customers's need for EthernetSSLClient makes this somewhat more difficult. Or do you have a concrete code-example on how to do what you suggest ?

thank you Whandall, this is already a tiny bit better than my original workarround. Still, I miss the niceness of a HTTPClient solution. Any idea on how to do so in combination with the EthernetSSLClient library ?

The code I provided was simply an instantiation of

applied to the corresponding part of your code, because you seemed to struggle with it


I have no experience with the SSL Arduino environment, but I would (naively) think that there
is a HTTPSClient object that would just give you the payload, as @johnwasser suggested.

Added: Seems that in an ESP32 environment the HTTPClient object is able to handle HTTPS also.

That library contains: EthernetHttpClient_SSL.h

I would start with the library example HTTPClient/SimpleGet

I finally found a nice abstraction for the HTTPClient: ArduinoHttpClient library

By adding the following to my code, I was able to get the responseBody out much more easily:

#include <ArduinoHttpClient.h>

const uint16_t  server_port = 443; // or whatever your connected server needs...
byte mac[] = { 0x12, 0x33, 0x4B, 0x2C, 0x34, 0x7D }; // MAC Adress of Arduino MKR ETH Shield
IPAddress client_ip(192, 168, 178, 23);   // client IP-address (Arduino)
IPAddress server_ip(192, 168, 178, 24);   // server IP-address (some server on the same LAN)

EthernetClient client;
EthernetSSLClient sslClient(client, TAs, (size_t)TAs_NUM);
HttpClient http(sslClient, server_ip, server_port);

With the above we can use http nicely abstracted from now on.

An example:

httpError = http.get("/Cmd/GetVersion");

if (httpError == 0) {          
  if (http.responseStatusCode() == 200) {  // takes at least the time defined under variable "kHttpWaitForDataDelay" (found in HttpClient.h)    
    String payload = http.responseBody();  // how nice !
    Serial.println(payload);
  } else {    
    Serial.print("Error: GET response failed. (statusCode = "); Serial.print(http.responseStatusCode()); Serial.println(")");
  }
} else {
  Serial.print("Error: connect failed. (error = "); Serial.print(httpError); Serial.println(")");
}  

One remark:
In my case, it was needed to change the ArduinoHttpClient library slightly. In particular, the sending-Header needed to inlcude Host: 192.168.178.24 in addition.

The following two changes to the original library made it work:

  1. In File "HttpClient.h", line 40: add the following:
    #define HTTP_HEADER_HOST "Host:"
  2. In File "HttpClient.cpp", line 173: add the following:
    sendHeader(HTTP_HEADER_HOST, iServerName);

(not sure if this change applies to all servers - but in my case, it helped. I can now connect my Arduino over Ethernet with SSL and have a nice httpClient-abstraction.