How to put a JSON response into a variable (Uno with Ethernet shield W5100)

Projects Networking, Protocols, and Devices
1

Hi all,

I'm currently working on a project what I thought would be fairly simple, but is already driving me nuts for a week searching all over the web trying to find a solution. It's about how to capture a stream (JSON answer) from an online API and converting it into a variable/array that can then be deserialized (in my case by the ArduinoJSON library). I found multiple questions like mine across various forums remaining unsolved.

Hardware:
Arduino Uno (R3) with an Ethernet shield (W5100) attached to it, hooked up to my router via LAN. Internet access and sending/retrieving data works beautifully. No problems there.

Libraries:

#include <SPI.h>
#include <Ethernet.h>
#include "ArduinoJson.h"

Code:
The Arduino makes an HTTP-request to an online API, which responds with a JSON formatted answer. Example of an actual response:

{
  "aral": {
    "e5": 1.759,
    "e10": 1.709,
    "diesel": 1.339
  },
  "shell": {
    "e5": 1.779,
    "e10": 1.719,
    "diesel": 1.349
  }
}

During testing I tracked the APIs JSON response by printing it to the serial monitor using:

while (client.available()) {
    char c = 0;
    client.readBytes(&c, 1);
    Serial.print(c);
}

This works great as the response shows up in full as expected without any problems. However client.readBytes is only able to capture the JSON response by each individual byte/character recieved.

What I would like is to put the full JSON response into a variable (let's say: const char json[] = "*the actual api json response*") so it can then be deserialized using the ArduinoJSON library for further use. I tried many things, of which this was the latest, but it didn't produce any results:

while (client.available()) {
    char c = client.read();
    readString += c; // store all individual characters into variable "readString"
}

I can get everything to work except for how to capture the JSON data stream sent by the API to the Ethernet shield and put it in some sort of variable/array for further use? I read a lot of ArduinoJSON documentation but the big hiatus remains the question above as the manual does not provide any insight on this. It basically starts from the point of having a filled array already present, but offers no intel on the step before of how to create this array from a JSON response received by the Arduino.

Can someone point me in the right direction on how to proceed?

(All my program is subsequently intended to do is outputting the petrol prices fetched from the API to a display. Extremely basic, nothing fancy. Done it a million times before; just not with a JSON API response like what I have to work with here.. That is actually my first time and I want to figure out how to do it.)

2
3

Thank you @Juraj . That was actually one of the posts I found while searching the web. However this only seems to work with a wifi network and I haven't been able to get it working using the Ethernet shield.

Below my updated code. It works like a charm getting the APIs JSON response but I'm stuck on the following:
I understand I have to somehow switch to HTTP 1.0 ( http.useHTTP10(true); ) and then capture the stream ( httpClient.getStream() ) to pass in deserializeJson(doc, httpClient.getStream());, but I can't figure out how to do this. What is it that I'm missing here?

// load libraries
#include <SPI.h>
#include <HttpClient.h> // https://github.com/amcewen/HttpClient/blob/master/examples/SimpleHttpExample/SimpleHttpExample.ino
#include <Ethernet.h>
#include <EthernetClient.h>

// Name of the server we want to connect to
const char kHostname[] = "date.jsontest.com";
// Path to download (this is the bit after the hostname in the URL that you want to download
const char kPath[] = "/";

byte mac[] = {0xDE, 0xAD, 0xBE, 0xEF, 0xFE, 0xED};

// Number of milliseconds to wait without receiving any data before we give up
const int kNetworkTimeout = 30*1000;
// Number of milliseconds to wait if no data is available before trying again
const int kNetworkDelay = 1000;

void setup(){
  Serial.begin(9600); 
  while (Ethernet.begin(mac) != 1){
    Serial.println("Error getting IP address via DHCP, trying again...");
    delay(15000);
  }
  Serial.print("DHCP assigned IP: ");
  Serial.println(Ethernet.localIP());
}

void loop(){
  int err =0;
  EthernetClient c;
  HttpClient http(c);
  // TO DO: downgrade to HTTP 1.0 somehow? Calling http.useHTTP10(true); returns an error: no such class..
  err = http.get(kHostname, kPath);
  if (err == 0){
    Serial.println("startedRequest ok");
    err = http.responseStatusCode();
    if (err >= 0){
      Serial.print("Got status code: ");
      Serial.println(err);

      // Print out whatever response we get
      err = http.skipResponseHeaders();
      if (err >= 0){
        int bodyLen = http.contentLength();
        Serial.print("Content length is: ");
        Serial.println(bodyLen);
        Serial.println();
        Serial.println("Body returned follows:");
      
        // Now we've got to the body, so we can print it out
        unsigned long timeoutStart = millis();
        char c;
        
        // Whilst we haven't timed out & haven't reached the end of the body
        while((http.connected() || http.available()) && ((millis() - timeoutStart) < kNetworkTimeout)){
            if (http.available()){
                c = http.read();
                // Print out this character
                Serial.print(c);
                // TO DO: capture the stream somehow...?
                bodyLen--;
                // We read something, reset the timeout counter
                timeoutStart = millis();
            }else{
                // We haven't got any data, so let's pause to allow some to arrive
                delay(kNetworkDelay);
            }
        }
      }else{
        Serial.print("Failed to skip response headers: ");
        Serial.println(err);
      }
    }else{    
      Serial.print("Getting response failed: ");
      Serial.println(err);
    }
  }else{
    Serial.print("Connect failed: ");
    Serial.println(err);
  }
  http.stop();
  // And just stop, now that we've tried a download
  while(1);
}

Are there perhaps any tutorials available somewhere which detail the process of using an Ethernet shield instead of wifi (ESP) for ArduinoJSON?

4

sorry. then use the ArduinoHttpClient library. it will return the json as String

5

What makes using Ethernet so different from ESP that things like stream capture are difficult/different? Is it simply that all Arduino Ethernet libraries lack this functionality or is there a more fundamental/technical reason that stream capture is easier with ESP?

6

the ESP8266HTTPClient library is bundled with the esp8266 platform.

I recommend using the String returned by responseBody(() because you json is small.
with ArduinoHttpClient library you would use the http object as Stream.

7

I am using in Ethernet library standard client.readString() function to which I save variable - full JSON payload. I have same code for ESP8266 or Arduino with Ethernet (client object is used for EthernetClient or WiFiClient class), below is output of Serial monitor on ESP8266 board. After receiving JSON payload, I am doing parsing using ArduinoJson library to read in my case value and volume of well. And then I am sending datas to MQTT Broker.

Fragment of code for reading payload
It is reading HTTP header (it is not printed in serial monitor), then there is blank row and then it will read, save and write JSON payload to serial monitor).

while (client.connected()) {
        String line = client.readStringUntil('\n'); //HTTP HEADER
        //Serial.println(line);
        if (line == "\r") {
          break;
        }
      }
      String line = client.readString(); //PAYLOAD
      Serial.println(line); //PAYLOAD (JSON) is printed to Serial monitor, it is stored in line variable

You can use HTTP 1.1 or 1.0, in this case because of JSON payload it will be not chunk encoded (if there is correct header of content-type set on side of webserver). If it is chunk encoded (probably because of text/html content type), use HTTP 1.0.

Full code if interested in testing working solution (you must set Ethernet library, there is set Ethernet2 at the moment for W5500 shield): hladinomer-studna-scripty/Ethernet_MQTT.ino at master · martinius96/hladinomer-studna-scripty · GitHub

It is reading JSON output at: http://arduino.clanweb.eu/studna_s_prekladom/json_output.php

8

Thank you @martinius96 , I can see your String approach there. After testing a lot I came up with what I think is in essence the same as yours, and it works nicely.

#include <SPI.h>
#include <HttpClient.h>
#include <Ethernet.h>
#include <EthernetClient.h>
#include <ArduinoJson.h>

const char kHostname[] = "date.jsontest.com";
const char kPath[] = "/";
const int kNetworkTimeout = 30000; // time to wait without receiving any data before giving up (ms)
const int kNetworkDelay = 1000;    // time to wait if no data is available before trying again (ms)

void getInfo(){
  int err = 0;                     // initially everything is ok
  EthernetClient c;
  HttpClient http(c);
  err = http.get(kHostname, kPath);
  if (err == 0){
    Serial.print(F("HTTP request sent"));
    err = http.responseStatusCode();
    if (err >= 0){
      Serial.print(F("Got status code: "));
      Serial.println(err);

      // Print out whatever response we get
      err = http.skipResponseHeaders();
      if (err >= 0){
        int bodyLen = http.contentLength();
        Serial.print(F("Content length is: "));
        Serial.println(bodyLen);
        Serial.println(F("Body returned the following:"));
      
        // Now we've got to the body, so we can print it out
        unsigned long timeoutStart = millis();
        char c;
        String JSONresponse = "";
        
        // Whilst we haven't timed out & haven't reached the end of the body
        while((http.connected() || http.available()) && ((millis() - timeoutStart) < kNetworkTimeout)){
            if (http.available()){
                c = http.read();
                // Print out this character
                Serial.print(c);
                JSONresponse += c;    // capture the JSON repsonse in a String for further use
                bodyLen--;
                // We read something, reset the timeout counter
                timeoutStart = millis();
            }else{
                // No data was received, so let's pause to allow some to arrive
                delay(kNetworkDelay);
            }
        }
        // print out the content of the JSON var
        Serial.println();
        Serial.println(F("Content of var JSONresponse: "));
        Serial.println(JSONresponse);

        //Use the ArduinoJson Assistant to calculate this:
        DynamicJsonDocument doc(192);
        DeserializationError error = deserializeJson(doc, JSONresponse);

        if(error){
          Serial.print(F("deserializeJson() failed: "));
          Serial.println(error.f_str());
        }else{
          Serial.println(F("Data successfully decoded!"));
          // further data processing...
        }
      }else{
        Serial.print(F("Failed to skip response headers: "));
        Serial.println(err);
      }
    }else{    
      Serial.print(F("Getting response failed: "));
      Serial.println(err);
    }
  }else{
    Serial.print(F("Connecting failed: "));
    Serial.println(err);
  }
  http.stop();
  Serial.println(F("HTTP connection closed."));
  Serial.println();
}
9

In my case I do the following:
1- I have the SCRIPT in a separate VOID (void scripts(EthernetClient &cscr))
2- I receive the JSON response in a function:

cscr.print(F("function update(){var xhttp;xhttp=new XMLHttpRequest();xhttp.onreadystatechange=function(){"));
    cscr.print(F("if (xhttp.readyState == 4 && xhttp.status == 200) {var json = JSON.parse(xhttp.responseText);"));
    cscr.print(F("document.getElementById(\"act_date\").innerHTML = json.arduino.act_date;"));
    cscr.print(F("document.getElementById(\"act_time\").innerHTML = json.arduino.act_time;"));
    cscr.print(F("document.getElementById(\"d_y_n\").innerHTML = json.arduino.d_y_n;"));
    cscr.print(F("document.getElementById(\"d_y_n_1\").className = \"marker \"+json.arduino.d_y_n;"));
    cscr.print(F("document.getElementById(\"v_tempc\").innerHTML = json.arduino.v_tempc;"));
    cscr.print(F("document.getElementById(\"v_tempf\").innerHTML = json.arduino.v_tempf;"));
    cscr.print(F("document.getElementById(\"v_hum\").innerHTML = json.arduino.v_hum;"));
    cscr.print(F("if (json.arduino.D13 == \"true\") {"));
    cscr.print(F("document.getElementById(\"D13\")).checked = true;} else {document.getElementById(\"D13\").checked = false;}"));
    cscr.print(F("document.getElementById(\"can_dump\").innerHTML = json.arduino.can_dump;"));
    cscr.print(F("document.getElementById(\"u_d\").innerHTML = json.arduino.u_d;"));
    cscr.print(F("document.getElementById(\"progress\").value = json.arduino.progress;"));
    cscr.print(F("if (json.arduino.light == \"true\") {"));
    cscr.print(F("document.getElementById(\"light\")).checked = true;} else {document.getElementById(\"light\").checked = false;}"));
    cscr.print(F("document.getElementById(\"status_system\").innerHTML = json.arduino.status_system;"));
    cscr.print(F("document.getElementById(\"threshold\").innerHTML = json.arduino.threshold;"));
    cscr.print(F("temp_btn(json.arduino.v_tempc)}};"));
    cscr.print(F("xhttp.open(\"GET\", \"http://")); //I print the IP of the arduino itself
    for (int i = 0; i < 4; i++) {
        cscr.print(ip[i]);
        if (i < 3) cscr.print(F("."));
    }
    cscr.print(F("/ajax1\", true);xhttp.send();}"));

3- Then I can manipulate in that function any JSON element

10

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.