parsing JSON Googlemap data from https chunk return

I am trying to parse the [“routes”][“legs”][0][“duration_in_traffic”][“text”] from the google maps API (https) I can get the whole json dataset to return and print via serial. but I cant figure out how to put it into a buffer for ArduinoJSON to deserialize…

here is the JSON data:

{
   "geocoded_waypoints" : [
      {
         "geocoder_status" : "OK",
         "place_id" : "ChIJYfryL-5gXIYRs-J4zdeHvLI",
         "types" : [ "postal_code" ]
      },
      {
         "geocoder_status" : "OK",
         "place_id" : "ChIJy9U5Yzj2XIYRRhV9xWIghyU",
         "types" : [ "postal_code" ]
      }
   ],
   "routes" : [
      {
         "bounds" : {
            "northeast" : {
               "lat" : 29.5670897,
               "lng" : -98.46781449999999
            },
            "southwest" : {
               "lat" : 29.3938896,
               "lng" : -98.6048464
            }
         },
         "copyrights" : "Map data ©2019",
         "legs" : [
            {
               "distance" : {
                  "text" : "18.2 mi",
                  "value" : 29285
               },
               "duration" : {
                  "text" : "23 mins",
                  "value" : 1399
               },
               "duration_in_traffic" : { 
                  "text" : "23 mins",   <-----This is what Im after
                  "value" : 1384
               },
               
            ---------------REMOVED FOR BREVITY--------------------         
   ],
   "status" : "OK"
}

and my IDE code:

#include <ESP8266WiFi.h>
#include <WiFiClientSecure.h>

#ifndef STASSID
#define STASSID "----SSID---"
#define STAPSK  "----KEY---"
#endif

const char* ssid = STASSID;
const char* password = STAPSK;

const char* host = "maps.googleapis.com";
const int httpsPort = 443;
// Current fingerprint for maps.googleapis.com

const char* fingerprint = "C6 3E 93 AB 25 2B 06 11 2F F9 D3 61 FB 2E 89 28 91 2C 27 46";


void setup()
{
  Serial.begin(115200);
  Serial.println();
  Serial.print("connecting to ");
  Serial.println(ssid);
  WiFi.mode(WIFI_STA);
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }
Serial.println("");
  Serial.println("WiFi connected");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());

  // Use WiFiClientSecure class to create TLS connection
  WiFiClientSecure client;
  Serial.print("connecting to ");
  Serial.println(host);

  Serial.printf("Using fingerprint '%s'\n", fingerprint);
  client.setFingerprint(fingerprint);

  if (!client.connect(host, httpsPort)) {
    Serial.println("connection failed");
    return;
  }


    String url = "/maps/api/directions/json?origin=78249&destination=78210&departure_time=now&traffic_model=best_guess&key=#####MYAPIKEY#####";
    Serial.print("requesting URL: ");
    Serial.println(url);
      client.print(String("GET ") + url + " HTTP/1.1\r\n" +
               "Host: " + host + "\r\n" +
               "User-Agent: ESP8266\r\n" +
               "Connection: close\r\n\r\n");

  Serial.println("request sent");
  while (client.connected()) {
    String line = client.readStringUntil('\n');
    if (line == "\r") {
      Serial.println("headers received");
      break;
    }
  }

  String chunk = "";
  int limit = 1;
  String response="";

Serial.println(response);
    do {
    if (client.connected()) {
      client.setTimeout(4000);
      chunk = client.readStringUntil('\n');
      response += chunk;
      Serial.println(chunk);
    }
  } while (chunk.length() > 0 && ++limit < 1000);
  const size_t capacity = 4*JSON_ARRAY_SIZE(0) + 4*JSON_ARRAY_SIZE(1) + JSON_ARRAY_SIZE(2) + JSON_ARRAY_SIZE(17) + 18*JSON_OBJECT_SIZE(1) + 76*JSON_OBJECT_SIZE(2) + 3*JSON_OBJECT_SIZE(3) + 3*JSON_OBJECT_SIZE(7) + 15*JSON_OBJECT_SIZE(8) + JSON_OBJECT_SIZE(10);
  DynamicJsonDocument doc(capacity + 8552 ); //add in extra bytes for string duplication
  deserializeJson(doc, chunk);
  char Buffer[550]; 
  strcpy(Buffer, doc["routes"]["legs"][0]["duration_in_traffic"]["text"].as<char*>());
  Serial.println(Buffer);    
}

void loop(){
}

why not use the ArduinoJson library?

I am using ArduinoJson… and just implemented the debug code

 DeserializationError error = deserializeJson(doc, chunk);
      if (error) 
      {
      Serial.print(F("deserializeJson() failed: "));
      Serial.println(error.c_str());
      return;
      }

it returns: deserializeJson() failed: InvalidInput .

which is what I already assumed. Im not sure how to place the chunk data returned one charater at a time into a buffer for ArduinoJson to deserialize.

this:

do {
    if (client.connected()) {
      client.setTimeout(4000);
      chunk = client.readStringUntil('\n');
      response += chunk;
      Serial.println(chunk);
    }
  } while (chunk.length() > 0 && ++limit < 1000);

from what I see is taking the get reply once character at a time, I need something more like the http clinet method of getString so I can put the entire json response into the buffer for parsing.

I just dont know how to do that.

my cut & paste missed the

#include <ArduinoJson.h>
#include <Arduino.h>

at the top of my code…

why readStringUntil('\n') are there some '\n' in json data?

the readStringUntil('\n') was in the tutorial i based the code on..

The following ArduinoJson 6 example does the HTTP GET, reads the HTTP status, skips the HTTP headers, then passes the TLS client object to the deserializer. The sketch does not have to buffer the JSON because the deserializer can read from the TLS client object.

I have a working example that I put together to get weather narratives from weather.gov but when I try to use the same methods for Google Map JSON data it wont even connect to the server. I double checked the fingerprint and tried a few different ways of troubleshooting which led me to the tutorial using the chunk method.

I guess I'm missing how to get the HTTPS GET into the ArduinoJson deserializer...

Tried the method with skipping the headers… now I’m getting a different Deserialize Error.

deserializeJson() failed: NotSupported

here is my code:

#include <ArduinoJson.h>
#include <Arduino.h>
#include <ESP8266WiFi.h>
#include <WiFiClientSecure.h>


#ifndef STASSID
#define STASSID "----ssid----"
#define STAPSK  "----key----"
#endif

const char* ssid = STASSID;
const char* password = STAPSK;

const char* host = "maps.googleapis.com";
const int httpsPort = 443;
// Current fingerprint for maps.googleapis.com

const char* fingerprint = "C6 3E 93 AB 25 2B 06 11 2F F9 D3 61 FB 2E 89 28 91 2C 27 46";


void setup()
{
  Serial.begin(115200);
  Serial.println();
  Serial.print("connecting to ");
  Serial.println(ssid);
  WiFi.mode(WIFI_STA);
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }
Serial.println("");
  Serial.println("WiFi connected");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());

  // Use WiFiClientSecure class to create TLS connection
  WiFiClientSecure client;
  Serial.print("connecting to ");
  Serial.println(host);

  Serial.printf("Using fingerprint '%s'\n", fingerprint);
  client.setFingerprint(fingerprint);

  if (!client.connect(host, httpsPort)) {
    Serial.println("connection failed");
    return;
  }

    // Send HTTP request
    String url = "/maps/api/directions/json?origin=78249&destination=78210&departure_time=now&traffic_model=best_guess&key=####KEY####";
    Serial.print("requesting URL: ");
    Serial.println(url);
      client.print(String("GET ") + url + " HTTP/1.1\r\n" +
               "Host: " + host + "\r\n" +
               "User-Agent: ESP8266\r\n" +
               "Connection: close\r\n\r\n");

      // Check HTTP status
      Serial.println("Check HTTP Status");
      char status[32] = {0};
      client.readBytesUntil('\r', status, sizeof(status));
      Serial.print("Status: ");
      Serial.println(status);
      // It should be "HTTP/1.0 200 OK" or "HTTP/1.1 200 OK"
      if (strcmp(status + 9, "200 OK") != 0) {
        Serial.print(F("Unexpected response: "));
        Serial.println(status);
        return;
       }

        // Skip HTTP headers
      char endOfHeaders[] = "\r\n\r\n";
      Serial.println("Skip Headers");
      if (!client.find(endOfHeaders)) {
        Serial.println(F("Invalid response"));
        return;
      }
      
  // Allocate the JSON doc   
  Serial.println("Allocate Json Doc"); 
  const size_t capacity = (4*JSON_ARRAY_SIZE(0) + 4*JSON_ARRAY_SIZE(1) + JSON_ARRAY_SIZE(2) + JSON_ARRAY_SIZE(15) + 16*JSON_OBJECT_SIZE(1) + 68*JSON_OBJECT_SIZE(2) + 3*JSON_OBJECT_SIZE(3) + 2*JSON_OBJECT_SIZE(7) + 14*JSON_OBJECT_SIZE(8) + JSON_OBJECT_SIZE(10)) + 8509;
  Serial.println("Capacity: " + capacity);
  DynamicJsonDocument doc(capacity);
  Serial.println("Begin Deserialize...");
  DeserializationError error = deserializeJson(doc, client);
      if (error) 
      {
      Serial.print(F("deserializeJson() failed: "));
      Serial.println(error.c_str());
      return;
      }
  Serial.println(doc["routes"]["legs"][0]["duration_in_traffic"]["text"].as<char*>());   
}

void loop(){
}

no need of arduino_json library , i use my own

uint16_t data_size = 30;  // expected array elements, 

 String myData[data_size];  // array that holds parsed data

    byte counter = 0 ;

    char delimiter = '\"';  // your delimiting symbol goes here

    for ( uint16_t  i = 0 ; i < received_string.length() ; i ++ ) {
      if ( received_string.charAt(i) == delimiter ) {
        counter++;
      }
      else {
        myData[counter] +=  received_string.charAt(i) ;
      }
    }


// here are the parsed/retrieved data

Serial.println(myData[0]);
Serial.println(myData[1]);
Serial.println(myData[2]);  // an so on

// make sure you set enough buffer size for myData String array

@Jmeredith

I can get the whole json dataset to return and print via serial.

Print everything coming back from the http server include HTTP headers. What is between the end of the HTTP headers and the JSON? Should be nothing but if there is something, the deserializer will choke.

Do you see "Transfer-Encoding: chunked" in the HTTP headers? If so, your code must handle the extra lines of chunk length. And if there are more than one chunk, your code needs a while loop to handle multiple JSON strings separated
by lines of chunk length.

I did not see “Transfer-Encoding: chunked” in the headers. it just started out with the JSON data.

I am trying to back track and simplify my code… this last attempt errors out and causes the Wemos D1 to reset… here is some of the error. maybe my GET reply is too big for my string?

13:08:34.909 -> [WiFi] Connecting to My Network ...1 
13:08:35.914 -> WiFi connected
13:08:35.914 -> IP address: 	
13:08:35.914 -> 192.168.1.119
13:08:35.914 -> [HTTPS] begin...
13:08:35.914 -> [HTTPS] GET...
13:08:37.664 -> [HTTPS] GET... code: 200
13:08:37.699 -> 
13:08:37.699 -> Exception (3):
13:08:37.699 -> epc1=0x4020873d epc2=0x00000000 epc3=0x00000000 excvaddr=0x40007459 depc=0x00000000
13:08:37.699 -> 
13:08:37.699 -> >>>stack>>>
13:08:37.699 -> 
13:08:37.699 -> ctx: cont
13:08:37.699 -> sp: 3ffffa50 end: 3fffffc0 offset: 01a0
13:08:37.699 -> 3ffffbf0:  40103766 00080000 3ffecb78 0000005c  
13:08:37.699 -> 3ffffc00:  00000046 4021a5a0 0000006c 40208834  
13:08:37.699 -> 3ffffc10:  4010324a 00000014 00000000 4021a5a0  
13:08:37.735 -> 3ffffc20:  0000011b 40103303 3ffed378 3ffef8c4  
13:08:37.735 -> 3ffffc30:  401020cb 00080000 00002200 402152fc  
13:08:37.735 -> 3ffffc40:  00000005 00000000 00000020 40100cfe  
13:08:37.735 -> 3ffffc50:  3ffe91b5 401040fb 3ffecc18 000001bb  
...deleted for brevity...
13:08:37.942 -> 3fffff70:  7701a8c0 00ffffff fe01a8c0 3ffeeb10  
13:08:37.942 -> 3fffff80:  3ffeea64 00000001 3ffeea98 402011ce  
13:08:37.976 -> 3fffff90:  402094c0 7701a8c0 feefeffe feefeffe  
13:08:37.976 -> 3fffffa0:  3fffdad0 00000000 3ffeeae0 40207008  
13:08:37.976 -> 3fffffb0:  feefeffe feefeffe 3ffe850c 40100459  
13:08:37.976 -> <<<stack<<<
13:08:37.976 -> 
13:08:37.976 ->  ets Jan  8 2013,rst cause:2, boot mode:(3,6)
13:08:37.976 -> 
13:08:37.976 -> load 0x4010f000, len 1384, room 16 
13:08:38.010 -> tail 8
13:08:38.010 -> chksum 0x2d
13:08:38.010 -> csum 0x2d
13:08:38.010 -> v8b899c12
13:08:38.010 -> ~ld

here is my code:

//#include <ArduinoJson.h>
//#include <Arduino.h>
#include <ESP8266WiFi.h>
#include <ESP8266HTTPClient.h>
#include <WiFiClientSecureBearSSL.h>

// SSL Fingerprint from Google Maps 
const char* fingerprint = "C6 3E 93 AB 25 2B 06 11 2F F9 D3 61 FB 2E 89 28 91 2C 27 46";

void setup()
{
  Serial.begin(115200);  //initialize serial
  WiFi.begin("---SSID---", "----KEY---");  //begin wifi connection
  Serial.print("[WiFi] Connecting to My Network ..."); //print connecting message
  
  int i = 0; //variable for while loop
  while (WiFi.status() != WL_CONNECTED) { // Wait for the Wi-Fi to connect
    delay(1000);  // one second delay
    Serial.print(++i); Serial.print(' ');  //print 1,2,3,4 while connecting to wifi
  }
    Serial.println(""); //print blank line
    Serial.println("WiFi connected");  //print connection message
    Serial.println("IP address: \t");  //print IP address preable
    Serial.println(WiFi.localIP());   //print IP
    gettraffic();  //call get traffic function
}

void loop()
{
}
//Get Traffic Function
void gettraffic()
{ 
  std::unique_ptr<BearSSL::WiFiClientSecure>client(new BearSSL::WiFiClientSecure); //envoke https client
  client->setFingerprint(fingerprint);  //set client fingerprint
  HTTPClient https;  //define client as "https"
  Serial.print("[HTTPS] begin...\n");  //print begin..
  //connect to google maps api
  if (https.begin(*client, "https://maps.googleapis.com/maps/api/directions/json?origin=78249&destination=78210&departure_time=now&traffic_model=best_guess&key=###KEY###")) 
  {
    Serial.print("[HTTPS] GET...\n"); //print GET...
    int httpCode = https.GET(); //set httpCode to whatever https.GET returned (header?)
    if (httpCode > 0) //if httpCode is greater than 0 then keep going
      {
      Serial.printf("[HTTPS] GET... code: %d\n", httpCode);  //print what the httpCode was
      if (httpCode == HTTP_CODE_OK || httpCode == HTTP_CODE_MOVED_PERMANENTLY)  //if our httpCode was "ok" keep going
        { 
        //pass GET result string to doc
        String payload = https.getString(); //is my payload too big?
        }
    } 
    else 
    {
    Serial.printf("[HTTPS] GET... failed, error: %s\n", https.errorToString(httpCode).c_str());  //print this if if GET fails
    }
  }
}