losing data on HTTPS connection... Can you please help me?

Hello, I’m trying to read JSON data from “api.weaher.gov” and display it on my e-paper module. I managed to connect to api.weather.gov and read JSON data using my Wemos d1 mini. However, when I printout on the JSON data on the serial monitor, random parts of the data are missing. When read data, it seems to read all the characters of the JSON data but when I print out completed string, there are missing parts. I tried google it but I couldn’t find anything about it… can you help me to find the problem? Thank you!!

Attached is the Serial output and below is the code.
On the Serial output, there should be more lines for number 7 but they are lost.

/*
    HTTP over TLS (HTTPS) example sketch

    This example demonstrates how to use
    WiFiClientSecure class to access HTTPS API.
    We fetch and display the status of
    esp8266/Arduino project continuous integration
    build.

    Limitations:
      only RSA certificates
      no support of Perfect Forward Secrecy (PFS)
      TLSv1.2 is supported since version 2.4.0-rc1

    Created by Ivan Grokhotkov, 2015.
    This example is in public domain.
*/

#include <ESP8266WiFi.h>
#include <WiFiClientSecure.h>
#include <ArduinoJson.h>

StaticJsonDocument<10000> doc;

#ifndef STASSID
#define STASSID "Sorry"
#define STAPSK  "Secret"
#endif

const char* ssid = STASSID;
const char* password = STAPSK;

//const char* host = "api.github.com";
const char* host = "api.weather.gov";
const int httpsPort = 443;

// Use web browser to view and copy
// SHA1 fingerprint of the certificate
const char fingerprint[] PROGMEM = "1ce610e06d392674ee443a469b449977aca3d472";

void setup() {
  Serial.flush();
  Serial.begin(115200);
  Serial.println();
  Serial.print("connecting to ");
  Serial.println(ssid);
  WiFi.mode(WIFI_STA);
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }
  Serial.println("");
  Serial.println("WiFi connected");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());

  WiFiClientSecure client;
  Serial.print("connecting to ");
  Serial.println(host);

  Serial.printf("Using fingerprint '%s'\n", fingerprint);
  client.flush();
  client.setFingerprint(fingerprint);

  if (!client.connect(host, httpsPort)) {
    Serial.println("connection failed");
    return;
  }

  String url = "/gridpoints/OKX/35,29/forecast";
  Serial.print("requesting URL: ");
  Serial.println(url);

  client.print(String("GET ") + url + " HTTP/1.1\r\n" +
               "Host: " + host + "\r\n" +
               "User-Agent: BuildFailureDetectorESP8266\r\n" +
               "Connection: close\r\n\r\n");

  Serial.println("request sent");

  long int time = millis();       // current time
  long int wait = 1000 * 100;      // wait 10 seconds before terminating the read process
  char c = 0;
  bool flag = false;
  String line = "";
  
  while ((time + wait) > millis())
  {
      while (client.available())
      {
          c = client.read();
          if (c < 0x20) {
            Serial.write('\\');
            Serial.write(c+ 0x20);
            Serial.write('\\');
            Serial.println();
          } else
            Serial.write(c);

          line += c;
          if (c == '\0') continue;
          
      }
  }    

  Serial.println("reply was:");
  Serial.println("==========");
  Serial.println(line);
  Serial.println("==========");
  Serial.println("closing connection");

  DeserializationError error = deserializeJson(doc, line);
  line = "";
  if (error) {
    Serial.print(F("deserializeJson() failed: "));
    Serial.println(error.c_str());
    return;
  }

  JsonVariant properties = doc["properties"];
  JsonArray periods = properties["periods"];
  JsonObject num1 = periods[0];
  String periodName = num1["name"];

  Serial.println(periodName);
}

void loop() {
}

Never post text output as images!

Although an ESP8266 (you Wemos D1 Mini is based on it) has about 50k of available RAM (which is much more than a standard Arduino has) you must keep an eye on it's usage. You cannot code like you do on a PC with gigabytes of RAM. Reading everything into a String object (byte by byte) use a bit of memory and fragments the available memory (it's not as bad as on the AVR boards but it's not neglectable. Reserving a big buffer for the JSON document doesn't help too.

Do you get an error? If you provided the actual text output we could see what you get but on the pictures nothing relevant is visible.

@ryanjoh did you ever solve this? I have been bouncing back and forth between using wunderground or weather.gov as my sources for weather updates.... one thing that I see is a huge difference in the number of characters returned by each API...

Wunderground 5day forecast returns around 5,500 characters with spaces.

Weather.gov forecast returns well over 11,000 characters with spaces.

is there a way to hold that string of JSON data in a more efficient manner on the esp8266 before its parsed?

-J

Arduino library for parsing potentially huge json streams on devices with scarce memory.

https://github.com/squix78/json-streaming-parser

@pylon Thank you for your advice! I was thinking that the memory limitation could be a problem too. But I didn't have any guesses to estimate the memory limitation. And since I'm new to arduino, I couldn't narrow the possible reasons that I don't even have a clue. So now I have one supporter for the memory limitation...lol Can I ask you if there are any kind of tools or method that I can keep track of the memory usage of arduino/wemos?

And sorry for the image. I couldn't copy and paste the Serial monitor and me being lazy. Is there any quick and easy way to copy and paste the Serial monitor?

@Jmeredith Yes I’ve got over 15000 characters including spaces too. I assumed this might caused by memory limitation but I couldn’t confirm it or had a solution so I went for a dirty quick way to solve it. I took only the part that I need from the read and added to string.

// mapping suggestion from Waveshare 2.9inch e-Paper to Wemos D1 mini
// BUSY → D2, RST → D4, DC → D3, CS → D8, CLK → D5, DIN → D7, GND → GND, 3.3V → 3.3V

// include library, include base class, make path known
#include <GxEPD.h>
#include <GxGDEW0154Z17/GxGDEW0154Z17.h> // 1.54" b/w/r 152x152
#include <GxIO/GxIO_SPI/GxIO_SPI.h>
#include <GxIO/GxIO.h>
GxIO_Class io(SPI, /CS=D8/ SS, /DC=D3/ 0, /RST=D4/ 2); // arbitrary selection of D3(=0), D4(=2), selected for default of GxEPD_Class
GxEPD_Class display(io, /RST=D4/ 2, /BUSY=D2/ 4); // default selection of D4(=2), D2(=4)

// FreeFonts from Adafruit_GFX
#include <Fonts/FreeSans9pt7b.h> // Sans
#include <Fonts/FreeSans12pt7b.h>
#include <Fonts/FreeSans18pt7b.h>
#include <Fonts/FreeSans24pt7b.h>
#include <Fonts/FreeMono9pt7b.h> // Mono
#include <Fonts/FreeMono12pt7b.h>
#include <Fonts/FreeMono18pt7b.h>
#include <Fonts/FreeMono24pt7b.h>
#include <Fonts/FreeMonoBold9pt7b.h> // Mono Bold
#include <Fonts/FreeMonoBold12pt7b.h>
#include <Fonts/FreeMonoBold18pt7b.h>
#include <Fonts/FreeMonoBold24pt7b.h>

#include <ESP8266WiFi.h> // wifi
const char* ssid = “ED2760”;
const char* password = “13571306”;

#include <WiFiClientSecure.h> // HTTPS
const char* host = “api.weather.gov”;
const int httpsPort = 443;
const char fingerprint PROGMEM = “1ce610e06d392674ee443a469b449977aca3d472”;

#include <ArduinoJson.h> // Weather.gov API
StaticJsonDocument<5000> doc;

#include <NTPClient.h> // time
#include <WiFiUdp.h>
#include <time.h>
const long utcOffsetInSeconds = -56060; // No Daylight Saving Time
WiFiUDP ntpUDP;
NTPClient timeClient(ntpUDP, “pool.ntp.org”, utcOffsetInSeconds);

bool timeFlag = false;

void setup()
{
Serial.begin(115200);
Serial.println();
Serial.println(“setup”);
display.init(115200); // enable diagnostic output on Serial
Serial.println(“setup done”);

connectWiFi();

// timeClient.begin();
// display.drawPaged(displayEPD);
// timeFlag = true;

bool b = https();
while(b == false)
{
b = https();
}

timeClient.begin();
display.drawPaged(displayEPD);
timeFlag = true;
}

void loop()
{
if (timeClient.getHours() < 1 && timeFlag == false) // No Daylight Saving Time
{
bool b = https();
while(b == false)
{
b = https();
}
display.drawPaged(displayEPD);
timeFlag = true;
}

if (timeClient.getHours() > 1)
{
timeFlag = false;
}

delay(1000* 20);
}

void displayEPD()
{
timeClient.update();

time_t rawtime = timeClient.getEpochTime();
struct tm * ti;
ti = localtime (&rawtime);

uint16_t year = ti->tm_year + 1900;
String yearStr = String(year);

//char month[12][12] = {“January”, “February”, “March”, “April”, “May”, “June”, “July”, “August”, “September”, “October”, “November”, “December”};
char month[12][12] = {“JAN”, “FEB”, “MAR”, “APR”, “MAY”, “JUN”, “JUL”, “AUG”, “SEP”, “OCT”, “NOV”, “DEC”};
String monthStr = month[ti->tm_mon];

//char daysOfTheWeek[7][12] = {“Sunday”, “Monday”, “Tuesday”, “Wednesday”, “Thursday”, “Friday”, “Saturday”};
char daysOfTheWeek[7][12] = {“SUN”, “MON”, “TUE”, “WED”, “THU”, “FRI”, “SAT”};
String dayOfTheWeek = daysOfTheWeek[timeClient.getDay()];

uint8_t day = ti->tm_mday;
String dayStr = day < 10 ? “0” + String(day) : String(day);

uint8_t hours = ti->tm_hour;
String hoursStr = hours < 10 ? “0” + String(hours) : String(hours);

uint8_t minutes = ti->tm_min;
String minuteStr = minutes < 10 ? “0” + String(minutes) : String(minutes);

uint8_t seconds = ti->tm_sec;
String secondStr = seconds < 10 ? “0” + String(seconds) : String(seconds);

JsonObject obj1 = doc[1];
JsonObject obj2 = doc[2];

String name1 = obj1[“name”];
String temp1 = obj1[“temperature”];
String weather1 = obj1[“shortForecast”];

String name2 = obj2[“name”];
String temp2 = obj2[“temperature”];
String weather2 = obj2[“shortForecast”];

delay(500);

const GFXfont* f9b = &FreeMonoBold9pt7b;
const GFXfont* f12b = &FreeMonoBold12pt7b;
const GFXfont* f18b = &FreeMonoBold18pt7b;
const GFXfont* f24b = &FreeMonoBold24pt7b;
const GFXfont* f9 = &FreeMono9pt7b;
const GFXfont* f12 = &FreeMono12pt7b;
const GFXfont* f18 = &FreeMono18pt7b;
const GFXfont* f24 = &FreeMono24pt7b;
const GFXfont* f9s = &FreeSans9pt7b;
const GFXfont* f12s = &FreeSans12pt7b;
const GFXfont* f18s = &FreeSans18pt7b;
const GFXfont* f24s = &FreeSans24pt7b;

display.fillScreen(GxEPD_WHITE);

//display.drawRect(11, 31, 130, 90, GxEPD_RED);
display.fillRect(/x,y/6, 5,/Length/55,/Height/51, GxEPD_RED);
display.drawRect(/x,y/6, 5,/Length/141,/Height/51, GxEPD_BLACK);
display.drawLine(/1st x,y/6, 30,/2nd x,y/60, 30, GxEPD_BLACK);
display.drawLine(/1st x,y/60, 5,/2nd x,y/60, 55, GxEPD_BLACK);

display.setTextColor(GxEPD_BLACK);
display.setFont(f12b);
display.setCursor(/Left/12,/Top/25);
display.println(dayOfTheWeek);

display.setCursor(/Left/12,/Top/50);
display.println(monthStr);

display.setFont(f24b);
if (day < 10) {
display.setCursor(/Left/87,/Top/45);
} else {
display.setCursor(/Left/75,/Top/45);
}
display.println(String(day));

display.setCursor(/Left/0,/Top/75);
display.setFont(f9b);
display.print(temp1);
display.print("/");
display.print(temp2+"F ");

display.setFont(f9);
display.print(weather1);
display.print("/");
display.print(weather2);
display.print(".");
}

boolean https()
{
WiFiClientSecure client;
client.setFingerprint(fingerprint);

if (!client.connect(host, httpsPort)) {
Serial.println(“connection failed”);
return false;
}

String url = “/gridpoints/OKX/35,29/forecast”;
Serial.print("requesting URL: ");
Serial.println(url);

client.print(String("GET “) + url + " HTTP/1.1\r\n” +
"Host: " + host + “\r\n” +
“User-Agent: BuildFailureDetectorESP8266\r\n” +
“Connection: close\r\n\r\n”);

Serial.println(“request sent”);
Serial.println();

char c = 0;
bool flag = false;
String line = “”;
int count = 0;
int count1 = 0;
int count2 = 0;
int counter = 0;

while (client.available())
{
c = client.read();

if( flag == false )
{ if ( c == ‘p’ )
{ c = client.read();
if ( c == ‘e’ )
{ c = client.read();
if ( c == ‘r’ )
{ c = client.read();
if ( c == ‘i’ )
{ c = client.read();
if ( c == ‘o’ )
{ c = client.read();
if ( c == ‘d’ )
{ c = client.read();
if ( c == ‘s’ )
{ c = client.read();
c = client.read();
c = client.read();
flag = true;
}
}
}
}
}
}
}
}

if ( flag == true )
{
line += c;
counter++;

if ( c == ‘}’ ) count++;
if ( count >= 3 )
{ flag = false;
line += “\r\n]”;
}
}
delay(1);
}

Serial.println(“reply was:”);
Serial.println("==========");
for (int i=0; i< line.length(); i++) {
Serial.print(line*);*

  • delay(1);*

  • }*

  • Serial.println();*

  • Serial.println("==========");*

  • Serial.println(“closing connection”);*

  • Serial.println(counter);*

  • DeserializationError error = deserializeJson(doc, line);*

  • if (error) {*

  • Serial.print(F("deserializeJson() failed: "));*

  • Serial.println(error.c_str());*

  • return false;*

  • }*

  • return true;*
    }
    void connectWiFi()
    { //Serial.println(WiFi.status());

  • WiFi.disconnect();*

  • WiFi.hostname(“Wemos”);*

  • WiFi.begin(ssid, password);*

  • Serial.println("-------------------------------------");*

  • Serial.print(“Connecting.”);*

  • while (WiFi.status() != WL_CONNECTED) {*

  • delay(500);*

  • Serial.print(".");*

  • }*

  • Serial.println(“Connected!!!”);*

  • Serial.print("IP address: ");*

  • Serial.println(WiFi.localIP());*

  • Serial.println();*
    }

@gbafamily Thank you so much! I will try it and let you know. Thank you!