ArduinoJson v7 deserialization problem

gilbert54 · April 4, 2024, 2:39pm

I'm getting data from my Google calendar in JSON format. Most of the time all goes well. But occasionally the deserialization produces erratic results. I tried with the payload as String or as c-string (const char *), but that didn't make any difference. It looks like the deserialize function is missing the terminating curly bracket and then picks up left-overs from a previous run. Which is odd, since I clear the String and the Json Document before I do the http GET().
When I copy/paste the payload string in the Json Assistant everything seems correct.
The problem only occurs when I get a shorter string as payload after getting a longer string (i.e. I first ask the calendar data for the coming 100 days and then again for the coming 20 days).

Here is part of my code:

        // Now get the calendar data from the Google calendar

    Serial.println("Getting Google Recycle calendar data...");

    http.begin(urlString);
    http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);                     // Google script always redirects to Google calendar API

    String payload;
    payload.clear();
    
    jsonDoc.clear();
    DeserializationError err;
    
    int httpResponseCode = http.GET();                                          // execute the GET request and wait for the response

    if (httpResponseCode > 0)                                                   //Check the http result code (https://http.dev/status)
    { 
      Serial.println("checking response...");
      payload = (http.getString());
      
      Serial.print("HTTP Response code: ");
      Serial.println(httpResponseCode);
      Serial.printf("Length of payload string: %lu\n", payload.length());
      Serial.print("Payload as String : ");
      Serial.println(payload);
      Serial.printf("Length of payload char[]: %lu\n", strlen(payload.c_str()));
      Serial.print("Payload as char [] : ");
      Serial.println(payload.c_str());

      err = deserializeJson(jsonDoc, payload);

      if(err)
      {
        Serial.print("deserializeJson() failed: ");
        Serial.println(err.c_str());
      }
      else
      {
        if(jsonDoc["data"].is<JsonArray>())
        {
          JsonArray j_dataArr = jsonDoc["data"].as<JsonArray>();
          const char * timeString;

          const char dateAsString [15] = {'\0'};
          uint8_t dateLen = 0;
          uint8_t eventIndex = 0;

          eventCount = 0;

          for(auto event : events)
          {
            event.type = "";
          }

          for(auto j_item : j_dataArr)
          {
            events[eventIndex].type = j_item["TYPE"].as<const char *>();          // event type
            timeString = j_item["DATE"].as<const char *>();                       // event time
            strptime(timeString, "%Y-%m-%dT%H:%M:%S", &events[eventIndex].calschedule);
            ++eventCount;

            if(++eventIndex >= sizeof(events)/sizeof(gCalEvent_t))                // if event table full
              break;                                                              // stop iteration
          }
          
          eventIndex = 1;
          for(auto event : events)
          {
            if((event.type == NULL) || strlen(event.type) == 0)
              break;

            Serial.print("Event #");
            Serial.print(eventIndex++);
            Serial.print(" - TYPE: ");
            Serial.print(event.type);
            Serial.print(", Date: ");
            Serial.println(&event.calschedule, "%A, %B %d %Y");
          }
        }
        else if(jsonDoc["data"].is<const char *>())                                 // if not an array it must be a c-string
        {
          const char * GCdata = jsonDoc["data"].as<const char *>();
          Serial.printf("Google Cal data = %s\n", GCdata);
        }
        else
        {
          Serial.println("Google Data don't make sense...");
        }
      }

It's not a killer problem, since the normal use case is to request only the calender data for the next day, but I'm puzzled why I'm getting this strange misbehaviour.
Hint: deserialization typically fails (does not report an error) if the lenght of the string is between 200 and 400 characters.

The results look like this:

This one is OK. I requested data for the coming 50 days. The payload contains 663 characters and deserialization produces a correct result. Note: I only keep the first 10 results to conserve memory
16:12:38.588 -> Getting Google Recycle calendar data...

16:12:44.019 -> checking response...

16:12:44.127 -> HTTP Response code: 200

16:12:44.127 -> Length of payload string: 663

16:12:44.127 -> Payload as String : {"status":"success","data":[{"TYPE":"RA","DATE":"2024-04-10T05:00:00.000Z"},{"TYPE":"PMD","DATE":"2024-04-10T05:00:00.000Z"},{"TYPE":"P/K","DATE":"2024-04-17T05:00:00.000Z"},{"TYPE":"GFT","DATE":"2024-04-17T05:00:00.000Z"},{"TYPE":"RA","DATE":"2024-04-24T05:00:00.000Z"},{"TYPE":"PMD","DATE":"2024-04-24T05:00:00.000Z"},{"TYPE":"GLAS","DATE":"2024-05-03T05:00:00.000Z"},{"TYPE":"GFT","DATE":"2024-05-03T05:00:00.000Z"},{"TYPE":"RA","DATE":"2024-05-08T05:00:00.000Z"},{"TYPE":"PMD","DATE":"2024-05-08T05:00:00.000Z"},{"TYPE":"GFT","DATE":"2024-05-15T05:00:00.000Z"},{"TYPE":"RA","DATE":"2024-05-22T05:00:00.000Z"},{"TYPE":"PMD","DATE":"2024-05-22T05:00:00.000Z"}]}

16:12:44.128 -> Length of payload char[]: 663

16:12:44.128 -> Payload as char [] : {"status":"success","data":[{"TYPE":"RA","DATE":"2024-04-10T05:00:00.000Z"},{"TYPE":"PMD","DATE":"2024-04-10T05:00:00.000Z"},{"TYPE":"P/K","DATE":"2024-04-17T05:00:00.000Z"},{"TYPE":"GFT","DATE":"2024-04-17T05:00:00.000Z"},{"TYPE":"RA","DATE":"2024-04-24T05:00:00.000Z"},{"TYPE":"PMD","DATE":"2024-04-24T05:00:00.000Z"},{"TYPE":"GLAS","DATE":"2024-05-03T05:00:00.000Z"},{"TYPE":"GFT","DATE":"2024-05-03T05:00:00.000Z"},{"TYPE":"RA","DATE":"2024-05-08T05:00:00.000Z"},{"TYPE":"PMD","DATE":"2024-05-08T05:00:00.000Z"},{"TYPE":"GFT","DATE":"2024-05-15T05:00:00.000Z"},{"TYPE":"RA","DATE":"2024-05-22T05:00:00.000Z"},{"TYPE":"PMD","DATE":"2024-05-22T05:00:00.000Z"}]}

16:12:44.179 -> Event #1 - TYPE: RA, Date: Wednesday, April 10 2024

16:12:44.179 -> Event #2 - TYPE: PMD, Date: Wednesday, April 10 2024

16:12:44.179 -> Event #3 - TYPE: P/K, Date: Wednesday, April 17 2024

16:12:44.210 -> Event #4 - TYPE: GFT, Date: Wednesday, April 17 2024

16:12:44.210 -> Event #5 - TYPE: RA, Date: Wednesday, April 24 2024

16:12:44.210 -> Event #6 - TYPE: PMD, Date: Wednesday, April 24 2024

16:12:44.210 -> Event #7 - TYPE: GLAS, Date: Friday, May 03 2024

16:12:44.210 -> Event #8 - TYPE: GFT, Date: Friday, May 03 2024

16:12:44.210 -> Event #9 - TYPE: RA, Date: Wednesday, May 08 2024

16:12:44.210 -> Event #10 - TYPE: PMD, Date: Wednesday, May 08 2024

Next, I did a request for the coming 20 days...

16:12:38.588 -> Getting Google Recycle calendar data...