Weird response in HTTPS request

Hi, I’m sometimes receiving some strange characters in the middle of the response in a https request. Someone of you maybe knows why and how to remove them in a good way?


/Create the return String from the response
String returnString = "";
bool hasOpened = false;
bool hasClosed = false;

while (httpsClient.available())
{

char c = httpsClient.read();

if (endpoint == "/ShareWebServices/Services/Publisher/ReadPublisherLatestGlucoseValues") {
if (c == '{') {
hasOpened = true;
}

//This remove anything outside the {}
if (hasOpened == true && hasClosed == false)
{
returnString.concat(c);
}

if (c == '}') {
hasClosed = true;
}

} else {
returnString.concat(c);
}

}

Serial.println(returnString );

{"WT":"Date(1638484694000)"
58
, "ST":"Date(1638484694000)", "DT":"Date(1638484694000+0100)", "Value":135, "Trend":"Flat"
}

{"WT":"Date(1638484694000)", "ST":"Date(1638484694000)"
72
,"DT":"Date(1638484694000+0100)", "Value":135, "Trend":"Flat"
}

{"WT":"Date(1638484694000)", "ST":"Date(1638484694000)", "DT":"Date(1638484694000+0100)", "Value":135, "Trend":"Flat"}

what are you printing? is that returnString once the while() has completed its work?

Yes, it's the returnString. I forgot to include it

are you sure endpoint matched "/ShareWebServices/Services/Publisher/ReadPublisherLatestGlucoseValues" ?

it is the http chunked transfer encoding. set the Content-length header on server

I'm sorry again. I updated the data i received in this code. The data i posted first was the raw response. So the endpoints match.

This is all the headers i receive.

HTTP/1.1 200 OK
Date: Thu, 09 Dec 2021 09:42:52 GMT
Content-Type: application/json
Transfer-Encoding: chunked
Connection: keep-alive
Vary: Origin
Vary: Access-Control-Request-Method
Vary: Access-Control-Request-Headers
X-Trace-Id: 862b535a-ae72-43a7-92aa-3118cdd76e5e
Strict-Transport-Security: max-age=15724800; includeSubDomains

{"WT":"Date(1639042699000)"
57
,"ST":"Date(1639042699000)","DT":"Date(1639042699000+0100)","Value":97,"Trend":"Flat"}

so where is it weird? is that the 57 in the answer?
who is building that answer?

I want to transform this string into a DynamicJsonDocument later, which means that it must be an valid json string. And the occasional occurrence of the numbers in the string makes it not valid.

It seems that this is really what you get back. I would try to understand what the sender of this answer is doing.

chunked transfer encoding is definitely something to look at (as suggested by @Juraj )

It's pretty hard to understand that since the data comes from a big company called Dexcom. I have a similar code in python and it works like a charm.

I assume that you let python decode the HTTP traffic ?

if chunked transfer encoding is active, you get your answer in "chunks" and the chunk size is transferred as a hexadecimal number followed by \r\n as a line separator

Yes i let python decode the HTTP traffic.

Is there anything similar i can do in the Arduino code?

if you look at

57
,"ST":"Date(1639042699000)","DT":"Date(1639042699000+0100)","Value":97,"Trend":"Flat"}

if you count the number of characters in the second line you'll find 87 of them and 57 is hexadecimal for 87

➜ it is likely chunked transfer

what's your arduino?

Do you mean what board i use?

yes ESP32, UNO with Ethernet shield, MKR, ....

ESP8266 with wifi

I would use GitHub - me-no-dev/ESPAsyncWebServer: Async Web Server for ESP8266 and ESP32

EDIT: are you the web server or are you just a client ?

ESP8266HTTPClient library decodes chunked transfer

How do I implement that in this code?

#include <ESP8266WiFi.h>
#include <WiFiClientSecure.h>
#include <ESP8266WebServer.h>
#include <ESP8266HTTPClient.h>
#include <ArduinoJson.h>

//=======================================================================
//                       Variables
//=======================================================================

//WiFi Credentials
const char *wifi_SSID = "";
const char *wifi_PASSWORD = "";

//Dexcom Account Credentials
const char *Dexcom_Username = "";
const char *Dexcom_Password = "";

//Fingerprint for Dexcom's Server
const char fingerprint[] PROGMEM = "9B BB 88 8F AE FB 98 5D 46 95 1C 6C 2A fE C9 98 17 60 3E 25";

//Endpoints for requests
const char *DEXCOM_AUTHENTICATE_ENDPOINT = "/ShareWebServices/Services/General/AuthenticatePublisherAccount";
const char *DEXCOM_LOGIN_ENDPOINT = "/ShareWebServices/Services/General/LoginPublisherAccountByName";
const char *DEXCOM_GLUCOSE_READINGS_ENDPOINT = "/ShareWebServices/Services/Publisher/ReadPublisherLatestGlucoseValues";

//Dexcom outside US host
const char *host = "shareous1.dexcom.com";

//Dexcom application id
const char *DEXCOM_APPLICATION_ID = "d89443d2-327c-4a6f-89e5-496bbb0317db";

//Default Dexcom session id
const char *DEFAULT_SESSION_ID = "00000000-0000-0000-0000-000000000000";


//Value to convert mg/dL to mmol/L
const float MMOL_L_CONVERTION_FACTOR = 0.0555;


//The current Dexcom sessionID
String currentSessionId = "";


//=======================================================================
//                     End of Variables
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
//                     Main Program Setup
//=======================================================================

void setup() {
  wifiSetup(wifi_SSID, wifi_PASSWORD);
  create_session(Dexcom_Username, Dexcom_Password);

}

//=======================================================================
//                   End of Main Program Setup
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
//                     Main Program Loop
//=======================================================================

void loop() {
  unsigned long beforeFetchTime = millis();
  DynamicJsonDocument sugarJsonObject = fetchDexcomData(Dexcom_Username, Dexcom_Password, currentSessionId);

  Serial.println("---------------------------------");
  Serial.println(float(sugarJsonObject["mmol_l"]));
  Serial.println("---------------------------------");
  unsigned long afterFetchTime = millis();
  Serial.print("The fetch took: ");
  Serial.println(String(afterFetchTime - beforeFetchTime));
  delay(3000);
}

//=======================================================================
//                   End of Main Program Loop
//=======================================================================



//=======================================================================
//               Main code for data fetching
//=======================================================================

DynamicJsonDocument fetchDexcomData(const char *username, const char *password, String currentSessionId)
{
  if (currentSessionId != "") {
    String minutes = "10";
    String max_count = "1";

    String payload = create_getSugar_json_string(currentSessionId, minutes, max_count);
    String jsonStringResponse = request(host, payload, DEXCOM_GLUCOSE_READINGS_ENDPOINT);

    if (jsonStringResponse.indexOf("InvalidArgument") > 0) {
      Serial.println("Something went terribly wrong");
    } else if (jsonStringResponse.indexOf("SessionIdNotFound") > 0) {
      Serial.println("The Dexcom sessionId was not found");
      create_session(username, password);
    }
    else {
      return (String_to_Json(jsonStringResponse));
    }
  } else {
    create_session(username, password);
  }
}






//=======================================================================
//            End of Main code for data fetching
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
//                  Create Dexcom session
//=======================================================================

void create_session(const char *username, const char *password)
{
  Serial.println("---------------------------------");
  Serial.println("Creating a new Dexcom sessionId");
  String session_id = "";
  String body = create_login_json_string(username, password);

  session_id = request(host, body, DEXCOM_AUTHENTICATE_ENDPOINT);
  session_id = request(host, body, DEXCOM_LOGIN_ENDPOINT);

  session_id.remove(0, 1);
  session_id.remove((session_id.length() - 1), 1);

  currentSessionId = session_id;
  Serial.println("New Dexcom sessionId is: " + session_id);
  Serial.println("---------------------------------");
}

//=======================================================================
//                End of Create Dexcom session
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
//                    Post Request function
//=======================================================================

String request(const char *host, String body, const char *endpoint)
{
  //Connecting to the server
  WiFiClientSecure httpsClient;
  httpsClient.setFingerprint(fingerprint);
  Serial.println("HTTPS Connecting");
  int r = 0;
  while ((!httpsClient.connect(host, 443)) && (r < 30)) {
    delay(100);
    Serial.print(".");
    r++;
  }
  if (r == 30) {
    //After trying 30 times the Connection has failed
    Serial.println("Connection failed");
  }
  else {
    //Else the Connection was successfull and the process can continue
    Serial.println("Connected to web");
    char postStr[120];
    sprintf(postStr, "POST %s HTTP/1.1", endpoint);

    //Sending all the headers
    httpsClient.println(postStr);
    httpsClient.print("Host: "); httpsClient.println(host);
    httpsClient.println("Content-Type: application/json");
    httpsClient.println("Accept: */*");
    httpsClient.print("Content-Length: "); httpsClient.println(body.length());
    httpsClient.println();
    //Sending the json string with the payload
    httpsClient.println(body);

    Serial.println("request sent");

    while (httpsClient.connected()) {
      String line = httpsClient.readStringUntil('\n');
      Serial.println(line);
      if (line == "\r") {
        Serial.println("headers received");
        break;
      }
    }
    //Create the return String from the response
    String returnString = "";
    bool hasOpened = false;
    bool hasClosed = false;

    while (httpsClient.available())
    {

      char c = httpsClient.read();

      if (endpoint == "/ShareWebServices/Services/Publisher/ReadPublisherLatestGlucoseValues") {
        if (c == '{') {
          hasOpened = true;
        }

        if (hasOpened == true && hasClosed == false) {
          returnString.concat(c);
        }

        if (c == '}') {
          hasClosed = true;
        }

      } else {
        returnString.concat(c);
      }

    }

    //Closing connection to the server
    Serial.println("closing connection");
    httpsClient.stop();

    Serial.println(returnString);

    return returnString;

  }
}

//=======================================================================
//                 End of Post Request function
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
//                    Creating json payloads
//=======================================================================

String create_getSugar_json_string(String dexcom_session, String minutes, String maxCount)
{
  DynamicJsonDocument doc(2048);

  doc["sessionId"] = dexcom_session;
  doc["minutes"] = minutes;
  doc["maxCount"] = maxCount;

  String json;
  serializeJson(doc, json);
  return json;
}


String create_login_json_string(String accountName, String password)
{
  DynamicJsonDocument doc(2048);

  doc["accountName"] = accountName;
  doc["password"] = password;
  doc["applicationId"] = DEXCOM_APPLICATION_ID;

  String json;
  serializeJson(doc, json);
  return json;
}

//=======================================================================
//                 End of Creating json payloads
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
//                        String to Json
//=======================================================================
//Packing up the fetched String and sends back a jsonArray with the values

DynamicJsonDocument String_to_Json(String input) {
  StaticJsonDocument<768> doc;

  DeserializationError error = deserializeJson(doc, input);

  if (error) {
    Serial.print(F("deserializeJson() failed: "));
    Serial.println(error.f_str());
  }
  else {
    String Trend = doc["Trend"];
    float Value = doc["Value"];
    String WT = doc["WT"];

    DynamicJsonDocument returnDoc(2048);

    int intValue = (int) round(Value * MMOL_L_CONVERTION_FACTOR * 10);

    returnDoc["mg_dl"] = Value;
    returnDoc["mmol_l"] = ((float)intValue / 10);
    returnDoc["trend"] = Trend;
    returnDoc["time"] = WT;

    return (returnDoc);
  }
}






void wifiSetup(const char *ssid, const char *password) {
  delay(1000);
  Serial.begin(115200);
  WiFi.mode(WIFI_OFF);
  delay(1000);
  WiFi.mode(WIFI_STA);

  WiFi.begin(ssid, password);


  Serial.print("Connecting");
  // Wait for connection
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }

  Serial.println("");
  Serial.print("Connected to ");
  Serial.println(ssid);
  Serial.print("IP address: ");
  Serial.println(WiFi.localIP());
}