Hi, I’m sometimes receiving some strange characters in the middle of the response in a https request. Someone of you maybe knows why and how to remove them in a good way?
/Create the return String from the response
String returnString = "";
bool hasOpened = false;
bool hasClosed = false;
while (httpsClient.available())
{
char c = httpsClient.read();
if (endpoint == "/ShareWebServices/Services/Publisher/ReadPublisherLatestGlucoseValues") {
if (c == '{') {
hasOpened = true;
}
//This remove anything outside the {}
if (hasOpened == true && hasClosed == false)
{
returnString.concat(c);
}
if (c == '}') {
hasClosed = true;
}
} else {
returnString.concat(c);
}
}
Serial.println(returnString );
{"WT":"Date(1638484694000)"
{"WT":"Date(1638484694000)", "ST":"Date(1638484694000)"
{"WT":"Date(1638484694000)", "ST":"Date(1638484694000)", "DT":"Date(1638484694000+0100)", "Value":135, "Trend":"Flat"}
J-M-L
December 9, 2021, 9:45am
2
what are you printing? is that returnString once the while() has completed its work?
Yes, it's the returnString. I forgot to include it
J-M-L
December 9, 2021, 9:47am
4
are you sure endpoint matched "/ShareWebServices/Services/Publisher/ReadPublisherLatestGlucoseValues" ?
Juraj
December 9, 2021, 9:47am
5
it is the http chunked transfer encoding. set the Content-length header on server
I'm sorry again. I updated the data i received in this code. The data i posted first was the raw response. So the endpoints match.
This is all the headers i receive.
HTTP/1.1 200 OK
{"WT":"Date(1639042699000)"
J-M-L
December 9, 2021, 9:56am
8
so where is it weird? is that the 57 in the answer?
I want to transform this string into a DynamicJsonDocument later, which means that it must be an valid json string. And the occasional occurrence of the numbers in the string makes it not valid.
harleyberglund:
{"WT":"Date(1639042699000)"
J-M-L
December 9, 2021, 10:04am
10
It seems that this is really what you get back. I would try to understand what the sender of this answer is doing.
chunked transfer encoding is definitely something to look at (as suggested by @Juraj )
It's pretty hard to understand that since the data comes from a big company called Dexcom. I have a similar code in python and it works like a charm.
J-M-L
December 9, 2021, 10:08am
12
I assume that you let python decode the HTTP traffic ?
if chunked transfer encoding is active, you get your answer in "chunks" and the chunk size is transferred as a hexadecimal number followed by \r\n as a line separator
Yes i let python decode the HTTP traffic.
Is there anything similar i can do in the Arduino code?
J-M-L
December 9, 2021, 10:11am
14
if you look at
57
if you count the number of characters in the second line you'll find 87 of them and 57 is hexadecimal for 87
➜ it is likely chunked transfer
what's your arduino?
Do you mean what board i use?
J-M-L
December 9, 2021, 10:17am
16
yes ESP32, UNO with Ethernet shield, MKR, ....
J-M-L
December 9, 2021, 10:19am
18
~~I would use https://github.com/me-no-dev/ESPAsyncWebServer~~
EDIT: are you the web server or are you just a client ?
Juraj
December 9, 2021, 10:30am
19
ESP8266HTTPClient library decodes chunked transfer
How do I implement that in this code?
#include <ESP8266WiFi.h>
#include <WiFiClientSecure.h>
#include <ESP8266WebServer.h>
#include <ESP8266HTTPClient.h>
#include <ArduinoJson.h>
//=======================================================================
// Variables
//=======================================================================
//WiFi Credentials
const char *wifi_SSID = "";
const char *wifi_PASSWORD = "";
//Dexcom Account Credentials
const char *Dexcom_Username = "";
const char *Dexcom_Password = "";
//Fingerprint for Dexcom's Server
const char fingerprint[] PROGMEM = "9B BB 88 8F AE FB 98 5D 46 95 1C 6C 2A fE C9 98 17 60 3E 25";
//Endpoints for requests
const char *DEXCOM_AUTHENTICATE_ENDPOINT = "/ShareWebServices/Services/General/AuthenticatePublisherAccount";
const char *DEXCOM_LOGIN_ENDPOINT = "/ShareWebServices/Services/General/LoginPublisherAccountByName";
const char *DEXCOM_GLUCOSE_READINGS_ENDPOINT = "/ShareWebServices/Services/Publisher/ReadPublisherLatestGlucoseValues";
//Dexcom outside US host
const char *host = "shareous1.dexcom.com";
//Dexcom application id
const char *DEXCOM_APPLICATION_ID = "d89443d2-327c-4a6f-89e5-496bbb0317db";
//Default Dexcom session id
const char *DEFAULT_SESSION_ID = "00000000-0000-0000-0000-000000000000";
//Value to convert mg/dL to mmol/L
const float MMOL_L_CONVERTION_FACTOR = 0.0555;
//The current Dexcom sessionID
String currentSessionId = "";
//=======================================================================
// End of Variables
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
// Main Program Setup
//=======================================================================
void setup() {
wifiSetup(wifi_SSID, wifi_PASSWORD);
create_session(Dexcom_Username, Dexcom_Password);
}
//=======================================================================
// End of Main Program Setup
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
// Main Program Loop
//=======================================================================
void loop() {
unsigned long beforeFetchTime = millis();
DynamicJsonDocument sugarJsonObject = fetchDexcomData(Dexcom_Username, Dexcom_Password, currentSessionId);
Serial.println("---------------------------------");
Serial.println(float(sugarJsonObject["mmol_l"]));
Serial.println("---------------------------------");
unsigned long afterFetchTime = millis();
Serial.print("The fetch took: ");
Serial.println(String(afterFetchTime - beforeFetchTime));
delay(3000);
}
//=======================================================================
// End of Main Program Loop
//=======================================================================
//=======================================================================
// Main code for data fetching
//=======================================================================
DynamicJsonDocument fetchDexcomData(const char *username, const char *password, String currentSessionId)
{
if (currentSessionId != "") {
String minutes = "10";
String max_count = "1";
String payload = create_getSugar_json_string(currentSessionId, minutes, max_count);
String jsonStringResponse = request(host, payload, DEXCOM_GLUCOSE_READINGS_ENDPOINT);
if (jsonStringResponse.indexOf("InvalidArgument") > 0) {
Serial.println("Something went terribly wrong");
} else if (jsonStringResponse.indexOf("SessionIdNotFound") > 0) {
Serial.println("The Dexcom sessionId was not found");
create_session(username, password);
}
else {
return (String_to_Json(jsonStringResponse));
}
} else {
create_session(username, password);
}
}
//=======================================================================
// End of Main code for data fetching
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
// Create Dexcom session
//=======================================================================
void create_session(const char *username, const char *password)
{
Serial.println("---------------------------------");
Serial.println("Creating a new Dexcom sessionId");
String session_id = "";
String body = create_login_json_string(username, password);
session_id = request(host, body, DEXCOM_AUTHENTICATE_ENDPOINT);
session_id = request(host, body, DEXCOM_LOGIN_ENDPOINT);
session_id.remove(0, 1);
session_id.remove((session_id.length() - 1), 1);
currentSessionId = session_id;
Serial.println("New Dexcom sessionId is: " + session_id);
Serial.println("---------------------------------");
}
//=======================================================================
// End of Create Dexcom session
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
// Post Request function
//=======================================================================
String request(const char *host, String body, const char *endpoint)
{
//Connecting to the server
WiFiClientSecure httpsClient;
httpsClient.setFingerprint(fingerprint);
Serial.println("HTTPS Connecting");
int r = 0;
while ((!httpsClient.connect(host, 443)) && (r < 30)) {
delay(100);
Serial.print(".");
r++;
}
if (r == 30) {
//After trying 30 times the Connection has failed
Serial.println("Connection failed");
}
else {
//Else the Connection was successfull and the process can continue
Serial.println("Connected to web");
char postStr[120];
sprintf(postStr, "POST %s HTTP/1.1", endpoint);
//Sending all the headers
httpsClient.println(postStr);
httpsClient.print("Host: "); httpsClient.println(host);
httpsClient.println("Content-Type: application/json");
httpsClient.println("Accept: */*");
httpsClient.print("Content-Length: "); httpsClient.println(body.length());
httpsClient.println();
//Sending the json string with the payload
httpsClient.println(body);
Serial.println("request sent");
while (httpsClient.connected()) {
String line = httpsClient.readStringUntil('\n');
Serial.println(line);
if (line == "\r") {
Serial.println("headers received");
break;
}
}
//Create the return String from the response
String returnString = "";
bool hasOpened = false;
bool hasClosed = false;
while (httpsClient.available())
{
char c = httpsClient.read();
if (endpoint == "/ShareWebServices/Services/Publisher/ReadPublisherLatestGlucoseValues") {
if (c == '{') {
hasOpened = true;
}
if (hasOpened == true && hasClosed == false) {
returnString.concat(c);
}
if (c == '}') {
hasClosed = true;
}
} else {
returnString.concat(c);
}
}
//Closing connection to the server
Serial.println("closing connection");
httpsClient.stop();
Serial.println(returnString);
return returnString;
}
}
//=======================================================================
// End of Post Request function
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
// Creating json payloads
//=======================================================================
String create_getSugar_json_string(String dexcom_session, String minutes, String maxCount)
{
DynamicJsonDocument doc(2048);
doc["sessionId"] = dexcom_session;
doc["minutes"] = minutes;
doc["maxCount"] = maxCount;
String json;
serializeJson(doc, json);
return json;
}
String create_login_json_string(String accountName, String password)
{
DynamicJsonDocument doc(2048);
doc["accountName"] = accountName;
doc["password"] = password;
doc["applicationId"] = DEXCOM_APPLICATION_ID;
String json;
serializeJson(doc, json);
return json;
}
//=======================================================================
// End of Creating json payloads
//=======================================================================
//-
//-
//-
//-
//-
//=======================================================================
// String to Json
//=======================================================================
//Packing up the fetched String and sends back a jsonArray with the values
DynamicJsonDocument String_to_Json(String input) {
StaticJsonDocument<768> doc;
DeserializationError error = deserializeJson(doc, input);
if (error) {
Serial.print(F("deserializeJson() failed: "));
Serial.println(error.f_str());
}
else {
String Trend = doc["Trend"];
float Value = doc["Value"];
String WT = doc["WT"];
DynamicJsonDocument returnDoc(2048);
int intValue = (int) round(Value * MMOL_L_CONVERTION_FACTOR * 10);
returnDoc["mg_dl"] = Value;
returnDoc["mmol_l"] = ((float)intValue / 10);
returnDoc["trend"] = Trend;
returnDoc["time"] = WT;
return (returnDoc);
}
}
void wifiSetup(const char *ssid, const char *password) {
delay(1000);
Serial.begin(115200);
WiFi.mode(WIFI_OFF);
delay(1000);
WiFi.mode(WIFI_STA);
WiFi.begin(ssid, password);
Serial.print("Connecting");
// Wait for connection
while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}
Serial.println("");
Serial.print("Connected to ");
Serial.println(ssid);
Serial.print("IP address: ");
Serial.println(WiFi.localIP());
}