Getting incorrect JSON from AsyncWebserver and HTTP_Post

Hello,

so I'm at a bit of a loss here. I'm trying to do a - rather simple - HTTP-Request to my ESP32. Most of them are working and my project so far has grown pretty big. However one of my HTTP_POST gives me a headache. I'm familiar with programming in general, but I'm not that good with C++. I broke my code down to a minimal example:

#include <WiFi.h>
#include <AsyncTCP.h>
#include <ESPAsyncWebServer.h>
#define ARDUINOJSON_USE_LONG_LONG 1
#include <ArduinoJson.h>


//Data
const char* ssid = "MySSID";
const char* password = "MySuperSafePassowrd";

struct Rule{
    int lightID; //ID of the corresponding light
    int ID; //starts with 1;
    String name; //
    unsigned long dateTime;
    byte frequency; //1:Once, 2:Daily, 3:Weekly
    byte deactivated;  //1:NONE, 2:ONCE, 3:PERMANENT
    byte action; //1:OFF, 2:ON, 3:SWITCH
};

//Variables
AsyncWebServer httpServer(80);

//Setup-Method
void setup() { 
  Serial.begin(115200);
  Serial.println("Setup");

  connectToWifi();
  Serial.println("connected");
  startHTTPServer();
  Serial.println("Server started");
}

//Loop-Methode: Ticker
void loop() {
  //check();
}

//Helpmethods
void connectToWifi() {
  Serial.println();
  Serial.print("Connecting to ");
  WiFi.disconnect(true);
  Serial.println(ssid);
  WiFi.begin(ssid, password);

  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }

  Serial.println("");
  Serial.println("WiFi connected");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());
}

void startHTTPServer() {
  //Add API-Methods
  add_postRule();

  // Start webserver
  httpServer.onNotFound(notFound);
  httpServer.begin();
}

void add_postRule() {
  httpServer.on("/api/postRule", HTTP_POST, [](AsyncWebServerRequest *request){}, NULL, [](AsyncWebServerRequest *request, uint8_t *data, size_t len, size_t index, size_t total){
    postRule(request, data);
  });
}

void postRule(AsyncWebServerRequest *request, uint8_t *data) {
  Rule r = parseRule(request, data);
  if (r.ID == -2)
    return;

  r.ID = 1; 

  //normally here would be logic to save it all and get a valid id

  request->send(200, "text/plain", "Rule added");
}

Rule parseRule(AsyncWebServerRequest *request, uint8_t *data) {
  JsonObject obj = deserializeData(request, data);
  Rule r;
  if (obj.isNull()) {
    r.ID = -2;
    return r;
  }
  r.lightID = obj["lightID"];
  r.ID = obj["ID"];
  Serial.println("ReadingName");
  r.name = obj["name"].as<String>();
  Serial.print("Regelname: ");
  Serial.println(r.name);
  r.dateTime = obj["dateTime"];
  r.frequency = obj["frequency"];
  r.deactivated = obj["deactivated"];
  r.action = obj["action"];
  printRule(r);
  return r;
}

void printRule(Rule& r) {
  Serial.print("Rule: LightID: ");
  Serial.println(r.lightID);
  Serial.print("ID: ");
  Serial.println(r.ID);
  Serial.print("Name: ");
  Serial.println(r.name);
  Serial.print("dateTime: ");
  Serial.println(r.dateTime);
  Serial.print("Frequency: ");
  Serial.println(r.frequency);
  Serial.print("Deaktivated: ");
  Serial.println(r.deactivated);
  Serial.print("Action: ");
  Serial.println(r.action);
}

JsonObject deserializeData(AsyncWebServerRequest *request, uint8_t *data) {
  Serial.println("Data: ");
  Serial.println((const char *)data); 
  Serial.println("Length: ");
  Serial.println(request->contentLength()); 
  DynamicJsonDocument doc(2048);
  DeserializationError error = deserializeJson(doc, (const char *)data,request->contentLength());
  if (error) {
    request->send(400, "text/plain", "Fehlerhafte JSON Struktur");
    doc.clear();
  }
  JsonObject obj = doc.as<JsonObject>();
  return obj;
}

void notFound(AsyncWebServerRequest *request) {
    request->send(404, "text/plain", "Not found");
}

I'm doing my Post via RESTer from Firefox (also tried another client) and get this as on the Serial Monitor

WiFi connected
IP address: 
192.168.137.212
connected
Server started
Data: 
{
"lightID":1,
"name":"Regelname",
"dateTime":1677427187,
"frequency":1,
"deactivated":2,
"action":3
}
Length: 
102
ReadingName
Regelname: null
Rule: LightID: 1
ID: 0
Name: null
dateTime: 0
Frequency: 0
Deaktivated: 0
Action: 0

also sometimes there are wierd thing in the data like in this example:

Data: 
{
"lightID":1,
"name":"Regelname 2",
"dateTime":1678427187,
"frequency":1,
"deactivated":2,
"action":3
}c-75e3-4847-ᆳ�xV��0�
Length: 
104
ReadingName
Regelname: null
Rule: LightID: 1
ID: 0
Name: null
dateTime: 0
Frequency: 0
Deaktivated: 0
Action: 0

The chars after the curly brackets closed are broken in Arduino IDE too.
As you can see my problem is, that it doesn't read out the data correct although I'm not sending it that way. I'm also not sure where exactly my problem is from, since Webserver as well as ArduinoJson seem to be pretty popular and well tested.

I would apprciate your help!
Thanks.

when you get this

you have size information (size_t len, size_t index, size_t total) that you totally disregard when calling postRule(request, data); and later parseRule(request, data);

so when it comes to calling deserializeData(request, data); you assume there is a well formed c-string pointed by data but this might not be the case and this will just print stuff until the next null char in memory

That definitely explains, why my Serial.print is giving so strange output. However I have tried a few things now and still can't seem to get it quite right.

I tried to pass the parameter len and total (what is the difference here?) to the deserializeJson. I also tried to convert the data to a char array, which sometimes seems to help with the Serial.print.

JsonObject deserializeData(AsyncWebServerRequest *request, uint8_t *data, size_t len) {
  char jsonData[len];
  for(int i = 0;i<len;i++) {
    jsonData[i] = (char)data[i]; 
  }
 
  Serial.println("Data: ");
  Serial.println(jsonData); 
  Serial.println("Length: ");
  Serial.println(len); 
  DynamicJsonDocument doc(2048);
  DeserializationError error = deserializeJson(doc, jsonData, len);
...

If I use

DeserializationError error = deserializeJson(doc, (char *)data, len);

I sometimes get the correct result, but I still breaks often.

Do you know what would be the best way to correctly cast it here?

I've not read your whole code but this is for multipart callbacks when you are being asked to provide chunks of data. so the len is how many bytes you have in the data and then it's the packet number and total size if I remember correctly

I'm not sure it makes sense in your context. what do you expect from

void add_postRule() {
  httpServer.on("/api/postRule", HTTP_POST, [](AsyncWebServerRequest *request){}, NULL, [](AsyncWebServerRequest *request, uint8_t *data, size_t len, size_t index, size_t total){
    postRule(request, data);
  });
}

the basic format is

    server.on("/post", HTTP_POST, [](AsyncWebServerRequest *request){
        // do stuff with the request, extract params etc
        request->send(200, "text/plain", "Rule added");
    });

So I'm doing a HTTP-Post-Request with this as a JSON-Body

{
"lightID":1,
"name":"Regelname 2",
"dateTime":1678427187,
"frequency":1,
"deactivated":2,
"action":3
}

In my Code I want to essentially get this:

httpServer.on("/api/postRule", HTTP_POST, [](AsyncWebServerRequest *request){}, NULL, [](AsyncWebServerRequest *request, uint8_t *data, size_t len, size_t index, size_t total){

  DynamicJsonDocument doc(2048);
  DeserializationError error = deserializeJson(doc, (char *)data, len); //deserialize the Body I get
  if (error) {//handling it}
  JsonObject obj = doc.as<JsonObject>();

  //and now do things with my JsonObject
  });

I hope it's a bit clearer now.

Edit: Since I have a body I can't use your basic format. What I use is from this comment Document how to parse JSON POSTed to the ESP · Issue #195 · me-no-dev/ESPAsyncWebServer · GitHub
Essentially there are 3 methods available and I use the one which is best, if you have a body.

OK yes, it was unclear to me that you were getting the JSON as part of the body.

can you try this to see if you receive the whole JSON in one chunk?

void add_postRule() {
  server.on("/test", HTTP_POST,
  [](AsyncWebServerRequest * request) {},
  [](AsyncWebServerRequest * request, const String & filename, size_t index, uint8_t *data, size_t len, bool final)  {},
  [](AsyncWebServerRequest * request, uint8_t *data, size_t len, size_t index, size_t total) {
    Serial.println("--------------");
    Serial.println("Body: ");
    Serial.write(data, len);
    Serial.println("--------------");
  });
}

I didn't try your exact code, but since I get all the console output in my example only once I'm pretty certain it is all at once.

However I tried a whole load of other things the past 2 days and I seem to have figured it out.
Not exactly sure why, but the DynamicJsonDocument(1024) is incorrect. I went through the Assistant Assistant | ArduinoJson 6 which gives you the minimal size of your JsonDocument and it suggested I use StaticJsonDocument<256> obj;
Since I switched to that the issue seems to have disappeared completely. Essentially I am now doing that:

void add_postRule() {
  httpServer.on("/api/postRule", HTTP_POST, [](AsyncWebServerRequest *request){}, NULL, [](AsyncWebServerRequest *request, uint8_t *data, size_t len, size_t index, size_t total){
    postRule(request, data, len);
    StaticJsonDocument<256> obj;
    DeserializationError error = deserializeJson(obj, (const char *)data,len);
    //now do stuff like
    int i = obj["ID"];
  });
}

Additionally: If somebody somehow stumbles upon this: You could also switch to adding parameters to your request with url/test?param=value&param2=value2
Though I think doing it via body is cooler and easer to read.

Good

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.