Json doc data to process fastled

Projects Programming
1

Hi, I would like to ask for some help with my project. I have a data set in json format where each variable holds a bundled string containing data for a single frame. It looks like this:

"totalFrames":"2",
"frame1":"0,0,255,0,0;0,1,255,0,0;0,2,255,255,0;0,3,255,255,0;",
"frame2":"0,0,255,255,0;0,2,255,255,0;",
...

The first two data is the x and y coordinate, the last three is the color in RGB. Each pixel of led is separated with a ";".

I wrote a script that opens up these bundles and processes them to fastled. My script works okay, but it's very slow. I tried to solve the issue but haven't been able make it faster.

StaticJsonDocument<1024> doc;
int currentFrame = 1;
#define DATA_PIN    D7
#define LED_TYPE    WS2811
#define COLOR_ORDER GRB
#define NUM_LEDS    36
CRGB leds[NUM_LEDS];
uint8_t Width  = 4;
uint8_t Height = 9;

void setup() {
  Serial.begin(115200);
  FastLED.addLeds<LED_TYPE,DATA_PIN,COLOR_ORDER>(leds, NUM_LEDS).setCorrection(TypicalLEDStrip);

}
uint16_t XY( uint8_t x, uint8_t y) {
  uint16_t i;
  if ( x & 0x01) {
    uint8_t reverseY = (Height - 1) - y;
    i = (x * Height) + reverseY;
  } else {
    i = (x * Height) + y;
  }
  return i;
}

void loop() {
  int totalFrames = doc["totalFrames"];
  if(currentFrame<=totalFrames){
       
    String frameData = doc["frame"+String(currentFrame)];
    int a=0;
 
    for (int i=0; i < frameData.length(); i++){ 
      if(frameData.charAt(i) == ';') { 
  
        String s = frameData.substring(a, i);
        s = s + ",";
        int sa[5], b=0, t=0;
        for (int j=0; j < s.length(); j++){ 
          if(s.charAt(j) == ',') { 
            sa[t] = s.substring(b, j).toInt(); 
            b=(j+1); 
            t++;
          }
        }
        leds[XY(sa[0], sa[1])] = CRGB(sa[2], sa[3], sa[4]);
       
        a=(i+1); 
      }
    }
    FastLED.show();
    delay(80); 
    currentFrame = currentFrame + 1;
  }
}
2

the parsing could be better without using the String class but what makes it slow is possibly acquiring the JSON. Your code does not show where it's coming from.

3

I've created a webserver that is running on an ESP8266. On this webserver i've made a platfrom in html where you can set colors of the pixels. Javascript processes the data. And when you save it, it bundels up the data, sends it to the ESP, deserialize it to the doc and also saves it in a txt file. This is where the data comes from.

4

how fast is that part ?

if you were to acquire the content line by line, you could parse with something like this

click to see the code 
char totalFrames[] = "\"totalFrames\":\"2\",";
char frame1[] = "\"frame1\":\"0,0,255,0,0;0,1,255,0,0;0,2,255,255,0;0,3,255,255,0;\",";
char frame2[] = "\"frame2\":\"0,0,255,255,0;0,2,255,255,0;\",";

void parseLine(char * line) {
  const size_t totalFrameLength = strlen("\"totalFrames\":\"");
  const size_t FrameIDLength = strlen("\"frame");

  if (strncmp(line, "\"totalFrames\":\"", totalFrameLength) == 0) {
    // totalFrames
    Serial.print("it's the total Frame count ==> ");
    char *endPtr;
    long n = strtol (line + totalFrameLength, &endPtr, 10); // https://cplusplus.com/reference/cstdlib/strtol/;
    if (*endPtr == '\"') {
      Serial.print("Expecting ");
      Serial.print(n);
      Serial.println(" frames.");
    } else {
      Serial.println("wrong format.");
    }
  }
  else if (strncmp(line, "\"frame", FrameIDLength) == 0) {
    Serial.print("it's a frame definition ==> ");
    char *endPtr;
    long n = strtol (line + FrameIDLength, &endPtr, 10);
    if (strncmp(endPtr, "\":\"", 3) == 0) {
      Serial.print("Frame ID ");
      Serial.println(n);
      char * startPtr = endPtr += 3; // go to the start of the [x,y,r,g,b;] repetition
      int x, y, r, g, b;
      while (*startPtr != '\"') {
        if (sscanf(startPtr, "%d,%d,%d,%d,%d;", &x, &y, &r, &g, &b) == 5) {
          Serial.print("point ("); Serial.print(x);
          Serial.print(", "); Serial.print(y);
          Serial.print(") - color ("); Serial.print(r);
          Serial.print(", "); Serial.print(g);
          Serial.print(", "); Serial.print(b);
          Serial.println(")");
        } else {
          Serial.println("wrong point format.");
          break;
        }
        startPtr = strchr(startPtr, ';') + 1;
      }
      Serial.println("------- END OF FRAME -------");
    } else {
      Serial.println("wrong frame format.");
    }
  }
  else {
    Serial.println("not recognized");
  }
}


void setup() {
  Serial.begin(115200);
  parseLine(totalFrames);
  parseLine(frame1);
  parseLine(frame2);
}

void loop() {}

with input read line by line in

"totalFrames":"2",
"frame1":"0,0,255,0,0;0,1,255,0,0;0,2,255,255,0;0,3,255,255,0;",
"frame2":"0,0,255,255,0;0,2,255,255,0;",

you would get

it's the total Frame count ==> Expecting 2 frames.
it's a frame definition ==> Frame ID 1
point (0, 0) - color (255, 0, 0)
point (0, 1) - color (255, 0, 0)
point (0, 2) - color (255, 255, 0)
point (0, 3) - color (255, 255, 0)
------- END OF FRAME -------
it's a frame definition ==> Frame ID 2
point (0, 0) - color (255, 255, 0)
point (0, 2) - color (255, 255, 0)
------- END OF FRAME -------

there is a minimal error check so not super robust probably. Assume the format of the line is mostly correct.

5

the html and javascript processing is kinda fast as it is happening on the smartphone (device). So untill it gets to the jsondocument it's working reliably.
When I deserialize the jsondocument, it creates seperate variables for the data and I can reach them something like this:

doc["totalFrames"] = 2;
doc["frame1"] = "0,0,255,0,0;0,1,255,0,0;0,2,255,255,0;0,3,255,255,0;";
doc["frame2"] = "0,0,255,255,0;0,2,255,255,0;";

so it is not packed into one single string.

6

Thanks for the code! By using your idea as guideline, I came up with the following script:

if(currentFrame<=totalFrames){

    const char *frameData = doc["frame"+String(currentFrame)];

    int x, y, r, g, b; 
    for (int i=0; i < strlen(frameData); i++){ 
      if (sscanf(frameData, "%d,%d,%d,%d,%d;", &x, &y, &r, &g, &b) == 5) {
        leds[XY(x, y)] = CRGB(r, g, b);
      }
      frameData = strchr(frameData, ';') + 1;
    }
    FastLED.show();
    delay(80); 
    currentFrame = currentFrame + 1;
}

Currently i'm testing it, but it looks promising.
Thanks again bro!

7

no but when you receive the JSON you can actually detect the new lines (assuming it's formatted as you presented) and call the line parser. Then you don't need the JSON package at all.

sscanf tells you (it's the test with == 5) if it managed to extract the 5 values matching the format you passed so you just want a while loop until sscanf fails for example (which means it would not have seen the trailing ; or the rest of the data so you know you need to stop)

8

Hi, I'm struggling to get this right. I have a json document looks like this:

{
"t":"2",
"f1":"255,0,0,0,1,4,5,8,9,12,13,16,17,20,21,24,25,28,29,32,33;0,110,255,2,3,6,7,10,11,14,15,18,19,22,23,30,31,34,35;",
"f2":"0,110,255,0,1,4,5,8,9,12,13,16,17,20,21,24,25,28,29,32,33;255,0,0,2,3,6,7,10,11,14,15,18,19,22,23,26,27,30,31,34,35;"
}

where 't' is the total frame number. The 'f1' and 'f2' are the frames data. The data is structured by color. So the first decimal is red, the second is green, the third is blue and the rest till the semicolon ';' is the number of the leds which has these rgb colors. After the semicolon ';' comes the next color with it's led numbers and so on.
The frame can also have an "x" value. This means this frame is the same as the previous one.

#include <stdio.h>
#include <stdlib.h> 
#include "LittleFS.h"
#include <ArduinoJson.h>

#define DATA_PIN    D7
#define LED_TYPE    WS2811
#define COLOR_ORDER GRB
#define NUM_LEDS    36
CRGB leds[NUM_LEDS];

int currentFrame = 1;
String savedAnim = "";
int firstLoad = 0;
int totalFrames = 0;

String load_from_file(String file_name) {
  String result = "";
  
  File this_file = LittleFS.open(file_name, "r");
  if (!this_file) { // failed to open the file, retrn empty result
    return result;
  }

  while (this_file.available()) {
      result += (char)this_file.read();
  }
  
  this_file.close();
  return result;
}

void setup() {
  Serial.begin(115200);
  LittleFS.begin();
  FastLED.addLeds<LED_TYPE,DATA_PIN,COLOR_ORDER>(leds, NUM_LEDS).setCorrection(TypicalLEDStrip);
	
}

void loop() {
  if(firstLoad == 0){
    savedAnim = load_from_file("custom.json");
    if (savedAnim == "") {
      Serial.println("The file was empty or not present");
    } else {
      DeserializationError error = deserializeJson(doc, savedAnim);
      if (error) {
        Serial.print(F("Failed: "));
        Serial.println(error.f_str());
        return;
      }else{
         Serial.println("Custom anim file loaded.");
         firstLoad = 1;
         currentFrame = 1;
    	 totalFrames = doc["t"];
      }
    }
  }else{
    FastLED.clear();  


    if(currentFrame<=totalFrames){
    
      const char *frameData = doc["f"+String(currentFrame)];
      cFrame = frameData;
      
      char *ptr = NULL;
      char f[1280];
      int r, g, b; 
      if(frameData == "x"){
        frameData = pFrame;
      }else{
        //Serial.println(frameData);
        if(sscanf(frameData, "%d,%d,%d,%s;", &r, &g, &b, &f)==4){
          Serial.println(f);
          byte index = 0;
          ptr = strtok(f, ",");
          while (ptr != NULL){
            int num = atoi(ptr);
            leds[num] = CRGB(r, g, b);
            index++;
            ptr = strtok(NULL, ",");
          }
          frameData = strchr(frameData, ';') + 1;
        } 
      }
  
      FastLED.show();
      delay(80); 
      currentFrame = currentFrame + 1;
    }else{
      currentFrame = 1;
    }
    pFrame = cFrame;
  }
}
9

I merged your post here as it's the same topic.

what are you struggling with exactly? parsing your weird format?

if you can influence the JSON that's generated you could make your life easier by having it structured

something like this would be easier to iterate through

{
  "frameCnt": 5,

  "frames": [
    {"RGB": [255, 0, 0], "indexes": [0, 1, 2]},
    {"RGB": [0, 255, 0], "indexes": [3, 4, 5]},
    {"RGB": [0, 0, 255], "indexes": [6, 7, 8]},
    {"RGB": [128, 128, 0], "indexes": [9, 10, 11]},
    {"RGB": [255, 128, 0], "indexes": [12, 13, 14]}
  ]
}

then you can use this type of code to walk through the arrays

#include <ArduinoJson.h>

const char* jsonString = R"(
{
  "frameCnt": 5,

  "frames": [
    {"RGB": [255, 0, 0], "indexes": [0, 1, 2]},
    {"RGB": [0, 255, 0], "indexes": [3, 4, 5]},
    {"RGB": [0, 0, 255], "indexes": [6, 7, 8]},
    {"RGB": [128, 128, 0], "indexes": [9, 10, 11]},
    {"RGB": [255, 128, 0], "indexes": [12, 13, 14]}
  ]
}
)";

void setup() {
  Serial.begin(9600);

  // Parse JSON
  DynamicJsonDocument doc(1024);
  deserializeJson(doc, jsonString);

  // Access frame data
  int frameCount = doc["frameCnt"];
  JsonArray frames = doc["frames"];

  // Print each frame
  for (int i = 0; i < frameCount; ++i) {
    JsonObject frame = frames[i];
    Serial.print("Frame ");
    Serial.print(i + 1);

    JsonArray rgb = frame["RGB"];
    Serial.print("\tRed: "); Serial.print(rgb[0].as<int>());
    Serial.print(", Green: "); Serial.print(rgb[1].as<int>());
    Serial.print(", Blue: "); Serial.print(rgb[2].as<int>());

    Serial.print("\t==> for Leds # ");
    JsonArray indexes = frame["indexes"];
    for (int j = 0; j < indexes.size(); ++j) {
      Serial.print(indexes[j].as<int>());
      if (j < indexes.size() - 1) {
        Serial.print(", ");
      }
    }
    Serial.println();
  }
}

void loop() {}

this will print

Frame 1	Red: 255, Green: 0, Blue: 0	==> for Leds # 0, 1, 2
Frame 2	Red: 0, Green: 255, Blue: 0	==> for Leds # 3, 4, 5
Frame 3	Red: 0, Green: 0, Blue: 255	==> for Leds # 6, 7, 8
Frame 4	Red: 128, Green: 128, Blue: 0	==> for Leds # 9, 10, 11
Frame 5	Red: 255, Green: 128, Blue: 0	==> for Leds # 12, 13, 14
10

Seems like a horribly inefficient way to transfer this type of data. Why ASCII / JSON? Why not a binary format?

11

The data is generated at a html, javascript based graphical editor (self made). Where you can set color for pixels, add frames, etc. Each frame stored in a hidden input and when you save it, it bundles them toghether in a string and saves it to file.

My previous attempt was to store the pixel data one-by-one (x,y,r,g,b;). But I takes up too much space after a few frames. So I tried to came up with a compression, what is why I tried different bundling. But that way it's eats up the memory and crashes.

Now I see, creating a good input data would be the first to get right. I never made binary format file. What would a binary format look like in this case?

12

If you create the data on a computer you’ll have to deal with type length for integers, it will have to match what you expect on your target arduino (use types with known length like uint8_t or uint16_t) and ensure it’s generated in little endian format.

I would suggest to first get your idea working.

As you control the generator, try to generate something that looks like the format I suggested. Then you can make use of the ArduinoJSON library to walk through the data as I demonstrated. (I made it very verbose you should obviously use one letter keys to minimize the info being transferred).

If this becomes too slow for your liking because of the amount of data transferred or eats up too much memory on your arduino then consider optimizing the format and possibly going binary.

JSON is verbose but contains its own coherence verification mechanism on the payload to some extent (matching {} and []) when you deserialize and this can make it easy to reject a wrong format. So if you go binary you’ll want to make the payload format robust too.

13

I tried to implement your code, but I realized it processes only one color for each frame. In my case there should be more than one color on each frame. Something like this:

const char* jsonString = R"(
{
  "frameCnt": 2,

  "frame1": [
    {"RGB": [255, 0, 0], "indexes": [0, 1, 2]},
    {"RGB": [0, 255, 0], "indexes": [3, 4, 5]}
  ],
  "frame2": [
    {"RGB": [0, 0, 255], "indexes": [6, 7, 8]},
    {"RGB": [128, 128, 0], "indexes": [9, 10, 11]},
    {"RGB": [255, 128, 0], "indexes": [12, 13, 14]}
  ]
}
)";
14

OK - yes makes sense

walking through the arrays is similar

you have a general loop to walk through the frames and for each frame a loop to walk through the individual color settings and an inner loop to set each led through its index

15

Thanks. According to your reply i've modified the code:

#include <ArduinoJson.h>

int currentFrame = 1;
int totalFrames = 1;
DynamicJsonDocument doc(1024);
  
const char* jsonString = R"(
{
  "t": 2,

  "frame1": [
    {"RGB": [255, 0, 0], "indexes": [0, 1, 2]},
    {"RGB": [0, 255, 0], "indexes": [3, 4, 5]}
  ],
  "frame2": [
    {"RGB": [0, 0, 255], "indexes": [6, 7, 8]},
    {"RGB": [128, 128, 0], "indexes": [9, 10, 11]},
    {"RGB": [255, 128, 0], "indexes": [12, 13, 14]}
  ]
}
)";
void setup() {
  Serial.begin(115200);
  deserializeJson(doc, jsonString);
  totalFrames = doc["t"];  
}
void loop() {

  if(currentFrame<=totalFrames){
    JsonArray frame = doc["frame"+currentFrame];
    Serial.print("Frame ");
    Serial.println(currentFrame);

    for (int i = 0; i < frame.size(); ++i) {
      JsonObject pixel = frame[i];
      Serial.print("Pixel ");
      Serial.print(i + 1);
  
      JsonArray rgb = pixel["RGB"];
      Serial.print("\tRed: "); Serial.print(rgb[0].as<int>());
      Serial.print(", Green: "); Serial.print(rgb[1].as<int>());
      Serial.print(", Blue: "); Serial.print(rgb[2].as<int>());
  
      Serial.print("\t==> for Leds # ");
      JsonArray indexes = pixel["indexes"];
      for (int j = 0; j < indexes.size(); ++j) {
        Serial.print(indexes[j].as<int>());
        if (j < indexes.size() - 1) {
          Serial.print(", ");
        }
      }
      Serial.println();
    }
    currentFrame++;    
  }else{
    currentFrame = 1;
  }
}

But it does not want to process it. It thinks that frame.size() is 0. So it does not starts the i loop. :frowning:

16

ok I found the problem in my code:

JsonArray frame = doc["frame"+currentFrame];

should be:

JsonArray frame = doc["frame"+String(currentFrame)];
17

If you had a couple [] around the frames to make that into an array, then you would not have to build dynamically the names. You would just iterate through the array as I did

18

Thanks, that's a great idea.
I've tested the code with a few animation and it works great. It can handle many frames.
Thanks again for your great help!

19

Good to hear

Have fun

20

Sure fun it is! :slight_smile:
You can see it in action here:
Flaming cyberpunk skull mask

Thanks again for your help.

1 Like