String Size limitation http.getString

Hi,

i´m starting with a new picture frame with a spectra 6 color display with a resolution of 1200x1600. The Display is slow, but the colors a not bad at all.
I like to update every hour or so. the data will come from a webserver.

if i take a picture with 850x1100 the variable "elementCount" in the function "void downloadImageData() {" returns the right value of 481526. If i take a bigger picture then it return is 1, so it seems that there is a limitation in the maximum string sitze. I didn´t found anything for the limitation. pictures with the full resolution of 1200x1600 will have 960000 values.

i´m using a Seed Xiao ESP32S3 with 8MB of PSRAM. the plan was:

get the string via http.getstring > count for ´,´ > alocate memory for the array in psram > fill the array with the hex values > send data to display framebuffer.

any idea?

the pictures are stored directly as array on the webserver:

// 6 Color Image Data 850*1133 
const unsigned char Image6color[481525] = {
0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,
0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,
0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,
0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,
0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,0x11,

// Image byte data is stored in PSRAM, so need to enable PSRAM.

#include "EPD_13in3e.h"
#include <WiFi.h>
#include <HTTPClient.h>
#define S_To_uS_Factor 1000000ULL  //Conversion factor for micro seconds to seconds

const char* ssid = "xxx";         // Your WiFi SSID
const char* password = "xxxx";  // Your WiFi Password
const char* getImageUrl = "http://192.168.2.12/demo_image.h";
int schlafen = 3600;

void setup() {
  Serial.begin(115200);
  connectWiFi();
  mainTask();
}

void loop() {
  delay(60000);
}

void connectWiFi() {
  int retryCount = 0;
  WiFi.mode(WIFI_STA);  // Set WiFi to station mode
  WiFi.begin(ssid, password);
  // Attempt to connect for a set number of retries
  while (WiFi.status() != WL_CONNECTED && retryCount < 10) {
    delay(1000);
    Serial.println("Connecting to WiFi...");
    retryCount++;
  }
  if (WiFi.status() == WL_CONNECTED) {
    Serial.println("Connected to WiFi!");
  } else {
    Serial.println("Failed to connect to WiFi after 10 attempts, stopping WiFi.");
    hibernate(3600);
  }
}

void mainTask() {
  if (WiFi.status() == WL_CONNECTED) {
    // Start downloading the image data
    downloadImageData();
    // Fetch the wakeup interval and go into deep sleep
    //  hibernate(schlafen);
  } else {
    //   hibernate(3600);  // Go to sleep for 1 hour if no WiFi connection
  }
}

void downloadImageData() {
  HTTPClient http;
  if (!http.begin(getImageUrl)) {
    Serial.println("Failed to initialize HTTP connection");
    return;
  }
  Serial.println("httpget");
  // Make the request
  int httpResponseCode = http.GET();

  if (httpResponseCode > 0 && httpResponseCode == HTTP_CODE_OK) {
    Serial.println("string payload");
    String payload = http.getString();
    // Count the number of elements based on commas in the payload
    long elementCount = 1;
    for (long i = 0; i < payload.length(); i++) {
      if (payload[i] == ',') elementCount++;
    }
    Serial.println(elementCount);

    // Allocate memory for the uint8_t array
    uint8_t* dataBuffer = (uint8_t*)ps_malloc(elementCount);
    if (dataBuffer == nullptr) {
      Serial.println("Failed to allocate memory in PSRAM");
      return;
    }
    // Parse the payload string and fill the array
    long index = 0;
    long startPos = 0;

    for (long i = 0; i < payload.length(); i++) {
      if (payload[i] == ',' || i == payload.length() - 1) {
        // Get the substring for the current hex value
        String hexValue = payload.substring(startPos, i);
        hexValue.trim();  // Trim whitespace around the hex string

        // Convert hex string to uint8_t and store it in the array
        dataBuffer[index++] = (uint8_t)strtol(hexValue.c_str(), nullptr, 16);
        // Update start position for the next hex value
        startPos = i + 1;
      }
    }

    DEV_Module_Init();
    EPD_13IN3E_Init();
    //  EPD_13IN3E_Clear(EPD_13IN3E_WHITE);
    delay(5000);
    Serial.println("print picture");
    // EPD_13IN3E_DisplayPart((const uint8_t*)dataBuffer, 175, 234, 850,1133);
    Serial.println("epd sleep");
    EPD_13IN3E_Sleep();
    Serial.println("epd power down");
    DEV_Module_Exit();


    free(dataBuffer);  // Free the allocated memory if no longer needed
    dataBuffer = nullptr;

  } else {
    Serial.printf("HTTP GET request failed, error: %s\n", http.errorToString(httpResponseCode).c_str());
  }

  http.end();  // Close connection
}

void hibernate(int interval) {
  WiFi.disconnect(true);  // Disconnect from any connection attempt
  WiFi.mode(WIFI_OFF);    // Turn off the WiFi
  delay(1000);
  esp_sleep_enable_timer_wakeup(interval * S_To_uS_Factor);  // Convert to microseconds
  esp_deep_sleep_start();
}

Trying to read into a String object first seems like it will be a problem given the limited contiguous RAM available (even on an ESP32). Why not read it directly into a PSRAM buffer? Perhaps the getStream() function from the HTTPClient library would be helpful?

For the record, the hard limit for String is

#ifdef BOARD_HAS_PSRAM
  enum {
    CAPACITY_MAX = 3145728
  };
#else
  enum {
    CAPACITY_MAX = 65535
  };
#endif

You must have PSRAM already enabled under the Tools menu; otherwise you would not have counted 480K commas, or used ps_malloc successfully. Once you exceed CAPACITY_MAX -- or encounter any memory allocation error -- the String becomes invalid, and among other things, length() will return zero.

Since you are sending actual "hex values" as comma-separated text, each one takes three or five bytes (are you sending the "0x"?) This is really inefficient. (BTW

cuts off the very last digit due to the way substring works; and strtol skips leading whitespace, so you don't have to trim.) Is your webserver written in C++? Is it also an Arduino?

What you should be doing is have the webserver return the raw binary data with the appropriate headers, like

Content-Type: application/octet-stream
Content-Length: 481525

There may be a more appropriate Content-Type for your image data. But if not, octet-stream will work: a bunch of bytes. With the Content-Length, you allocate that size. Then using getStream, you can read each byte and fill the dataBuffer, without needing strtol.

Hi,

it is working now. removed the overhead (0x). My webserver runs on windows, it is only for testing. i´m not finished with finding the right way for me.
My other idea was to read the stream from a sd card. But it i have errors in the stream.

  SD.begin(43);
  listDir(SD, "/", 0);
  Serial.println("open file");
  dataFile = SD.open(F("/8.txt"));
  String payload = dataFile.readString();

When i fill the buffer and print the current value with the following code

    int index = 0;
    int startPos = 0;
    for (int i = 0; i < payload.length(); i++) {
      if (payload[i] == ',' || i == payload.length() - 1) {
        // Get the substring for the current hex value
        String hexValue = payload.substring(startPos, i);
        Serial.println(hexValue);
        // Convert hex string to uint8_t and store it in the array
        dataBuffer[index++] = (uint8_t)strtol(hexValue.c_str(), nullptr, 16);
        // Update start position for the next hex value
        startPos = i + 1;
      }
    }

the output shows (seems there are missing signs if i put it here as code, please see attached picture):

00
2
52
33
63
65
3013
30
11
32
53
1
36
32

The stream from the webserver as source looks as expected:

10
05
23
03
01
20
11
50

21
20
02
10
22
10
16
13
20

Is there a problem with the readString from sd card function or is this the wrong way doing it?

I still don't understand why you insist on using a String object.

I also don't understand why insist on representing binary data as ASCII Hex. Even without the "0x", it's still horribly inefficient.

1 Like

the pictures have to be resized, color indexed and dithered. After this the bitmap is converted to the array. this is the way the display library support to upload them.

i´m an absolute beginner. until now my knowlege was enough for my usage for messuring and regulation temperature, build a logger and other easy things.
any help is welcome.

If you're talking about this

That is of course C code, so the way that is supposed to be used is as unsigned char data, not a text file full of hex values. (Text is just char. unsigned char is a separate type; often the alias uint8_t is used instead, which is more explicit that it's 8-bit and less confused with char).

I modified the Examples | "Examples for {your ESP32 board} "| WebServer | HelloServer

#include <WiFi.h>
#include <WebServer.h>
#include "arduino_secrets.h"

WebServer server(80);

void handleRoot() {
  server.send(200, "text/plain", "hello from esp32!");
}

void handleNotFound() {
  server.send(404, "text/plain", "not found");
}

const uint8_t imageData[] = {
  'B','E','G','I','N',0x0A,
  0x68,0x65,0x6C,0x6C,0x6F,0x0A,
  0x00,'E','N','D',
};

void handleData() {
  String data(reinterpret_cast<const char *>(imageData), sizeof(imageData));
  server.send(200, "application/octet-stream", data);
}

void setup(void) {
  Serial.begin(115200);
  WiFi.begin(SECRET_SSID, SECRET_PASS);
  for (int i = 0; WiFi.status() != WL_CONNECTED; i++) {
    Serial.print(i % 10 ? "." : "\n.");
    delay(100);
  }
  Serial.println();
  Serial.println(WiFi.localIP());

  server.onNotFound(handleNotFound);
  server.on("/", handleRoot);
  server.on("/data", handleData);

  server.begin();
  Serial.println("HTTP server started");
}

void loop(void) {
  server.handleClient();
  delay(2);  //allow the cpu to switch to other tasks
}

Then trying from desktop

$ curl -i 192.168.1.15/data
HTTP/1.1 200 OK
Content-Type: application/octet-stream
Content-Length: 16
Connection: close

Warning: Binary output can mess up your terminal. Use "--output -" to tell 
Warning: curl to output it to your terminal anyway, or consider "--output 
Warning: <FILE>" to save to a file.
$ curl 192.168.1.15/data | hexdump -C
00000000  42 45 47 49 4e 0a 68 65  6c 6c 6f 0a 00 45 4e 44  |BEGIN.hello..END|
00000010

server.send takes either a String or const char *, aka "C-string", for the data. A literal string in quotes, like in handleRoot, is the latter. The Content-Length is returned, as I had suggested earlier. The trick is that

  • real binary data, like for an image, might have a zero in it
  • C-strings are terminated with a binary zero, aka NUL character
    • e.g. "abc" is four bytes: 0x61,0x62,0x63,0x00
  • you can easily construct a String with a C-string
    • but then it will stop at the first NUL it encounters

So instead, you create a String with a pointer and an explicit length. Apparently curl does a little look-ahead and says, "this looks like binary data, are you sure you want me to print it?". Instead with hexdump you can see the text and the NUL: complete binary data.

You can even avoid the copy by using send_P instead, which has an overload that takes a length. Since on ESP32, PROGMEM does nothing

#define PROGMEM
#define PGM_P        const char *

it's an easy conversion

void handleData() {
  server.send_P(200, "application/octet-stream",
                reinterpret_cast<PGM_P>(imageData), sizeof(imageData));
}

These are peculiarities with using an ESP32 with C++ as a web server. But whatever you're using -- Node, Python, PHP, Go -- it should be serving binary data with a Content-Length. If the web server is not C, then it will need to convert the header file content to binary instead of just compiling it. It would be similar to your "count the commas" text-to-binary conversion, but done server-side, where you are less constrained.

You could even generate a binary file, and store that on SD. If you really want to, we can also talk about your SD/printing issue.

Hi,

i begin to understand what you wrote, also on your first answer. i can configure the webserver.

at the moment:

PS C:\Users\Markus> Invoke-WebRequest -UseBasicParsing -Uri http://192.168.2.12/demo_image.h


StatusCode        : 200
StatusDescription : OK
Content           : // 6 Color Image Data 1200*1600
                    const unsigned char Image6color[960000] = {
                    11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,
                    11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,
                    11,11,11,11,11,11,11,...
RawContent        : HTTP/1.1 200 OK
                    Accept-Ranges: bytes
                    Content-Length: 3000083
                    Content-Type: text/plain
                    Date: Thu, 13 Feb 2025 08:30:37 GMT
                    ETag: "d701567317ddb1:0"
                    Last-Modified: Wed, 12 Feb 2025 09:35:07 GMT
                    S...
Forms             :
Headers           : {[Accept-Ranges, bytes], [Content-Length, 3000083], [Content-Type, text/plain], [Date, Thu, 13 Feb
                    2025 08:30:37 GMT]...}
Images            : {}
InputFields       : {}
Links             : {}
ParsedHtml        :
RawContentLength  : 3000083

after:

StatusCode        : 200
StatusDescription : OK
Content           : {47, 47, 32, 54...}
RawContent        : HTTP/1.1 200 OK
                    Accept-Ranges: bytes
                    Content-Length: 3000083
                    Content-Type: application/octet-stream
                    Date: Thu, 13 Feb 2025 08:34:01 GMT
                    ETag: "d701567317ddb1:0"
                    Last-Modified: Wed, 12 Feb 2025 0...
Headers           : {[Accept-Ranges, bytes], [Content-Length, 3000083], [Content-Type, application/octet-stream], [Date, Thu, 13 Feb 2025 08:34:01 GMT]...}
RawContentLength  : 3000083

But there is nothing more i can configure on server side.

I´m still struggling with the sd card read and the wrong data in the string.

Rows and columns of pixel values get translated into text that gets transmitted serially for reasons I won't go into, but have to do with transmission errors.

That text... you don't have to and really should not treat as string (or String) but as individual text that should be read and translated in small groups back into pixel values that get held and stored into rows and columns as a picture.

There are a few fundamentals you need to learn to do that but you should be able to do it after you learn more than you now know has to be.

In the meantime I suggest that you drop the whole string thing as that's only going to hold you up and send you down false paths and waste your time.

What you need to learn will stand by you not just for this goal but many others as well. Fundamental is like that. When you see the outside of a house, foundations on up are not visible but without all that the rest falls down.

When you know foundations and build on those, you won't have many mysteries. When you have mysteries, that's a sign that you need to look and learn what's behind and under them in order to master your craft.

You don't need to create a String object. It's odd that 'WebServer.h' doesn't have an overload that simply takes a pointer and data size, but you can use this one:

  void send_P(int code, PGM_P content_type, PGM_P content, size_t contentLength);

because PGM_P is a macro that simply evaluates to const char *. Using this overload, allows the data to be in PSRAM and not managed by a String object.

Yeah, I made an edit an hour after the initial post to add the use of send_P. Left the manual binary String stuff in, since it might be relevant in some other situation.

What kind of server is this?

Looks like the Type has been changed, but not the Length. And if you look up in (or have memorized some of) an ASCII table, those four numbers that start the Content are the code points for // 6, the same as before. This is a change only in that one Content-Type header, which slightly changed the output of the Invoke-WebRequest, but not much else. For the actual binary data defined using C in demo_image.h, it

  • wouldn't include the comment,
  • wouldn't include the C type and variable name, and
  • the Content-Length would be one-third the size, because each of the two hex digits and comma would be converted to the corresponding single-byte value.

It confusing because in post #4, the code matches the screenshot, except for the unprintable box character; but in post #5, they don't match each other, nor post #4.

That box character is what you get in the Serial Monitor (on Windows at least) when trying to print a control character that is not one of a few recognized whitespace characters.

void setup() {
  Serial.begin(115200);
}

void loop() {
  Serial.println(static_cast<char>(-1));
  for (int i = 0; i < ' '; i++) {
    Serial.print(" 0x");
    if (i < 0x10) Serial.print("0");
    Serial.print(i, HEX);
    Serial.print(" -(");
    Serial.print(static_cast<char>(i));
    Serial.print(")- ");
    if (i % 4 == 0) Serial.println();
  }
  Serial.println();
  delay(1000);
}

prints

�
 0x00 -()- 
 0x01 -()-  0x02 -()-  0x03 -()-  0x04 -()- 
 0x05 -()-  0x06 -()-  0x07 -(a)-  0x08 -()- 
 0x09 -(	)-  0x0A -(
)-  0x0B -()-  0x0C -()- 
 0x0D -(
)-  0x0E -()-  0x0F -()-  0x10 -()- 
 0x11 -()-  0x12 -()-  0x13 -()-  0x14 -()- 
 0x15 -()-  0x16 -()-  0x17 -()-  0x18 -()- 
 0x19 -()-  0x1A -()-  0x1B -(e)-  0x1C -()- 
 0x1D -()-  0x1E -()-  0x1F -()- 

My copy-to-code looks more successful than yours -- let's see if it survives past the preview

  • The first (tiny) question mark in a diamond is the Unicode replacement character, which you will get for bytes that comprise an invalid UTF-8 sequence. -1, same as 0xFF, is never part of a valid sequence.
  • In the Serial Monitor I see a box for 0x00 NUL, but not here
  • 0x0D CR moves to the next line here, but has no visual effect (and no box) in the Serial Monitor; you just see -()-
  • 0x0C FF (form feed/page break) has no effect
  • 0x09 HT (horizontal tab, aka just plain tab) and 0x0A LF have the expected whitespace effect, and no box
  • ETA: most of the boxes made it, but two display differently than in preview: 0x07 -(a)- and 0x1B -(e)-

Based on this, you might nave NUL bytes in your file. You can confirm this with hexdump -C (or whatever the equivalent on Windows) on the file directly. You can also print the numeric values of each character.

  String hexValue("30\00013", 5);
  Serial.print(hexValue);
  for (int i = 0; i < hexValue.length(); i++) {
    Serial.print(i ? ' ' : '\t');
    Serial.print(static_cast<int>(hexValue.charAt(i)), HEX);
  }
  Serial.println();

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.