Randomly unable to read/open in SDFat

In SDFat. randomly I cannot read or open files, and sd.exists(path) would always be false after about 5 minutes of runtime. I use exFAT

#include <ESPAsyncWebServer.h>
#include <BufferedPrint.h>
#include <FreeStack.h>
#include <MinimumSerial.h>
#include <RingBuf.h>
#include <SdFat.h>
#include <SdFatConfig.h>
#include <sdios.h>
#include <WiFi.h>

// SD card chip select pin
const uint8_t SD_CHIP_SELECT = 5;

// Create SdFat instance
SdFat sd;

// Replace with your network credentials
const char* ssid = "ASUS_C8_2G_plus";
const char* password = "Seyitabdullah1440";

// Create a web server object that listens for HTTP requests on port 80
AsyncWebServer server(80);

// Helper function to check for the existence of an index file
String findIndexFile(const char* path) {
  String indexFiles[] = {"/index.html", "/index.htm", "/index.txt"};
  for (String indexFile : indexFiles) {
    String fullPath = String(path) + indexFile;
    if (sd.exists(fullPath.c_str())) {
      return fullPath;
    }
  }
  return "";
}

// Function to handle directory requests by generating an auto-index page
void handleDirectoryRequest(String path) {
  SdFile dir;

  // Attempt to open the directory
  if (!dir.open(path.c_str(), O_READ)) {
    server.send(500, "text/html", "<h1>500 - Error Opening Directory</h1>");
    return;
  }

  String output = "<html><head><title>Index of " + path + "</title></head><body>";
  output += "<h1>Index of " + path + "</h1><ul>";

  SdFile entry;
  char fileName[64];

  // Iterate over the directory entries
  while (entry.openNext(&dir, O_READ)) {
    entry.getName(fileName, sizeof(fileName));
    
    // Append a slash for directories
    if (entry.isDir()) {
      output += "<li><a href=\"" + path + String(fileName) + "/\">" + String(fileName) + "/</a></li>";
    } else {
      output += "<li><a href=\"" + path + String(fileName) + "\">" + String(fileName) + "</a></li>";
    }
    
    entry.close();
  }

  dir.close();
  output += "</ul></body></html>";

  // Send the generated page
  server.send(200, "text/html", output);
}

// Updated logic in handleFileRequest()
void handleFileRequest() {
  String path = server.uri();

  if (path == "/software") {
    handleSoftwareRequest();
    return;
  }

  if (path.endsWith("/")) {
    // If the path is a directory, check for an index file
    String indexPath = findIndexFile(path.c_str());
    if (!indexPath.isEmpty()) {
      path = indexPath;  // Use the found index file if it exists
    } else {
      handleDirectoryRequest(path);  // Generate auto-index only if no index file is found
      return;
    }
  }

  // Check if the file exists on the SD card
  if (!sd.exists(path.c_str())) {
    server.send(404, "text/html", "<h1>404 - File Not Found</h1>");
    return;
  }

  // Attempt to open the file for reading
  SdFile file;
  if (!file.open(path.c_str(), O_RDONLY)) {
    server.send(500, "text/html", "<h1>500 - Error Opening File</h1>");
    return;
  }

  // Determine content type and send the file
  String contentType = "application/octet-stream";
  if (path.endsWith(".html") || path.endsWith(".htm")) {
    contentType = "text/html";
  } else if (path.endsWith(".css")) {
    contentType = "text/css";
  } else if (path.endsWith(".png")) {
    contentType = "image/png";
  } else if (path.endsWith(".ico")) {
    contentType = "image/x-icon";
  } else if (path.endsWith(".jpg") || path.endsWith(".jpeg")) {
    contentType = "image/jpeg";
  } else if (path.endsWith(".gif")) {
    contentType = "image/gif";
  } else if (path.endsWith(".txt")) {
    contentType = "text/plain";
  } else if (path.endsWith(".exe")) {
    contentType = "application/vnd.microsoft.portable-executable";
  } else if (path.endsWith(".iso")) {
    contentType = "application/x-iso9660-image";
  } else if (path.endsWith(".zip")) {
    contentType = "application/zip";
  } else if (path.endsWith(".apk")) {
    contentType = "application/vnd.android.package-archive";
  }

  server.setContentLength(file.fileSize());
  server.send(200, contentType, "");

  uint8_t buffer[1024];
  size_t bytesRead;
  while ((bytesRead = file.read(buffer, sizeof(buffer))) > 0) {
    server.client().write(buffer, bytesRead);
  }
  file.close();
}

// Function to handle /software requests
void handleSoftwareRequest() {
  server.send(200, "text/html", "<h1>Software page - currently empty</h1>");
}

// Function to handle not found requests
void notFound() {
  server.send(404, "text/html", "<h1>404 - Not Found</h1>");
}

void setup() {
  Serial.begin(115200);

  // Initialize SD card
  if (!sd.begin(SD_CHIP_SELECT, SD_SCK_MHZ(10))) {
    Serial.println("SD card initialization failed!");
    return;
  }
  Serial.println("SD card initialized.");

  // Connect to Wi-Fi
  WiFi.begin(ssid, password);
  Serial.println("Connecting to Wi-Fi...");
  while (WiFi.status() != WL_CONNECTED) {
    delay(1000);
    Serial.print(".");
  }

  // Print IP address when connected
  Serial.println("\nConnected to Wi-Fi");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());

  // Handle all file requests
  server.onNotFound(handleFileRequest);

  // Start the server
  server.begin();
  Serial.println("HTTP server started");
}

void loop() {
  // Handle incoming client requests
  server.handleClient();
}

Not sure if this is the issue but it's one you need to fix : you don't use ESPAsyncWebServer correctly.

The callbacks take a parameter and you don't call server.send()but request->send()

for example

should be

void notFound(AsyncWebServerRequest *request) {
  request-> send(404, "text/plain", "<h1>404 - Not Found</h1>");
}

same goes for the other callbacks.

make sure you enable all warnings when compiling and check those warnings out.

also there is no server.handleClient(); with ESPAsyncWebServer, there is a dedicated task to handle the server... Seems you are totally mixing up web server codes...

yeah. it's certaintly another problem but I don't think theyre related

Well, calling a function with a parameter when it has been declared without one (i.e., with an empty parameter list) results in undefined behavior in C++.

This is because you are violating the function's signature, which explicitly defines the types and number of parameters the function expects.

When you declare a function like this:

void notFound();  // No parameters expected

You are stating that notFound takes no arguments. Attempting to call it with any argument like this:

notFound(aRequest);  // Incorrect call with a parameter

will not match the function's signature and can lead to unpredictable behavior.

So I'd start to fix that before going any further and make claims it's unrelated. You can't keep a code with undefined behavior and expect a normal behavior...

1 Like

oh wait. I didn't think about the runtime part of this... we are too used to compiler errors aren't we :sweat_smile:

are we ? :cold_face:

#include <ESPAsyncWebServer.h>
#include <WiFi.h>
#include <SdFat.h>
#define oldcode
// SD card chip select pin
const uint8_t SD_CHIP_SELECT = 5;

// Create SdFat instance
SdFat sd;
String soft[255]; // Adjust the size as needed
String lnk[255]; // Adjust the size as needed
// Create AsyncWebServer object on port 80
AsyncWebServer server(80);

// Replace with your network credentials
const char* ssid = "ASUS_C8_2G_plus";
const char* password = "Seyitabdullah1440";

// Helper function to check for the existence of an index file
String findIndexFile(const char* path) {
  String indexFiles[] = {"/index.html", "/index.htm", "/index.txt"};
  for (String indexFile : indexFiles) {
    String fullPath = String(path) + indexFile;
    if (sd.exists(fullPath.c_str())) {
      return fullPath;
    }
  }
  return "";
}

// Function to handle directory requests and generate an auto-index page if no index file is found
void handleDirectoryRequest(AsyncWebServerRequest *request, String path) {
  SdFile dir;

  if (!dir.open(path.c_str(), O_READ)) {
    request->send(500, "text/html", "<h1>500 - Error Opening Directory</h1>");
    return;
  }

  String output = "<html><head><title>Index of " + path + "</title></head><body>";
  output += "<h1>Index of " + path + "</h1><ul>";

  SdFile entry;
  char fileName[64];

  while (entry.openNext(&amp;dir, O_READ)) {
    entry.getName(fileName, sizeof(fileName));
    if (entry.isDir()) {
      output += "<li><a href=\"" + path + String(fileName) + "/\">" + String(fileName) + "/</a></li>";
    } else {
      output += "<li><a href=\"" + path + String(fileName) + "\">" + String(fileName) + "</a></li>";
    }
    entry.close();
  }

  dir.close();
  output += "</ul></body></html>";

  request->send(200, "text/html", output);
}
String replacePlaceholder(String input, const char* placeholder, const String&amp; replacement) {
  input.replace(placeholder, replacement);
  return input;
}
void handleFileRequest(AsyncWebServerRequest *request) {
  String path = request->url();

  if (path == "/software") {
    //request->send(200, "text/html", "<h1>Software page - currently empty</h1>");
    //return;
    path="/software.htm";
  }

  if (path.endsWith("/")) {
    String indexPath = findIndexFile(path.c_str());
    if (!indexPath.isEmpty()) {
      path = indexPath;
    } else {
      handleDirectoryRequest(request, path);
      return;
    }
  }

  if (!sd.exists(path.c_str())) {
    request->send(404, "text/html", "<h1>404 - File Not Found</h1>");
    return;
  }

  SdFile file;
  Serial.println("Attempting to open file: " + path);
  if (!file.open(path.c_str(), O_RDONLY)) {
    request->send(500, "text/html", "<h1>500 - Error Opening File: " + String(path.c_str()) + "</h1>");
    Serial.println("Error opening file: " + String(path.c_str()));
    return;
  }
  Serial.println("File opened successfully");

  // Determine content type
  String contentType = "application/octet-stream";
  if (path.endsWith(".html") || path.endsWith(".htm")) {
    contentType = "text/html";
  } else if (path.endsWith(".css")) {
    contentType = "text/css";
  } else if (path.endsWith(".png")) {
    contentType = "image/png";
  } else if (path.endsWith(".ico")) {
    contentType = "image/x-icon";
  } else if (path.endsWith(".jpg") || path.endsWith(".jpeg")) {
    contentType = "image/jpeg";
  } else if (path.endsWith(".gif")) {
    contentType = "image/gif";
  } else if (path.endsWith(".txt")) {
    contentType = "text/plain";
  } else if (path.endsWith(".exe")) {
    contentType = "application/vnd.microsoft.portable-executable";
  } else if (path.endsWith(".iso")) {
    contentType = "application/x-iso9660-image";
  } else if (path.endsWith(".zip")) {
    contentType = "application/zip";
  } else if (path.endsWith(".apk")) {
    contentType = "application/vnd.android.package-archive";
  }

  // Only modify content if it is HTML
  if (contentType == "text/html") {
    // Read file content into a String
   String fileContent;
char tempBuffer[128];  // Use a small, fixed-size buffer for chunk reading
int bytesRead;

// Read the file in chunks and append to fileContent
while ((bytesRead = file.read(tempBuffer, sizeof(tempBuffer) - 1)) > 0) {
    tempBuffer[bytesRead] = '\0';  // Null-terminate to safely use as a string
    fileContent += tempBuffer;     // Append the chunk to the fileContent
}

// Print the content before modification

// Replace placeholder with "just junk here"
fileContent.replace("?SOFT0?", soft[0]);
fileContent.replace("?SOFT1?", soft[1]);
fileContent.replace("?SOFT2?", soft[2]);
fileContent.replace("?SOFT3?", soft[3]);
fileContent.replace("?SOFT4?", soft[4]);
fileContent.replace("?SOFT5?", soft[5]);
fileContent.replace("?SOFT6?", soft[6]);
fileContent.replace("?SOFT7?", soft[7]);

fileContent.replace("?LNK0?", lnk[0]);
fileContent.replace("?LNK1?", lnk[1]);
fileContent.replace("?LNK2?", lnk[2]);
fileContent.replace("?LNK3?", lnk[3]);
fileContent.replace("?LNK4?", lnk[4]);
fileContent.replace("?LNK5?", lnk[5]);
fileContent.replace("?LNK6?", lnk[6]);
fileContent.replace("?LNK7?", lnk[7]);
// Print the modified content

// Send modified content
request->send(200, contentType, fileContent);

  } else {
    // For non-HTML content, send directly as a stream
    AsyncWebServerResponse *response = request->beginResponseStream(contentType);
    response->addHeader("Content-Length", String(file.fileSize()));
    response->setCode(200);
    request->send(response);

     uint32_t fileSize = file.fileSize();
 uint8_t *buffer = new uint8_t[fileSize];
  size_t bytesRead = file.read(buffer, fileSize);
  
  //Send the file response
if (bytesRead > 0) {
  request->send_P(200, contentType, buffer, fileSize);
 } else {
   request->send(500, "text/html", "<h1>500 - Error Reading File</h1>");
 }
  }

  // Clean up
  file.close();
}


// Function to handle not found requests
void notFound(AsyncWebServerRequest *request) {
  request->send(404, "text/html", "<h1>404 - Not Found</h1>");
}

void setup() {
  Serial.begin(115200);

  if (!sd.begin(SD_CHIP_SELECT, SD_SCK_MHZ(8))) {
   Serial.println("SD card initialization failed!");
    return;
  }
  Serial.println("SD card initialized.");

  WiFi.begin(ssid, password);
  Serial.println("Connecting to Wi-Fi...");

  while (WiFi.status() != WL_CONNECTED) {
    delay(1000);
    Serial.print(".");
  }

  Serial.println("\nConnected to Wi-Fi");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());

  server.onNotFound(handleFileRequest);
  server.on("/software", HTTP_GET, handleFileRequest);
  server.begin();
  Serial.println("HTTP server started");


  SdFile file;
if (!file.open("/soft.csv", O_RDONLY)) {
  Serial.println("Failed to open file");
  return;
}
char line[128]; // Buffer for reading lines
int softIndex = 0;

while (file.fgets(line, sizeof(line)) &amp;&amp; softIndex < 255) {
  String lineStr = String(line);
  Serial.println("Read line: " + lineStr); // Debug: Print the read line

  int commaIndex = lineStr.indexOf(','); // Find the first comma

  if (commaIndex != -1) {
    soft[softIndex] = lineStr.substring(0, commaIndex); // Extract first column
    lnk[softIndex] = lineStr.substring(commaIndex + 1); // Extract the rest after the first comma

    Serial.println("First column: " + soft[softIndex]); // Debug: Print extracted first column
    Serial.println("Second column: " + lnk[softIndex]); // Debug: Print extracted second column

    softIndex++;
  } else {
    Serial.println("No valid columns found in this line."); // Debug: Print if no valid columns are found
  }
}

file.close();

// Print the soft array for verification
//for (int i = 0; i < softIndex; i++) {
//  Serial.println("soft[" + String(i) + "]: " + soft[i]);
//  Serial.println("lnk[" + String(i) + "]: " + lnk[i]);
//}




}

void loop() {}

Kinda updated the code a little bit. still having same problem

if you try to work from that example

but use SdFat

does it work ?

server.serveStatic("/", SPIFFS, "/").setDefaultFile("index.htm");
I don't think we can do easily turn stuff like these with SDFat. But I will try adding a heap check as maybe that is the reason.

I've not read the code in details but

is a lot of Strings and depending how you fill them up, might end up as a lot of memory

I wish we had a proper way to make dynamic arrays. yeah arduino strings absolutely eat RAM. i guess i'll lower it to 128 and char* but problem is. same issue happened before i implemented it

I checked heap. we have RAM. It happened again. hard coded stuff like /heap or /software worked but when its a file, all files are 404/dirs are 500. It seems like a problem with SDFat

Maybe it's the off-brand microSD that I don't even know it's true clock speed

did you try lowering the clock speed ?

at 4 MhZ it doesn't work. so does in 8. works in 10-12. doesnt work in above too.