ESP32-S3 SD Module / Library not working properly

Projects Programming
1

Hello,

i am currently writing the code for a project. The device (ESP32-S3) should connect to my phone via BLE and respond to commands, change states, etc. I am currently testing the device on a breadboard which also has a MicroSD-Module on it. The connection is correct.

The Problem:
The SD-Card seems to be connected/disconnected at random times.

Main.cpp

#include <Arduino.h>
#include <BleHandler.h>
#include <SDHandler.h>
#include <StateManager.h>
#include <VMD.h>

// Global objects
BluetoothManager bleManager;
SDHandler sdHandler;
StateManager stateManager;

// Global variable for the BLE command
BleCommand currentCommand = BleCommand::INVALID_COMMAND;

// Command callback: Store the received command
void handleCommand(BleCommand cmd)
{
    currentCommand = cmd;
}

// State handlers
void handleInit()
{
    Serial.println("Initializing...");
    // Initialize BLE

    if (sdHandler.init())
    {
        Serial.println("SD card initialized!");
    }
    else
    {
        Serial.println("Error initializing SD card!");
        stateManager.setState(State::ERROR);
        return;
    }

    bleManager.init("VMD_Device");
    bleManager.setCommandCallback(handleCommand);
    // Move to advertising state
    stateManager.setState(State::ADVERTISING);
}

void handleAdvertising()
{
    if (bleManager.isConnected())
    {
        Serial.println("Device connected!");
        stateManager.setState(State::WAITING);
    }
}

// Function which gets called after "GET:" and sends the data set from the SD-card
void handleDatasetRequest(const char *timestamp)
{
    static char datasetList[256]; // Smaller buffer
    if (sdHandler.getDatasetList(datasetList, sizeof(datasetList)))
    {
        bleManager.sendDatasetList(datasetList);
    }
    else
    {
        bleManager.sendResponse("Error Sending Dataset!");
        Serial.println("Error Sending Dataset!");
        stateManager.setState(State::ERROR);
    }
}

void handleWaiting()
{
    // Nothing to do, just wait for commands
}

void handleMeasuring()
{
    // Nothing to do for now
}

void handleError()
{
    Serial.println("In ERROR state");
    delay(5000);
    // Add error recovery logic if needed
}

void setup()
{
    Serial.begin(115200);

    // Register state handlers
    stateManager.setStateHandler(State::INIT, handleInit);
    stateManager.setStateHandler(State::ADVERTISING, handleAdvertising);
    stateManager.setStateHandler(State::WAITING, handleWaiting);
    stateManager.setStateHandler(State::MEASURING, handleMeasuring);
    stateManager.setStateHandler(State::ERROR, handleError);
    bleManager.setDatasetRequestCallback(handleDatasetRequest);

    // Start with INIT state
    stateManager.setState(State::INIT);
}

void loop()
{
    stateManager.update();

    // Check if the BLE device is still connected
    if (!bleManager.isConnected() && stateManager.getCurrentState() != State::ADVERTISING && stateManager.getCurrentState() != State::ERROR)
    {
        if (stateManager.getCurrentState() == State::WAITING)
        {
            stateManager.setState(State::ADVERTISING);
            return;
        }

        Serial.println("Device disconnected, transitioning to ERROR state");
        stateManager.setState(State::ERROR);
        return;
    }

    // Handle the command in the main loop
    if (bleManager.isCommandReceived())
    {
        Serial.println(int(currentCommand));
        switch (currentCommand)
        {
        case BleCommand::START_MEASURE:
            Serial.println("Received START_MEASURE command");
            stateManager.setState(State::MEASURING);
            break;

        case BleCommand::STOP_MEASURE:
            Serial.println("Received STOP_MEASURE command");
            if (stateManager.getCurrentState() == State::MEASURING)
            {
                stateManager.setState(State::WAITING);
            }
            break;

        case BleCommand::GET_DATASET_LIST:
            Serial.println("Received GET_DATASET_LIST command");
            char datasetList[1024];
            if (sdHandler.getDatasetList(datasetList, sizeof(datasetList)))
            {
                if (strlen(datasetList) == 0)
                {
                    Serial.println("Dataset list is empty");
                    bleManager.sendDatasetList("empty");
                }
                else
                {
                    Serial.print("The dataset list is: ");
                    Serial.println(datasetList);
                    bleManager.sendDatasetList(datasetList);
                }
            }
            break;

        case BleCommand::GET_BATTERY:
        {
            Serial.println("Received GET_BATTERY command");
            // Read battery level
            int battery_level = static_cast<int>(2 * analogRead(BATTERY_ADC));

            if (bleManager.isConnected())
            {
                bleManager.sendResponse(battery_level);
                Serial.printf("Sent battery level: %u\n", battery_level);
            }
            else
            {
                Serial.println("BLE not connected, can't send battery level");
            }

            stateManager.setState(State::WAITING);
            break;
        }

        case BleCommand::GET_DATASET:
            Serial.println("Received GET_DATASET command");
            break;

        case BleCommand::INVALID_COMMAND:
            Serial.println("Received INVALID_COMMAND, going back");
            stateManager.setState(State::WAITING);
            break;
        }
        bleManager.resetCommandReceived();
    }

    delay(10);
}

SDHandler.cpp

#include <SDHandler.h>
#include <VMD.h>

const char *SDHandler::CONFIG_FILE = "/config.csv";
const char *SDHandler::DATA_DIRECTORY = "/data";
const char *SDHandler::FILE_EXTENSION = ".csv";

SDHandler::SDHandler() : cardPresent(false) {}

bool SDHandler::init()
{
    // Setup SPI pins only once
    SPI.begin(SD_SCLK, SD_MISO, SD_MOSI, SD_CS);

    // Add a power-up delay
    delay(1000);

    // Initialize SD card with CS pin
    if (!SD.begin(SD_CS))
    {
        Serial.println("SD card initialization failed");
        return false;
    }

    // Check card type
    uint8_t cardType = SD.cardType();
    if (cardType == CARD_NONE)
    {
        Serial.println("No SD card attached");
        return false;
    }

    // Create data directory if it doesn't exist
    if (!checkCreatePath(DATA_DIRECTORY))
    {
        Serial.println("Failed to create data directory");
        return false;
    }

    cardPresent = true;
    return true;
}

bool SDHandler::startNewLog()
{
    if (!SD.exists(DATA_DIRECTORY))
    {
        SD.mkdir(DATA_DIRECTORY);
    }

    // Count the number of existing log files
    File root = SD.open(DATA_DIRECTORY);
    if (!root || !root.isDirectory())
        return false;

    int fileCount = 0;
    File file;
    while (file = root.openNextFile())
    {
        if (!file.isDirectory() && String(file.name()).endsWith(FILE_EXTENSION))
        {
            fileCount++;
        }
        file.close();
    }
    root.close();

    // If the limit is reached, delete the file named "MeasurementLog1.csv"
    if (fileCount >= 100)
    {
        String oldestFileName = String(DATA_DIRECTORY) + "/MeasurementLog1.csv";
        SD.remove(oldestFileName);

        // Rename remaining files to maintain the sequence
        for (int i = 2; i <= fileCount; i++)
        {
            String oldName = String(DATA_DIRECTORY) + "/MeasurementLog" + String(i) + ".csv";
            String newName = String(DATA_DIRECTORY) + "/MeasurementLog" + String(i - 1) + ".csv";
            SD.rename(oldName, newName);
        }
    }

    // Create a new log file
    String filename = createFileName(generateTimestamp());
    currentLogFile = SD.open(filename.c_str(), FILE_WRITE);
    if (!currentLogFile)
        return false;

    return writeHeader(currentLogFile);
}

bool SDHandler::logData(float *data, int dataSize)
{
    if (!currentLogFile)
        return false;

    // Write timestamp
    currentLogFile.print(generateTimestamp());

    // Write data values
    for (int i = 0; i < dataSize; i++)
    {
        currentLogFile.print(",");
        currentLogFile.print(data[i]);
    }
    currentLogFile.println();

    // Ensure data is written
    currentLogFile.flush();
    return true;
}

bool SDHandler::getDatasetList(char *buffer, size_t bufferSize)
{
    File root = SD.open(DATA_DIRECTORY);
    if (!root || !root.isDirectory())
        return false;

    String list = "";
    File file;
    while (file = root.openNextFile())
    {
        if (!file.isDirectory() && String(file.name()).endsWith(FILE_EXTENSION))
        {
            // Extract timestamp from filename
            String filename = file.name();
            String timestamp = filename.substring(0, filename.lastIndexOf('.'));
            list += timestamp + ";";
        }
        file.close();
    }
    root.close();

    if (list.length() >= bufferSize)
        return false;
    strcpy(buffer, list.c_str());
    return true;
}

bool SDHandler::readDataset(const char *timestamp, char *buffer, size_t *dataSize)
{
    String filename = createFileName(timestamp);
    File file = SD.open(filename.c_str(), FILE_READ);
    if (!file)
        return false;

    *dataSize = file.size();
    if (*dataSize > 0)
    {
        file.read((uint8_t *)buffer, *dataSize);
    }
    file.close();
    return true;
}

bool SDHandler::getDatasetInfo(const char *timestamp, uint32_t *size, uint32_t *records)
{
    String filename = createFileName(timestamp);
    File file = SD.open(filename.c_str(), FILE_READ);
    if (!file)
        return false;

    *size = file.size();

    // Count number of lines (records)
    *records = 0;
    while (file.available())
    {
        if (file.read() == '\n')
            (*records)++;
    }

    file.close();
    return true;
}

String SDHandler::generateTimestamp()
{
    // Scan data directory for existing logs
    File root = SD.open(DATA_DIRECTORY);
    if (!root || !root.isDirectory())
    {
        return "MeasurementLog1";
    }

    int maxIndex = 0;
    File file;

    // Look through all files
    while (file = root.openNextFile())
    {
        if (!file.isDirectory())
        {
            String filename = file.name();

            // Check if filename matches our pattern
            if (filename.startsWith("MeasurementLog"))
            {
                // Extract the number after "MeasurementLog"
                String numberPart = filename.substring(14); // "MeasurementLog" is 14 characters
                // Remove the .csv extension if present
                numberPart = numberPart.substring(0, numberPart.indexOf('.'));

                // Convert to integer and update maxIndex if larger
                int currentIndex = numberPart.toInt();
                if (currentIndex > maxIndex)
                {
                    maxIndex = currentIndex;
                }
            }
        }
        file.close();
    }
    root.close();

    // Return next number in sequence
    return "MeasurementLog" + String(maxIndex + 1);
}

String SDHandler::createFileName(const String &timestamp)
{
    return String(DATA_DIRECTORY) + "/" + timestamp + FILE_EXTENSION;
}

bool SDHandler::writeHeader(File &file)
{
    return file.println("timestamp,value1,value2,value3"); // Adjust headers as needed
}

bool SDHandler::checkCreatePath(const char *path)
{
    if (!SD.exists(path))
    {
        if (!SD.mkdir(path))
        {
            Serial.printf("Failed to create directory: %s\n", path);
            return false;
        }
    }
    return true;
}

bool SDHandler::closeCurrentLog()
{
    if (currentLogFile)
    {
        currentLogFile.close();
        return true;
    }
    return false;
}

bool SDHandler::isCardPresent() const
{
    return cardPresent;
}

BleHandler.cpp

#include "BleHandler.h"

// Define UUIDs
const char *BluetoothManager::SERVICE_UUID = "4fafc201-1fb5-459e-8fcc-c5c9c331914b";
const char *BluetoothManager::COMMAND_CHAR_UUID = "beb5483e-36e1-4688-b7f5-ea07361b26a8";
const char *BluetoothManager::RESPONSE_CHAR_UUID = "beb5483f-36e1-4688-b7f5-ea07361b26a8";
const char *BluetoothManager::DATA_CHAR_UUID = "beb54840-36e1-4688-b7f5-ea07361b26a8";

BluetoothManager::BluetoothManager() : deviceConnected(false),
                                       hasErrorFlag(false),
                                       commandReceived(false),
                                       onCommandReceived(nullptr),
                                       onDatasetRequested(nullptr)
{
    memset(lastError, 0, sizeof(lastError));
}

void BluetoothManager::init(const char *deviceName)
{
    // Initialize BLE device
    BLEDevice::init(deviceName);

    // Create server
    pServer = BLEDevice::createServer();
    pServer->setCallbacks(this);

    // Create service
    pService = pServer->createService(SERVICE_UUID);

    // Create characteristics
    pCommandCharacteristic = pService->createCharacteristic(
        COMMAND_CHAR_UUID,
        BLECharacteristic::PROPERTY_WRITE);
    pCommandCharacteristic->setCallbacks(this);

    pResponseCharacteristic = pService->createCharacteristic(
        RESPONSE_CHAR_UUID,
        BLECharacteristic::PROPERTY_NOTIFY | BLECharacteristic::PROPERTY_READ);
    pResponseCharacteristic->addDescriptor(new BLE2902());

    pDataCharacteristic = pService->createCharacteristic(
        DATA_CHAR_UUID,
        BLECharacteristic::PROPERTY_NOTIFY | BLECharacteristic::PROPERTY_READ);
    pDataCharacteristic->addDescriptor(new BLE2902());

    // Start service and advertising
    pService->start();
    startAdvertising();
}

void BluetoothManager::startAdvertising()
{
    BLEAdvertising *pAdvertising = BLEDevice::getAdvertising();
    pAdvertising->addServiceUUID(SERVICE_UUID);
    pAdvertising->setScanResponse(true);
    pAdvertising->setMinPreferred(0x06); // functions that help with iPhone connections issue
    pAdvertising->setMinPreferred(0x12);
    BLEDevice::startAdvertising();
}

void BluetoothManager::stopAdvertising()
{
    BLEDevice::stopAdvertising();
}

void BluetoothManager::onConnect(BLEServer *pServer)
{
    deviceConnected = true;
    clearError();
}

void BluetoothManager::onDisconnect(BLEServer *pServer)
{
    deviceConnected = false;
    // Start advertising again
    startAdvertising();
}

void BluetoothManager::onWrite(BLECharacteristic *pCharacteristic)
{
    if (pCharacteristic == pCommandCharacteristic)
    {
        std::string value = pCharacteristic->getValue();
        if (value.length() > 0)
        {
            processCommand(value.c_str());
        }
    }
}

void BluetoothManager::processCommand(const char *command)
{
    if (!command)
        return;

    BleCommand cmd = parseCommand(command);
    commandReceived = true;

    Serial.print("In processCommand: ");
    Serial.println(int(cmd));
    // Use static buffer instead of stack
    static char errorMsg[32];

    if (onCommandReceived)
        onCommandReceived(cmd);

    switch (cmd)
    {
    case BleCommand::INVALID_COMMAND:
        snprintf(errorMsg, sizeof(errorMsg), "ERROR: Invalid command");
        sendResponse(errorMsg);
        break;

    case BleCommand::GET_DATASET:
        if (onDatasetRequested)
        {
            const char *timestamp = strchr(command, ':');
            if (timestamp)
                onDatasetRequested(timestamp + 1);
        }
        break;

    default:
        break;
    }
}

BleCommand BluetoothManager::parseCommand(const char *cmd)
{
    Serial.print("In parseCommand: ");
    Serial.println(cmd);
    if (!cmd || strlen(cmd) == 0)
        return BleCommand::INVALID_COMMAND;

    else if (strcmp(cmd, "START") == 0)
        return BleCommand::START_MEASURE;
    else if (strcmp(cmd, "STOP") == 0)
        return BleCommand::STOP_MEASURE;
    else if (strcmp(cmd, "BATTERY") == 0)
        return BleCommand::GET_BATTERY;
    else if (strcmp(cmd, "LIST") == 0)
        return BleCommand::GET_DATASET_LIST;
    else if (strncmp(cmd, "GET:", 4) == 0 && strlen(cmd) > 4)
        return BleCommand::GET_DATASET;

    else
    {
        Serial.print("In parseCommand/else: ");
        Serial.println("Invalid");
        return BleCommand::INVALID_COMMAND;
    }
}

void BluetoothManager::sendResponse(const char *data)
{
    if (!deviceConnected || strlen(data) == 0)
    {
        return;
    }

    std::string convertedString(data);

    pResponseCharacteristic->setValue(convertedString);
    pResponseCharacteristic->notify();
    delay(RESPONSE_DELAY);
}

void BluetoothManager::sendResponse(u16_t value)
{
    if (!deviceConnected)
    {
        return;
    }

    pResponseCharacteristic->setValue(value); // Send the integer
    pResponseCharacteristic->notify();
    delay(RESPONSE_DELAY);
}

void BluetoothManager::sendDatasetList(const char *list)
{
    if (deviceConnected)
    {
        pResponseCharacteristic->setValue((uint8_t *)list, strlen(list));
        pResponseCharacteristic->notify();
        delay(RESPONSE_DELAY);
    }
}

void BluetoothManager::sendMeasurementData(const char *data, size_t length)
{
    if (deviceConnected)
    {
        // Send data in chunks if necessary (MTU size consideration)
        const size_t maxChunkSize = 512;
        size_t sent = 0;

        while (sent < length)
        {
            size_t chunkSize = min(maxChunkSize, length - sent);
            pDataCharacteristic->setValue((uint8_t *)(data + sent), chunkSize);
            pDataCharacteristic->notify();
            sent += chunkSize;
            delay(RESPONSE_DELAY);
        }
    }
}

void BluetoothManager::setCommandCallback(CommandCallback callback)
{
    onCommandReceived = callback;
}

void BluetoothManager::setDatasetRequestCallback(DatasetRequestCallback callback)
{
    onDatasetRequested = callback;
}

bool BluetoothManager::isConnected() const
{
    return deviceConnected;
}

bool BluetoothManager::hasError() const
{
    return hasErrorFlag;
}

const char *BluetoothManager::getLastError() const
{
    return lastError;
}

void BluetoothManager::setError(const char *error)
{
    hasErrorFlag = true;
    strncpy(lastError, error, sizeof(lastError) - 1);
    lastError[sizeof(lastError) - 1] = '\0';
}

void BluetoothManager::clearError()
{
    hasErrorFlag = false;
    lastError[0] = '\0';
}

bool BluetoothManager::isCommandReceived()
{
    return commandReceived;
}

void BluetoothManager::resetCommandReceived()
{
    if (commandReceived)
    {
        commandReceived = false;
    }
}

My problem arises when:
Connect to device -> Send "LIST" (works) -> Send "BATTERY" (works) -> Send "LIST"

I get this on the serial port:

Initializing...
SD card initialized!
State change: INIT -> ADVERTISING
Device connected!
State change: ADVERTISING -> WAITING

Seems good, device has connected
Then:

In parseCommand: LIST
In processCommand: 3
3
Received GET_DATASET_LIST command
Dataset list is empty

This seems to work, the first 4 outputs are for debugging. I also get "empty" on my phone.
Then:

In parseCommand: BATTERY
In processCommand: 2
2
Received GET_BATTERY command
Sent battery level: 7846

Again, this Command works

In parseCommand: LIST
In processCommand: 3
3
Received GET_DATASET_LIST command
[ 20765][E][sd_diskio.cpp:126] sdSelectCard(): Select Failed
[ 20770][E][sd_diskio.cpp:624] ff_sd_status(): Check status failed
[ 21276][E][sd_diskio.cpp:126] sdSelectCard(): Select Failed
[ 21781][E][sd_diskio.cpp:126] sdSelectCard(): Select Failed
[ 21786][E][vfs_api.cpp:105] open(): /sd/data does not exist, no permits for creation

This is the problem.
Somehow, the SD Card wont work correctly. More Context:
When I reset/start the ESP, the SD Card wont mount sometimes. I need to press Reset a few times so that it works. Could this be the problem?

Since i am working on the breadboard, the problem could also lie at the SPI bus, which could have problems because of the cables (jumpers) and the module.

I tried the SD test example and it works, but only sometimes, like i explained. I need to press reset sometimes until it works.

2

This where you must start.
Why build a complex tower on quicksand ?

2 Likes
3

I have found the problem:

int battery_level = static_cast<int>(2 * analogRead(BATTERY_ADC));

The ADC had some effect on the SPI communication which maybe lead to an incorrect state of the bus.

Therefore i added some lines to reinitialize the SPI bus after reading the Pin:

            // Re-initialize SPI bus after ADC operation
            SPI.end();
            delay(10);
            SPI.begin(SD_SCLK, SD_MISO, SD_MOSI, SD_CS);
            delay(10);