Writing data in ESP32 (High speed)

Hello every one . I want to write ADXl345 accelerometer module data in SD card via the esp32-s3-wroom-1 . I have to choose 1600Hz data rate in my accelerometer and I have to write all data in SD card. There are two problems with this. First of all I can't use these two with each other because when I try this my accelerometer didn't measure. The second problem that I have the speed of writing in SD card in last try I achieve 200 byte per second!
this is my test code to measuring the writing speed in SD card:

#include "FS.h"
#include "SD.h"
#include <SPI.h>
String dataMessage;
int x = 0;
void setup() {
  //Initiate an SPI communication instance.
  SPI.begin();
  initSDCard();
  // If the data.txt file doesn't exist
  // Create a file on the SD card and write the data labels
  File file = SD.open("/data.txt");
  if(!file) {
    writeFile(SD, "/data.txt", " x\r\n");
  }
  else {
  }
  file.close();
}

void loop() {
  x++;
  dataMessage = String(x)+"\r\n";
  //dataMessage = String(epochTime) + "," + String(temp) + "," + String(hum) + "," + String(pres)+ "\r\n";
  
  //Append the data to file
  appendFile(SD, "/data.txt", dataMessage.c_str());
}

void initSDCard(){
   if (!SD.begin()) {
    return;
  }
  uint8_t cardType = SD.cardType();

  if(cardType == CARD_NONE){
    return;
  }
  if(cardType == CARD_MMC){
  } else if(cardType == CARD_SD){
  } else if(cardType == CARD_SDHC){
  } else {
  }
  uint64_t cardSize = SD.cardSize() / (1024 * 1024);
}

// Write to the SD card
void writeFile(fs::FS &fs, const char * path, const char * message) {
  File file = fs.open(path, FILE_WRITE);
  if(!file) {
    return;
  }
  if(file.print(message)) {
  } else {
  }
  file.close();
}

// Append data to the SD card
void appendFile(fs::FS &fs, const char * path, const char * message) {

  File file = fs.open(path, FILE_APPEND);
  if(!file) {
    return;
  }
  if(file.print(message)) {
    //Serial.println("Message appended");
  } else {
    //Serial.println("Append failed");
  }
  file.close();
}``` 
thanks for your attention

In the loop you are calling appendFile(), so opening and closing the file all the time. That's too slow and unnecessary. You have to keep the file open while recording and close at the end.

And would be better to write bytes instead of strings, in case that it is not fast enough.

thank you for your nice advice :pray:

and how about adxl345 dataloggering? because this two modules use spi and because of the data rete I have to use HSPI for both!
and when I try to use them it didn't work
I try to use RTOS but it steal not working properly
here is the code:

#include "FS.h"
#include "SD.h"
#include <SPI.h>
//Assign the Chip Select signal to pin 10.
int CS1=10;
int CS2=9;
int btn=17;
//This is a list of some of the registers available on the ADXL345.
//To learn more about these and the rest of the registers on the ADXL345, read the datasheet!
char POWER_CTL = 0x2D;	//Power Control Register
char DATA_FORMAT = 0x31;
char DATAX0 = 0x32;	//X-Axis Data 0
char DATAX1 = 0x33;	//X-Axis Data 1
char DATAY0 = 0x34;	//Y-Axis Data 0
char DATAY1 = 0x35;	//Y-Axis Data 1
char DATAZ0 = 0x36;	//Z-Axis Data 0
char DATAZ1 = 0x37;	//Z-Axis Data 1
char sample = 0x2C; //Sample rate
//This buffer will hold values read from the ADXL345 registers.
char values[10];
//These variables will be used to hold the x,y and z axis accelerometer values.
int16_t x,y,z;
float x1;
float y2;
float z1;
String dataMessage;
//int x = 32767 ;
unsigned long time1 ;
//unsigned long time2,time3;
void setup() {
  Serial.begin(115200);
  //Initiate an SPI communication instance.
  SPI.begin();
  //Set up the Chip Select pin to be an output from the Arduino.
  pinMode(CS1, OUTPUT);
  pinMode(CS2, OUTPUT);
  pinMode(btn,OUTPUT);
  digitalWrite(btn,LOW);
  //Before communication starts, the Chip Select pin needs to be set high.
  digitalWrite(CS1, HIGH);
  digitalWrite(CS2, HIGH);
  //Put the ADXL345 into +/- 8G range by writing the value 0x01 to the DATA_FORMAT register.
  writeRegister(DATA_FORMAT, 0x02);
  //Put the ADXL345 into Measurement Mode by writing 0x08 to the POWER_CTL register.
  writeRegister(POWER_CTL, 0x08);  //Measurement mode
  writeRegister(sample, 0x0F);
  digitalWrite(CS1, LOW);
  initSDCard();
  // If the data.txt file doesn't exist
  // Create a file on the SD card and write the data labels
  File file = SD.open("/data.txt");
  if(!file) {
    writeFile(SD, "/data.txt", "Time, x, y, z \r\n");
  }
  else {
  }
  file.close();
  digitalWrite(CS1, HIGH);
  time1 = millis();
	xTaskCreate(
        task1,    // Function that should be called
        "Task 1", // Name of the task (for debugging)
        1000,     // Stack size (bytes)
        NULL,     // Parameter to pass
        1,        // Task priority
        NULL      // Task handle
  );

	xTaskCreate(
        task2,    // Function that should be called
        "Task 2", // Name of the task (for debugging)
        1000,     // Stack size (bytes)
        NULL,     // Parameter to pass
        1,        // Task priority
        NULL      // Task handle
  );
}
void loop() {
/*
  digitalWrite(CS1, LOW);
  Serial.println("Start");
  dataMessage = String(time1) + "," + String(x1) + "," + String(y2) + "," + String(z1)+ "\r\n";
  File file = SD.open("/data.txt", FILE_APPEND);
  if(!file) {
    return;
  }
  if (digitalRead(btn)==0){
  file.print(dataMessage);
  }else if(digitalRead(btn)==1){
  file.close();
  Serial.println("end");
  }
  //time2 = millis();
  //time3 = time2-time1;
  //Serial.println(time3);
  digitalWrite(CS1, HIGH);
*/
}

void initSDCard(){
   if (!SD.begin()) {
    return;
  }
  uint8_t cardType = SD.cardType();

  if(cardType == CARD_NONE){
    return;
  }
  if(cardType == CARD_MMC){
  } else if(cardType == CARD_SD){
  } else if(cardType == CARD_SDHC){
  } else {
  }
  uint64_t cardSize = SD.cardSize() / (1024 * 1024);
}

// Write to the SD card
void writeFile(fs::FS &fs, const char * path, const char * message) {
  File file = fs.open(path, FILE_WRITE);
  if(!file) {
    return;
  }
  if(file.print(message)) {
  } else {
  }
  file.close();
}
//This function will write a value to a register on the ADXL345.
//Parameters:
//  char registerAddress - The register to write a value to
//  char value - The value to be written to the specified register.
void writeRegister(char registerAddress, char value){
  //Set Chip Select pin low to signal the beginning of an SPI packet.
  digitalWrite(CS2, LOW);
  //Transfer the register address over SPI.
  SPI.transfer(registerAddress);
  //Transfer the desired register value over SPI.
  SPI.transfer(value);
  //Set the Chip Select pin high to signal the end of an SPI packet.
  digitalWrite(CS2, HIGH);
}

//This function will read a certain number of registers starting from a specified address and store their values in a buffer.
//Parameters:
//  char registerAddress - The register addresse to start the read sequence from.
//  int numBytes - The number of registers that should be read.
//  char * values - A pointer to a buffer where the results of the operation should be stored.
void readRegister(char registerAddress, int numBytes, char * values){
  //Since we're performing a read operation, the most significant bit of the register address should be set.
  char address = 0x80 | registerAddress;
  //If we're doing a multi-byte read, bit 6 needs to be set as well.
  if(numBytes > 1)address = address | 0x40;
  
  //Set the Chip select pin low to start an SPI packet.
  digitalWrite(CS2, LOW);
  //Transfer the starting register address that needs to be read.
  SPI.transfer(address);
  //Continue to read registers until we've read the number specified, storing the results to the input buffer.
  for(int i=0; i<numBytes; i++){
    values[i] = SPI.transfer(0x00);
  }
  //Set the Chips Select pin high to end the SPI packet.
  digitalWrite(CS2, HIGH);
}
void task1(void * parameters){
  //Reading 6 bytes of data starting at register DATAX0 will retrieve the x,y and z acceleration values from the ADXL345.
  //The results of the read operation will get stored to the values[] buffer.
  for(;;){
  readRegister(DATAX0, 6, values);

  //The ADXL345 gives 10-bit acceleration values, but they are stored as bytes (8-bits). To get the full value, two bytes must be combined for each axis.
  //The X value is stored in values[0] and values[1].
  x = ((int)values[1]<<8)|(int)values[0];
  x = (x*100)/64;
  x1 = x;
  x1/=100;
  //The Y value is stored in values[2] and values[3].
  y = ((int)values[3]<<8)|(int)values[2];
  y = (y*100)/64;
  y2 = y;
  y2/=100;
  //The Z value is stored in values[4] and values[5].
  z = ((int)values[5]<<8)|(int)values[4];
  z = (z*100)/64;
  z1 = z;
  z1/=100;
  }
}
void task2(void * parameters){
  for(;;){
  Serial.println("Start");
  dataMessage = String(time1) + "," + String(x1) + "," + String(y2) + "," + String(z1)+ "\r\n";
  File file = SD.open("/data.txt", FILE_APPEND);
  if(!file) {
    return;
  }
  if (digitalRead(btn)==0){
  file.print(dataMessage);
  }else if(digitalRead(btn)==1){
  file.close();
  Serial.println("end");
  }
  //time2 = millis();
  //time3 = time2-time1;
  //Serial.println(time3);
  digitalWrite(CS1, HIGH);
  }
}

If you connect both devices in the same SPI channel, with CS (chip select) you set which one is using the channel, only one at a time.

Are you sure that your tasks running in parallel are not using the channel at the same time?
Maybe would be better to use two separated SPI channels.
Or maybe you could check other alternatives, some ESP32-S3 modules have up to 16MB of PSRAM. Or send the data via WIFI or ESPNow.

fortunately I use esp32-s3-wroom-1 .yes there are two spi in this board , HSPI & VSPI but the VSPI speed is low. I have to collect data for about 5 min then send the text file witch I saved it in SD card . for that I think in 5 min the data size will be greater than 16MB PSRAM memory. do you think that RTOS is a good choice?

But 5 minutes would be: 5 x 60s x 1600Hz = 480K samples.
If each sample is 6 bytes (3 axis of 2 bytes), then: 480K x 6 = 2.88M (2.74 MB) of memory.

You could store the values in a memory array and after the 5 minutes send all to a text file at once.

I don't think you need to use RTOS for this. And your code is incomplete or incorrect. I don't know how you control the frequency, tasks are not sync, time is not updated.
And you still open and close the file all the time in the task. You should open the file only once, in the setup, and close it only at the end, after the 5 mins.

I searched for PSRAM and I saw that there is a limitation for writing in PSRAM (between 10,000 up to 100,000) do you have any idea with out using PSRAM?
and do you have tutorial or link that help me for in this project?

Thank you for your help :pray:

I don't think so. But Ok, I would start with a very simple code, without RTOS, PSRAM or ADXl345, just writing numbers to the SD at 1600Hz.
Once this is working and you are sure that you understand the code, then add the sensor and the rest, step by step.

Something like this (not tested):

// ...
// needed declarations

File file;
uint16_t x,y,z;
uint32_t start_time;
const uint32_t TOTAL_TIME = 5 * 60 * 1000*1000; // 5 minutes, in µ seconds
bool done = false; // will be set to true after 5 minuts. 

void setup() 
{
  // initializations ...
  Serial.begin(115200);
  SPI.begin();
  initSDCard();

  file = fs.open( path, FILE_WRITE); // open the file
  start_time = micros(); // in µ seconds
}

uint32_t current_time=0;  // in µ seconds
uint32_t previous_time=0; // in µ seconds
uint32_t count=0; // samples counter

char dataMessage[50]; // max 50 chars per sample/line

void loop() 
{
  current_time = micros();
  if( !done && current_time - previous_time >= 625 ) { // enter only after 625 µseconds => 1600Hz
    
      x = 12; // sensor values, hardcoded
      y = 34;
      z = 56;

      sprintf( dataMessage, "%i,%i,%i" , x, y, z); // build the char array
      file.println( dataMessage); // write to SD card
      count++; // samples counter

      if( current_time - start_time >= TOTAL_TIME){ // 5 minuts passed
        Serial.print( "Start time: ");    Serial.println( start_time); 
        Serial.print( "End time: ");      Serial.println( current_time); 
        Serial.print( "Total time (µs): ");Serial.println( current_time - start_time); 
        Serial.print( "Total samples: "); Serial.println( count); 
        file.close();
        done = true;
        Serial.println("end");
      }

      previous_time = current_time;
  }
}

This is not tested, probably you need to adjust and complete it.
Be careful with Serial.println inside the loop, it is slow. In each loop execution you have less than 625 micro seconds. If you print many characters it will take longer and reduce the frequency.
With the summary, check if it was able to write the 5 * 60 * 1600 samples in the expected time.
Once this is working add the sensor readings and the rest, step by step.

thank you very much
thank you for your attention

rather than writing text would writing raw binary data to the SD card improve the thruput?