Did some small modification and a bit of clean up.
Since i am also including a 32 bit timecode as part of the frame, it seemed wasteful to send that as a separate part of the Queue, so instead i made the SdBlock static to the data transmission function.
And it won't be sent until it's full. I send the remaining when the function is called with a len == 0, Other solutions for that exist.
In effect this just becomes an extra part of the Queue, which is somehow what i originally was looking for. So i'm happy.
With the Queue at 32 blocks, there seems to be no moment when the writing slows down the processor due to the actual writing to the SD. Basically it should now be as fast as memcpy-twice (which is for sure quicker than the SD-card) so it should free up quite a bit of processing power.
Don't know what it means for the WiFi if it is also sharing that core, but testing may show up something or not.
The advantage of passing the name once and not with every block is in some way space.
This is tailored for a specific program, a slightly different method sending a name and Append or write
might be more appropriate.
Not closing the file while there is blocks in the buffer is an efficiency thing, and closing it when there is time, also makes sense.
Priorities might be something to fiddle with a bit as well.
I am considering to do the same thing for reading streams as well. Reading from the SD is also seems to be a bit of a bottleneck, though less than writing since no extra processing is required (no data creation or reception). The Queue can be used in either direction.
The error Queue is a must have. You need something to communicate between the cores. The main thread is in charge of what that other thread is supposed to e doing, and status reports are required or all hell might break lose.
#include "FS.h"
#include "SD.h"
#define HSPI_SCK 17
#define HSPI_MISO 16
#define HSPI_MOSI 4
#define SD_CS 15 // Added 10K pullup for v2 (strapping pin must be HIGH)
#define LED_WHITE 14
#define SPI_FRQ 80000000
#define SERIAL_OUTPUT_BAUD 500000
#define PIXEL_BYTES (680 * 3)
#define PORTS 4
#define FRAMES (40 * 60 * 1) // 40 Hz, 60 Seconds, 1 Minute
#define SD_TASK_PRIORITY 2 //
#define FILE_NAME "TestxTaskSd_7"
#define FILENAME_LENGTH 30
#define BLOCK_SIZE 512
#define SD_QUEUE_SIZE 32 // maybe only 8
#define SD_ERR_QUEUE_SIZE 16
#define NO_ERROR 0
#define UNEXP_SDQ 1
#define UNEXP_SDQ_EMPTY 2
#define UNEXP_FILENAME_EMPTY 3
#define UNEXP_FILENAME_NULL 4
#define FILE_NOT_CREATED 5
#define NAME_RECEIVED 6
#define EMPTY_QUEUE 7
#define WRITE_ERROR 8
#define HEADER_WRITTEN 9
#define WRITE_STOP 10
#define NAME_NOT_EXP 11
#define Q_NOT_EMPTY 12
#define STACK_SIZE 13
#define WAIT_FOR_NAME 0
#define WAIT_FOR_DATA 1
typedef struct {
uint16_t len = 0;
uint8_t block[BLOCK_SIZE];
} SdBlock;
QueueHandle_t SdQ;
QueueHandle_t SdFileName;
QueueHandle_t SdError;
SPIClass * hspi = NULL;
uint8_t stripixels[PORTS][PIXEL_BYTES];
uint16_t lengths[PORTS] = {PIXEL_BYTES, PIXEL_BYTES, PIXEL_BYTES, PIXEL_BYTES};
uint16_t totalLengths = 0;
uint32_t expSize;
void setup() {
Serial.begin(500000);
delay(1000);
Serial.println();
Serial.println();
Serial.println("SD Task on core 0 test");
pinMode(LED_WHITE, OUTPUT);
QueueCreation();
if (InitSdCard()) {
Serial.println(F("SD card initialized"));
}
TaskCreation();
uint8_t b = 0;
for (uint8_t i = 0; i < PORTS; i++) {
for (uint16_t j = 0; j < PIXEL_BYTES; j++) {
stripixels[i][j] = b;
b++;
}
totalLengths += lengths[i];
}
Serial.println(F("Arrays Filled."));
Serial.print(F("Total Lengths : "));
Serial.println(totalLengths, DEC);
expSize = FRAMES * (totalLengths + 4);
delay(1000);
if (CreateSdFileToTask(FILE_NAME)) {
Serial.println(F("Created SD Ok."));
uint32_t frames = FRAMES;
uint32_t lastFrame = millis();
uint32_t firstFrame = lastFrame;
int pinstate = HIGH;
while (frames) {
digitalWrite(LED_WHITE, pinstate);
pinstate = HIGH - pinstate;
frames--;
while (millis() - lastFrame < 25); // blocking time limiter
//lastFrame += 25; // i use these to see if a frame might be delayed
lastFrame = millis();
uint32_t moment = lastFrame - firstFrame; // calculate the elapsed time from 1st frame
uint8_t tc[4]; // timecode format i have been using, Big-endian, regardless of what
uint8_t nr_bytes = 4; // the processor uses.
while (nr_bytes) {
nr_bytes--;
tc[3 - nr_bytes] = (moment >> (8 * nr_bytes)) & 0xFF; //edit not 4 - nr_bytes, but 3
}
if (!SendDataBlockToSdQueue(tc, 4)) {
frames = 0;
}
for (uint8_t i = 0; i < PORTS; i++) {
if (!SendDataBlockToSdQueue(stripixels[i], lengths[i])) {
frames = 0;
}
}
} // frames
if (StopSdFileToTask()) {
uint32_t totalTime = millis() - firstFrame;
Serial.println(F("Filename Cleared, Writing Stopped"));
Serial.print(F("Total elapsed time = "));
Serial.println(totalTime, DEC);
}
}// file Created
PrintFileSize(FILE_NAME);
}
void TaskCreation() {
xTaskCreatePinnedToCore(
HandleQueueToSD,
"Task Write To SD", // A name just for humans
3000, // The stack size can be checked by calling `uxHighWaterMark = uxTaskGetStackHighWaterMark(NULL);`
NULL, // No parameter is used
SD_TASK_PRIORITY, // Priority, with 3 (configMAX_PRIORITIES - 1) being the highest, and 0 being the lowest.
NULL, // Task handle is not used here
0 // pinned to core
);
}
void QueueCreation() {
SdQ = xQueueCreate(SD_QUEUE_SIZE, sizeof(SdBlock));
SdFileName = xQueueCreate(1, FILENAME_LENGTH + 1); // filename Queue should be only 1 so the main thread can be blocked
uint8_t err = 0;
SdError = xQueueCreate(SD_ERR_QUEUE_SIZE, sizeof(err)); // error codes are only 1 byte
if (SdQ == NULL) {
Serial.println(F("Error creating the SD queue"));
}
if (SdFileName == NULL) {
Serial.println(F("Error creating the finename queue"));
}
if (SdError == NULL) {
Serial.println(F("Error creating the SD Error queue"));
}
}
void loop() {}
//----------------------------------------------------------------------------------
void PrintFileSize(String filename) {
filename = "/" + filename;
filename += ".pst";
File sFile = SD.open(filename, FILE_READ);
uint32_t filesize = sFile.size();
sFile.close();
Serial.print(F("File Size = "));
Serial.println(filesize, DEC);
Serial.print(F("Expected Size = "));
Serial.println(expSize, DEC);
}
void PrintAllErrorQueue() {
int ret = pdPASS;
uint8_t err = NO_ERROR;
while (uxQueueMessagesWaiting(SdError)) {
ret = xQueueReceive(SdError, &err, portMAX_DELAY);
Serial.print(F("Error : "));
Serial.println(err, DEC);
}
}
bool StopSdFileToTask() {
if (SendDataBlockToSdQueue(stripixels[0], 0)) {
delay(1); // send the final block and wait a ms.
Serial.println(F("Last Block sent !"));
}
else {
Serial.println(F("Last Block not sent !"));
}
PrintAllErrorQueue(); // flush the error Queue
char nameString[FILENAME_LENGTH + 1] = {'\0'};
xQueueSend(SdFileName, &nameString, portMAX_DELAY); // send the noname
while (!uxQueueMessagesWaiting(SdError)) { // wait for response
delayMicroseconds(10); // stability
}
uint8_t err;
int ret = xQueueReceive(SdError, &err, portMAX_DELAY);
if (err != WRITE_STOP) {
Serial.println(F("'Write Stop' not received"));
Serial.print(F("Error : "));
Serial.println(err, DEC);
PrintAllErrorQueue();
return false;
}
return true;
}
bool CreateSdFileToTask(String filename) {
PrintAllErrorQueue(); // flush the error Queue
char nameString[FILENAME_LENGTH + 1] = {'\0'};
filename = "/" + filename;
filename += ".pst"; // adding the extension here is better
uint8_t len = filename.length();
if (len > FILENAME_LENGTH) len = FILENAME_LENGTH;
filename.toCharArray(nameString, len + 1);
xQueueSend(SdFileName, &nameString, portMAX_DELAY); // send the name
while (!uxQueueMessagesWaiting(SdError)) { // wait for response
delay(10); // stability
}
uint8_t err;
int ret = xQueueReceive(SdError, &err, portMAX_DELAY);
if (err != NAME_RECEIVED) {
Serial.println(F("Name not received"));
Serial.print(F("Error : "));
Serial.println(err, DEC);
PrintAllErrorQueue();
return false;
}
char header[] = "DmagiX PixelStream vx.xx\rColorSequence=X\rFrameSize=0000\rDmxSize=000\rID=000000#";
SdBlock qBlock;
memcpy (qBlock.block, header, sizeof(header) - 1);
qBlock.len = sizeof(header) - 1;
expSize += sizeof(header) - 1;
xQueueSend(SdQ, &qBlock, portMAX_DELAY); // send the header to the SD Queue
ret = xQueueReceive(SdError, &err, portMAX_DELAY);
if (err != HEADER_WRITTEN) {
Serial.println(F("Header not written"));
Serial.print(F("Error : "));
Serial.println(err, DEC);
PrintAllErrorQueue();
return false;
}
ret = pdPASS;
err = NO_ERROR;
while (uxQueueMessagesWaiting(SdError) && (ret == pdPASS)) {
ret = xQueueReceive(SdError, &err, portMAX_DELAY);
Serial.print(F("Error : "));
Serial.println(err, DEC);
}
if (err != NO_ERROR) return false;
delay(10);
return true; // success
}
bool SendDataBlockToSdQueue(uint8_t * block, uint16_t len) {
const uint16_t blockMax = 512;
static SdBlock qBlock;
uint16_t ndx = 0;
if (len) {
while ((len) && (!uxQueueMessagesWaiting(SdError))) { // break also on error message waiting
uint16_t blockSize = blockMax - qBlock.len; // space available in the block
if (blockSize > len) { // if we have more available in the qBlock than what we have to copy.
blockSize = len; // don't copy what we don't have
}
memcpy (qBlock.block + qBlock.len, block + ndx, blockSize); // copy what needs to be copied
qBlock.len += blockSize;
len -= blockSize;
ndx += blockSize;
if (qBlock.len >= blockMax) { // whole block
xQueueSend(SdQ, &qBlock, portMAX_DELAY); // send the block to the SD Queue
qBlock.len = 0;
}
}
}
else { // no 'len', send what we already have remaining in the qBlock.
Serial.println(F("No 'len'"));
xQueueSend(SdQ, &qBlock, portMAX_DELAY); // send the block to the SD Queue
qBlock.len = 0;
}
int ret = pdPASS;
uint8_t err = NO_ERROR;
while (uxQueueMessagesWaiting(SdError) && (ret == pdPASS)) {
ret = xQueueReceive(SdError, &err, portMAX_DELAY);
Serial.print(F("Error : "));
Serial.println(err, DEC);
}
if (err != NO_ERROR) return false;
return true; // success
}
bool InitSdCard() {
if (hspi == NULL) {
hspi = new SPIClass(HSPI);
}
hspi->begin(HSPI_SCK, HSPI_MISO, HSPI_MOSI, SD_CS);
hspi->setFrequency(SPI_FRQ);
if (!SD.begin(SD_CS, *hspi, SPI_FRQ)) {
Serial.println(F("No SD card found"));
return false;
}
return true;
}
//-----------------------------------------------------------------------------------
void HandleQueueToSD(void *pvParameters) {
SdBlock taskBlock;
char nameString[FILENAME_LENGTH + 1] = {'\0'};
uint8_t taskState = WAIT_FOR_NAME;
File sFile;
for (;;) {
vTaskDelay(1 / portTICK_PERIOD_MS); // when running a higher priority
if ((SdQ != NULL) && (SdFileName != NULL) && (SdError != NULL)) { // Sanity check
// the Queues need to exist or we won't do anything
switch (taskState) {
case WAIT_FOR_NAME: {
if (nameString[0] == '\0') { // only if we have no name
if (uxQueueMessagesWaiting(SdQ)) { // shouldn't happen, but we need to read them regardless
int ret = xQueueReceive(SdQ, &taskBlock, (TickType_t) 0); // it should be there, if it isn't we can continue
if (ret == pdPASS) { // and send error messages back
uint8_t err = UNEXP_SDQ;
xQueueSend(SdError, &err, (TickType_t) 0); // don't wait
}
else if (ret == pdFALSE) {
uint8_t err = UNEXP_SDQ_EMPTY;
xQueueSend(SdError, &err, (TickType_t) 0); // don't wait
}
}
if (uxQueueMessagesWaiting(SdFileName)) {
int ret = xQueueReceive(SdFileName, &nameString, portMAX_DELAY); // read the filename
if (ret == pdPASS) {
if (nameString[0] == '\0') {
uint8_t err = UNEXP_FILENAME_NULL;
xQueueSend(SdError, &err, (TickType_t) 0); // should not be empty
}
else {
sFile = SD.open(nameString, FILE_WRITE); // open the file
if (!sFile) {
uint8_t err = FILE_NOT_CREATED;
xQueueSend(SdError, &err, (TickType_t) 0); // error
}
else {
uint8_t err = NAME_RECEIVED;
xQueueSend(SdError, &err, (TickType_t) 0); // No error !!
int ret = xQueueReceive(SdQ, &taskBlock, portMAX_DELAY); // the first block should always be sent
if (ret == pdFALSE) {
uint8_t err = EMPTY_QUEUE;
xQueueSend(SdError, &err, (TickType_t) 0);
sFile.close();
nameString[0] = '\0';
}
else {
uint16_t bytesWritten = sFile.write(taskBlock.block, taskBlock.len);
if (bytesWritten != taskBlock.len) { // error
uint8_t err = WRITE_ERROR;
xQueueSend(SdError, &err, (TickType_t) 0);
nameString[0] = '\0';
}
else {
uint8_t err = HEADER_WRITTEN; // no error
xQueueSend(SdError, &err, (TickType_t) 0);
taskState = WAIT_FOR_DATA;
}
sFile.close(); // close file anyway
}
}
}
}
else if (ret == pdFALSE) {
uint8_t err = UNEXP_FILENAME_EMPTY;
xQueueSend(SdError, &err, (TickType_t) 0); // don't wait
}
}
}
}
case WAIT_FOR_DATA: {
if (uxQueueMessagesWaiting(SdQ)) {
sFile = SD.open(nameString, FILE_APPEND); // open the file
int ret = pdPASS;
while (ret == pdPASS) {
ret = xQueueReceive(SdQ, &taskBlock, (TickType_t) 0); //portMAX_DELAY);
if (ret == pdPASS) {
uint16_t bytesWritten = sFile.write(taskBlock.block, taskBlock.len);
if (bytesWritten != taskBlock.len) { // error
uint8_t err = WRITE_ERROR;
xQueueSend(SdError, &err, (TickType_t) 0); // send the error
// send specifics, the error was showing up until the data was processed
// Still a mystery.
err = bytesWritten >> 8;
xQueueSend(SdError, &err, (TickType_t) 0);
err = bytesWritten & 0xFF;
xQueueSend(SdError, &err, (TickType_t) 0);
err = taskBlock.len >> 8;
xQueueSend(SdError, &err, (TickType_t) 0);
err = taskBlock.len & 0xFF;
xQueueSend(SdError, &err, (TickType_t) 0);
nameString[0] = '\0'; // clear the name
taskState = WAIT_FOR_NAME; // back to the name task
ret = pdFALSE; // Stop write loop
int fl = pdPASS;
while (fl == pdPASS) { // flush the Queue
fl = xQueueReceive(SdQ, &taskBlock, (TickType_t) 0);
}
}
}
} // read Queue loop
sFile.close();
} // if SdQ available
if (uxQueueMessagesWaiting(SdFileName)) { // check if there is a name available
int ret = xQueueReceive(SdFileName, &nameString, portMAX_DELAY); // only then read the filename
if (ret == pdPASS) {
if (nameString[0] == '\0') {
uint8_t err = WRITE_STOP;
xQueueSend(SdError, &err, (TickType_t) 0); // should be empty
}
else { // we were in a stream, so the name shoyuld have been empty
uint8_t err = NAME_NOT_EXP;
xQueueSend(SdError, &err, (TickType_t) 0); // not empty
nameString[0] = '\0';
}
taskState = WAIT_FOR_NAME; // task switches back either way
int q = pdPASS;
bool notEmpty = false;
while (q == pdPASS) { // flush the Queue just in case (should be empty)
q = xQueueReceive(SdQ, &taskBlock, (TickType_t) 0);
if (q == pdPASS) {
notEmpty = true;
}
}
if (notEmpty) { // the queue really should be empty
uint8_t err = Q_NOT_EMPTY;
xQueueSend(SdError, &err, (TickType_t) 0); // should be empty
}
}
}
} // wait for data
}
} // Sanity check
} // Infinite loop
}
Who knows who else might want this, but i do !
-- verbatim, I just looked it up). How much slower is it if two tasks contend to read the same memory?