Unreliable file writing on connected flash drive

For about 3 weeks I've been fighting with my Giga trying to get it to write files to an attached flash drive on the USB1 port. It will either hard crash or return a -22 error code and then it refuses to write files anymore. The confusing thing is that it works fine for a while but then stops working after a short pause. It logs it's own attempted server touches (HEAD requests just to keep the connection alive and available). After the first log, subsequent logging fails.

Here's my H and CPP files: Logs.h & Logs.cpp

Logs.h:

#pragma once
//------------------------------------------------------------------------------------------
#include "Arduino.h"
#include "WString.h"
#define __FILENAME__ (__builtin_strrchr(__FILE__, '\\') ? __builtin_strrchr(__FILE__, '\\') + 1 : __FILE__)


using namespace arduino;
//------------------------------------------------------------------------------------------
void DataLogStart(void);
void DataLog(const String &Text);
void DataLogEnd(void);
void QuickFormatUSB(void);
String uriEncode(const String &Text);
//------------------------------------------------------------------------------------------

Logs.cpp:

#include "Logs.h"
#include "ValidateRoutines.h"
#include <DigitalOut.h>
#include <FATFileSystem.h>
#include <Arduino_USBHostMbed5.h>
#include "lvgl.h"
#include "Times.h"
#include <WiFi.h>
#include <WiFiSSLClient.h>
#include "BUSYBomb.h"
//------------------------------------------------------------------------------------------
using namespace mbed;
USBHostMSD msd;
FATFileSystem usb("usb");
volatile bool usb_mounted=false;

void driveAttached();
void driveDettached();
void ResetUSB(void);

unsigned long timeoutStart;
unsigned long timeoutEnd;
signed long timeout;
#define timeoutLoop(timeout_test,timeout_length,timeout_failure_action) \
timeoutStart=millis(); \
timeoutEnd=timeoutStart+timeout_length; \
timeout=timeoutEnd-timeoutStart; \
while(!(timeout_test) && timeout>0) \
{ \
  timeoutStart=millis(); \
  timeout=timeoutEnd-timeoutStart; \
} \
if(timeout<=0){timeout_failure_action};


#define db Serial.print(String(__FILENAME__)+" ");Serial.println(__LINE__);
#define dbt(STRING) Serial.println(STRING);
//------------------------------------------------------------------------------------------
void DataLogStart(void)
{
  BUSYBomb

  db
  if(usb_mounted)
  {
    db
    ResetUSB();
    return;
  }
  db
  usb_mounted=false;
  db
  digitalWrite(PA_15, LOW);
  db
  delay(200);
  db
  digitalWrite(PA_15, HIGH);
  db
  /*if(!msd.attach_detected_callback(driveAttached))
  {
    dbt("DataLogStart: USBHostMSD::attach_detected_callback(...) call failed");
  }
  if(!msd.attach_removed_callback(driveDettached))
  {
    dbt("DataLogStart: USBHostMSD::attach_removed_callback(...) call failed");
  }
  */

  db
  timeoutLoop(msd.connect(),1000,dbt("Timeout during \"msd::connect\" loop.");return;)
  db
  timeoutLoop(msd.connected(),1000,dbt("\"msd::connected(...)\" timeout.");)

  // Attempt to mount the USB file system
  db
  if(usb.mount(&msd) != 0)
  {
    dbt("\"usb::mount(...)\" failed.");
    return;
  }
  db
  usb_mounted = true;
}
//------------------------------------------------------------------------------------------
void DataLog(const String &Text)
{
  BUSYBomb
  dbt("DataLog function entered.");
  dbt("String to log: "+Text);
  if (!usb_mounted)
  {
    db
    DataLogStart();
    db
    if (!usb_mounted)
    {
      dbt("DatalogStart(...) failed during DataLog call.");
      return;
    }
  }

  String CurTime = getLocaltime();
  String Line;
  Line.reserve(CurTime.length() + Text.length() + 3); // Preallocate memory
  Line += CurTime;
  Line += "\t";
  Line += Text;
  Line += "\n";

  db
  if (!msd.connected())
  {
      dbt("USB (usb_mounted) flagged as mounted but msd::connected(...) returned false.");
      return;
  }

  db
  FILE *f = fopen("/usb/agsmac.log", "a+");
  if (f == NULL)
  {
    dbt("\"fopen\" failed.");
    return;
  }
  db
  char *line = const_cast<char*>(Line.c_str());
  size_t bytesToWrite=Line.length();
  size_t bytesWritten=0;
  bool writeProgress=true;
  size_t writeLoopCount=0;
  do
  {
    writeLoopCount++;
    bytesWritten=fprintf(f, "%s", line);
    if(bytesWritten>0)
    {
      bytesToWrite-=bytesWritten;
      line+=bytesWritten;
      writeProgress=true;
    }
    else if(bytesWritten<0)
    {
      fclose(f);
      dbt("File write operation (fprintf) failed.");
      return;
    }
    else if(bytesWritten==0)
    {
      fflush(f);
      dbt("File write operation 0 bytes written.  (fflush) called.");
      if(!writeProgress)
      {
        dbt("No write progress 2 iterations in a row, haulting write operations.");
        break;
      }
      writeProgress=false;
    }
  }
  while(bytesToWrite>0 && bytesWritten>0 && writeProgress);
  dbt("Write operation loop iteration count: "+String(writeLoopCount));
  //db
  //timeoutLoop(fflush(f)==0,10000,dbt("File flush operation (fflush) timeout.");)
  db
  timeoutLoop(fclose(f)==0,10000,dbt("File close operation (fclose) timeout."););
  dbt("DataLog function leaving.");
}
//------------------------------------------------------------------------------------------    line 167
void DataLogEnd(void)
{
  BUSYBomb

  dbt("DataLogEnd: Preparing drive for safe removal...");

  if (usb_mounted)
  {
    dbt("DataLogEnd: Unmounting the filesystem...");
    if (usb.unmount() != 0)
    {
      dbt("DataLogEnd: Failed to unmount the filesystem.");
    }
    else
    {
      dbt("DataLogEnd: Filesystem unmounted successfully.");
    }
    usb_mounted = false;
  }

  dbt("DataLogEnd: Resetting USB connection...");
  delay(500);
  db
  pinMode(PA_15, OUTPUT);  // Control USB power
  db
  digitalWrite(PA_15, LOW); // Turn off USB power
  db
  delay(10);  // Initial short delay
  db
  if (!msd.connected())
  {
      Serial.println("USB disconnected successfully.");
  }
  else
  {
      Serial.println("USB is still active after power-off.");
  }
  delay(990);  // Remaining delay
  db
  digitalWrite(PA_15, HIGH); // Turn USB power back on
  db
  delay(500);              // Allow time for the device to reinitialize

  dbt("DataLogEnd: Drive is ready for safe removal.");
}
//------------------------------------------------------------------------------------------
void QuickFormatUSB(void)
{
	BUSYBomb

	dbt("QuickFormatUSB: Starting quick format...");

	// Ensure the USB drive is connected
	if (!msd.connected())
	{
		dbt("QuickFormatUSB: USB drive not connected. Attempting reset.");
		ResetUSB();
    db
		if (!msd.connected())
		{
			dbt("QuickFormatUSB: Reset failed. USB drive is still not connected.");
			return;
		}
	}

	// If the filesystem is mounted, unmount it
  db
	if (usb_mounted)
	{
		dbt("QuickFormatUSB: Unmounting the filesystem...");
		if(usb.unmount()!=0)
		{
			dbt("QuickFormatUSB: Failed to unmount the filesystem. Attempting reset.");
			ResetUSB();
      
      db
			if (!msd.connected())
			{
				dbt("QuickFormatUSB: Reset failed. USB drive is not connected.");
				return;
			}
		}
    db
		usb_mounted = false;
	}

	// Perform quick format by initializing the filesystem
	dbt("QuickFormatUSB: Initializing the filesystem...");
  int usbReformat=usb.reformat(&msd);
	if (usbReformat != 0)
	{
		dbt("QuickFormatUSB: Failed to format the USB drive: Error code "+String(usbReformat,HEX)+". Attempting reset.");
		ResetUSB();

		db
    if (!msd.connected())
		{
			dbt("QuickFormatUSB: Reset failed. USB drive is still not connected.");
			return;
		}

		// Retry format after reset
    db
    usbReformat=usb.reformat(&msd);
		if (usbReformat != 0)
		{
			dbt("QuickFormatUSB: Retry failed: Error code "+String(usbReformat,HEX)+" USB drive format unsuccessful.");
			return;
		}
	}

	// Remount the filesystem
	dbt("QuickFormatUSB: Mounting the filesystem...");
  int usbMount=usb.mount(&msd);
	if (usbMount != 0)
	{
		dbt("QuickFormatUSB: Failed to remount the filesystem: Error code "+String(usbMount,HEX)+".");
		return;
	}

	db
  usb_mounted = true;
	dbt("QuickFormatUSB: Format completed successfully.");
}
//------------------------------------------------------------------------------------------
void ResetUSB()
{
  BUSYBomb

  dbt("ResetUSB: Entered");
	// Step 1: Unmount the file system
	if (usb_mounted)
	{
		int result = usb.unmount();
		if (result == 0)
		{
      dbt("\"usb::unmount(...)\" failed.");
		}
		else
		{
      dbt("\"usb::unmount(...)\" succeeded.");
		}
		usb_mounted = false;
	}

  // Step 2: Reset the USB connection
  db
	pinMode(PA_15, OUTPUT);  // Set the USB power control pin
	digitalWrite(PA_15, LOW); // Turn off USB port power
	delay(1000);              // Wait for a second
	digitalWrite(PA_15, HIGH); // Turn USB port power back on

	// Step 3: Reinitialize the USB device
  db
  timeoutLoop(msd.connect(),1000,dbt("Timeout during \"msd::connect(...)\".");)
	// Step 4: Mount the file system
  db
  timeoutLoop(msd.connected(),1000,dbt("Timeout during \"msd::connected(...)\".");)
}
//------------------------------------------------------------------------------------------
void driveAttached()
{
  ResetUSB();
  //DataLogStart();
}
//------------------------------------------------------------------------------------------
void driveDettached()
{
  ResetUSB();
  //DataLogStart();
  usb_mounted=false;
}
//------------------------------------------------------------------------------------------
String uriEncode(const String &Text)
{
  const char *safeChars = "$-_.+!*'(),?/~"; // Allowed characters
  String Res;
  char buffer[4]; // Buffer for percent-encoding
  
  for (size_t i = 0; i < Text.length(); i++)
  {
    char c = Text.charAt(i);
    // Check if the character is alphanumeric or a safe character
    if (isalnum(c) || strchr(safeChars, c))
    {
      Res += c; // Append safe character as-is
    }
    else
    {
      // Percent-encode the character
      snprintf(buffer, sizeof(buffer), "%%%02X", (unsigned char)c);
      Res += buffer;
    }
  }
  return Res;
}//------------------------------------------------------------------------------------------
void SendLogs(const String &Server)
{
  //usb.rename("/usb/agsmac.log","/usb/agsmac_topush.log");
}
//------------------------------------------------------------------------------------------

I've been "conversing" with ChatGPT 4o trying to troubleshoot this but to no avail. I've ordered a couple of compatible Ethernet Shields as a backup workaround but I'd really rather not to have to rely on the SD Card functionality as from my previous experience with the Uno it's rather slow.

It should be noted that I'm using older versions of the board libraries as using newer version causes renders using the LVGL library to malfunction and is unusable.

I've also tried using version regressing the Arduino_USBHostMbed5, also to no avail.

Here's the current versions of the regressed libraries I'm on:

/*
  use version 4.1.5 of the [Arduino Mbed OS Giga Boards] library.
  use version 8.3.11 of the [lvgl] library.
  use version 0.2.0 of the [Arduino_USBHostMbed5] library.
*/

Hi @alphathinktink

With 0.3.1 of Arduino_USBHostMbed5 I'm having no stability problems with flash on USB1 however that was not always the case. I found the platform very sensitive to the drive being used and had to try a few before finding something that worked reliably.

I'm now using the SanDisk Ultra Fit and everything is stable. However, if speed is important then you may find SD faster. The best I see is 50-60KB/sec write (1KB chunks), 576KB/sec read (64KB chunks).

I also experienced blocking during writes of double-digit ms that I couldn't tolerate so moved the usb functionality to the M4 and use a shared memory area (D3) to pass data. As I'm using the usb as a black box recorder the separation was a good idea in itself.

It is telling that mbed decided not to carry over the USBhostMSD functionality from version 5 to 6. It will be interesting to see what speeds the GIGA can achieve with Zephyr OS.

Just to clarify, not my code? I thought it would be with how cryptic the documentation can be.
...

...

Which size are you using. I want to use the exact same one.
...

...

Was that with the Arduino official compatible Ethernet Shield or something else?
...

...

Never even heard of M4 or D3. Can you enlighten me or point me in the direction of a web page that would help?

I haven't had chance to review your code, hopefully in the next day or two (just sharing my experiences for the moment)

I'm using the 32GB as that's more than sufficient for my needs

Those figures are the best speeds I've seen for the USB. I've not tried SD personally but there are multiple reports of slow USB speeds GIGA USB Read speed is very slow! Something like 4 times slower than SDFat on SD drives. · Issue #59 · arduino-libraries/Arduino_USBHostMbed5 · GitHub

The GIGA is dual-core, the main core (M7) and a co-processor (M4, running at half speed). They can be programmed independently Guide to GIGA R1 Dual Cores | Arduino Documentation
By moving my USB functionality from the main core to the M4 meant any blocking wouldn't impact the running of the main core.

D3 refers to memory domain 3. It is 64KB and is shareable between cores. The methods that can be used for comms between cores is covered here.
https://www.st.com/resource/en/application_note/an5617-stm32h745755-and-stm32h747757-lines-interprocessor-communications-stmicroelectronics.pdf
There is an RPC library (discussed in the dual core guide linked above) however I found it too resource intensive so implemented my own methods. I suggest starting with the RPC lib if you decide to utilise the second core.

1 Like

Thanks, I'll start with this first. I'll report back my findings as soon as I can.

Had a quick scan of your code and can't see any obvious problems but I'll have a closer look soon. I would suggest opening/closing the file once per session rather than once per write. I think it has to seek the file end each time otherwise.

Interesting theory. I'd have to have a way to remove the drive live though so I would need to make some kind of function to close the file before removal. I am worried about the resources always being used while the file is open. Do you foresee an issue with resource management on an always open file?

I don't see (or have, as this is how I do it) a problem. The default is to flush buffs to disk on a new sector (512 bytes typically) so that's the most you'd loose on a hot pull. You'd need to detect removal/insertion and close/open as needed. Resources no problem as it's little more than a pointer.

Damn, leaving the file handle open didn't work. However, the rework and testing revealed I was using the wrong integer type for the return value from fprintf which is a signed int, I was using size_t, which is unsigned and would never test negative (for error) and would cause the board to infinitely loop on an error. The value returned from fprintf on the failure was 0xFFFFFFFF (not a very helpful error code (I think it's invalid parameter or something like that)).

This means I'll wait for those flash drives to come in, unless you have any other ideas in the meantime.

Thanks anyways.

hmm, damn, also thought I'd scored a winner there too! It's a crude test (and an even cruder sketch) but does the FileWrite sketch in the Arduino_USBHostMbed5 examples run successfully?

Those flash drives came in. Testing one now it seems to be working far more reliably. I will leave it run for a while to make sure it's solid.

1 Like

Current File System

  Type                 : FAT32
  Allocation Unit Size : 16K
  Flags : 00000000

This is how the flash drives that came in were formatted. Maybe it's relevant, maybe not.

So far I've not seen any issues. This may have solved it.

I always reformat with those default settings, so sounds promising :crossed_fingers:

"This is the way."

I let it run overnight and then pulled the logs and checked the sequence numbers. Everything is there.

Excellent news. Arduino should include one in the box with the GIGA and I should buy some shares in SanDisk

So, uh the Ultra Fit worked for me also. But thing is after a couple months heat in the Giga's port the plastic flash drive warped and then began shredding. Photo below.
So far, Verbatim's Metal Exec 16GB has been working for me serving up JPEGs to the display and doing some testing with ... datalogging to a flash drive (doh!).

Thanks for that head's up. I don't feel any heat on the port but shall monitor

If the case for the flash drive falls apart but it's all inside an enclosure, what's the issue? Does the flash drive not function anymore outside of it's casing?

It will prolly still work. Just moved on to different disk.
Side note. Have some 10-yo cruzer glides that work really well, but they full size thumbs.
Another option is Verbatim model below, which so far is good.
I have yet to really stress test these disks.