Go Down

Topic: [SD FAT] problem when moving files (Read 2301 times) previous topic - next topic

stanzani

hello
I am using https://github.com/greiman/SdFat and have problem when moving files.
I write files in the SD root, the renaming to the new desination myfile.ext -> destdir/myfile.ext as following

   uint32_t CAutomaSD::SD_move(const char* sourceFileName, const char * dirName)
   {
      uint32_t retVal = ASD_OK;
     char destFileName[32]; //TODO:  "SENT/8.3" string -->(size 17+\n min?)
     snprintf(destFileName, sizeof(destFileName), "%s%s", dirName, sourceFileName);

     if(SD.exists(destFileName))
     {
      retVal |= ASD_FILEPREEXIST;
           SD.remove(destFileName); //remove from destination dir if present

     }
     bool isRenamed = false;
     uint8_t tryCnt = 0;
     while(!isRenamed)
     {
      isRenamed = SD.rename(sourceFileName, destFileName);
      tryCnt++;
      if(tryCnt == 10)
      {
         retVal |= ASD_CANTMOVEFILE;
         break;
      }
     }

   return retVal;
}


as you can see I implemented a retry mechanism just in case. What sometimes happens is that it takes mor than one try for successfully move (rename) a file
In the worst case, and as soon as the number of files in the SD memory gets higher and higher, these epsiodes thends to be higher and higher

My question is about the maturity of this libreary (the rename is a new method compared to the Arduino SD original library) and if renaming files is a safe operation with this library (did anybody tried?)

thanks much in advance for your reply

fat16lib

#1
Mar 16, 2015, 08:38 pm Last Edit: Mar 16, 2015, 08:39 pm by fat16lib
Try running this test program.  It creates files in root and moves them to /DIR.

It runs three passes.  The first pass just moves files from root to /DIR.  The next two passes remove the existing file from /DIR and then moves the new file.  It creates and moves 100 files on each pass.
Code: [Select]

/*
 * This program tests rename().
 */
#include <SPI.h>
#include <SdFat.h>

// SD chip select pin
const uint8_t chipSelect = SS;

// file system
SdFat sd;

// Serial print stream
ArduinoOutStream cout(Serial);
//------------------------------------------------------------------------------
#define error(s) sd.errorHalt(s)
//------------------------------------------------------------------------------
void setup() {
  File file;
  char dname[20], rname[13];
  Serial.begin(9600);
  while (!Serial) {}  // wait for Leonardo

  cout << F("Insert an empty SD.  Type any character to start.") << endl;
  while (Serial.read() <= 0) {}

  // initialize the SD card at SPI_HALF_SPEED to avoid bus errors with
  // breadboards.  use SPI_FULL_SPEED for better performance.
  if (!sd.begin(chipSelect, SPI_HALF_SPEED)) {
    sd.initErrorHalt();
  }
  if (sd.exists("dir")) error("dir exists");
  
  if (!sd.mkdir("dir")) error("mkdir failed");
 
  for (int ipass = 0; ipass < 3 ; ipass++) {
    for (int ifile = 0; ifile < 100; ifile++) {
      snprintf(rname, sizeof(rname), "file%d.ext", ifile);
      snprintf(dname, sizeof(dname), "dir/file%d.ext", ifile);
      if (sd.exists(dname)) {
        if (!sd.remove(dname)) error("remove failed");
      }
      if (!file.open(rname, O_CREAT | O_WRITE)) error("open failed");
      file.println(rname);
      file.close();
      if (!sd.rename(rname, dname)) error("rename failed");
    }
  }
  sd.ls(LS_R|LS_SIZE);

  cout << F("Done") << endl;
}
void loop() {}


It should print the following:
Quote
Insert an empty SD.  Type any character to start.
DIR/          
  FILE0.EXT      11
  FILE1.EXT      11
  FILE2.EXT      11
  FILE3.EXT      11
  FILE4.EXT      11
  FILE5.EXT      11
  FILE6.EXT      11
  FILE7.EXT      11
  FILE8.EXT      11
  FILE9.EXT      11
  FILE10.EXT     12
  FILE11.EXT     12
  FILE12.EXT     12
  FILE13.EXT     12
  FILE14.EXT     12
  FILE15.EXT     12
  FILE16.EXT     12
  FILE17.EXT     12
  FILE18.EXT     12
  FILE19.EXT     12
  FILE20.EXT     12
  FILE21.EXT     12
  FILE22.EXT     12
  FILE23.EXT     12
  FILE24.EXT     12
  FILE25.EXT     12
  FILE26.EXT     12
  FILE27.EXT     12
  FILE28.EXT     12
  FILE29.EXT     12
  FILE30.EXT     12
  FILE31.EXT     12
  FILE32.EXT     12
  FILE33.EXT     12
  FILE34.EXT     12
  FILE35.EXT     12
  FILE36.EXT     12
  FILE37.EXT     12
  FILE38.EXT     12
  FILE39.EXT     12
  FILE40.EXT     12
  FILE41.EXT     12
  FILE42.EXT     12
  FILE43.EXT     12
  FILE44.EXT     12
  FILE45.EXT     12
  FILE46.EXT     12
  FILE47.EXT     12
  FILE48.EXT     12
  FILE49.EXT     12
  FILE50.EXT     12
  FILE51.EXT     12
  FILE52.EXT     12
  FILE53.EXT     12
  FILE54.EXT     12
  FILE55.EXT     12
  FILE56.EXT     12
  FILE57.EXT     12
  FILE58.EXT     12
  FILE59.EXT     12
  FILE60.EXT     12
  FILE61.EXT     12
  FILE62.EXT     12
  FILE63.EXT     12
  FILE64.EXT     12
  FILE65.EXT     12
  FILE66.EXT     12
  FILE67.EXT     12
  FILE68.EXT     12
  FILE69.EXT     12
  FILE70.EXT     12
  FILE71.EXT     12
  FILE72.EXT     12
  FILE73.EXT     12
  FILE74.EXT     12
  FILE75.EXT     12
  FILE76.EXT     12
  FILE77.EXT     12
  FILE78.EXT     12
  FILE79.EXT     12
  FILE80.EXT     12
  FILE81.EXT     12
  FILE82.EXT     12
  FILE83.EXT     12
  FILE84.EXT     12
  FILE85.EXT     12
  FILE86.EXT     12
  FILE87.EXT     12
  FILE88.EXT     12
  FILE89.EXT     12
  FILE90.EXT     12
  FILE91.EXT     12
  FILE92.EXT     12
  FILE93.EXT     12
  FILE94.EXT     12
  FILE95.EXT     12
  FILE96.EXT     12
  FILE97.EXT     12
  FILE98.EXT     12
  FILE99.EXT     12
Done

stanzani

thanks for you reply first
my code is behaving as the example I think ... the only difference is in the file size: I am renaming files of 20-30K and smaller file (<1K). Noticeably the larger files suffer of the issue I reported, while the smaller are renamed ok ...

ciao

stanzani

I am experiment the following now

...\SdFat\SdSpiSAM3X.cpp
#define USE_SAM3X_DMAC 0
/** Use extra Bus Matrix arbitration fix if nonzero */
#define USE_SAM3X_BUS_MATRIX_FIX 0
/** Time in ms for DMA receive timeout */
#define SAM3X_DMA_TIMEOUT (100*3)
/** chip select register number */
#define SPI_CHIP_SEL 3
/** DMAC receive channel */
#define SPI_DMAC_RX_CH  1
/** DMAC transmit channel */
#define SPI_DMAC_TX_CH  0

i.e. NOT using DMA (DMA enable and DMA channels is something that should be decided at system level right? and I do not know if there are other players in my system using DMA channels which may confinct with this

fat16lib

#4
Mar 17, 2015, 02:33 pm Last Edit: Mar 17, 2015, 03:37 pm by fat16lib
Quote
I am renaming files of 20-30K
The data is not moved so size does not matter.  Only the directory entry moves.  You must close the file before you move a file or you may have problems.

Quote
DMA enable and DMA channels is something that should be decided at system level right?
Not with Arduino.  By default I do use DMA on Due.  Some SPI libraries may conflict with DMA use.

Don't "play" with the Due DMA unless you know about the SAM3X DMA channels and bus matrix.

Try disabling DMA.  For the current SdFat on GitHub edit SdFatConfig.h.  At about line 109 set USE_ARDUINO_SPI_LIBRARY nonzero like this.

Code: [Select]
#define USE_ARDUINO_SPI_LIBRARY 1
This will use the standard Arduino SPI.h library at slow speed.

Please remove the retry loop from your code.  It will never work correctly since you may cause a corrupt file system.  If rename fails it is generally because of a hardware problem and this must be fixed.  If rename fails because of a bug, retry won't help either.

You need very clean wiring to your SD card since my implementation of DMA SPI on Due runs at 42 MHz.

You could try SdFat-beta, it is a total rewrite.  You can disable DMA in the beta by setting SD_SPI_CONFIGURATION to one in SdFatConfig.h.

fat16lib

#5
Mar 17, 2015, 03:29 pm Last Edit: Mar 17, 2015, 03:34 pm by fat16lib
I increased the file size to 100 KiB (102,400 bytes) and the number of files to 500.  I ran this test on a Due with SD_SPI_CONFIGURATION zero to use DMA and one to use the standard Arduino SPI library.

Code: [Select]

/*
 * This program tests rename()
 */
#include <SPI.h>
#include <SdFat.h>

// SD chip select pin
const uint8_t chipSelect = SS;

// file system
SdFat sd;
uint8_t block[2048];
// Serial print stream
ArduinoOutStream cout(Serial);
//------------------------------------------------------------------------------
#define error(s) sd.errorHalt(s)
//------------------------------------------------------------------------------
void setup() {
  File file;
  char dname[20], rname[13];
  Serial.begin(9600);
  while (!Serial) {}  // wait for Leonardo

  cout << F("Insert an empty SD.  Type any character to start.") << endl;
  while (Serial.read() <= 0) {}

  // initialize the SD card at SPI_HALF_SPEED to avoid bus errors with
  // breadboards.  use SPI_FULL_SPEED for better performance.
  if (!sd.begin(chipSelect, SPI_HALF_SPEED)) {
    sd.initErrorHalt();
  }
  if (sd.exists("dir")) error("dir exists");
 
  if (!sd.mkdir("dir")) error("mkdir failed");
 
  for (int ipass = 0; ipass < 3 ; ipass++) {
    for (int ifile = 0; ifile < 500; ifile++) {
      snprintf(rname, sizeof(rname), "file%d.ext", ifile);
      snprintf(dname, sizeof(dname), "dir/file%d.ext", ifile);
      Serial.print(ipass);
      Serial.write(' ');
      Serial.println(rname); 
      if (sd.exists(dname)) {
        if (!sd.remove(dname)) error("remove failed");
      }
      if (!file.open(rname, O_CREAT | O_WRITE)) error("open failed");
      for (int k = 0; k < 50; k++) {
        if (sizeof(block) != file.write(block, sizeof(block))) {
          error("write failed");
        }
      }
      file.close();
      if (!sd.rename(rname, dname)) error("rename failed");
    }
  }
  sd.ls(LS_R|LS_SIZE);

  cout << F("Done") << endl;
}
void loop() {}



I ran the test on SdFat-beta which supports long file names so names are lower case.  The test ran without error.  Here is the start of the directory listing.
Quote
0 dir/
      102400 file0.ext
      102400 file1.ext
      102400 file2.ext
      102400 file3.ext
      102400 file4.ext
      102400 file5.ext
      102400 file6.ext
The test prints pass and file name while running:
Quote
2 file495.ext
2 file496.ext
2 file497.ext
2 file498.ext
2 file499.ext
This test runs at half speed, 21 MHz for DMA.  I also tested it at full speed and it works.


Go Up