SDFat LIB , using O_READ +file.openNext and O_WRITE + file.rename

Hello , I'm trying to build a small data loguer that self upload csv file to a remote FTP server . So far this work great

But ,I've been puzzling around a little bug I got .

One of the first thing I would like my code to do is to look to the sd card , and find all .CSV files that are there , upload them // when upload is successful , rename the file to .BAK.

I've been able to do this all with the basic Arduino SD Library , except the renaming Part

Seems simple with SDFat lib, But the file.openNext function need to open files in O_READ mode. and the rename funtion need to open the file in O_WRITE mode.

If I open the file first in O_WRITE mode , file.openNext doest find any file.

If I open a new instance of the files in O_WRITE mode (is this evend a good idea ?) of the file found before in O_READ mode , file.openNext doesn't open the next files (example code does this) ..... I'm shurly doing something wrong.

Rewinding after each renamed file could resolve the issue, and allow a new search for not renamed csv. But may hang up if infinit loop if rename fail, always finding the same file, alway not being able to rename it.

As qty of csv files migth grow up to huge amount, I dont think that building a index of the name of csv files to rename, and after renaming them all , a good idea either

Idea ?

Included code is a bit striped down, but is compiling ok

Hardware: Arduino 2560 ( waiting for arduino due to ship to try to increase ftp speed>18kbs) + Ethernet Sheild + 2gb sdcard

#include <SdFat.h>
#include <SdFatUtil.h>
#include <SPI.h>
#include <Ethernet.h>
 
#define error(s) error_P(PSTR(s))
#define FTP_BUFFER_SIZE 64      /// on atmega 2560 FTP speed   64 = 17.5 kbps     512 = 19kbps

byte mac[] = { 0x90, 0xA2, 0xDA, 0x00, 0x59, 0x67 };  

unsigned char dhcp = 0;    
IPAddress ip        ( 192, 168, 81 , 177 );    
IPAddress gateway   ( 192, 168, 81 , 254 );
IPAddress subnet    ( 255, 255, 255, 0   );
IPAddress ftpserver ( 192, 168,  81, 102 );
 
Sd2Card card;       
SdFat sd;
SdBaseFile file;   

EthernetClient client;
EthernetClient dclient;

char outBuf[128];
char outCount;
char fileNameFTP[13] = "00000000.TXT"; //  file to upload // file name will be renamed later 
char fileNameLOG[13] = "00000000.CSV"; //  file to logto  // file name will be renamed later 

unsigned char sd_pin_cs = 4;
unsigned char eth_pin_cs = 10;


//...............errors..................
ArduinoOutStream cout(Serial);
void error_P(const char* str) {
  PgmPrint("error: ");
  SerialPrintln_P(str);
  if (card.errorCode()) {
    PgmPrint("SD error: ");
    Serial.print(card.errorCode(), HEX);
    Serial.print(',');
    Serial.println(card.errorData(), HEX);
  }
  while(1);
}


/**************************** SETUP ********************************/ 
void setup(){
  Serial.begin(9600);   

   pinMode     (eth_pin_cs,OUTPUT);
   digitalWrite(eth_pin_cs,HIGH);

   if(sd.begin(sd_pin_cs,SPI_HALF_SPEED) == 0)  {  //on mega2560 SPI_HALF_SPEED or SPI_FULL_SPEED  both got 17 -18 kbps FTP
     Serial.println(F("SD init fail"));          
   }
 
   Serial.print(F("Get DHCP : "));
   if (Ethernet.begin(mac) == 0)   Serial.println(F("Failed  "));
   else                            Serial.println(Ethernet.localIP());
 
  digitalWrite(eth_pin_cs,HIGH);
  
  delay(2000); 
}







/*************************** MAIN ************************/ 
void loop(){
  byte inChar;
  Serial.println(F("----MAIN-----"));

  if(dhcp==1) Ethernet.maintain();  //be sure to call Ethernet.maintain() regularly if dhcp. 
  uploadAllFiles();

  Serial.print(F("KAMIKAZEeeee....AAARRRrrggggg.... " ));
  while(1);
}

code part2

/*************************** FTP  ************************/ 
void uploadAllFiles(void) {
  unsigned char  i; 
  unsigned char  isDifferent=0; 
  unsigned char  isDotCsv=0; 
  char fileNameNEW[13] = "01234567.BAK"; 
  

//   find files
//    while(  file is csv) 
//       upload file   
//       rename file .bak 

  while (file.openNext(sd.vwd(), (O_READ ))) {  // O_WRITE cant find any file at first    O_READ and O_WRITE after , cant find any file after first 
    file.getFilename(fileNameFTP);
    Serial.print(F("Found this file on SD  :" ));  Serial.println(fileNameFTP);
    Serial.print(F("Using this file to log :" ));  Serial.println(fileNameLOG);

    //compare fileNameFTP to fileNameLOG we dont want  to upload the file we are using right now 
    isDifferent=0;
    for(i=0;i<12;i++){
      if(fileNameFTP[i]!= fileNameLOG[i]){
        isDifferent=1;
      }
    }

    if (isDifferent)       Serial.println(F("File is not the one i'm using " ));      
    else                   Serial.println(F("File is the one i'm using " ));      

    if((fileNameFTP[0] !='\n')&&
       (fileNameFTP[1] !='\n')&&   
       (fileNameFTP[2] !='\n')&&   
       (fileNameFTP[3] !='\n')&&   
       (fileNameFTP[4] !='\n')&&   
       (fileNameFTP[5] !='\n')&&   
       (fileNameFTP[6] !='\n')&&   
       (fileNameFTP[7] !='\n')&&   
       (fileNameFTP[8]  =='.')&&   
       ((fileNameFTP[9] =='C' )||(fileNameFTP[9] =='c'))&&   
       ((fileNameFTP[10]=='S' )||(fileNameFTP[10]=='s'))&&                   
       ((fileNameFTP[11]=='V' )||(fileNameFTP[11]=='v'))){
         isDotCsv=1; 
         Serial.println(F("File is  looking like    XXXXXXXX.CSV " ));      
       }
     else{
         isDotCsv=0;
         Serial.println(F("File is not looking like  XXXXXXXX.CSV " ));      
     }
 
    if((isDifferent==1) && (isDotCsv==1)){
         Serial.print(F("Uploading file :" ));         Serial.println(fileNameFTP);
         if(doFTP()){
          Serial.println(F("FTP Upload OK"));
           
          renameFileDotBak();
                 
         }
         else Serial.println(F("FTP Upload FAIL"));
     }
     else{
        Serial.println(F("Not uploading file " ));      
     }
    file.close();
  }
  Serial.println(F("No more files  " ));     
  
}

void renameFileDotBak(void ){
  char fileNameNEW[13] = "01234567.BAK";  

    fileNameNEW[0]=fileNameFTP[0]; //get the same 8 first char 
    fileNameNEW[1]=fileNameFTP[1];
    fileNameNEW[2]=fileNameFTP[2];
    fileNameNEW[3]=fileNameFTP[3];
    fileNameNEW[4]=fileNameFTP[4];
    fileNameNEW[5]=fileNameFTP[5];
    fileNameNEW[6]=fileNameFTP[6];
    fileNameNEW[7]=fileNameFTP[7];

    Serial.print("Renaming file : ");      
    Serial.print(fileNameFTP);      
    Serial.print(" To : ");      
    Serial.print(fileNameNEW);      

    SdFile filetorename(fileNameFTP, O_WRITE);
    Serial.println(F(" Open SD File to rename "));          

    Serial.println(F(" Rename file "));          
    if (filetorename.rename(sd.vwd(), fileNameNEW))
     Serial.println(F(" Succes"));          
    else 
     Serial.println(F(" Failed"));      

   Serial.println(F(" Close renamed SD File "));          
   if (!filetorename.close()) Serial.println("Error file.close");
    
}


/*************************** FTP  ************************/ 
byte doFTP(){
     Serial.print(F(" Connection..."));
     if (client.connect(ftpserver,21)) {
       Serial.println(F(" Succes "));
    } 
    else {
      file.close();
      Serial.println(F(" Failed"));
      return 0;
    }


  if(!eRcv()) return 0;
  client.write("USER arduino\r\n");  // change your username here 
  if(!eRcv()) return 0;
  client.write("PASS arduino\r\n");  // change your password here 
  if(!eRcv()) return 0;
  client.write("SYST\r\n");
  if(!eRcv()) return 0;
  client.write("PASV\r\n");
  if(!eRcv()) return 0;

  char *tStr = strtok(outBuf,"(,");
  int array_pasv[6];
  for ( int i = 0; i < 6; i++) {
    tStr = strtok(NULL,"(,");
    array_pasv[i] = atoi(tStr);
    if(tStr == NULL)
    {
      Serial.println(F("Bad PASV Answer"));    
    }
  }

  unsigned int hiPort,loPort;
  hiPort = array_pasv[4] << 8;
  loPort = array_pasv[5] & 255;

  Serial.print(F("Data port: "));
  hiPort = hiPort | loPort;
  Serial.println(hiPort);

  if (dclient.connect(ftpserver,hiPort)) {
    Serial.println(F("Data connected"));
  } 
  else {
    Serial.println(F("Data connection failed"));
    client.stop();
    file.close();
    return 0;
  }

  Serial.println(F("Opening file on SD Card"));
  file.open(fileNameFTP,O_READ);

  client.write("STOR ");
  client.println(fileNameFTP);

  if(!eRcv())
  {
    dclient.stop();
    return 0;
  }

  Serial.println(F("Uploading..."));
  byte clientBuf[FTP_BUFFER_SIZE];
  int clientCount = 0;
  int counting_64bytes = 0;
  int counting_Kbytes = 0;
  
  // default 17 - 18kbps on mega 2560
  while(file.available())
  {
    clientBuf[clientCount] = file.read();
    clientCount++;

    if(clientCount > FTP_BUFFER_SIZE-1)    {
      dclient.write(clientBuf,FTP_BUFFER_SIZE);
      clientCount = 0;
      counting_64bytes++;
    }
    
    if(counting_64bytes > (32 *16) ){
      counting_64bytes=0;
      counting_Kbytes=counting_Kbytes+32;
      Serial.print(counting_Kbytes);
      Serial.println(F("Kb "));
    }
  }

  if(clientCount > 0) dclient.write(clientBuf,clientCount);

  dclient.stop();
  Serial.println(F("Data FTP Chanel disconnected"));

  if(!eRcv()) return 0;
  client.write("QUIT\r\n");
  if(!eRcv()) return 0;

  client.stop();
  Serial.println(F("Command FTP Chanel disconnected"));

  file.close();
  Serial.println(F("SD closed"));
  return 1;
}




byte eRcv(){
  byte respCode;
  byte thisByte;
  while(!client.available()) delay(1);
  respCode = client.peek();
  outCount = 0;
  while(client.available())  {  
    thisByte = client.read();    
    Serial.write(thisByte);

    if(outCount < 127)    {
      outBuf[outCount] = thisByte;
      outCount++;      
      outBuf[outCount] = 0;
    }
  }
  if(respCode >= '4')  {
    efail();
    return 0;  
  }
  return 1;
}


void efail(){
  byte thisByte = 0;

  client.write("QUIT\r\n");
  while(!client.available()) delay(1);
  while(client.available())  {  
    thisByte = client.read();    
    Serial.write(thisByte);
  }
  client.stop();
  Serial.println(F("Command disconnected"));
  file.close();
  Serial.println(F("SD closed"));
}

bump