manage files from SD card over internet

Hello, this is my first post on this forum even if i used a lot this tool.

I'm working on an arduino UNO with an ethernet shield, and i have to manage files from SD card by using a web page.
Functions i have to implement are the following :

  • explore existing files of the SD card
  • import files from a computer to SD card
  • delete files.

The first step is working, i can print repertories and each files on an html page and download them.

My problem is that i really don't no how to delete a choosen file and how to import file from the computer (text file for example). After a lot of research with a large documentation on this subject, i don't find help with my problem.

This is my actual code, i hope someone could bring me a little help !

#include <SPI.h>
#include <SD.h>
#include <Ethernet.h>  // Using WIZ5500 Ethernet chip so Adafruit Ethernet2 library is required
/************ ETHERNET STUFF ************/
byte mac[] = { 0xDE, 0xAD, 0xBE, 0xEF, 0xFE, 0xED };  // change if necessary
byte ip[] = { 10, 8, 0, 2 };                     // change if necessary
EthernetServer server(80);

/************ SDCARD STUFF ************/
#define SDCARD_CS 4
File root;


#define WIZ_CS 10

// store error strings in flash to save RAM
#define error(s) error_P(PSTR(s))

void error_P(const char* str) {
  Serial.print(F("error: "));
  Serial.println(str);

  while (1);
}

void setup() {
  Serial.begin(115200);
  while (!Serial);      // For 32u4 based microcontrollers like 32u4 Adalogger Feather

  Serial.print(F("Free RAM: ")); Serial.println(FreeRam());

  if (!SD.begin(SDCARD_CS)) {   //verification que la carte est bien detéctée
    error("card.init failed!");
  }

  printDirectory(root, 0);
  root = SD.open("/");

  // listing des dossiers + fichiers avec la fonction printDirectory
  Serial.println(F("Fichiers trouves dans le repertoire:"));
  printDirectory(root, 0);

  Serial.println();
  Serial.println(F("Done"));

  // lancement du serveur
  Serial.println(F("Initialisation..."));
  Ethernet.init(WIZ_CS);
  // on donne le temps a l'initialisation
  delay(1000);
  // lancement de la connexion ethernet avec les adresses mac et ip specifiees
  //adresse mac donnee par nous et ip est une adresse libre dans le reseau uha filaire
  Ethernet.begin(mac, ip);


  // affichage de l'ip utilisee sur le port serie pour verification
  Serial.print(F("L'adresse IP est : "));
  Serial.println(Ethernet.localIP());


  server.begin(); //lancement serveur
}

void ListFiles(EthernetClient client, uint8_t flags, File dir) { //fonction de listing des fichiers pour affichage html
  client.println("<ul>");

  while (true) { //on prefere utiliser une boucle infinie
    File entry = dir.openNextFile(); //parcours le repertoire en lisant successivement les fichiers

    // fin de chaine : plus de fichiers suivants
    if (! entry) {
      break;
    }

    // identation et listing html
    client.print("<li><a href=\"");
    client.print(entry.name()); //on prend le nom du fichier entry declare precedemment
    if (entry.isDirectory()) { //si c'est un repertoire, affichage special
      client.println("/");
    }
    client.print("\">");

    // si on clique sur un dossier, il affiche de nouveau le contenu (on change de repertoire en quelque sorte)
    //le nouveau repertoire est le repertoire fils et on re test la presence de contenu
    client.print(entry.name());
    if (entry.isDirectory()) {
      client.println("/");
    }

    client.print("</a>");

    client.println("<input type=button>cliquez pour supprimer</>");
    //client.println("<input type=button OnClick=SD.remove(entry.name())>          cliquez pour supprimer</>");
    client.println("</li>");
    entry.close(); //fermeture du fichier entry car traitement effectue
  }


  client.println("</ul>");
}

// How big our line buffer should be. 100 is plenty!
#define BUFSIZ 100

void loop()
{
  char clientline[BUFSIZ];
  char name[17];
  int index = 0;

  EthernetClient client = server.available();
  if (client) {
    // an http request ends with a blank line
    boolean current_line_is_blank = true;

    // reset the input buffer
    index = 0;

    while (client.connected()) {
      if (client.available()) {
        char c = client.read();

        // If it isn't a new line, add the character to the buffer
        if (c != '\n' && c != '\r') {
          clientline[index] = c;
          index++;
          // are we too big for the buffer? start tossing out data
          if (index >= BUFSIZ)
            index = BUFSIZ - 1;

          // continue to read more data!
          continue;
        }

        // got a \n or \r new line, which means the string is done
        clientline[index] = 0;

        // Print it out for debugging
        Serial.println(clientline);

        // Look for substring such as a request to get the file
        if (strstr(clientline, "GET /") != 0) {
          // this time no space after the /, so a sub-file!
          char *filename;

          filename = clientline + 5; // look after the "GET /" (5 chars)  *******
          // a little trick, look for the " HTTP/1.1" string and
          // turn the first character of the substring into a 0 to clear it out.
          (strstr(clientline, " HTTP"))[0] = 0;

          if (filename[strlen(filename) - 1] == '/') { // Trim a directory filename
            filename[strlen(filename) - 1] = 0;      //  as Open throws error with trailing /
          }

          Serial.print(F("Web request for: ")); Serial.println(filename);  // print the file we want

          File file = SD.open(filename, O_READ);
          if ( file == 0 ) {  // Opening the file with return code of 0 is an error in SDFile.open
            client.println("HTTP/1.1 404 Not Found");
            client.println("Content-Type: text/html");
            client.println();
            client.println("<h2>File Not Found!</h2>");
            client.println("
<h3>Couldn't open the File!</h3>");
            break;
          }

          Serial.println("File Opened!");

          client.println("HTTP/1.1 200 OK");
          if (file.isDirectory()) {
            Serial.println("is a directory");
            //file.close();
            client.println("Content-Type: text/html");
            client.println();
            client.print("<h2>Files in /");
            client.print(filename);
            client.println(":</h2>");
            ListFiles(client, LS_SIZE, file);
            file.close();
          } else { // Any non-directory clicked, server will send file to client for download
            client.println("Content-Type: application/octet-stream");
            client.println();

            char file_buffer[16];
            int avail;
            while (avail = file.available()) {
              int to_read = min(avail, 16);
              if (to_read != file.read(file_buffer, to_read)) {
                break;
              }
              // uncomment the serial to debug (slow!)
              //Serial.write((char)c);
              client.write(file_buffer, to_read);
            }
            file.close();
          }
        } else {
          // everything else is a 404
          client.println("HTTP/1.1 404 Not Found");
          client.println("Content-Type: text/html");
          client.println();
          client.println("<h2>File Not Found!</h2>");
        }
        break;
      }
    }
    // give the web browser time to receive the data
    delay(1);
    client.stop();
  }
}



void printDirectory(File dir, int numTabs) {
  while (true) {
    File entry =  dir.openNextFile();
    if (! entry) {
      // no more files
      break;
    }
    for (uint8_t i = 0; i < numTabs; i++) {
      Serial.print('\t');
    }
    Serial.print(entry.name());
    if (entry.isDirectory()) {
      Serial.println("/");
      printDirectory(entry, numTabs + 1);
    } else {
      // files have sizes, directories do not
      Serial.print("\t\t");
      Serial.println(entry.size(), DEC);
    }
    entry.close();
  }
}

You can use the SD.remove(filename); command from the SD library to remove a file from your SD card connected to your Arduino. The function must be called from your program running in SRAM on your microcontroller.

    //client.println("<input type=button OnClick=SD.remove(entry.name())>          cliquez pour supprimer</>");

Unless you have a javascript function called SD.remove(filename) that communicates with your microcontroller to likewise call SD.remove(filename) from Arduino's SD library you cannot use the above method to do this.

When a computer connects to an HTTP server that computer's browser interprets the HTML and runs any javascript on the local computer and displays the webpage. It can only make requests from the HTTP server. You would need to make a request from the server at an endpoint. Then have the server's endpoint handle the request.

Your server running on your Arduino (that also has the SD card connected to it) then can handle the delete by calling SD.remove(filename).

Does it mean that i just can operate in one direction and i can't doing 2nd and 3rd step without programming an endpoint ? (I don't really know what is an endpoint, i'm not a master on web coding).

Do you have a track for my next researches ? I'm turning around with this.

Thanks for the response !

So you are already handling one endpoint in your code so you understand the details. Because you've got this one, I am confident you can make a handler for another endpoint. You can think of an endpoint consisting of a handler residing at the end of a URI. So when an HTTP client requests a specific URI, your HTTP server at that location handles the request in the expected way.

userlucasm:
Does it mean that i just can operate in one direction ... ?

No, you can definitely do this in both directions the way you have specified. We just need to add some code to your server.

userlucasm:
Does it mean ... i can't doing 2nd and 3rd step without programming an endpoint ?

So you already have an HTTP server running and have created an endpoint to handle a root request, you just don't have the code in place to handle the specific request you want in the way you have specified.

This is your handler for the root of your HTTP server. Everything that happens inside there is what your server is doing if an HTTP client (in this case a web browser such as Chrome) has requested yourserver.com/. You are using this endpoint to list the files.

if (strstr(clientline, "GET /") != 0){...}

Now, you can make another endpoint to remove the files, by creating another handler on your server. I suggest something like yourserver.com. You can make that an index that lists all the files to remove. You can then dynamically handle /remove/[filename] to remove specific filenames.

What I would do first to understand the concept would be to make a static endpoint /remove/file1.txt and have that use the SD.remove("file2.txt"); Then you can make it dynamic to handle all the files on your card the same way you did to list all the files on your card.

Note: yourserver.com can also be /directory/file.ext

Hi,
if i resume i could create a link to go to /myserveur/remove and then list files and delete them by an action (mouse click for example).

But how can i choose parameters and conditions in the removing case ?
I don't know how to translate this "pseudo-code", i wish you could help me and tell me if it's a good solution :

void remove_files(file, ...){
if(file in directory is clicked){
filename=file;
send to serveur SD.remove(filename);
delay and refreshing}}

Keep this in mind when reading the following answers to the questions:

This looks like a whole lot of stuff to read and that it could be overwhelming. It is not far off from what you have coded, it's just helps you understand scoping and where code can be executed.

I am describing something similar to REST (Representational State Transfer). You do not have to solve this problem with REST, I just think it is a good way to do it because you want to interface your Arduino with a web page. HTTP using a subset of REST is excellent at this.

The idea is you want to make a request of the server through a URI as the interface between the client and server. The server will respond to your request and your client program will process the response for your output.

In your specific circumstance you want the client to be a web browser and you want your server to be your Arduino. You want this because the web browser is a good shell for a graphical user interface for your user and your Arduino has the files on it's SD card to be accessed by the user.

userlucasm:
if i resume i could create a link to go to /myserveur/remove and then list files and delete them by an action (mouse click for example).

Yes, but specifically in the HTTP server REST method of mouse clicking. You would "mouse click" on a link in your web browser. (You could call javascript and send an AJAX HTTP DELETE request. This is something I would do but it will certainly muddy the water in making this as simple as possible. Also, we need to think of resources on the server side, it is a microcontroller.)

So maybe you served up an unordered list of files from your server root, /.

<!--This is HTML (text) that is served up in the body of your HTTP response.
You should client.print() this out.-->

<ul>
 <li>file1.txt</li>
 <li>file2.txt</li>
 <li>file3.txt</li>
</ul>

This is read only. So let's add the ability to request a delete function. We do this by requesting a different URI. I previously suggested that we use /remove/file1.txt. If we are to use this as the endpoint we need to do two things. Use this endpoint in our client and handle this endpoint on our server. I might generally say that our endpoint is in the form /arg1/arg2/...

Let's add the endpoint to our client using a link (the easiest and most straight forward way).

<!--This is HTML (text) that is served up in the body of your HTTP response.
You should client.print() this out.-->

<ul>
 <li>file1.txt <a href="<myserver>/remove/file1.txt">remove</a></li>
 <li>file2.txt <a href="<myserver>/remove/file2.txt">remove</a></li>
 <li>file3.txt <a href="<myserver>/remove/file3.txt">remove</a></li>
</ul>

Now that we have the client using the endpoint, let's add the endpoint to our server. Because our client is going to be a web browser, it is probably easiest if we only use REST methods GET and POST. In this if block in your code you are reading the HTTP request and handling all GET requests in here.

if (strstr(clientline, "GET /") != 0){...}

In the following code you go on to parse the HTTP request further to retrieve the first argument you store in char *filename. I would maybe name this variable arg1 because it does not always need to be a file name. You can then parse arg2 anything after the next '/' and so on when needed later.

if (filename[strlen(filename) - 1] == '/') { // Trim a directory filename
 filename[strlen(filename) - 1] = 0;      //  as Open throws error with trailing /
}

Once you implement the further parsing of the URI, you will then be able to use arg1, arg2 and so on to select which handler you want to call for the different endpoints.

userlucasm:
But how can i choose parameters and conditions in the removing case ?

So now you have parameters of arg1 and arg2. I would again suggest arg1 be remove for your delete handler. /remove/arg2

I would then suggest that arg2 be the file you would like to have removed. /remove/file1.txt

userlucasm:
I don't know how to translate this "pseudo-code", i wish you could help me and tell me if it's a good solution :

void remove_files(file, ...){

if(file in directory is clicked){
filename=file;
send to serveur SD.remove(filename);
delay and refreshing}}

So the pseudo code does not work with the paradigm we are exploring because the click is happening in the web browser (your client program) and the deleting is happening on the Arduino (your server program).

The pseudo code for this paradigm would be more like this: Each code block is pseudo code that happens 1st on the client, then the next on the server, then client... back and forth until the client makes no more requests of the server.

In between each of the following code blocks the HTTP request data is being transmitted through your network from client to server. Then the reverse happens between the next code blocks.

This is the natural flow of REST. Client does something then sends data. Data is received by server does something and responds back to client. The natural break up creates the interface between the two programs (client and server).

//User requests your root server through a web browser
//Enters your URI into the address bar of their favorite web browser
//The web browser requests the root of your server
location: <myserver>/
//Your server receives the request for <myserver>/
readRequest();

//Your server parses the request and sees it is the root '/'
parseRequest();

//Your handler for the root is called
if(arg1 == '/' || arg1 == '' || arg1 == '/index.html'){
 handleRoot();
}

//List files on SD card as well as actions you can take (Upload, Remove, ect.)
//The handler then prints it's HTTP response to send HTML back to the web browser
//with links containing URI's to other endpoints (actions you can take [Upload, Remove, ect.])
//The web browser receives the response and renders the HTML for you automatically
//It displays the lists of URI's (endpoints) of actions the user can take
/**********************
*Upload Files - Link to Endpoint <myserver>/upload/
*Remove Files - Link to Endpoint <myserver>/remove/
*Edit Files - Link to Endpoint <myserver>/edit/
*
*File Index
*file1.txt - Link to Endpoint <myserver>/view/file1.txt
*file2.txt - Link to Endpoint <myserver>/view/file2.txt
*file3.txt - Link to Endpoint <myserver>/view/file3.txt
*
**********************/

//The user "mouse clicks" on a link, let's say the Remove Files link
//It makes an HTTP request to the server at endpoint <myserver>/remove/
//Your server now handles the request in the same way
readRequest();
parseRequest();

//but now the request has arg1 == remove so it calls your remove handler
if(arg2 == "" && arg1 == "remove"){
 handleRemove();
}
//The web browser receives the response and renders the HTML for you automatically
//It displays the lists of URI's (endpoints) of actions the user can take
/**********************
*Upload Files - Link to Endpoint <myserver>/upload/
*Remove Files - Link to Endpoint <myserver>/remove/
*Edit Files - Link to Endpoint <myserver>/edit/
*
*File Index
*file1.txt - Link to Endpoint <myserver>/remove/file1.txt
*file2.txt - Link to Endpoint <myserver>/remove/file2.txt
*file3.txt - Link to Endpoint <myserver>/remove/file3.txt
*
**********************/

//The user "mouse clicks" on a link, let's say the Remove file3.txt link
//It makes an HTTP request to the server at endpoint <myserver>/remove/file3.txt
//Your server now handles the request in the same way
readRequest();
parseRequest();

//but now the request has arg1 == remove
//and arg2 == file3.txt
//so it calls your remove file handler
if(arg1 == "remove" && arg2 != ""){
 handleRemoveFile(arg2);
}

void handleRemoveFile(file_name){
  SD.remove(file_name); //Wooo! We finally got the Arduino to call the line of code we have been waiting for.
}