next topic »

[on:]

I've got an Arduino Uno and an Ethernet Shield with imbedded SD card, and am able to successfully serve images and videos from the card to a client browser.  But when I load my full website to the card, it only partially launches the site, apparently stalling on some javascript (.js files) imbedded in subdirectories.

My site, as loaded in the SD card, contains the following files in the root directory:

feed.xml
Home_files(folder)
Home.htm
Images(folder)
index.htm
Music_and_Videos_files(folder)
Music_and_Videos.html
Projects_files(folder)
Projects.html
Scripts(folder)

The code I'm using is largely from Ladyada's tutorial, modified to include additional MIME types as follows:

Code:

#include <SPI.h>
#include <SdFat.h>
#include <SdFatUtil.h>
#include <Ethernet.h>

/************ ETHERNET STUFF ************/
byte mac[] = { 0xDE, 0xAD, 0xBE, 0xEF, 0xFE, 0xED };
byte ip[] = { 192, 168, 1, 177 };
char rootFileName[] = "index.htm";
Server server(80);

/************ SDCARD STUFF ************/
Sd2Card card;
SdVolume volume;
SdFile root;
SdFile file;

// store error strings in flash to save RAM
#define error(s) error_P(PSTR(s))

void error_P(const char* str) {
  PgmPrint("error: ");
  SerialPrintln_P(str);
  if (card.errorCode()) {
    PgmPrint("SD error: ");
    Serial.print(card.errorCode(), HEX);
    Serial.print(',');
    Serial.println(card.errorData(), HEX);
  }
  while(1);
}

void setup() {
  Serial.begin(115200);
 
  PgmPrint("Free RAM: ");
  Serial.println(FreeRam());  
  
  // initialize the SD card at SPI_HALF_SPEED to avoid bus errors with
  // breadboards.  use SPI_FULL_SPEED for better performance.
  pinMode(10, OUTPUT);                       // set the SS pin as an output (necessary!)
  digitalWrite(10, HIGH);                    // but turn off the W5100 chip!

  if (!card.init(SPI_HALF_SPEED, 4)) error("card.init failed!");
  
  // initialize a FAT volume
  if (!volume.init(&card)) error("vol.init failed!");

  PgmPrint("Volume is FAT");
  Serial.println(volume.fatType(),DEC);
  Serial.println();
  
  if (!root.openRoot(&volume)) error("openRoot failed");

  // list file in root with date and size
  PgmPrintln("Files found in root:");
  root.ls(LS_DATE | LS_SIZE);
  Serial.println();
    
  // Recursive list of all directories
  PgmPrintln("Files found in all dirs:");
  root.ls(LS_R);
  
  Serial.println();
  PgmPrintln("Done");
    
  // Debugging complete, we start the server!
  Ethernet.begin(mac, ip);
  server.begin();
}

// How big our line buffer should be. 100 is plenty!
#define BUFSIZ 100

void loop()
{
 
  char clientline[BUFSIZ];
  char *filename;
  int index = 0;
  int image = 0;
  
  Client client = server.available();
    if (client) {
    // an http request ends with a blank line
    boolean current_line_is_blank = true;
    
    // reset the input buffer
    index = 0;
    
      while (client.connected()) {
        if (client.available()) {
          char c = client.read();
          
        // If it isn't a new line, add the character to the buffer
        if (c != '\n' && c != '\r') {
          clientline[index] = c;
          index++;
          // are we too big for the buffer? start tossing out data
          if (index >= BUFSIZ)
            index = BUFSIZ -1;
            
          // continue to read more data!
          continue;
        }
          
        // got a \n or \r new line, which means the string is done
        clientline[index] = 0;
        filename = 0;
        
        // Print it out for debugging
        Serial.println(clientline);
        
        // Look for substring such as a request to get the root file
        if (strstr(clientline, "GET / ") != 0) {
          filename = rootFileName;
        }
        if (strstr(clientline, "GET /") != 0) {
          // this time no space after the /, so a sub-file
          
          if (!filename) filename = clientline + 5; // look after the "GET /" (5 chars)
          // a little trick, look for the " HTTP/1.1" string and
          // turn the first character of the substring into a 0 to clear it out.
          (strstr(clientline, " HTTP"))[0] = 0;
          
          // print the file we want
          Serial.println(filename);
          
          if (! file.open(&root, filename, O_READ)) {
            client.println("HTTP/1.1 404 Not Found");
            client.println("Content-Type: text/html");
            client.println();
            client.println("<h2>File Not Found!</h2>");
            break;
          }
          
          Serial.println("Opened!");
          
          client.println("HTTP/1.1 200 OK");
          if (strstr(filename, ".htm") != 0)
             client.println("Content-Type: text/html");
         else if (strstr(filename, ".css") != 0)
             client.println("Content-Type: text/css");
         else if (strstr(filename, ".png") != 0)
             client.println("Content-Type: image/png");
          else if (strstr(filename, ".jpg") != 0)
             client.println("Content-Type: image/jpeg");
         else if (strstr(filename, ".gif") != 0)
             client.println("Content-Type: image/gif");
         else if (strstr(filename, ".3gp") != 0)
             client.println("Content-Type: video/mpeg");
         else if (strstr(filename, ".pdf") != 0)
             client.println("Content-Type: application/pdf");
         else if (strstr(filename, ".js") != 0)
             client.println("Content-Type: application/x-javascript");
         else if (strstr(filename, ".xml") != 0)
             client.println("Content-Type: application/xml");
         else
             client.println("Content-Type: text");

          client.println();
          
          int16_t c;
          while ((c = file.read()) >= 0) {
              // uncomment the serial to debug (slow!)
              //Serial.print((char)c);
              client.print((char)c);
          }
          file.close();
        } else {
          // everything else is a 404
          client.println("HTTP/1.1 404 Not Found");
          client.println("Content-Type: text/html");
          client.println();
          client.println("<h2>File Not Found!</h2>");
           }
        break;
      }
    }
    // give the web browser time to receive the data
    delay(1);
    client.stop();
  }
}

Note that, in order to get my site to launch at all, I changed names of 2 files to fit DOS 8.3 format; in particular, I renamed index.html to index.htm, Home.html to Home.htm, and then I modified the index.htm file to call Home.htm rather than Home.html as it originally did.  So I'm thinking the 8.3 format issue is hitting me elsewhere in the file chain.

The serial monitor window shows the following activity, initiated when I enter http://192.168.1.177 into any browser:

GET / HTTP/1.1
index.htm
Opened!
GET /Home.htm HTTP/1.1
Home.htm
Opened!
GET /Scripts/iWebSite.js HTTP/1.1
Scripts/iWebSite.js
GET /Scripts/Widgets/SharedResources/WidgetCommon.js HTTP/1.1
Scripts/Widgets/SharedResources/WidgetCommon.js
GET /Scripts/Widgets/Navbar/navbar.js HTTP/1.1
Scripts/Widgets/Navbar/navbar.js
GET /Home_files/Home.js HTTP/1.1
Home_files/Home.js
GET /Home_files/about_me.png HTTP/1.1
Home_files/about_me.png
GET /Home_files/Home.css HTTP/1.1
Home_files/Home.css
GET /Home_files/shapeimage_4.png HTTP/1.1
Home_files/shapeimage_4.png
GET /Home_files/shapeimage_3.png HTTP/1.1
Home_files/shapeimage_3.png
GET /Home_files/shapeimage_1.png HTTP/1.1
Home_files/shapeimage_1.png
GET /Home_files/shapeimage_2.png HTTP/1.1
Home_files/shapeimage_2.png

I suspect my error is obvious to someone out there.  Can anybody help?

[Reply #1 on:]

Code:

GET /[glow]Home_files[/glow]/[glow]shapeimage_4[/glow].png HTTP/1.1

One, two, three,... Oh, darn, that's more than 8.

If the issue really is an 8.3 thing, you have a bunch of renaming to do.

[Reply #2 on:]

Okay.  As I stated in my original post, "I'm thinking the 8.3 format issue is hitting me elsewhere in the file chain."  And  root.ls(LS_DATE | LS_SIZE);  indeed suggests that all of my nice long filenames have been truncated to lowly 8.3, as shown in another snippet from the serial monitor window:

Volume is FAT32

Files found in root:
_~1.TRA       2010-11-20 08:55:58 4096
README.TXT    2010-11-19 09:23:44 6552
TRASHE~1/     2010-11-20 08:55:58
SPOTLI~1/     2010-11-20 08:55:58
FSEVEN~1/     2010-12-03 13:55:38
FKDATPDF.PDF  2010-11-20 10:09:20 13651
_FKDAT~1.TXT  2010-11-20 11:08:40 4096
FKDATTAB.TXT  2010-11-20 10:08:38 217

But you can also see from the output of   Serial.println(volume.fatType(),DEC);  that the card partition is formatted FAT32, which can ostensibly support long file names.  And the long names show up just fine when I mount the card on the computer I used to format and populate it.

And then I found the explanation, in the form of a key SdFat library limitation described at http://code.google.com/p/sdfatlib/: "SdFat only supports short 8.3 file names."

So before I drop this effort altogether, can someone point me to site builder software (Mac, Linux or Windows) that will implement the DOS 8.3 filename protocol?

[Reply #3 on:]

Living with the 8.3 format means that YOU have to give your files short names. When you create a file in whatever web design software you are using, you assign the name, not the software. Don't assign the files names with 42 characters, or spaces or other invalid characters. You don't need special software.

[Reply #4 on:]

Thanks for bearing with me on this, Paul.

But I've been using iWeb 3.0.2 on a Mac, and it auto-generates lots of long filenames that are contributing to my grief.  I can certainly control the names of the images, songs, videos etc. that I incorporate into my site, but my web builder is creating working files with names like navbar_0_normal.png and shapeimage_1.png.  In your 1st reply to my original post, you noticed that my SD card read was choking on /Home_files/shapeimage_4.png, a non-8.3 filename in a non-8.3 folder, neither of which was named or created by me.

So if you, or anyone else, has successfully built a site that uses short protocol exclusively, can you share with me the platform and software title?  'Cause it appears I need something more special than Mac iWeb...

[Reply #5 on:]

If you spent one or two evenings with python you should be able to write a script that converts all filenames to 8.3 format and all references in the html files too.

1) You first scan the dir tree for all filenames and write them into a file together with a 8.3 replacement string.

2) With the use of this file you rename all files

3) With the use of this file you check all html (etc) files to rename the links.

(easier said than) done

[Reply #6 on:]

I suspect the reason SdFat only implements 8.3 filenames is that long file names are a bit of a nasty hack and quite complex to code.  But there may be a library out there that supports long filenames, worth having a look around.

For grizzly details see wikipedia

[Reply #7 on:]

Hi,
I am currently using the same base sketch to do the same thing.
Most of the examples I have seen use the same type of 'print' to the ethernet device as your sketch...

          int16_t c;
          while ((c = file.read()) >= 0) {
              // uncomment the serial to debug (slow!)
              //Serial.print((char)c);
              client.print((char)c);
          }
          file.close();
.
This sends byte at a time so one byte of actual data=about 60 bytes in an ethernet frame.

If instead of the above you allocate an array in ram as a buffer (I am using a mega board so I'm using 200 bytes), you can send blocks of data instead of byte by byte. The difference is amazing!
Use something like...

 int16_t c;
          bufindex=0;
          while ((c = file.read()) >= 0) {
            buf[bufindex++](char)c);
            if (bufindex==maxbytes){
              client.write(buf,maxbytes);
              bufindex=0;
            }
          }
          // file has been completely read so can now be closed
          file.close();
          // It is unlikely that the file length will be an exact multiple of
          // 'maxbytes' so this sends the last partialy filled buffer (if any)
          if (bufindex>0){
            client.write(buf,bufindex);
          }
        }
 
This is my 1st attempt at arduino so my coding is pretty awful. Long way to go yet.

[Reply #8 on:]

You'll get the best performance if you size the array to be somewhat smaller that the payload of a packet. That's somewhere around 56 bytes, maybe a bit less. A 60 byte array will result in 2 packets, as will a 100 byte array.