MKR GSM 1400 - WiFi101 OTA library

Hello Juraj

I did a implementation with gprs on a mkr1400.
It works with small sketches.
With large sketches, it does not find a proper end to read out the tcp socket.

It reads out the sockets for thousends of times like:

15:24:08.677 -> +UUSORD: 0,7162
15:24:08.715 -> AT+USORD=0,512

15:24:08.715 -> +USORD: 0,512,"5448207061636B65742072656365697665643A20436C6173733A2030780070726F636573733A2041434B2072656365697665642077697468202E6C656E20213D20323A20436C6173733A20307800206C656E3A2000537563636573730047656E6572616C204661696C75726500435243204661696C0054696D656F757400436F6D6D616E64206E6F742061636B6E6F776C656467656420284E41434B29004F7574206F662072616E676500496E76616C69642041726700496E76616C6964206F7065726174696F6E004D656D6F7279204572726F72004861726477617265204572726F7200446174612053656E7400446174612052656365697665640049324320436F6D6D204661696C7572650044617461205061636B6574204F7665727772697474656E0000000000003E5B4F5B3E3E6B4F6B3E1C3E7C3E1C183C7E3C181C577D571C1C5E7F5E1C00183C1800FFE7C3E7FF0018241800FFE7DBE7FF30483A060E2629792926407F050507407F05253F5A3CE73C5A7F3E1C1C08081C1C3E7F14227F22145F5F005F5F06097F017F006689956A606060606094A2FFA29408047E040810207E201008082A1C08081C2A08081E101010100C1E0C1E0C30383E3830060E3E0E06000000000000005F00000007000700147F147F14242A7F2A12231308646236495620500008070300001C2241000041221C002A1C7F1C2A08083E"
15:24:08.785 -> 
15:24:08.785 -> OK
15:24:08.785 -> 
15:24:08.785 -> +UUSORD: 0,6650
15:24:08.785 -> AT+USORD=0,512

15:24:08.785 -> +USORD: 0,512,"080800807030000808080808000060600020100804023E5149453E00427F400072494949462141494D331814127F1027454545393C4A49493141211109073649494936464949291E0000140000004034000000081422411414141414004122140802015909063E415D594E7C1211127C7F494949363E414141227F4141413E7F494949417F090909013E414151737F0808087F00417F41002040413F017F081422417F404040407F021C027F7F0408107F3E4141413E7F090909063E4151215E7F09192946264949493203017F01033F4040403F1F2040201F3F4038403F631408146303047804036159494D43007F4141410204081020004141417F04020102044040404040000307080020545478407F284444383844444428384444287F385454541800087E090218A4A49C787F0804047800447D40002040403D007F1028440000417F40007C047804787C080404783844444438FC1824241818242418FC7C08040408485454542404043F44243C4040207C1C2040201C3C4030403C44281028444C9090907C4464544C4400083641000000770000004136080002010204023C2623263C1EA1A161123A4040207A385454555921555579412254547842215554784020545579400C1E5272123955555559395454545939555454580000457C410002457D420001457C407D1211127DF0282528F07C545545002054547C54"
15:24:08.853 -> 
15:24:08.853 -> OK

The modem loads data, the sockets is reading, the bytes waiting on the sockets lesser, thousands of bytes, but then, at the end of the stream:


15:24:09.793 -> +UUSORD: 0,506
15:24:09.793 -> AT+USORD=0,512

15:24:09.793 -> +USORD: 0,506,"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000043000000000000000000000000000000000000000000000000000000000000004300000000000000000000000000000000000000000000000000000000000000430000000000000000000000000000000000000000000000000000000000000043000000000000000000000000000000000000000000000000000000000000004300000000000000000000000000000000000000000000000000000000000000430000000000000000000000000000000000000000000000000000000000000043000000000000000000000000000000000000000000000000000000000000003D2C02007D1102000000000090420300424B0200E34D0200E34D0200E34D0200E34D0200E34D0200E34D0200E34D0200E34D0200E34D0200FFFFFFFFFFFFFFFFFFFFFFFFFFFF00000100415343494900000000000000000000000000000000000000000000000000000041534349490000000000000000000000000000000000000000000000000000000000DD600000A5EF0000DD0601002D080100890B01005911010031120100D5120100ED120100F51C01001D1F010051240100DD8D01006D90010015A90100DDB90100B560000000000000"
15:24:09.860 -> 
15:24:09.860 -> OK
15:24:09.895 -> AT+USORD=0,512

15:24:10.107 -> ERROR
15:24:10.141 -> 
15:24:10.141 -> +UUSOCL: 0
15:24:10.141 -> AT+USOCL=0

15:24:10.141 -> ERROR
15:24:10.921 -> Timeout downloading update file at 160 bytes. Can't continue with update.

Your code in this section is

while (length > 0) {
    if (!client2.readBytes(&b, 1)) // reading a byte with timeout
      break;
    file.write(b);
    length--;
  }
  file.close();
  client2.stop();
  if (length > 0) {
    SD.remove(BIN_FILENAME);
    Serial.print("Timeout downloading update file at ");
    Serial.print(length);
    Serial.println(" bytes. Can't continue with update.");
    return;
  }

It breaks at a random position, sometimes
15:00:06.661 -> Timeout downloading update file at 203154 bytes. Can't continue with update.
or
15:02:52.748 -> Timeout downloading update file at 512 bytes. Can't continue with update.

I can't see the bug, the while condition is fullfiled, do you have a thought ?

Thx

try client2.setTimeout(2000); it is in millis. default is 1000. it sets how long readBytes waits for the next byte.

It's something else:


16:27:00.424 -> +UUSORD: 0,1018
16:27:00.457 -> AT+USORD=0,512

16:27:00.457 -> +USORD: 0,512,"D9794378EA4405000000190000007D000000232D302B2000686C4C006566674546470000000000000000000000000000803F0000A0420000A0420000604100003041000060410000304100008C4200008C4200008C4200004842000016C30000803F0000A0420000A0420000484200004842696E7465726E65740000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030430000C8420000B4420000B4420000704200007042010000000000BE420000A0400000484200000040000020410000803F320000002B34313738363032353030300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064000000F04902000024744800002042A7200300AF200300B9200300C1200300CC200300D6200300002000420024004200280042002C0042003000420034004240420F000A0000000A00000008000000FFFFFFFFFFFFFFFF00366E01FFFFFFFF00C2010000000800FFFFFFFF01000000BC01002000000000C0430300E0430300A043030000000000000000000000"
16:27:00.490 -> 
16:27:00.490 -> OK
16:27:00.490 -> 
16:27:00.490 -> +UUSORD: 0,506
16:27:00.523 -> AT+USORD=0,512

16:27:00.523 -> +USORD: 0,506,"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000043000000000000000000000000000000000000000000000000000000000000004300000000000000000000000000000000000000000000000000000000000000430000000000000000000000000000000000000000000000000000000000000043000000000000000000000000000000000000000000000000000000000000004300000000000000000000000000000000000000000000000000000000000000430000000000000000000000000000000000000000000000000000000000000043000000000000000000000000000000000000000000000000000000000000003D2C02007D1102000000000090420300424B0200E34D0200E34D0200E34D0200E34D0200E34D0200E34D0200E34D0200E34D0200E34D0200FFFFFFFFFFFFFFFFFFFFFFFFFFFF00000100415343494900000000000000000000000000000000000000000000000000000041534349490000000000000000000000000000000000000000000000000000000000DD600000A5EF0000DD0601002D080100890B01005911010031120100D5120100ED120100F51C01001D1F010051240100DD8D01006D90010015A90100DDB90100B560000000000000"
16:27:00.561 -> 
16:27:00.561 -> OK
16:27:00.597 -> AT+USORD=0,512

16:27:00.834 -> ERROR
16:27:00.834 -> 
16:27:00.834 -> +UUSOCL: 0
16:27:00.834 -> AT+USOCL=0

16:27:00.834 -> ERROR
16:27:02.613 -> Timeout downloading update file at 49 bytes. Can't continue with update.

The socket is empty after the last 506 bytes. Then mrkgsm wants to read the next 512 bytes with

16:27:00.597 -> AT+USORD=0,512

This should not be a problem, normaly it goes like:


16:27:06.809 -> AT+USORD=0,512

16:27:06.809 -> +USORD: 0,0,""
16:27:06.809 -> 
16:27:06.809 -> OK

But here the modem responds with a error:


OK
16:32:59.783 -> AT+USORD=0,512

16:33:00.030 -> ERROR
16:33:00.065 -> 
16:33:00.065 -> +UUSOCL: 0
16:33:00.065 -> AT+USOCL=0

16:33:00.065 -> ERROR

Also the closing of the socket ends with a error.

The question is: Why is lenght 209 bytes while the socket is empty in this example:

16:32:59.783 -> AT+USORD=0,512

16:33:00.030 -> ERROR
16:33:00.065 -> 
16:33:00.065 -> +UUSOCL: 0
16:33:00.065 -> AT+USOCL=0

16:33:00.065 -> ERROR
16:33:01.822 -> Timeout downloading update file at 209 bytes. Can't continue with update.

So while counting down all the large number at the end the lenght and the avaiable bytes at the socket differs but it doesn't matter.. With


 while (length > 0) {
    if (!client2.readBytes(&b, 1)) // reading a byte with timeout
      break;
    file.write(b);
    length--;
  }
  file.close();
  client2.stop();
  if (length > 0) {
   // SD.remove(BIN_FILENAME);
    Serial.print("Timeout downloading update file at ");
    Serial.print(length);
    //dev
    file.close();
    client2.stop();
    Serial.println(" bytes. Can't continue with update.");
    return;
  }

it works perfectly.
IMHO:

  • I have to find the cause of the countdown dif
  • or i have to rewrite your error handler

What do you think about to compute a checksum of the stored file on the sd and compare with stored value on the server?

Do you see a problem to remove your check like i did in my hack? It seems that with some extra rounds of

16:27:06.809 -> AT+USORD=0,512

16:27:06.809 -> +USORD: 0,0,""

with NULL readouts of the sockets at the end, no more bytes goes to the file, so i see no problem.

I think i found it out: While downloading, the buffer reads out the socket buffer to empty and the socket is empty. Now every
AT+USORD=0,512 the 'lenght' variable in your code is -- but no bytes from the socket. So we see at the end of the procedure the amount of zero readouts of the socket.

`length' is sent by the upload tool. if the final received length doesn't match, some bytes are lost. they can be lost in the middle if buffer overflows happen.

You are right. This is vimdiff of the downloaded file and the compiled file:

The diffs seem to happen randomly.

I see no way for me to handle it properly with this method. ;=(

I also tried with the example from httpclient, more or less:

 char c;
        // Whilst we haven't timed out & haven't reached the end of the body
        while ( (http.connected() || http.available()) &&
               ((millis() - timeoutStart) < kNetworkTimeout) )
        {
            if (http.available())
            {
                c = http.read();
        // save this to the sd card          
                bodyLen--;
                // We read something, reset the timeout counter
                timeoutStart = millis();
            }
            else
            {
                // We haven't got any data, so let's pause to allow some to
                // arrive
                delay(kNetworkDelay);
            }
        }

With worser result.

Most of the downloaded file is identical to the one on the server,
with the Juraj method about 50 bytes of a 200 kB files differ, with the char method from the example the same but much more timeouts.

Questions:

  • Why these diffs?
  • Caused by?
  • Why is this flaky behaviour?
  • I agree but which buffer and why out of a sudden?
  • Am I the first to try this on the mkr1400?

I don't think you are first to try, but I can't find the older forum thread or issue report.

So if someone finds this thread AND a working solution, let us know!! Thx

I am also interested in a solution, unfortunately i am not able to do it myself.

The topic you are looking for is maybe this one:

@Modiot could you share your code please?

I've got an idea:
Use the internal modem http stack, save the file to the internal memory of the modem, check size, copy to the SD card and reboot.

Here's the sketch from the MKRGSM lib for the undocumented(???) GSMFileUtils class for the download to the internal file system:


/*
  Download a large file and store it into the GSM module filesystem.
 This sketch connects to a website through a MKR GSM 1400 board and
 downloads a large file and stores it into the filesystem of the GSM
 module.
 The file is processed in blocks of 512 bytes in order to save RAM.
 A block of data is read from the GSM module and the appended to a
 file created by the sketch.
 Circuit:
 * MKR GSM 1400 board
 * Antenna
 * SIM card with a data plan
 created 19 June 2020
 by Giampaolo Mancini
*/

// libraries
#include <MKRGSM.h>

GSMFileUtils fileUtils(false);

#include "helpers.h"


// Please enter your sensitive data in the Secret tab or arduino_secrets.h
// PIN Number
const char PINNUMBER[] = "";
// APN data
const char GPRS_APN[] = "internet";
const char GPRS_LOGIN[] = "";
const char GPRS_PASSWORD[] = "";

// initialize the library instance
GSMClient client;
GPRS gprs;
GSM gsmAccess;

// URL, path and port (for example: example.org)

void setup()
{
    // initialize serial communications and wait for port to open:
    Serial.begin(9600);
    while (!Serial) {
        ; // wait for serial port to connect. Needed for native USB port only
    }

    Serial.println("Starting Arduino web client.");

    fileUtils.begin();

    // List files on the GSM module's filesystem
    auto numberOfFiles = fileUtils.fileCount();
    Serial.print("Number of Files: ");
    Serial.println(numberOfFiles);
    Serial.println();

    printFiles(fileUtils);

    auto server = promptAndReadLine("Please, enter server name:", "arduino.cc");
    auto port = promptAndReadInt("Please, enter server port:", 80);
    auto filename = promptAndReadLine("Please, enter file name:", "asciilogo.txt");
    auto filesize = promptAndReadInt("Please, enter file size:", 2263);
    Serial.println("Connecting...");

    // connection state
    bool connected = false;

    // After starting the modem with GSM.begin()
    // attach the shield to the GPRS network with the APN, login and password
    while (!connected) {
        if ((gsmAccess.begin(PINNUMBER) == GSM_READY) && (gprs.attachGPRS(GPRS_APN, GPRS_LOGIN, GPRS_PASSWORD) == GPRS_READY)) {
            connected = true;
        } else {
            Serial.println("Not connected");
            delay(1000);
        }
    }

    // if you get a connection, report back via serial:
    if (client.connect(server.c_str(), port)) {
        Serial.println("connected");
        // Make a HTTP request:
        client.print("GET /");
        client.print(filename);
        client.println(" HTTP/1.1");
        client.print("Host: ");
        client.println(server);
        client.println("Connection: close");
        client.println();
    } else {
        // if you didn't get a connection to the server:
        Serial.println("Connection failed");
    }

    // Download and store block-by-block
    storeFileBuffered(filename, filesize);

    auto updateBinSize = fileUtils.listFile(filename);
    Serial.print(filename);
    Serial.print(" downloaded size: ");
    Serial.println(updateBinSize);

    numberOfFiles = fileUtils.fileCount();
    Serial.print("Number of Files: ");
    Serial.println(numberOfFiles);
    Serial.println();

    printFiles(fileUtils);

}

void loop()
{
    // if there are incoming bytes available
    // from the server, read them and print them:
    if (client.available()) {
        char r = client.read();
        if (r < 16)
            Serial.print(0);
        Serial.print(r, HEX);

    }

    // if the server's disconnected, stop the client:
    if (!client.available() && !client.connected()) {
        Serial.println();
        Serial.println("disconnecting.");
        client.stop();

        // do nothing forevermore:
        for (;;)
            ;
    }
}

void storeFileBuffered(String filename, uint32_t totalLen)
{
    Serial.print("Ready to download \"");
    Serial.print(filename);
    Serial.print("\" - len: ");
    Serial.print(totalLen);
    Serial.println(" bytes.");

    constexpr uint32_t len { 512 };

    uint32_t cycles = totalLen / len;
    uint32_t spares = totalLen % len;

    int totalRead { 0 };

    fileUtils.deleteFile(filename);

    Serial.print("Saving file in ");
    Serial.print(cycles + 1);
    Serial.print(" blocks. [");
    Serial.print(cycles);
    Serial.print(' ');
    Serial.print(len);
    Serial.print(" -bytes blocks and ");
    Serial.print(spares);
    Serial.println(" bytes].");

    bool is_header_complete = false;
    String http_header;

    // Skip the HTTP header
    while (!is_header_complete) {
        while (client.available()) {
            const char c = client.read();
            http_header += c;
            if (http_header.endsWith("\r\n\r\n")) {
                Serial.println("Header Complete");
                is_header_complete = true;
                break;
            }
        }
    }

    // Define download and save lambda
    auto downloadAndSaveTrunk = [filename](uint32_t len) {
        char buf[len] { 0 };
        uint32_t written { 0 };

        if (client.available())
            written = client.readBytes(buf, len);

        fileUtils.appendFile(filename, buf, written);
        return written;
    };

    // Define wrapper function
    auto saveTrunk = [&totalRead, downloadAndSaveTrunk](size_t iter, uint32_t len) {
        Serial.print("Block ");
        if (iter < 10) Serial.print(' '); if (iter < 100) Serial.print(' '); 
        Serial.print(iter);

        totalRead += downloadAndSaveTrunk(len);

        Serial.print(": ");
        Serial.print(len);
        Serial.print(" - ");
        Serial.print(totalRead);
        Serial.println();
    };

    // Download and save complete trunks + spares
    for (auto c = 0; c <= cycles; c++)
        saveTrunk(c, len);

    Serial.println();        

}

The source (also helper.h) can be found in the official MKRGSM lib.

Then, with the SerialGSMPasstrough you can play around:

So,there is plenty of memory on the modem (bytes):

10:50:40.901 -> AT+ULSTFILE=1

10:50:40.901 -> +ULSTFILE: 19562496

I tried with a 207 kB file - works flawless. The BIG advantage of this method is that eveything is done by the bsd os within the modem, the memory of the mkr is not affected and AFTER the http download we have the file present in one piece in a internal memory to copy in a row to the SD card.

Next steps are:
-Check the size of the downloaded file, maybe a checksum check? Any hints?
-Copy the file in stream to the SD card. Any elegant code proposals?