Serial.print to Ethernet, instead of Serial Monitor

Greetings;

I have an Arduino Mega 2560 R3 with a W5500 and a bunch of sensors to monitor pool/home water quality. I use ArduinoOTA to upload the sketches via ethernet (way faster than USB, BTW!).

The project has a one-page web server, where I display sensor data every 5 seconds. I write to an SD card every 15mins and send an email with that file to myself once a day...

All this works, but at least during the beta testing, I'd like to also observe the Serial monitor output. Obviously, a 50m USB wire is out of the question, so my thought was to print to a second web page instead...

After setting up my web server (reserved IP by router):

void setupEthernetServer() {
  //SETUP ETHERNET WEB-SERVER
  // start the Ethernet connection and the server:
  //Ethernet.begin(mac, ip);  //set up with fixed IP
  Serial.print(F("Ethernet IP Request  "));
  tft.print(F("\nEthernet IP Req "));
  Ethernet.init(EthernetChipSelect);
  if (Ethernet.begin(mac) == 0) { // Start Ethernet with dynamic IP
    Serial.println(F("---> FAILED: no DHCP"));
    tft.setTextColor(RED);
    tft.print(F("---> FAILED: no DHCP"));
    tft.setTextColor(WHITE);

    Serial.print(F("Ethernet Shield      "));
    tft.print(F("\nEthernet Shield "));
    if (Ethernet.hardwareStatus() == EthernetNoHardware) { // Check for Ethernet hardware present
      Serial.println(F("---> FAILED: no Hardware\n"));
      tft.setTextColor(RED);
      tft.print(F("---> FAILED: no Hardware"));
      tft.setTextColor(WHITE);
    } else {
      Serial.println(F("---> OK"));
      tft.print(F("---> OK"));

      Serial.print(F("Ethernet Cable       "));
      tft.print(F("\nEthernet Cable  "));
      if (Ethernet.linkStatus() == LinkOFF) {
        Serial.println(F("---> FAILED: no Cable\n")); // not connected
        tft.setTextColor(RED);
        tft.print(F("---> FAILED: no Cable\n"));
        tft.setTextColor(WHITE);
      } else {
        Serial.println(F("---> OK"));
        tft.print(F("---> OK\n"));
      }
    }
    errorSetup = true;
    ErrorSetup = ErrorSetupDelay;
    return; // or not!
  } //eof (Ethernet.begin(mac) == 0)

  server.begin();   // start the server
  Serial.print(F("---> OK at: ")); Serial.println(Ethernet.localIP());
  tft.print(F("---> OK "));
  tft.print(Ethernet.localIP());
  tft.print(F("\n"));
  ArduinoOTA.begin(Ethernet.localIP(), "Orientes", "I5tanbul", InternalStorage); // start the OTEthernet library with internal (flash) based storage
} //eof setupEthernetServer()

my current 5 second web update looks like this:

void listenForClient() {
  client = server.available();
  if (client) {
    //Serial.println(F("new client"));
    boolean currentLineIsBlank = true;    // an http request ends with a blank line
    while (client.connected()) {
      if (client.available()) {
        char c = client.read();
        // Serial.write(c);
        // if you've gotten to the end of the line (received a newline
        // character) and the line is blank, the http request has ended,
        // so you can send a reply
        // if (c == 'Z' && !programOTA && !internetOTA) internetOTA = true; // if a 'Z' was input, that means we are in OTA programming mode...

        if (c == '\n' && currentLineIsBlank) {
          // send a standard http response header
          client.println(F("HTTP/1.1 200 OK"));
          client.println(F("Content-Type: text/html"));
          client.println(F("Connection: close"));  // the connection will be closed after completion of the response
          client.println(F("Refresh: 5"));  // refresh the page automatically every 5 sec
          client.println();
          client.println(F(""));
          client.println(F(""));

          //send time stamp to web page
          client.print(day()); client.print(F("-"));
          client.print(monthShortStr(month()));  client.print(F("-"));
          client.print(year());  client.print(F(" "));
          client.print(hour());
          webDigits(minute());
          webDigits(second());
          client.print(F("
"));
          client.print(F("
"));

          // send House values to web page
          client.print(F("Rain Water: "));
          client.print(pReading[rainTank - 8], 2);
          client.print(F(" m3"));
          client.print(F("
"));

          client.print(F("Line Pressure: "));
          client.print(pReading[houseLine - 8], 2);
          client.print(F(" bars"));
          client.print(F("
"));

          client.print(F("Recirc Pressure: "));
          client.print(pReading[recircFilter - 8], 2);
          client.print(F(" bars"));
          client.print(F("
"));
          client.print(F("
"));

          // send Pool Values to Web Page
          client.print(F("Pool EC: "));
          client.print(ezoReading[eC], 0);
          client.print(F(" uS"));
          client.print(F("
"));

          client.print(F("Pool Temp: "));
          client.print(ezoReading[rTD], 1);
          client.print(F(" degC"));
          client.print(F("
"));

          client.print(F("Pool pH: "));
          client.print(ezoReading[pH], 2);
          client.print(F(" pH"));
          client.print(F("
"));

          client.print(F("Pool Filter: "));
          client.print(pReading[poolFilter - 8], 2);
          client.print(F(" bars"));
          client.print(F("
"));

          client.print(F("Balance Tank: "));
          client.print(pReading[houseLine - 8], 0);
          client.print(F(" Percent"));
          client.print(F("
"));

          client.println(F(""));
          break;
        }
        if (c == '\n') currentLineIsBlank = true; // you're starting a new line
        else if (c != '\r') currentLineIsBlank = false; // you've gotten a character on the current line
      } //eof if(client.available)
    } //eof while
    // give the web browser time to receive the data
    delay(1);
    // close the connection:
    client.stop();
    //Serial.println(F("client disconnected"));
  }   //eof if(client)
} //eof listenForClient

void webDigits(int digits) {      // utility for digital clock display: prints preceding colon and leading 0
  client.print(F(":"));
  if (digits < 10)  client.print('0');
  client.print(digits);
}

The idea is to print ALL Serial.print and Serial.println statements to an ethernet serial monitor. I'm guessing that this would take a form similar to what this above 5secs page does.

But other than that, I'm completely lost:

1- how do I specify writing to a second page of 192.168.x.xxx ? 2- Does EVERY mention of "Serial.print" run through a version of the above, or can I drop some of those lines after the first call? 3- will there be a history of the Serial.prints available, or would the page start from where I point a browser to it? 4- if no to the above, I would have to write into another log file and dump this onto the web page upon browser pointer, correct? 5- How would I decide at compile time to either compile with Serial.print (lab/office setting) or Ethernet.print (remote setting)? The trigger would be PORT = ComX -> Serial.Print, PORT = ArduinoOTA(x.x.x.x) -> Ethernet.print... 6- This is slightly off topic, but still relates to Ethernet. I use Smtp2Go to send the daily email. Many on the Forum think that it is unsafe to do so, due to hackability. What else could I do? a) send the file to a server behind my firewall every day (I have an old NAS sitting there)? b) Would the Arduino Cloud be any safer and potentially help me display the information too? c) something else?

Some are very basic questions. Sorry about that. Any of the "HTML Basics" links I found on the web seem to overstretch my axons...

Cheers

If you feel ambitious and want to try a different approach take a look at the ESP32, you will get about 1 Meg usable program memory and about 500 Meg RAM. There are many pieces of code on the web doing OTA, however all I have seen is WiFi. You would also get 2 cores to do your processing allowing threading in your code. A 50 meter wire would be OK, just use an appropriate interface such as RS232, RS485 etc. There are USB to CAT x adapters to extend the USB connection, I have a set I purchased about two years ago and they work great. Look a little deeper into what you want to accomplish and break the questions down into something more digestible. No clue as to what overstretch my axons... indicates as you did not define axons.

Thanks Gil,

I looked long and hard at Ethernet vs WiFi and decided at the time that given the noisy environment (several pumps, water ionizers, relays, etc), that a good solid, shielded Cat6A connection would likely perform better than WiFi. We're sitting 3m underground and redundant Cat6A cables go from there to my IT closet... Hence W5500.

Having a hard enough time programming one core, let alone two...

Will take your advice and parse out the post into separate sections, to see, if I get more bites.

https://en.wikipedia.org/wiki/Axon, kinda like that 50m cable connecting two cores...

Cheers

in order to support another page, you have to start parsing the headers you receive from the client. The first line will have the 'file' that the client wants to receive.

For a page request, it will look like:

GET /home.html HTTP/1.1

I wrote a simple web server for Arduino that runs on a MEGA, you could look at that for ideas. It's in the Library Manager as YAAWS - Yet Another Arduino Web Server. It serves up files from an SD card, and it handles the GET parsing.

my TelnetStream library

just two comments based on that code:

          client.println(F("HTTP/1.1 200 OK"));
          client.println(F("Content-Type: text/html"));
          client.println(F("Connection: close"));  // the connection will be closed after completion of the response
          client.println(F("Refresh: 5"));  // refresh the page automatically every 5 sec

a) you shouldn't use client.print with the F-Makro. Instead use Juraj's StreamLib Library to buffer the client print. it will reduce network overhead and make the result in the browser faster https://github.com/jandrassy/StreamLib

b) instead of a refresh every 5 seconds, consider to use fetch API and update the values on the page only.

noiasca:
just two comments based on that code:

          client.println(F("HTTP/1.1 200 OK"));

client.println(F(“Content-Type: text/html”));
         client.println(F(“Connection: close”));  // the connection will be closed after completion of the response
         client.println(F(“Refresh: 5”));  // refresh the page automatically every 5 sec




a) you shouldn't use client.print with the F-Makro. Instead use Juraj's StreamLib Library to buffer the client print.
it will reduce network overhead and make the result in the browser faster
https://github.com/jandrassy/StreamLib

b) instead of a refresh every 5 seconds, consider to use fetch API and update the values on the page only.

clarification for a). the F macro is ok if you use buffering. without buffering every character from F macro string is sent as single TCP packet

Thanks MHotchin, Juraj (yet again) and noiasca (also). Your suggestions will give me something to chew on for the near future. It's rainy season here in Medellin and Covid everywhere!

Cheers;

it is simple to add BufferedPrint. it wraps the client object (or other Print derived type).

        char buff[RESPONSE_BUFFER_SIZE];
        BufferedPrint response(client, buff, sizeof(buff));
        response.println(F("HTTP/1.1 404 Not Found"));
        response.println(F("Connection: close"));
        response.print(F("Content-Length: "));
        response.println(strlen(" not found") + strlen(fn));
        response.println();
        response.print(fn);
        response.print(F(" not found"));
        response.flush();

don't forget flush()

gilshultz:
…take a look at the ESP32, you will get about 1 Meg usable program memory and about 500 Meg RAM.

“500 Meg RAM”?? Color me skeptical…

Juraj: my TelnetStream library

Thanks Juraj; Just ran some tests on this and it seems to be exactly what I'm looking for.

Before I proceed with implementation, point #5 of my initial post comes up again, slightly re-formulated:

5- How would I decide at compile time to either compile with Serial.print (lab/office setting) or Telnet.print (remote setting)? The TEST would be PORT = ComX -> Serial.Print, PORT = ArduinoOTA(x.x.x.x) -> Telnet.print...

Reading in https://www.deviceplus.com/arduino/arduino-preprocessor-directives-tutorial/ it seems like I can add the following define

#define Serial TelnetPrint

at the beginning of the code and then compile. That would keep it manual, of course, having to remember to comment this out in the lab.

I'm thinking of something like this, although I have no Idea on how to define PORT

#if (PORT == COMxx)

  #define Serial TelnetPrint
  #warning "compiled for TELNET"

#elif (PORT == ArduinoOTA(x.x.x.x))

 #warning "compiled for COMxx"

#else

 //compile this entirely different code

#endif

Hoping someone can direct me.

Cheers.

trilife: Thanks Juraj; Just ran some tests on this and it seems to be exactly what I'm looking for.

Before I proceed with implementation, point #5 of my initial post comes up again, slightly re-formulated:

5- How would I decide at compile time to either compile with Serial.print (lab/office setting) or Telnet.print (remote setting)? The TEST would be PORT = ComX -> Serial.Print, PORT = ArduinoOTA(x.x.x.x) -> Telnet.print...

I would print to Serial only until the network starts and the only print to TelnetStream.

Juraj: I would print to Serial only until the network starts and the only print to TelnetStream.

Yes, just figured that out!

trilife: Yes, just figured that out!

but it depends. If you inspect my Regulator project you will see that I print to both. but it is in a 'central' logging function called at the end of the loop(). the function prints a complete set of relevant values as one line. before I implemented logging to SD card, telnet was a way how to create a log file. I redirected the telnet client on the PC to a file to inspect the values later in Excel

Juraj: but it depends. If you inspect my Regulator project you will see that I print to both. but it is in a 'central' logging function called at the end of the loop(). the function prints a complete set of relevant values as one line. before I implemented logging to SD card, telnet was a way how to create a log file. I redirected the telnet client on the PC to a file to inspect the values later in Excel

That's similar to what I have now. On file to log Housewater values, the other Poolchemistry. I email these to myself once a day. Without Telnet, I would probably have had a 3rd file for error/process logs. But for debugging, Telnet seems to be the way to go. I can OTA flash the project in situ and observe from the comfort of my office

noiasca: b) instead of a refresh every 5 seconds, consider to use fetch API and update the values on the page only.

Hello @noiasca; Thanks for the comments. Now, I started looking at your recommendation about fetch-API and cannot find an example implemented in Arduino. I'm not fluent enough in HTML or XML (my Mandarin is way better!) to modify what's out there from. Would you be able to point me in the right direction please? Cheers.

I don't have that information in English

but you would need following:

  • a HTML file/resource wherein you have placeholders marked with id= to make them accessible from a JavaScript
  • using a fetch API to get new data from the Arduino periodically (more or less a GET reques to your Arduino)
  • a resource ("file") on your webserver which responds the new data. I prefer JSON as data format, because it makes parsing of the data components very easy.

these a the 3 needed components to make an automatic reload of data

noiasca: I don't have that information in English

Thanks Noiasca. You can send it to me in French, German, Spanish or Turkish....

not as simple as an example, but maybe you can modify it for your data. https://github.com/jandrassy/IsgModbusTcpSG/tree/master/IsgModbusTcpSG see the WebServer.ino and the content of the folder 'data' it uses SD card for static files, but it works too if you open the index.html on a computer. it will get the data from Arduino. (set the IP address of the Arduino in script.js)

trilife: You can send it to me in French, German, Spanish or Turkish....

wow, impressive. I still struggle with English, but ok.

Here is a quick try of a HTML, a JSON and a JavaScript using the fetch API: https://werner.rothschopf.net/microcontroller/202011_arduino_webserver_fetch_api_en.htm