NTP conversion error using timelib.h

Hi,
I’ve been fighting with updating my rtc-module from ntp-server.
My code gives a correct timestamp back (can be checked on various conversion sites) but when using timelib.h to convert from unix epoch to human readable time I get very strange values.

I want 2018-12-31 10:10:15
but I get 2018-8-173 10:10:15

I have stared at this problem long now, but I can’t figure out whats wrong. Down code contains a lot of logging to serial due to my efforts to figure out the problem, but basically, at startup it retreives a NTP timestamp, converts it to unix epoch and also to y2k epoch (to be able to compare timestamp wiht local arduino time) and when it finds a difference I want to set rtc to ntp time. But the conversion from unix epoch to human readable time using timelib gets all screwed up…

The code is

#include <RtcDS3231.h>
#include <SoftwareSerial.h> // Since we want to use hw-serial for debugging
#include <Wire.h>

#include <TimeLib.h>
#include <WiFiEsp.h>
#include <WifiEspUdp.h>

const char ssid[] = "myssid";  //  your network SSID (name)
const char pass[] = "mypass";       // your network password

// NTP settings
char timeServer[] = "se.pool.ntp.org";
unsigned int localPort = 2390;  // local port to listen for UDP packets
const int NTP_PACKET_SIZE = 48;  // NTP timestamp is in the first 48 bytes of the message
const int UDP_TIMEOUT = 2000;    // timeout in miliseconds to wait for an UDP packet to arrive
byte packetBuffer[NTP_PACKET_SIZE]; // buffer to hold incoming and outgoing packets
const int timeZone = 1;     // Central European Time

// RTC module object
RtcDS3231<TwoWire> rtcModule(Wire);

// Wifi client object
WiFiEspClient client;

// SoftwareSerial object for Wifi module on pin 6 (RX) and 7 (TX).
// Wifi card needs to be preset to 9600b I.E. SoftwareSerials speed limit.
SoftwareSerial Serial1(6, 7);

// Set radio status
int status = WL_IDLE_STATUS;

// A UDP instance to let us send and receive packets over UDP (for NTP)
WiFiEspUDP Udp;

// Set up timelib
tmElements_t tm;
int Year, Month, Day, Hour, Minute, Second;

void setup()
{
  Serial.println();
  Serial.println("TimeNTP Lab");

  // Enable IC2 communication
  Wire.begin();

  Serial.begin(115200);

  // Start rtc module
  rtcModule.Begin();

  // Initialise serial for ESP module
  Serial1.begin(9600);
  WiFi.init(&Serial1);

  // Check for the presence of the shield
  if (WiFi.status() == WL_NO_SHIELD) {
    Serial.println("WiFi shield not present");

    // Dont continue
    while (true);
  }

  Serial.print("Connecting to ");
  Serial.println(ssid);
  WiFi.begin(ssid, pass);

  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }

  Serial.print("IP number assigned by DHCP is ");
  Serial.println(WiFi.localIP());
  Serial.println("Starting UDP");
  Udp.begin(localPort);

  Serial.println("waiting for sync");
  setSyncProvider(getNTPTime);
  setSyncInterval(300); // 300 sec = 5 min

  // Set clock to compile time to have a starting point
  RtcDateTime compiled = RtcDateTime(__DATE__, __TIME__);
  rtcModule.SetDateTime(compiled);

  setRTCTime();
}

time_t prevDisplay = 0; // when the digital clock was displayed

void loop()
{
  if (timeStatus() != timeNotSet) {
    if (now() != prevDisplay) { //update the display only if time has changed
      prevDisplay = now();
      Serial.print("NTP time is ");
      digitalClockDisplay();

      RtcDateTime localNow = rtcModule.GetDateTime();
      char localTimeString[19];
      sprintf(localTimeString, "%04d-%02d-%02d %02d:%02d:%02d", localNow.Year() , localNow.Month(), localNow.Day(), localNow.Hour(), localNow.Minute(), localNow.Second());
      Serial.print("Local time is ");
      Serial.println(localTimeString);
      
      Serial.println();
    }
  }
}

void setRTCTime() {
  // Compare local time with NTP time
  unsigned long currentTime = now(); // unix epoch
  //time_t currentTime = now(); // unix epoch
  if (currentTime > 0) {
    Serial.print("NTP Time in unix epoch is ");
    Serial.println(currentTime);
    unsigned long epoch2k = currentTime - 946684800;
    Serial.print("NTP time in epoch2k is ");
    Serial.println(epoch2k);

    RtcDateTime localNow = rtcModule.GetDateTime();
    Serial.print("Local Time in epoch2k is ");
    Serial.println(localNow);

    if (localNow != epoch2k) {
      Serial.println();
      Serial.println("Setting time...");

      Serial.print("NTP time is ");
      digitalClockDisplay();
      char timeString[19];
      //sprintf(timeString, "%02d/%02d/%04d %02d:%02d:%02d", day(), month(), year(), hour(), minute(), second());
      sprintf(timeString, "%04d-%02d-%02d %02d:%02d:%02d", year(), month(), day(), hour(), minute(), second());
      
      char localTimeString[19];
      sprintf(localTimeString, "%04d-%02d-%02d %02d:%02d:%02d", localNow.Year() , localNow.Month(), localNow.Day(), localNow.Hour(), localNow.Minute(), localNow.Second());
      Serial.print("Local time is ");
      Serial.println(localTimeString);

      Serial.print("Trying to set local time to ");
      Serial.println(timeString);
      createElements(timeString);
      setTime(makeTime(tm));
      Serial.println();
      
      //setTime(currentTime);
      setTime(currentTime);
      //rtcModule.SetDateTime(timeString);
    } else {
      Serial.println("Fick inget svar från NTP-tjänsten. Provar senare...");
    }
  }
}

void digitalClockDisplay()
{
  // digital clock display of the time
  Serial.print(year());
  Serial.print("-");
  Serial.print(month());
  Serial.print("-");
  Serial.print(day());
  Serial.print(" ");
  Serial.print(hour());
  printDigits(minute());
  printDigits(second());
  Serial.println();
}

void printDigits(int digits)
{
  // utility for digital clock display: prints preceding colon and leading 0
  Serial.print(":");
  if (digits < 10)
    Serial.print('0');
  Serial.print(digits);
}

long getNTPTime() {
  sendNTPpacket(timeServer); // send a NTP packet to the time server

  // wait for a reply for UDP_TIMEOUT miliseconds
  unsigned long startMs = millis();
  while (!Udp.available() && (millis() - startMs) < UDP_TIMEOUT) {}

  Serial.println(Udp.parsePacket());
  if (Udp.parsePacket()) {
    Serial.println("packet received");
    // We've received a packet, read the data from it and put it into the buffer
    Udp.read(packetBuffer, NTP_PACKET_SIZE);

    // the timestamp starts at byte 40 of the received packet and is four bytes,
    // or two words, long. First, extract the two words:
    unsigned long highWord = word(packetBuffer[40], packetBuffer[41]);
    unsigned long lowWord = word(packetBuffer[42], packetBuffer[43]);
    // combine the four bytes (two words) into a long integer
    // this is NTP time (seconds since Jan 1 1900):
    unsigned long ntp_timestamp = highWord << 16 | lowWord;

    unsigned long epoch = ntp_timestamp - 2208988800UL + timeZone * SECS_PER_HOUR;

    return epoch;
  }
  return 0; // Failed to get time or parse the packet
}

void sendNTPpacket(char *ntpSrv)
{
  // set all bytes in the buffer to 0
  memset(packetBuffer, 0, NTP_PACKET_SIZE);

  // Initialize values needed to form NTP request
  packetBuffer[0] = 0b11100011;   // LI, Version, Mode
  packetBuffer[1] = 0;     // Stratum, or type of clock
  packetBuffer[2] = 6;     // Polling Interval
  packetBuffer[3] = 0xEC;  // Peer Clock Precision
  // 8 bytes of zero for Root Delay & Root Dispersion
  packetBuffer[12]  = 49;
  packetBuffer[13]  = 0x4E;
  packetBuffer[14]  = 49;
  packetBuffer[15]  = 52;

  // all NTP fields have been given values, now send a packet requesting the timestamp:
  Udp.beginPacket(ntpSrv, 123); //NTP requests are to port 123
  Udp.write(packetBuffer, NTP_PACKET_SIZE);
  Udp.endPacket();
}

void createElements(const char *str)
{
  sscanf(str, "%d-%d-%d %d:%d:%d", &Year, &Month, &Day, &Hour, &Minute, &Second);
  tm.Year = CalendarYrToTm(Year);
  tm.Month = Month;
  tm.Day = Day;
  tm.Hour = Hour;
  tm.Minute = Minute;
  tm.Second = Second;
}

Good job using code tags with your first post.

But the conversion from unix epoch to human readable time using timelib gets all screwed up...

Is your problem with the timelib and NTC time stamp or with the RTC local time?

Does this produce the correct time with the timelib functions?

setSyncProvider(getNTPTime);
if (now() != prevDisplay) { //update the display only if time has changed
      prevDisplay = now();
      Serial.print("NTP time is ");
      digitalClockDisplay();

Or is the problem with the RTC and its display?

Serial.print("Local time is ");
      Serial.println(localTimeString);

Your problem statement doesn't make clear to me if the issue is with "NTP time is" or "Local time is".
Or both?

My code gives a correct timestamp back (can be checked on various conversion sites) but when using timelib.h to convert from unix epoch to human readable time I get very strange values.

Have you run the TimeNTP_ESP8266WiFi_TimeLib library example successfully?

Why use NTP and RTC? Just use ‘setSyncProvider(getNtpTime);’ to keep the internal millis-based time trued up and forget the RTC.

gfvalvo:
Why use NTP and RTC? Just use 'setSyncProvider(getNtpTime);' to keep the internal millis-based time trued up and forget the RTC.

yeah but what if you want the time when there is no internet connection ?

Deva_Rishi:
yeah but what if you want the time when there is no internet connection ?

Then you're going to have a hard time updating the RTC with NTP, which was the stated purpose of the code in the original post.

gfvalvo:
Then you're going to have a hard time updating the RTC with NTP, which was the stated purpose of the code in the original post.

if you get rid of the RTC all together this will be easier ? i figured the goal was to update the RTC and have it available offline updated.

Guess it depends on the project. An IoT project (for example) is kind of useless without an internet connection. So, the time is moot if there isn't one and can be updated when the connection returns. The internal time will freewheel without updates. But, it won't go away. Just lose accuracy.

Hi,

Thanks :slight_smile:

Sorry I wasn't clear. The problem is when converting the timestamp from ntp to "normal" time. The month and day of month get wrong. Year and time is fine.

I have a bit bigger project that I want to do, but I want it to be able to keep time after reboots even without internet. I extracted the parts regarding ntp to try to find the problem, so maybe the code here seems a bit confusing, but it's meant to fetch time from ntp and update the rtc with it, (using tons of logging..) so when I figured that out I can reuse relevant parts in the other project.

So when I set the rtc using compile time and extract time there are no issues. When getting that time and present it in readable form, it works fine.
When getting the timestamp from ntp and convert it to unix epoch and verifying via external site, the time stamp seems ok.

But when, using timelib, converting the very same unix epoch timestamp from ntp, I get that weird month and day. So there has to be some problem with my way of using the lib. The now() function in timelib returns a correct timestamp as I see it.

Hmmm, maybe a bug in the class RtcDateTime? Seems like the class does not convert the remaining days (month 8-12).

Please post an MCVE that simply demonstrates the problem without all the extraneous stuff in your current code.

Sorry I wasn’t clear. The problem is when converting the timestamp from ntp to “normal” time. The month and day of month get wrong. Year and time is fine.

Does this time library example run correctly?

/*
 * Time_NTP.pde
 * Example showing time sync to NTP time source
 *
 * This sketch uses the ESP8266WiFi library
 */
 
#include <TimeLib.h> 
#include <ESP8266WiFi.h>
#include <WiFiUdp.h>

const char ssid[] = "*************";  //  your network SSID (name)
const char pass[] = "********";       // your network password

// NTP Servers:
IPAddress timeServer(132, 163, 4, 101); // time-a.timefreq.bldrdoc.gov
// IPAddress timeServer(132, 163, 4, 102); // time-b.timefreq.bldrdoc.gov
// IPAddress timeServer(132, 163, 4, 103); // time-c.timefreq.bldrdoc.gov


const int timeZone = 1;     // Central European Time
//const int timeZone = -5;  // Eastern Standard Time (USA)
//const int timeZone = -4;  // Eastern Daylight Time (USA)
//const int timeZone = -8;  // Pacific Standard Time (USA)
//const int timeZone = -7;  // Pacific Daylight Time (USA)


WiFiUDP Udp;
unsigned int localPort = 8888;  // local port to listen for UDP packets

void setup() 
{
  Serial.begin(9600);
  while (!Serial) ; // Needed for Leonardo only
  delay(250);
  Serial.println("TimeNTP Example");
  Serial.print("Connecting to ");
  Serial.println(ssid);
  WiFi.begin(ssid, pass);
  
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }

  
  Serial.print("IP number assigned by DHCP is ");
  Serial.println(WiFi.localIP());
  Serial.println("Starting UDP");
  Udp.begin(localPort);
  Serial.print("Local port: ");
  Serial.println(Udp.localPort());
  Serial.println("waiting for sync");
  setSyncProvider(getNtpTime);
}

time_t prevDisplay = 0; // when the digital clock was displayed

void loop()
{  
  if (timeStatus() != timeNotSet) {
    if (now() != prevDisplay) { //update the display only if time has changed
      prevDisplay = now();
      digitalClockDisplay();  
    }
  }
}

void digitalClockDisplay(){
  // digital clock display of the time
  Serial.print(hour());
  printDigits(minute());
  printDigits(second());
  Serial.print(" ");
  Serial.print(day());
  Serial.print(".");
  Serial.print(month());
  Serial.print(".");
  Serial.print(year()); 
  Serial.println(); 
}



void printDigits(int digits){
  // utility for digital clock display: prints preceding colon and leading 0
  Serial.print(":");
  if(digits < 10)
    Serial.print('0');
  Serial.print(digits);
}

/*-------- NTP code ----------*/

const int NTP_PACKET_SIZE = 48; // NTP time is in the first 48 bytes of message
byte packetBuffer[NTP_PACKET_SIZE]; //buffer to hold incoming & outgoing packets

time_t getNtpTime()
{
  while (Udp.parsePacket() > 0) ; // discard any previously received packets
  Serial.println("Transmit NTP Request");
  sendNTPpacket(timeServer);
  uint32_t beginWait = millis();
  while (millis() - beginWait < 1500) {
    int size = Udp.parsePacket();
    if (size >= NTP_PACKET_SIZE) {
      Serial.println("Receive NTP Response");
      Udp.read(packetBuffer, NTP_PACKET_SIZE);  // read packet into the buffer
      unsigned long secsSince1900;
      // convert four bytes starting at location 40 to a long integer
      secsSince1900 =  (unsigned long)packetBuffer[40] << 24;
      secsSince1900 |= (unsigned long)packetBuffer[41] << 16;
      secsSince1900 |= (unsigned long)packetBuffer[42] << 8;
      secsSince1900 |= (unsigned long)packetBuffer[43];
      return secsSince1900 - 2208988800UL + timeZone * SECS_PER_HOUR;
    }
  }
  Serial.println("No NTP Response :-(");
  return 0; // return 0 if unable to get the time
}

// send an NTP request to the time server at the given address
void sendNTPpacket(IPAddress &address)
{
  // set all bytes in the buffer to 0
  memset(packetBuffer, 0, NTP_PACKET_SIZE);
  // Initialize values needed to form NTP request
  // (see URL above for details on the packets)
  packetBuffer[0] = 0b11100011;   // LI, Version, Mode
  packetBuffer[1] = 0;     // Stratum, or type of clock
  packetBuffer[2] = 6;     // Polling Interval
  packetBuffer[3] = 0xEC;  // Peer Clock Precision
  // 8 bytes of zero for Root Delay & Root Dispersion
  packetBuffer[12]  = 49;
  packetBuffer[13]  = 0x4E;
  packetBuffer[14]  = 49;
  packetBuffer[15]  = 52;
  // all NTP fields have been given values, now
  // you can send a packet requesting a timestamp:                 
  Udp.beginPacket(address, 123); //NTP requests are to port 123
  Udp.write(packetBuffer, NTP_PACKET_SIZE);
  Udp.endPacket();
}