UDP NTP Update Error [SOLVED]

Hi

I have some code here which is mostly working really well but then - sometimes, for some reason, it sets the clock to 2036. And it reliably sets it wrongly, so to speak, so I think it's a programming error.

This is the output from the serial monitor:-

Time to set the clock initially
RTC before: 2012-02-18  16:28:03  
RTC after : 2012-02-18  16:28:05  
Time update successful

Time to set the clock at 16:28:10
RTC before: 2012-02-18  16:28:10  
RTC after : 2036-02-07  06:28:17  
Time update successful

And here the code:-

#include <Bounce.h>
#include <Wire.h>
#include "RTClib.h"
#include <Servo.h>
#include <SPI.h>         
#include <Ethernet.h>
#if ARDUINO < 100
#include <UDP.h>
#else
#include <EthernetUdp.h>	// New from IDE 1.0
#endif
byte mac[] = { 
  0x00, 0x11, 0x22, 0x33, 0xFB, 0x11 }; 
EthernetClient client;
unsigned int localPort = 8888;             
byte timeServer[] = {
  193, 79, 237, 14};    // ntp1.nl.net NTP server  
const int NTP_PACKET_SIZE= 48;             // NTP time stamp is in the first 48 bytes of the message
byte pb[NTP_PACKET_SIZE];                  // buffer to hold incoming and outgoing packets 
#if ARDUINO >= 100
EthernetUDP Udp;
#endif		
Servo servo1;
#define ledPin1 8      		// the number of the LED pin
#define buttonPin 2		// the number of the button pin
const int pinSpeaker = 7;

int buttonState = LOW;		// used to set the button state
int LedState1 = LOW;		// used to set the LED state
int LedState2 = LOW;	        // used to set the LED state if LED was already on
long LedOnDuration = 10000;	// time to keep the LED on for (10s)
long FlashRate = 400;		// time to keep the LED on for (0.4s)
long previousMillis1 = 0;	// will store last time LED was updated for press 1
long previousMillis2 = 0;       // will store last time LED was updated for press 2

//The pips
unsigned int pip1 = 0;
unsigned int pip2 = 0;
unsigned int pip3 = 0;
unsigned int pip4 = 0;
unsigned int pip5 = 0;
unsigned int pip6 = 0;
unsigned int RTCUD = 0;
unsigned int INITIALSET = 0;


// Instantiate a Bounce object with a 20 millisecond debounce time
Bounce nursery = Bounce( buttonPin, 100);
int nurseryvalue;

RTC_DS1307 RTC;

enum {
  OFF, ON, FLASH};

void setup() 
{
  pinMode(buttonPin,INPUT);      
  pinMode(ledPin1,OUTPUT);      
  pinMode(pinSpeaker, OUTPUT);
  digitalWrite(ledPin1, OFF);
  servo1.attach(6);
  LedState1 = OFF;
  LedState2 = OFF;
  Serial.begin(57600);
  Wire.begin();
  RTC.begin();
  Serial.println("Annunciator Panel v0.6");
  Ethernet.begin(mac);	   
  Udp.begin(localPort);
  Serial.println();
  if (! RTC.isrunning()) {
    Serial.println("The RTC is NOT running/set!");

  }
}

void loop() {
  DateTime now = RTC.now();
  nursery.update ( ); 
  nurseryvalue = nursery.read();
  unsigned long currentMillis = millis();


  if (millis() > 30000 && INITIALSET < 1)   
  {
    Serial.print("Time to set the clock initially");
    Serial.println();
    RTCUD = 1;
  }
  //  if (now.hour() == 17 && now.minute() == 50 && now.second() == 10) 
  if (now.hour() == 16 && now.minute() == 28 && now.second() == 10)  
  {
    if (RTCUD < 1)  
    {
      Serial.print("Time to set the clock at 16:28:10");
      Serial.println();
      RTCUD = 1;
    }
  }
  if (now.hour() == 16 && now.minute() == 28 && now.second() == 50)  
  {
    if (RTCUD < 1)  
    {
      Serial.print("Time to set the clock at 16:28:50");
      Serial.println();
      RTCUD = 1;
    }
  }

  if (RTCUD == 1)
  {
    Serial.print("RTC before: ");
    PrintDateTime(RTC.now());
    Serial.println();
    sendNTPpacket(timeServer);
    delay(1000);
    if ( Udp.available() ) {

#if ARDUINO < 100
      Udp.readPacket(pb, NTP_PACKET_SIZE);
#else
      Udp.read(pb, NTP_PACKET_SIZE);      // New from IDE 1.0,
#endif	


      unsigned long t1, t2, t3, t4;
      t1 = t2 = t3 = t4 = 0;
      for (int i=0; i< 4; i++)
      {
        t1 = t1 << 8 | pb[16+i];      
        t2 = t2 << 8 | pb[24+i];      
        t3 = t3 << 8 | pb[32+i];      
        t4 = t4 << 8 | pb[40+i];
      }


      float f1,f2,f3,f4;
      f1 = ((long)pb[20] * 256 + pb[21]) / 65536.0;      
      f2 = ((long)pb[28] * 256 + pb[29]) / 65536.0;      
      f3 = ((long)pb[36] * 256 + pb[37]) / 65536.0;      
      f4 = ((long)pb[44] * 256 + pb[45]) / 65536.0;
      const unsigned long seventyYears = 2208988800UL;
      t1 -= seventyYears;
      t2 -= seventyYears;
      t3 -= seventyYears;
      t4 -= seventyYears;
      t4 += 1;               
      if (f4 > 0.4) t4++;    
      RTC.adjust(DateTime(t4));

      Serial.print("RTC after : ");
      PrintDateTime(RTC.now());
      Serial.println();

      Serial.println("Time update successful");
      RTCUD = 0;      // RTC has been set for the morning / evening
      INITIALSET = 1; // RTC has been initially set
    }
    else
    {
      Serial.println("Time update unsuccessful");
    }
  }


  if (nurseryvalue == HIGH)
  {
    if (buttonState == OFF)
    {
      // button pressed
      //===============
      buttonState = ON;
      if (LedState1 == OFF)
      {
        // the button's pressed and LED is OFF
        // turn it ON
        //====================================
        Serial.print("button 1 pressed @");
        Serial.print(now.hour(), DEC);
        Serial.print(':');
        Serial.print(now.minute(), DEC);
        Serial.print(':');
        Serial.print(now.second(), DEC);
        Serial.println();
        digitalWrite(ledPin1, ON);
        LedState1 = ON;
        /*servo1.write(230);
         delay(1500);              // wait for a second
         */
        previousMillis1 = currentMillis;
      }
      else if (LedState1 == ON)
      { 		
        // if the button's pressed and LED is ON
        // start FLASHing
        //======================================
        digitalWrite(ledPin1, OFF);
        LedState1 = FLASH;
        /*
        servo1.write(230);
         delay(1500);
         */
        Serial.print("button 2 pressed @");
        Serial.print(now.hour(), DEC);
        Serial.print(':');
        Serial.print(now.minute(), DEC);
        Serial.print(':');
        Serial.print(now.second(), DEC);
        Serial.println();
        previousMillis1 = currentMillis;	
      }
    }
  }
  else
  {
    buttonState = OFF;
    servo1.write(30);
  }

  if (previousMillis1 + LedOnDuration < currentMillis) 
  {		
    // If the predefined interval has elapsed for the first button press
    // turn led OFF
    //==================================================================
    digitalWrite(ledPin1, OFF);
    LedState1 = OFF;
  }

  if (LedState1 == FLASH && previousMillis2 + FlashRate < currentMillis) 
  {        	
    // If the predefined interval has elapsed for the second button press
    // make the LED 'flash' by toggling it
    //===================================================================
    digitalWrite(ledPin1, !digitalRead(ledPin1));
    previousMillis2 = currentMillis;
  }

  
}

void PrintDateTime(DateTime t)
{
  char datestr[24];
  sprintf(datestr, "%04d-%02d-%02d  %02d:%02d:%02d  ", t.year(), t.month(), t.day(), t.hour(), t.minute(), t.second());
  Serial.print(datestr);  
}


// send an NTP request to the time server at the given address 
unsigned long sendNTPpacket(byte *address)
{
  // set all bytes in the buffer to 0
  memset(pb, 0, NTP_PACKET_SIZE); 
  // Initialize values needed to form NTP request
  // (see URL above for details on the packets)
  pb[0] = 0b11100011;   // LI, Version, Mode
  pb[1] = 0;     // Stratum, or type of clock
  pb[2] = 6;     // Polling Interval
  pb[3] = 0xEC;  // Peer Clock Precision
  // 8 bytes of zero for Root Delay & Root Dispersion
  pb[12]  = 49; 
  pb[13]  = 0x4E;
  pb[14]  = 49;
  pb[15]  = 52;

  // all NTP fields have been given values, now
  // you can send a packet requesting a timestamp: 		   
#if ARDUINO < 100
  Udp.sendPacket( pb,NTP_PACKET_SIZE,  address, 123); //NTP requests are to port 123
#else
  // IDE 1.0 compatible:
  Udp.beginPacket(address, 123); //NTP requests are to port 123
  Udp.write(pb,NTP_PACKET_SIZE);
  Udp.endPacket(); 
#endif	  
}

Any idea why the second setting (i.e. the timed one, not the initial set) sets the time to 2036? I think 2036 is one of the values returned in the UDP message...

THANKS

If t1 is 0, how much do you suppose t1 << 8 is?

What are f1, f2, f3 and f4 for?

The time actually used to set the clock is the value in t4 only. t1, t2, and t3 are not used.

In other words, you are extracting the data from the packet incorrectly.

PaulS:
If t1 is 0, how much do you suppose t1 << 8 is? What are f1, f2, f3 and f4 for?

Are you asking because you don't know or because you do but you want me to do some thinking? Either is fine but I'm unclear over your intention.
I don't know what << does. Is it a check to find out if t1 is of smaller order than 8?

PaulS:
The time actually used to set the clock is the value in t4 only. t1, t2, and t3 are not used.
In other words, you are extracting the data from the packet incorrectly.

Well that's very odd. Because as you see, the exact same packet extraction successfully sets the time initially.

Are you asking because you don't know or because you do but you want me to do some thinking?

I know the answer (0). I want you to do some thinking/research.

I don't know what << does.

Then you need to learn. << - Arduino Reference

Is it a check to find out if t1 is of smaller order than 8?

Not even close.

Because as you see, the exact same packet extraction successfully sets the time initially.

Actually, that may not be true. You are asking for a response for the NTP server, ans assuming that you get one. You should not be parsing the packet unless you got one. You need to add some more if statements, and Serial.print() statements to confirm that the same code is used.

I would guess that the time is not actually adjust in the first case, and IS adjusted, incorrectly, in the second case.

PaulS:

Because as you see, the exact same packet extraction successfully sets the time initially.

Actually, that may not be true. You are asking for a response for the NTP server, ans assuming that you get one. You should not be parsing the packet unless you got one. You need to add some more if statements, and Serial.print() statements to confirm that the same code is used.

I would guess that the time is not actually adjust in the first case, and IS adjusted, incorrectly, in the second case.

Interesting. Thanks for the pointers.

However, the time must be getting updated in the first case as it can correctly set the time back from 2036 to 2012.

You see, if I press 'reset' on the board once the time has been incorrectly set to 2036, the time will get set to 2012. So that's definitely coming from the UDP response. But then the second setting of it messes it up, meaning the second "if time..." check won't return a true value. I will add some serial.print() statements but I strongly suspect the wrong packet is being extracted because it's too much of a coincidence that 2036 is one of the values sent in the UDP response:-

NTP2RTC 0.5b
network ...
rtc ...

RTC before: 2036-02-07  06:36:32  
T1 .. T4 && fractional parts
2036-02-07  06:28:46  0.3137
2012-02-18  18:46:12  0.4521
2036-02-07  06:28:16  0.0000
2012-02-18  18:46:26  0.7903

RTC after : 2012-02-18  18:46:28  
done ...

Thanks, Paul.

I'd investigate what is in the packet that you get, and how big that packet is. Is it the same size, every time?

Any idea why the second setting (i.e. the timed one, not the initial set) sets the time to 2036? I think 2036 is one of the values returned in the UDP message...

2036-02-07 is the day that the second counter in 32 bits NTP will overflow. so somewhere there is a value 0xFFFF FFFF.

possible causes:

  1. The RTC resets itself to that value.
  2. local variable is reset to -1 == 0xFFFF FFFF
  3. NTP packet returned contains 0xFFFF FFFF for some (un)known reason
  4. the inital packet content = 0x0000 0000 which is decreased somewhere (there is no packet received,)
    e.g. local timezone correction ==> causing an underflow rsulting in 0xFFFF xxxx which represents 7 febr 2036, a few hours before overflow.

Hopes this helps,

robtillaart:

Any idea why the second setting (i.e. the timed one, not the initial set) sets the time to 2036? I think 2036 is one of the values returned in the UDP message...

2036-02-07 is the day that the second counter in 32 bits NTP will overflow. so somewhere there is a value 0xFFFF FFFF.

possible causes:

  1. The RTC resets itself to that value.
  2. local variable is reset to -1 == 0xFFFF FFFF
  3. NTP packet returned contains 0xFFFF FFFF for some (un)known reason
  4. the inital packet content = 0x0000 0000 which is decreased somewhere (there is no packet received,)
    e.g. local timezone correction ==> causing an underflow rsulting in 0xFFFF xxxx which represents 7 febr 2036, a few hours before overflow.

Hopes this helps,

Hi Rob. When I run the below code (which is 99% yours, I've just modified it to get an address via DHCP - nothing to do with the time function), I get the below result.

/*
 * NAME: NTP2RTC
 * DATE: 2012-02-19
 *  URL: http://www.arduino.cc/playground/Main/DS1307OfTheLogshieldByMeansOfNTP
 *
 * PURPOSE:
 * Get the time from a Network Time Protocol (NTP) time server
 * and store it to the RTC of the adafruit logshield
 *
 * NTP is described in:
 * http://www.ietf.org/rfc/rfc958.txt (obsolete)
 * http://www.ietf.org/rfc/rfc5905.txt 
 *
 * based upon Udp NTP Client, by Michael Margolis, mod by Tom Igoe
 * uses the RTClib from adafruit (based upon Jeelabs)
 * Thanx!
 * mod by Rob Tillaart, 10-10-2010
 * 
 * This code is in the public domain.
 * 
 */

// libraries for ethershield
#include <SPI.h>         
#include <Ethernet.h>

#if ARDUINO >= 100
#include <EthernetUdp.h>	// New from IDE 1.0
#else
#include <UDP.h>
#endif	

// libraries for realtime clock
#include <Wire.h>
#include <RTClib.h>

RTC_DS1307 RTC;


// Enter a MAC address for your controller below.
// Newer Ethernet shields have a MAC address printed on a sticker on the shield
byte mac[] = { 0x00, 0xAA, 0xBB, 0xCC, 0xDE, 0x02 };
// Initialize the Ethernet client library
EthernetClient client;

unsigned int localPort = 8888;             // local port to listen for UDP packets

// find your local ntp server http://www.pool.ntp.org/zone/europe or 
// http://support.ntp.org/bin/view/Servers/StratumTwoTimeServers
// byte timeServer[] = {192, 43, 244, 18}; // time.nist.gov NTP server
byte timeServer[] = {193, 79, 237, 14};    // ntp1.nl.net NTP server  

const int NTP_PACKET_SIZE= 48;             // NTP time stamp is in the first 48 bytes of the message

byte pb[NTP_PACKET_SIZE];                  // buffer to hold incoming and outgoing packets 

#if ARDUINO >= 100
// An EthernetUDP instance to let us send and receive packets over UDP
EthernetUDP Udp;		// New from IDE 1.0
#endif	


///////////////////////////////////////////
//
// SETUP
// 
void setup() 
{
  Serial.begin(19200);
  Serial.println("NTP2RTC 0.5");

  // start Ethernet and UDP

  Ethernet.begin(mac);	   // For when you are directly connected to the Internet.
  Udp.begin(localPort);
  Serial.println("network ...");

  // init RTC
  Wire.begin();
  RTC.begin();
  Serial.println("rtc ...");
  Serial.println();
}

///////////////////////////////////////////
//
// LOOP
// 
void loop()
{
  Serial.print("RTC before: ");
  PrintDateTime(RTC.now());
  Serial.println();

  // send an NTP packet to a time server
  sendNTPpacket(timeServer);

  // wait to see if a reply is available
  delay(1000);

  if ( Udp.available() ) {
    // read the packet into the buffer
#if ARDUINO >= 100
    Udp.read(pb, NTP_PACKET_SIZE);      // New from IDE 1.0,
#else
    Udp.readPacket(pb, NTP_PACKET_SIZE);
#endif	

    // NTP contains four timestamps with an integer part and a fraction part
    // we only use the integer part here
    unsigned long t1, t2, t3, t4;
    t1 = t2 = t3 = t4 = 0;
    for (int i=0; i< 4; i++)
    {
      t1 = t1 << 8 | pb[16+i];      
      t2 = t2 << 8 | pb[24+i];      
      t3 = t3 << 8 | pb[32+i];      
      t4 = t4 << 8 | pb[40+i];
    }

    // part of the fractional part
    // could be 4 bytes but this is more precise than the 1307 RTC 
    // which has a precision of ONE second
    // in fact one byte is sufficient for 1307 
    float f1,f2,f3,f4;
    f1 = ((long)pb[20] * 256 + pb[21]) / 65536.0;      
    f2 = ((long)pb[28] * 256 + pb[29]) / 65536.0;      
    f3 = ((long)pb[36] * 256 + pb[37]) / 65536.0;      
    f4 = ((long)pb[44] * 256 + pb[45]) / 65536.0;

    // NOTE:
    // one could use the fractional part to set the RTC more precise
    // 1) at the right (calculated) moment to the NEXT second! 
    //    t4++;
    //    delay(1000 - f4*1000);
    //    RTC.adjust(DateTime(t4));
    //    keep in mind that the time in the packet was the time at
    //    the NTP server at sending time so one should take into account
    //    the network latency (try ping!) and the processing of the data
    //    ==> delay (850 - f4*1000);
    // 2) simply use it to round up the second
    //    f > 0.5 => add 1 to the second before adjusting the RTC
    //   (or lower threshold eg 0.4 if one keeps network latency etc in mind)
    // 3) a SW RTC might be more precise, => ardomic clock :)


    // convert NTP to UNIX time, differs seventy years = 2208988800 seconds
    // NTP starts Jan 1, 1900
    // Unix time starts on Jan 1 1970.
    const unsigned long seventyYears = 2208988800UL;
    t1 -= seventyYears;
    t2 -= seventyYears;
    t3 -= seventyYears;
    t4 -= seventyYears;

    
    Serial.println("T1 .. T4 && fractional parts");
    PrintDateTime(DateTime(t1)); Serial.println(f1,4);
    PrintDateTime(DateTime(t2)); Serial.println(f2,4);
    PrintDateTime(DateTime(t3)); Serial.println(f3,4);
    PrintDateTime(DateTime(t4)); Serial.println(f4,4);
    Serial.println();

    // Adjust timezone and DST... in my case substract 4 hours for Chile Time
    // or work in UTC?
    t4 -= (3 * 3600L);     // Notice the L for long calculations!!
    t4 += 1;               // adjust the delay(1000) at begin of loop!
    if (f4 > 0.4) t4++;    // adjust fractional part, see above
    RTC.adjust(DateTime(t4));

    Serial.print("RTC after : ");
    PrintDateTime(RTC.now());
    Serial.println();

    Serial.println("done ...");
    // endless loop 
    while(1);
  }
  else
  {
    Serial.println("No UDP available ...");
  }
  // wait 1 minute before asking for the time again
  // you don't want to annoy NTP server admin's
  delay(60000L); 
}

///////////////////////////////////////////
//
// MISC
// 
void PrintDateTime(DateTime t)
{
    char datestr[24];
    sprintf(datestr, "%04d-%02d-%02d  %02d:%02d:%02d  ", t.year(), t.month(), t.day(), t.hour(), t.minute(), t.second());
    Serial.print(datestr);  
}


// send an NTP request to the time server at the given address 
unsigned long sendNTPpacket(byte *address)
{
  // set all bytes in the buffer to 0
  memset(pb, 0, NTP_PACKET_SIZE); 
  // Initialize values needed to form NTP request
  // (see URL above for details on the packets)
  pb[0] = 0b11100011;   // LI, Version, Mode
  pb[1] = 0;     // Stratum, or type of clock
  pb[2] = 6;     // Polling Interval
  pb[3] = 0xEC;  // Peer Clock Precision
  // 8 bytes of zero for Root Delay & Root Dispersion
  pb[12]  = 49; 
  pb[13]  = 0x4E;
  pb[14]  = 49;
  pb[15]  = 52;

  // all NTP fields have been given values, now
  // you can send a packet requesting a timestamp: 
#if ARDUINO >= 100
  // IDE 1.0 compatible:
  Udp.beginPacket(address, 123); //NTP requests are to port 123
  Udp.write(pb,NTP_PACKET_SIZE);
  Udp.endPacket(); 
#else
  Udp.sendPacket( pb,NTP_PACKET_SIZE,  address, 123); //NTP requests are to port 123
#endif	  

}
///////////////////////////////////////////
//
// End of program
//

What do you get? Or anyone else for that matter?

The only thing unusual about my setup is that my arduino is getting served internet via an iMac... but I don't see how that could make a difference. But just in case.

Q: Have you read the specs of the (S)NTP protocol? e.g. - RFC 2030 - Simple Network Time Protocol (SNTP) Version 4 for IPv (RFC2030) - (especially chapter 3 & 4) -

In short the NTP protocol uses 4 timestamps: See - http://www.eecis.udel.edu/~mills/database/brief/seminar/ntp.pdf - page 12

  1. Reference Timestamp: This is the time at which the local clock was last set or corrected, in 64-bit timestamp format. starts at byte 16
  2. Originate Timestamp: This is the time at which the request departed the client for the server, in 64-bit timestamp format. starts at byte 24
  3. Receive Timestamp: This is the time at which the request arrived at the server, in 64-bit timestamp format. starts at byte 32
  4. Transmit Timestamp: This is the time at which the reply departed the server for the client, in 64-bit timestamp format. starts at byte 40

From those 4 time stamps one can calculate the round trip time which is useful for ignore the answer if it took too long.
Furthermore one can calculate the network latency part of the handshake.

To simplify the Arduino code a lot it only uses T4 and F4 and the rest is ignored (for future experiments). In fact from the fractional part only 2 bytes are used.

As the buffer used for the UDP packet contains 0x0000 0000 in the timestamp places these are interpreted by the code as a timestamp in 2036-02-07 .

This explains the two 2036-02-07 timestamps (T1 & T3).

The two other timestamps make more sense (although I can't explain the 8 seconds difference at the moment)
(T2) 2012-02-19 16:01:43
(T4) 2012-02-19 16:01:51

There is a three hour difference with the RTC clock and that is exactly the timezone coded in the example ( 3 x 3600L )

And yes, it is true the code does not check if the value returned in T4/F4 makes sense. It also does not check the roundtrip time, the network delay, the leap second correction or all 32 bits of the fractional part and more. (in fact it is a lousy NTP implementation!) But that is done on purpose to keep the footprint of the code small. The code can even be smaller, without floating point math, (should be on TODO-list). It is not too difficult to add all mentioned items but the codesize would at least double (assumption) and it would hardly increase the precission. In fact all the extra math would cost extra time. But for those investigating NTP it would be very informative.

So in conclusion, the NTP code discussed only uses the transmit timestamp and the other timestamps are to be ignored as they are menaingless unless you are investigating the (S)NTP protocol details.

Hopes this helps to understand the code and design choices better,

robtillaart:
Hopes this helps to understand the code and design choices better,

Thanks, Rob. I was vaguely aware what the four timestamps were.

The question of why they're over or underflowing, is of course still unresolved. This isn't something I'm ordering you to do (of course), I need to do some testing of my own. I've tried two different NTP servers.

I wonder if it could be to to do with the fact that my Arduino is connected to my iMac via Ethernet, which is connected via WiFi to the AP which is connected to the router by Ethernet. Unusual hoppage in the home.

The question of why they're over or underflowing, is of course still unresolved. This isn't something I'm ordering you to do (of course), I need to do some testing of my own. I've tried two different NTP servers.

I explained that, the buffer used for the data contains 0x0000 0000 and it is interpreted by the code as a day in februari 2036.

If you really want to understand what is happening I propose you implement the full handshake of NTP, read the time from the RTC and put it in the appropiate place in the UDP packet before sending. Take a time measurement - micros() - when the packet is send and received to determine network latency and determine the server delay.

Then print the values in the received packet. These should not include "2036" then anymore.

If you can do this you are one big step closer to an Arduino based NTP server, which would be a great achievement...

I wonder if it could be to to do with the fact that my Arduino is connected to my iMac via Ethernet, which is connected via WiFi to the AP which is connected to the router by Ethernet. Unusual hoppage in the home.

No, UDP packets don't care to hop - in fact they do that for a living :wink:

robtillaart:
If you really want to understand what is happening I propose you implement the full handshake of NTP...if you can do this you are one big step closer to an Arduino based NTP server, which would be a great achievement...

Thanks - but I can't. It's well and truly beyond my current abilities.

My project is essentially simple - I was hoping to add your code to my project as my DS1307 loses ~10 seconds a day!

What's so odd is that the code reliably works on the one hand, and reliably doesn't on the other.

When it always works:-
When the call to update the time is made for the first time in the loop.
When it sets the time to 2036:-
When run for a second time in the loop, (e.g. 12hrs later or 2mins later)

See the annotated screenshot.

In the first call to the NTP server, T1 and T3 are corrupt, but T2 and T4 are not.

In the second call, now T3 is the only correct one!

I believe I'm somehow making people think this issue is more complicated to resolve than it potentially needs to be.

robtillaart:
What is true is that the code does not check if the returned values make sense. That should be added, but it is very difficult to define unambiguously "the returned value make sense" (I wait for your proposal :wink:

As when this code fails, it sets the clock to 2036 or 2055, one could implement a check for if the returned value is in or above the year 2036. 2036 is the year of the NTP rollover, so I think it would be safe to assume that by the time this code won't still be in use. I expect I am somehow exposing myself to ridicule for that assumption, but for our purposes, I think it's fair enough.

Does anyone have any idea why this might be happening? It's driving me slightly nuts.

Hi,
I suggest to get the return value of the UDP read function and check if the received packet size is as expected:

#if ARDUINO < 100
      int ret = Udp.readPacket(pb, NTP_PACKET_SIZE);
#else
      int ret = Udp.read(pb, NTP_PACKET_SIZE);      // New from IDE 1.0,
#endif	

if( ret != NTP_PACKET_SIZE )
{
   // check.
}

ea123:
I suggest to get the return value of the UDP read function and check if the received packet size is as expected:

I implemented your code, and it turns out the packet size IS as expected, even when it produces the odd date.

See this screenshot.

Updated code, which produced the screenshot, is here.

#include <Twitter.h>
#include <Bounce.h>
#include <Wire.h>
#include "RTClib.h"
#include <Servo.h>
#include <SPI.h>         
#include <Ethernet.h>
#include <EthernetUdp.h>	

byte mac[] = { 
  0x00, 0x11, 0x22, 0x33, 0xFB, 0x11 }; 
EthernetClient client;
Twitter twitter("");
unsigned int localPort = 8888;             
byte timeServer[] = {
  193, 79, 237, 14};    // ntp1.nl.net NTP server  
const int NTP_PACKET_SIZE= 48;             
byte pb[NTP_PACKET_SIZE];                  
#if ARDUINO >= 100
EthernetUDP Udp;
#endif		
Servo servo1;
#define ledPin1 8      		
#define buttonPin 2		
const int pinSpeaker = 7;
int buttonState = LOW;		
int LedState1 = LOW;		
int LedState2 = LOW;	        
long LedOnDuration = 10000;
long FlashRate = 400;		
long previousMillis1 = 0;	
long previousMillis2 = 0;    

//The pips
unsigned int pip1 = 0;
unsigned int pip2 = 0;
unsigned int pip3 = 0;
unsigned int pip4 = 0;
unsigned int pip5 = 0;
unsigned int pip6 = 0;
unsigned int RTCUD = 0;
unsigned int INITIALSET = 0;

// Instantiate a Bounce object with a 20 millisecond debounce time
Bounce nursery = Bounce( buttonPin, 500);
int nurseryvalue;

RTC_DS1307 RTC;

enum {
  OFF, ON, FLASH};

void setup() 
{
  pinMode(buttonPin,INPUT);      
  pinMode(ledPin1,OUTPUT);      
  pinMode(pinSpeaker, OUTPUT);
  digitalWrite(ledPin1, OFF);
  servo1.attach(6);
  LedState1 = OFF;
  LedState2 = OFF;
  Serial.begin(57600);
  Wire.begin();
  RTC.begin();
  Serial.println("Annunciator Panel v0.6");
  Ethernet.begin(mac);	   
  Udp.begin(localPort);
  Serial.println();
  if (! RTC.isrunning()) {
    Serial.println("Error!: The RTC is NOT running/set!");

  }
}

void loop() {
  DateTime now = RTC.now();
  nursery.update(); 
  nurseryvalue = nursery.read();
  unsigned long currentMillis = millis();

  if (currentMillis > 25000 && INITIALSET < 1)
  {
    Serial.println("25s have elapsed since power on/reset, so");
    Serial.println("time to sync the clock initially.");
    Serial.println();
    RTCUD = 1;
  }

  if (now.hour() == 14 && now.minute() == 32 && now.second() == 0)  
  {
    if (RTCUD < 1)  
    {
      Serial.print("Time to set the clock at: ");
      PrintDateTime(RTC.now());
      Serial.println();
      RTCUD = 1;
    }
  }

  if (RTCUD == 1)
  {
    RTCUD = 0;         
    INITIALSET = 1;   
    Serial.print("RTC before: ");
    PrintDateTime(RTC.now());
    Serial.println();
    sendNTPpacket(timeServer);
    delay(1000);
    if ( Udp.available() ) {

      int ret = Udp.read(pb, NTP_PACKET_SIZE);      // New from IDE 1.0,

      if( ret != NTP_PACKET_SIZE )
      {
        Serial.println();
        Serial.println("Error: NTP response was of an unexpected size");
        Serial.println();
      }
      else if ( ret = NTP_PACKET_SIZE )
      {
        Serial.println("NTP response was of the expected size");
      }      
      unsigned long t1, t2, t3, t4;
      t1 = t2 = t3 = t4 = 0;
      for (int i=0; i< 4; i++)
      {
        t1 = t1 << 8 | pb[16+i];      
        t2 = t2 << 8 | pb[24+i];      
        t3 = t3 << 8 | pb[32+i];      
        t4 = t4 << 8 | pb[40+i];
      }

      float f1,f2,f3,f4;
      f1 = ((long)pb[20] * 256 + pb[21]) / 65536.0;      
      f2 = ((long)pb[28] * 256 + pb[29]) / 65536.0;      
      f3 = ((long)pb[36] * 256 + pb[37]) / 65536.0;      
      f4 = ((long)pb[44] * 256 + pb[45]) / 65536.0;
      const unsigned long seventyYears = 2208988800UL;
      t1 -= seventyYears;
      t2 -= seventyYears;
      t3 -= seventyYears;
      t4 -= seventyYears;
      t4 += 1;               
      if (f4 > 0.4) t4++;    
      RTC.adjust(DateTime(t4));
      Serial.print("RTC after : ");
      PrintDateTime(RTC.now());
      Serial.println();
      Serial.println("NTP Time Sync successful");
      Serial.println();
    }
    else
    {
      Serial.println("Error: NTP Time Sync unsuccessful");
      Serial.println();
      RTCUD = 0;           
    }
  }


void PrintDateTime(DateTime t)
{
  char datestr[24];
  sprintf(datestr, "%04d-%02d-%02d  %02d:%02d:%02d  ", t.year(), t.month(), t.day(), t.hour(), t.minute(), t.second());
  Serial.print(datestr);  
}


// send an NTP request to the time server at the given address 
unsigned long sendNTPpacket(byte *address)
{
  // set all bytes in the buffer to 0
  memset(pb, 0, NTP_PACKET_SIZE); 
  // Initialize values needed to form NTP request
  // (see URL above for details on the packets)
  pb[0] = 0b11100011;   // LI, Version, Mode
  pb[1] = 0;     // Stratum, or type of clock
  pb[2] = 6;     // Polling Interval
  pb[3] = 0xEC;  // Peer Clock Precision
  // 8 bytes of zero for Root Delay & Root Dispersion
  pb[12]  = 49; 
  pb[13]  = 0x4E;
  pb[14]  = 49;
  pb[15]  = 52;

  // all NTP fields have been given values, now
  // you can send a packet requesting a timestamp: 		   
#if ARDUINO < 100
  Udp.sendPacket( pb,NTP_PACKET_SIZE,  address, 123); //NTP requests are to port 123
#else
  // IDE 1.0 compatible:
  Udp.beginPacket(address, 123); //NTP requests are to port 123
  Udp.write(pb,NTP_PACKET_SIZE);
  Udp.endPacket(); 
#endif	  
}

Perhaps I should software reset my sketch every two hours??? I still think that because this code ALWAYS works on the first sync, it's a problem with the code. Isn't that logical? Especially now that we've established the packet is the expected size.

Thank all.

OK,

Some NTP providers block sites or send faulty values to clients that ask the time to often.

See this thread - http://arduino.cc/forum/index.php/topic,52018.0.html -

can you modify the code so that the whole content of the packet is dumped in HEX ?

int i= 0;
while (i < PACKETSIZE)
{
  Serialprint(pb[i], HEX);
  i++;
  if ((i % 4) ==0) Serial.println();
}

I would like to see the contents of the header of teh packet.

TIA,

robtillaart:
Some NTP providers block sites or send faulty values to clients that ask the time to often.

But we've ruled that out because I've tried different NTP servers.

robtillaart:
can you modify the code so that the whole content of the packet is dumped in HEX ?

Yep. Here is the first one

RTC before: 2012-02-20 19:08:03
C14FEDE
07B030
2416ED
0000
00016
5050530
D2ED188B
1173C742
0000
0000
D2ED1893
72243DF6
RTC after : 2012-02-20 19:08:05
done ...

and the second:

RTC before: 2012-02-20 19:09:10
D2ED1893
722619DC
C14FEDE
07B030
2416ED
0000
00019
5050530
D2ED18CC
12EE29BC
0000
0000
RTC after : 2036-02-07 06:28:17
done ...

If the hex dump was of no use, could this explain a lot?

The hexdump cannot be mapped upon the NTP packet layout as the borders between bytes are unknown. This modified code adds a . after every byte

int i= 0;
while (i < PACKETSIZE)
{
  Serialprint(pb[i], HEX);
  Serial.print(".");
  i++;
  if ((i % 4) ==0) Serial.println();
}

That said, it is clear that even from this faulty hexdump that the header of the UDP packet has shifted 8 bytes (2 lines).
This means the packet is parsed incorrectly.

could this explain a lot?

Definitely, I was not aware of that one. The issue 669 you found is closely related and seems to explain the bug completely - I did not dive in the details - . You can apply the patch proposed and see if it fixes the bug. Thank you for finding it, I will apply a note on the playground.

Thanx,

robtillaart:
The hexdump cannot be mapped upon the NTP packet layout as the borders between bytes are unknown.

could this explain a lot?

The issue 669 you found is closely related and seems to explain the bug

Here are the new hex dumps:-

RTC before: 2012-02-22 22:22:38
C1.4F.ED.E.
0.7B.0.30.
24.1.6.ED.
0.0.0.0.
0.0.0.1D.
50.50.53.0.
D2.EF.E9.1F.
9C.B.1E.69.
0.0.0.0.
0.0.0.0.
D2.EF.E9.2F.
29.C6.EC.6.
RTC after: 2012-02-22 22:22:40

RTC before: 2012-02-22 22:23:10
D2.EF.E9.2F.
29.C8.C3.B7.
C1.4F.ED.E.
0.7B.0.30.
24.1.6.ED.
0.0.0.0.
0.0.0.1C.
50.50.53.0.
D2.EF.E9.40.
9C.D8.72.10.
0.0.0.0.
0.0.0.0.
RTC after: 2036-02-07 06:28:17

N.B. Strangely, we can't know if fix 669 fixes this bug, because it causes the sketch to report "UDP not available" (by serial) as if ( Udp.available() returns a negative result. It is therefore not possible to set the time even initially! I wonder why that is. In case you're wondering, the above hex dumps were taken with the Arduino 1.0 ethernet library.

In all the cases you posted where the time is set incorrectly, the last 8 bytes are all 0. It isn't rocket surgery to figure out to stop setting the time to an invalid value. Simply ignore the packet if the last 8 bytes are all 0.