CRC-16-CCITT checksum of telemetry string via toCharArray

Hi, i have an issue with the CRC16-CCITT calculation of a telemetry string. I have a string of telemetry that im transmitting to a receiver, that acts as a gateway to an online hub which requires a CRC checksum, the checksum is favourably the CRC16-CCITT algorithm. I have been halted by this final hurdle and would appreciate any help. Please see below the code i have used.... I am not a professional software engineer so apologies for any obvious mistakes.

#include <TinyGPS++.h>
#include <SoftwareSerial.h>
#include <SPI.h>
#include <LoRa.h>
#include <SD.h>
#include <OneWire.h>
#include <util/crc16.h>

static const int RXPin = 7, TXPin = 6;
const int chipSelect = 4;
int sensorPin = 6;
int counter = 0;
String telemetry;
char telemetry_copy[100];

#define GPS_BAUD 9600
#define gpsPort ssGPS

OneWire  ds(8);
TinyGPSPlus tinyGPS;
SoftwareSerial ssGPS(RXPin, TXPin);

void setup()
{
 pinMode(9, OUTPUT);
 Serial.begin(9600);
 gpsPort.begin(GPS_BAUD);
 
//  ****** LoRa Module Check ****** 

 if (!LoRa.begin(433.525E6)){
   Serial.println("Starting LoRa failed!");}

//  ****** SD Card Module Check ******  

 if (!SD.begin(chipSelect)){
   Serial.println("SD Card failed, or not present");}

//  ****** LoRa Module Settings ******

//  LoRa.setSPIFrequency(6);  
 LoRa.setTxPower(17); // 2-17 
 LoRa.setSpreadingFactor(11); //6-12
 LoRa.setCodingRate4(8); //5-8  
 LoRa.setSignalBandwidth(62.5E3);  //7.8E3, 10.4E3, 15.6E3, 20.8E3, 31.25E3, 41.7E3, 62.5E3, 125E3, and 250E3.
   
/// ****** Prepare SD Card File ******     

 File Data = SD.open("Data.txt", FILE_WRITE);
 Data.println("HAB ID, Counter, Time, Lat, Long, Altitude, Sats, Int Temp, Ext Temp");  
 Data.close(); 

/// *** Buzzer Confirm Setup ***  

 delay(200);
 tone(9, 2000); 
 delay(100);
 noTone(9);
 delay(50);
 tone(9, 2000); 
 delay(100);
 noTone(9);
  
}

 // ****** CRC Checksum ******
 
uint16_t gps_CRC16_checksum (char *string)
{
 size_t i;
 uint16_t crc;
 uint8_t c;

 crc = 0xFFFF;

 // Calculate checksum ignoring the first two $s
 for (i = 2; i < strlen(string); i++)
 {
   c = string[i];
   crc = _crc_xmodem_update (crc, c);
 }

 return crc;
}
void loop()
{
 Main();
 SmartDelay(20000); 
}
void Main()
{
    
// ****** Temperature Sensor ****** Bus Upto 50 Sensors ******

 byte i;
 byte present = 0;
 byte type_s;
 byte data[12];
 byte addr[8];
 float celsius;      
 if ( !ds.search(addr)) {     
   ds.reset_search();    
   return;  }  
 ds.reset();
 ds.select(addr);
 delay(1000);
 ds.write(0x44, 1);  
 delay(1000);  
 present = ds.reset();
 ds.select(addr);    
 ds.write(0xBE); 

// ****** Internal Temperaure Data Conversion to Celcius ******

 for ( i = 0; i < 9; i++) {
   data[i] = ds.read();    }
 int16_t raw = (data[1] << 8) | data[0];
 if (type_s) {
   raw = raw << 3;
   if (data[7] == 0x10) {
     raw = (raw & 0xFFF0) + 12 - data[6];  }  }
   else {
   byte cfg = (data[4] & 0x60);    
   if (cfg == 0x00) raw = raw & ~7; 
   else if (cfg == 0x20) raw = raw & ~3; 
   else if (cfg == 0x40) raw = raw & ~1;   }    
 celsius = (float)raw / 16.0;  

// ****** External Temperature Sensor ******

 delay(500);
 int reading = analogRead(sensorPin);
 float voltage = reading * 5.0;  //Reference voltage 5.0 for pefect 5v line
 voltage /= 1024.0;
 float temperatureC = (voltage - 0.5) * 100;

// ***** String Edit ******

 float latt = (tinyGPS.location.lat());
 float lngg = (tinyGPS.location.lng());
 float alt = (tinyGPS.altitude.meters());

 String telemetry = "$CornOnTheBranch,";

 telemetry += counter; telemetry += ",";
   if (tinyGPS.time.hour() < 10) telemetry += "0";
 telemetry += (tinyGPS.time.hour());
 telemetry += ":";  
   if (tinyGPS.time.minute() < 10) telemetry += "0";
 telemetry += (tinyGPS.time.minute()); 
 telemetry += ":";
   if (tinyGPS.time.second() < 10) telemetry += "0";
 telemetry += (tinyGPS.time.second());
 telemetry += ","; 
 telemetry += String(latt, 6); telemetry += ","; 
 telemetry += String(lngg, 6); telemetry += ","; 
 telemetry += String(alt, 1); telemetry += ","; 
 telemetry += (tinyGPS.satellites.value()); telemetry += ",";
 telemetry += celsius; telemetry += ",";
 telemetry += temperatureC;
 telemetry += "*";

 telemetry.toCharArray(telemetry_copy, 100);

// *********** Debug **************
 
 Serial.println(telemetry);
 Serial.println(telemetry_copy);
 Serial.println(gps_CRC16_checksum(telemetry_copy), HEX);

// ****** Transmit Packet ******

 LoRa.beginPacket();
 LoRa.print(telemetry); 
 LoRa.println(gps_CRC16_checksum(telemetry_copy), HEX);
 LoRa.endPacket();

// ****** Buzzer Confirm ******    
  
 tone(9, 2000);
 delay(100); 
 noTone(9);
 delay(500);
 
// ****** Save To SD Card ****** 

 File Data = SD.open("Data.txt", FILE_WRITE);
 if (Data) {
   Data.println(telemetry);
   Data.close();    }   
   delay(500); 

// ****** Plus 1 to Counter ******   

 counter++;  

}

// ****** Timing ******

static void SmartDelay(unsigned long ms)
{
 unsigned long start = millis();
 do
 {
   while (gpsPort.available())
     tinyGPS.encode(gpsPort.read());
 } while (millis() - start < ms);
}

The telemetry is successfully received and uploaded to the online hub, i would like confirmation from someone more experienced that the checksum calculated is correct(or)incorrect and maybe some guidance as to how to rectify this issue. Below is an example of the first loop output via serial monitor of the String, CharArray and calculated checksum in that order. This loop was whilst the tracker was inside a building so im not concerned about the GPS data not being there, I have many hours of successful data logging behind this build already.

$CornOnTheBranch,0,00:00:00,0.000000,0.000000,0.0,0,27.38,25.68*
$CornOnTheBranch,0,00:00:00,0.000000,0.000000,0.0,0,27.38,25.68*
E31E

I have visited...

https://www.scadacore.com/tools/programming-calculators/online-checksum-calculator/

...and inputting the same string results with the following... (I attempted to put this into a code box but wouldnt behave!)

CRC-16-CCITT

Generator Type Big Endian (ABCD) Little Endian (DCBA)
Normal 0x1021 20 19 19 20
Reversed 0x8408 8E 7B 7B 8E
Reversed Reciprocal 0x8810 24 B4 B4 24

FYI the telemetry consists of the following data....

$HAB name,packet count,time,lat,long,altitude,satalites,internal temp,external temp*

I am receiving errors from the Hub as follows,

'Exception in UKHAS main parse: ValueError: Invalid CRC16-CCITT checksum'

I understand this could be a question for the Hub. But i would still appreciate pointers into better understanding what im doing wrong, as im completely stuck!

Guidelines of the required checksum can be found at the following link...

http://habitat.readthedocs.io/en/latest/ukhas_parser.html#checksum-algorithms

Thankyou for any advice in advance,

Best regards, L

P.S Im using an Arduino Nano 328, im not sure if the other hardware is relevant as i have the various modules all working as expected and already spent time range testing and parsing NMEA data all successfully. However further details can be provided if required.

Please edit your post to add code tags. See "How to use this forum" for instructions.

I get different answers:

CRC-16-CCITT
(X.25, V.41, HDLC, XMODEM, Bluetooth, SD, many others; known as CRC-CCITT)

Generator Type Big Endian (ABCD) Little Endian (DCBA)
Normal 0x1021 73 0C 0C 73
Reversed 0x8408 89 21 21 89
Reversed Reciprocal 0x8810 D1 83 83 D1

Remember that your checksum function is starting at index 2 so it is skipping the first two characters of the message. Do any of those match the desired value?

I have some code which does the same CRC and it agrees with what you have shown.
CRC of
"CornOnTheBranch,0,00:00:00,0.000000,0.000000,0.0,0,27.38,25.68*"
is 0xE31E

My code also agrees with the website you linked to: the string "habitat" should checksum to 0x3EFB
But the website is not clear whether the terminating asterisk is included in the CRC. Try calculating the CRC without the trailing asterisk. Maybe that will work.

Do you have a link to a site which shows a known good message with its CRC?

The problem with some of those online CRC calculator sites is that they don't specify exactly how they are calculating the CRC.
This site
allows you to specify any of the variations on a theme.

Select CRC-16
First select Predefined and from the dropdown select CRC16_XMODEM
THEN select Custom - this will enable entries in the next box of detailed parameters
Change the initial value to 0xffff
For Input Data select String and then type the word habitat in the text box.
Now click on Calculate CRC.
The result will be 0x3EFB.

You can also try the string "CornOnTheBranch,0,00:00:00,0.000000,0.000000,0.0,0,27.38,25.68*" which will give the result 0xE31E.

Pete

You dont have to send the payload all the way through Habhub to check the CRC.

Print the payload, all of it, to Serial console and copy the characters after the last $ and before the * to here;

https://www.lammertbies.nl/comm/info/crc-calculation.html

Oh, and I doubt you need to use LoRa at 17dBm and SF11, 10dBm and SF8 is good enough for 250-400km+.

el_supremo:
My code also agrees with the website you linked to: the string "habitat" should checksum to 0x3EFB
But the website is not clear whether the terminating asterisk is included in the CRC.

Its not.

There is more than one way to calculate a 16-bit CRC. How do you know that the online calculators are using the same method?

Well, it's not exactly a different method. The calculation is the same, just some different parameters that shuffle and organize the bits differently. The selection of parameters is a personal preference or a religious debate, depending on where you stand.

MorganS:
There is more than one way to calculate a 16-bit CRC. How do you know that the online calculators are using the same method?

The online calculator that I provided a link for, when selected for the correct CRC-CCITT (0xFFFF), produces a checksum that is accepted by Habhub.

As I mentioned, the CRC site I linked to also gets it correct.

@srnet: do you have any link to docs which state that the asterisk is not part of the CRC? It certainly looks like it can't be.

Pete

el_supremo:
@srnet: do you have any link to docs which state that the asterisk is not part of the CRC? It certainly looks like it can't be.

I dont have a link, and I dont need one. I do know the * is not included, in the same was as it is not in NMEA sentences.

If you want a known working example take a look at my own LoRa (and FSK RTTY) HAB tracker software, its been working with Habhub for the last 3 years;

it is not in NMEA sentences.

Yeah, I knew I had come across a situation where asterisk wasn't included in the CRC but couldn't remember what it was. That was it.

Pete

Thanks for your response all!

jremington, Ive edited as requested.

johnwasser, i cant seem to replicate your results. I understand the first 2 characters in the chararray are skipped for the calculation. The checksum will only calculate after the second $ and before the *. (turns out i have included the asterix when it shouldnt be in there)

johnwasswer/el_supremo/srnet, i have had some progress, using the following calculator.....

https://www.lammertbies.nl/comm/info/crc-calculation.html

The calculated checksum agrees with my code of hex value, E31E. However this was including the asterix. I do not want the asterix included in the calculated checksum. I believe this is likely to be the issue, an oversight on my behalf! Didnt help that the calculator at....

https://www.scadacore.com/tools/programming-calculators/online-checksum-calculator/

was giving me incorrect results therefore sending me in circles as i couldnt replicate them with any of the variations of inclusions of the $ characters or * character. Im glad that my understanding was sound but a mere oversight and dodgy calculator was the weak link!

While i have your attention, any feedback regarding the code i have written will be received with thanks, constructive criticism is welcome! I have been 'tinkering' for around 18 months on various data protocols and micro-controllers.

Thanks for your responses, fresh eyes! I will confirm these findings tomorrow either way, thanks again chaps...

Best Regards,

L

ps srnet Thanks for your input regarding power levels, code and spreading factor, I have merely used a number of previous flights from others as guidelines to assign the parameters. The maximum distance will be 35km altitude and dependant on wind speed within the stratosphere could potentially exceed 200Km from base, unlikely but possible. Thought id play it safe... i have only managed to test at a range of 10Km so far with a very satisfactory snr using various parameters. Are you a seasoned HAB enthusiast?

In your CRC function, after this statement:

  c = string[i];

add this:

  if (c == '*') return crc;

That will exclude the asterisk from the calculation.

Pete
[pedantic] A CRC is not a checksum [/pedantic]

Brilliant cheers el_supremo,

L

// Roger crc vs checksum clarity...

CornOnTheBranch:
ps srnet Thanks for your input regarding power levels, code and spreading factor, I have merely used a number of previous flights from others as guidelines to assign the parameters. The maximum distance will be 35km altitude and dependant on wind speed within the stratosphere could potentially exceed 200Km from base, unlikely but possible. Thought id play it safe... i have only managed to test at a range of 10Km so far with a very satisfactory snr using various parameters. Are you a seasoned HAB enthusiast?

I carried out, together with another HAB guy in the UK, the first of the long distance testing of LoRa in late 2014. I was considering LoRa for use in another satellite and high altitude balloons seemed an ideal way of carrying out real long range testing.

Read about it here;

Semtech LoRa Transceivers – a KISS approach to Long Range Data Telemetry - January 2015

I initially wrote the HAB tracker software for PICAXE but then produced a version for Arduino as my first foray into the Arduino world and C.

My HAB2 software has more features than most tracker programs;

Dual LoRa rates, one for normal in flight use, a shortened long distance location only payload for ground searching.
LoRa telemetry and FSK RTTY.
Remote control of the distant balloon tracker.
Based on low cost Arduino Pro Mini.
Extensive power management, multi week operation possible with only a pair of AAA batteries.
Very low parts count tracker transmitter.
Supports Ublox GPS for 18000m+ operation.
Link test functions built in.
Receivers can automatically bind to a tracker transmitter, frequencies and LoRa settings.
Over the air (in flight) configuration changes to settings are possible.
Can use the ATMEGA328 CPU for voltage and temperature measuments for a minimal setup.
Test and simulation modes, an active GPS is not required for testing.

Matching, very low cost LoRa receivers, with various display options and audio uplink to Habitat and Spacenear.

srnet

This is an excellent write up, i take my hat off to you for your work on LoRa. I had planned to get some field work analysis on the various bandwidths and s/f at greater ranges, you've saved me a lot of time there! I have also built a receiver using a pro mini 3v3 8Mhz to good effect, however i have been using a program by David Akerman to send the telemetry to the Habitat which hasnt caused me any issues but has its limits. Your version looks alot more flexible, i may make good use of it in the future, i haven't the skill set to write such programs!

I will take your advice and extensive research and reduce the s/f to 9, thanks again for your input.

Best regards, L. 73

Just to formally confirm, including the asterix in the CRC calculation to produce the 16bit HEX Checksum was indeed the issue.

Thanks again for your input.

Best regards,

L

For the record... Here is the first confirmed successful parse notification from the Habitat!

[2018-06-08 08:56:34,363] INFO habitat.parser MainThread: UKHAS parsed data from CornOnTheBranch successfully

CornOnTheBranch:
srnet

This is an excellent write up, i take my hat off to you for your work on LoRa. I had planned to get some field work analysis on the various bandwidths and s/f at greater ranges, you've saved me a lot of time there! I have also built a receiver using a pro mini 3v3 8Mhz to good effect, however i have been using a program by David Akerman to send the telemetry to the Habitat which hasnt caused me any issues but has its limits. Your version looks alot more flexible, i may make good use of it in the future, i haven't the skill set to write such programs!

I will take your advice and extensive research and reduce the s/f to 9, thanks again for your input.

Best regards, Luke. 73

2E0LJL

I did suggest to the UKHAS guys that it would be an idea to have FLDIGI allow for a direct upload (of a HAB packet) from the serial terminal into Habitat\Spacenear. I was told it would be easy, but its outside the scope of my ability.

As I recall the favoured solution is to use a Raspberry Pi and a LoRa HAT.

The KISS approach I took with my own receivers was to use AFSK RTTY generated by the receiver and connected to the sound card for upload by FLDIGI. This allows for very simple and low cost receivers based on Arduino Pro Mini's. Indeed you can use any of the tracker boards I do for this purpose.

These simple receivers will also do a Bluetooth upload of the trackers position into an Android mapping application (Alpinquest) so you can see on screen where your balloon is, so you can track the position of your balloon without the need for an Internet connection to Habitat\Spacenear.

There is no compelling reason to do an analysis of the range you can get with various spreading factors and bandwidths. Although the receiver sensitivity figures in the LoRa data sheet do not relate to the real world, the Semtech LoRa calculator tool gives results that are a very good relative indication of the effectiveness various settings by comparing the so called not real world 'Receiver sensitivity' figures in the calculator.

What is vital in any flight, is to record the SNR and RSSI readings of the LoRa packet receipts, then you can decide for yourself how effective your transmissions are. The SNR reading in particular is a very good indication of approaching link failure (RSSI is definetly not). If your using SF8, which operates down to -10dB SNR, and you receive a signal of SNR 0dB, you have around 10dB of link margin left which is around 3 times more distance.