client.print not appearing in database

I've been posting JSON to my REST API server using the following:

http.begin("http://my-api.herokuapp.com/classes/myClass/");  //Specify Request Destinations

  http.addHeader("Content-Type", "application/json");
  http.addHeader("X-Parse-Application-Id", "my_api");

  char json[115 + 17 + 1] = "{\"ReceiptBoolean\":true,\"Device\":{\"__type\":\"Pointer\",\"className\":\"Device\",\"objectId\":\"";
  strcat(json, objectID);
  strcat(json, "\"},\"MAC_Address\":\"");
  uint8_t MAC[6];
  WiFi.macAddress(MAC);
  char MACstr[17 + 1];
  sprintf(MACstr, "%02X:%02X:%02X:%02X:%02X:%02X", MAC[5], MAC[4], MAC[3], MAC[2], MAC[1], MAC[0]);
  strcat(json, MACstr);
  strcat(json, "\"}");
int httpCode = http.POST(json);

This has worked well, but I've come across this option which is a lot cleaner:

const char* host = "my-api.herokuapp.com";
  const int httpsPort = 443;
  WiFiClientSecure client;
String url = "/classes/myClass/";
client.print(String("POST ") + url + " HTTP/1.1\r\n" +
                 "Host: " + host + "\r\n" +
                 "Content-Type: application/json\r\n" +
                 "X-Parse-Application-Id: my_api\r\n" +
                 "ReceiptBoolean: true\r\n" +
                 "Device:{__type:Pointer,className:Device,objectId:"+ objectID +"}\r\n" +
                 "MAC_Address:" + WiFi.macAddress() + "\r\n" +
                 "Connection: close\r\n\r\n");

In the former, the API (Parse API) successfully receives & logs the stated fields. In the latter (desired) example, the API registers the post with a new row in the database & responds with an id for the row (which is how the parse API recognises each line). It doesn't however post any of the necessary fields into the database.

Why would this be?

In the first piece of code, you are sending the data in the request body, in the second snippet, you are sending it in the headers. Why?
Also, you didn’t put quotes in your JSON string in the second snippet

The use of Strings is not recommended, they cause memory fragmentation.
The following might not look as nice, but at least it won’t crash your program if you leave it running for prolonged periods of time.

#define HOST "my-api.herokuapp.com"  // The host name of the server (without "http://" and without any slashes)
#define URI "/classes/myClass/"  // The URI of the resource you wish to send the request to
#define PORT 443  // 80 for HTTP or 443 for HTTPS

const char* httpHeaders =    "POST " URI " HTTP/1.1\r\n"
                             "Host: " HOST "\r\n"
                             "Content-Type: application/json\r\n"
                             "X-Parse-Application-Id: my_api\r\n"
                             "Connection: close\r\n"
                             "Content-Length: "; // + bodyLength + "\r\n"
                          // "\r\n"
const char* body_1 =         "{\"ReceiptBoolean\":true,\"Device\":{\"__type\":\"Pointer\",\"className\":\"Device\",\"objectId\":\"";
char objectIDstr[11];     // + objectID        (4 294 967 295 + null)
const char* body_2 =         "\"},\"MAC_Address\":\"";
char MACstr[17 + 1];      // + MACstr          (6 hexadecimal bytes + 5 colons + null)
const char* body_3 =         "\"}";

size_t fixedBodyLength;

void setup() {
  uint8_t MAC[6];
  WiFi.macAddress(MAC);
  sprintf(MACstr, "%02X:%02X:%02X:%02X:%02X:%02X", MAC[5], MAC[4], MAC[3], MAC[2], MAC[1], MAC[0]);

  fixedBodyLength = strlen(body_1) + strlen(body_2) + strlen(body_3) + strlen(MACstr);

  // ...
}

void loop() {
  // ...
  if(sendRequest(objectID)) {
    // request was successful
  } else {
    // request failed
  }
}

bool sendRequest(unsigned int objectID) {
  /* Convert the objectID to a string */
  size_t bodyLength = fixedBodyLength;
  bodyLength += sprintf(objectIDstr, "%u", objectID);

  /* Connect to the server */
  WiFiClient client;  // Create a WiFiClient object
  if (!client.connect(HOST, PORT)) {  // Try to connect to the server
    Serial.println("\r\nError:\tConnection failed");
    return false;  // return false if it failed
  }
  
  /* Send the request to the server, 
     write headers, lenght of the body and body over the TCP connection 
     and print them to the Serial Monitor as well */
  sendWithDebug(httpHeaders, client);
  sendWithDebug(bodyLength, client);
  sendWithDebug("\r\n\r\n", client);
  sendWithDebug(body_1, client);
  sendWithDebug(objectIDstr, client);
  sendWithDebug(body_2, client);
  sendWithDebug(MACstr, client);
  sendWithDebug(body_3, client);

  client.flush(); // wait for the transmission to finish

  /* Check the server's response to see if the request was successful */
  bool HTTPstatusOK = checkHTTPstatusOK(client);

  client.stop(); // Close the connection

  return HTTPstatusOK;
}

template <class T> void sendWithDebug(T dataToSend, WiFiClient& client) {
  client.print(dataToSend);
  Serial.print(dataToSend);
}

/*
   Check the server's response and check the HTTP status code to see if the request was successful.
   Returns true on success.
*/
bool checkHTTPstatusOK(WiFiClient& client) {
  unsigned long timeout = millis();

  /* Read until the first space (SP) of the response. */
  char data = 0;
  while (data != ' ') {  // wait for a space character
    if (client.available() > 0) {  // if theres a character to read from the response
      data = client.read();  // read it
    }
    // if the space doesn't arrive within 5 seconds, timeout and return
    if (millis() - timeout > 5000) {  
      Serial.println("Error: Client Timeout !");
      return false;
    }
    yield();
  }
  
  /* The three next characters are the HTTP status code.
      Wait for 3 characters to be received, and read them into a buffer. */
  while (client.available() < 3) {
    if (millis() - timeout > 5000) {
      Serial.println("\r\nError:\tClient Timeout !");
      return false;
    }
    yield();
  }
  char statusCodeStr[4];  // three digits + null
  client.read((uint8_t*) statusCodeStr, 3);  // read the status code into the buffer
  statusCodeStr[3] = '\0';  // add terminating null character, needed for strcmp()

  /* Check if the status code was "200". If so, the request was successful. */
  if (strcmp(statusCodeStr, "200") != 0) {
    Serial.print("\r\nError:\tHTTP status ");
    Serial.println(statusCodeStr);
    return false;
  }
  
  return true;
}

Pieter

Edit: I haven’t tried it with TLS, you probably need to use ‘WiFiClientSecure’ instead of WiFiClient.
It seems like you never actually connect to the server in the second snippet. Is it because you didn’t post all of your code, or are you writing to a non-existing TLS socket?
Please post all of your code.

To use HTTPS (TLS), you can use the following code:

WiFiClientSecure client; // Use WiFiClientSecure class to create TLS (HTTPS) connection
  if (!client.connect(HOST, PORT)) {   // Connect to the server on port 443
    Serial.println("Connection failed");    // If the connection fails, stop and return
    return false;
  }

  if (!client.verify(fingerprint, HOST)) {   // Check the SHA-1 fingerprint of the SSL (TLS) certificate
    Serial.println("Certificate doesn't match");
    return false; // if it doesn't match, it's not safe to continue
  }

You can use the following command to get the fingerprint of the SSL certificate:

openssl s_client -connect my-api.herokuapp.com:443 < /dev/null 2>/dev/null | openssl x509 -fingerprint -noout -in /dev/stdin | sed 's/:/ /g'

Pieter,

Thankyou for your very concise answer which is a bit mind-blowing trying to fit it into my existing code!

I'll come back later with more constructive reply once I've fully understood whats going on!

Hmmm. sendWithDebug isn't declared anywhere. What type of variable should it be?

You have to change the second parameter from WiFiClient to WiFiClientSecure.

Here’s the full code for TLS instead of TCP:

#include <WiFiClientSecure.h>   // Include the HTTPS library
#include <ESP8266WiFi.h>        // Include the Wi-Fi library

#define HOST "my-api.herokuapp.com"  // The host name of the server (without "http://" and without any slashes)
#define URI "/classes/myClass/"  // The URI of the resource you wish to send the request to
#define PORT 443  // port 443 for HTTPS

#define FINGERPRINT "08 3B 71 72 02 43 6E CA ED 42 86 93 BA 7E DF 81 C4 BC 62 30"

const char* httpHeaders =    "POST " URI " HTTP/1.1\r\n"
                             "Host: " HOST "\r\n"
                             "Content-Type: application/json\r\n"
                             "X-Parse-Application-Id: my_api\r\n"
                             "Connection: close\r\n"
                             "Content-Length: "; // + bodyLength + "\r\n"
                          // "\r\n"
const char* body_1 =         "{\"ReceiptBoolean\":true,\"Device\":{\"__type\":\"Pointer\",\"className\":\"Device\",\"objectId\":\"";
char objectIDstr[11];     // + objectID        (4 294 967 295 + null)
const char* body_2 =         "\"},\"MAC_Address\":\"";
char MACstr[17 + 1];      // + MACstr          (6 hexadecimal bytes + 5 colons + null)
const char* body_3 =         "\"}";

size_t fixedBodyLength;

void setup() {
  uint8_t MAC[6];
  WiFi.macAddress(MAC);
  sprintf(MACstr, "%02X:%02X:%02X:%02X:%02X:%02X", MAC[5], MAC[4], MAC[3], MAC[2], MAC[1], MAC[0]);

  fixedBodyLength = strlen(body_1) + strlen(body_2) + strlen(body_3) + strlen(MACstr);

  // ...
}

void loop() {
  // ...
  static int objectID = 0;
  if(sendRequest(objectID)) {
    // request was successful
    objectID++;
  } else {
    // request failed
  }
}

bool sendRequest(unsigned int objectID) {
  /* Convert the objectID to a string */
  size_t bodyLength = fixedBodyLength;
  bodyLength += sprintf(objectIDstr, "%u", objectID);

  /* Connect to the server */
  WiFiClientSecure client; // Use WiFiClientSecure class to create TLS (HTTPS) connection
  if (!client.connect(HOST, PORT)) {   // Connect to the server on port 443
    Serial.println("Connection failed");    // If the connection fails, stop and return
    return false;
  }

  if (!client.verify(FINGERPRINT, HOST)) {   // Check the SHA-1 fingerprint of the SSL (TLS) certificate
    Serial.println("Certificate doesn't match");
    client.stop();
    return false; // if it doesn't match, it's not safe to continue
  }
  
  /* Send the request to the server, 
     write headers, lenght of the body and body over the TCP connection 
     and print them to the Serial Monitor as well */
  sendWithDebug(httpHeaders, client);
  sendWithDebug(bodyLength, client);
  sendWithDebug("\r\n\r\n", client);
  sendWithDebug(body_1, client);
  sendWithDebug(objectIDstr, client);
  sendWithDebug(body_2, client);
  sendWithDebug(MACstr, client);
  sendWithDebug(body_3, client);

  client.flush(); // wait for the transmission to finish

  /* Check the server's response to see if the request was successful */
  bool HTTPstatusOK = checkHTTPstatusOK(client);

  client.stop(); // Close the connection

  return HTTPstatusOK;
}

template <class T> void sendWithDebug(T dataToSend, WiFiClientSecure& client) {
  client.print(dataToSend);
  Serial.print(dataToSend);
}

/*
   Check the server's response and check the HTTP status code to see if the request was successful.
   Returns true on success.
*/
bool checkHTTPstatusOK(WiFiClientSecure& client) {
  unsigned long timeout = millis();

  /* Read until the first space (SP) of the response. */
  char data = 0;
  while (data != ' ') {  // wait for a space character
    if (client.available() > 0) {  // if theres a character to read from the response
      data = client.read();  // read it
    }
    // if the space doesn't arrive within 5 seconds, timeout and return
    if (millis() - timeout > 5000) {  
      Serial.println("Error: Client Timeout !");
      return false;
    }
    yield();
  }
  
  /* The three next characters are the HTTP status code.
      Wait for 3 characters to be received, and read them into a buffer. */
  while (client.available() < 3) {
    if (millis() - timeout > 5000) {
      Serial.println("\r\nError:\tClient Timeout !");
      return false;
    }
    yield();
  }
  char statusCodeStr[4];  // three digits + null
  client.read((uint8_t*) statusCodeStr, 3);  // read the status code into the buffer
  statusCodeStr[3] = '\0';  // add terminating null character, needed for strcmp()

  /* Check if the status code was "200". If so, the request was successful. */
  if (strcmp(statusCodeStr, "200") != 0) {
    Serial.print("\r\nError:\tHTTP status ");
    Serial.println(statusCodeStr);
    return false;
  }
  
  return true;
}

Pieter

Still getting the issue. My repo is here: https://github.com/pantri/A-Bit-Pushy/tree/master/pb01_sketch . I’m not sure if its fully functioning as I cannot compile it at present.

Thanks again for your help.

The Arduino IDE automatically creates function prototypes for you, but not if you work across multiple files. You need to move the sendWithDebug function above the sendRequest function, or create a prototype for sendWithDebug at the top of the file.

Pieter

Hmmm. OK. I think I understand you...

I've added line 22 to the httpPost file in the repo. I'm getting a different series of errors now:

pb01b_httpPost:22: error: expected constructor, destructor, or type conversion before '(' token
 sendWithDebug(httpHeaders, bodyLength, body_1, objectIDstr, body_2, MACstr, body_3);
              ^
/Users/thomascooper/Documents/GitHub/A-Bit-Pushy/pb01_sketch/pb01b_httpPost.ino: In function 'bool sendRequest(unsigned int)':
pb01b_httpPost:45: error: 'sendWithDebug' was not declared in this scope
   sendWithDebug(httpHeaders, client);
                                    ^
pb01b_httpPost:57: error: 'checkHTTPstatusOK' was not declared in this scope
   bool HTTPstatusOK = checkHTTPstatusOK(client);
                                               ^
/Users/thomascooper/Documents/GitHub/A-Bit-Pushy/pb01_sketch/pb01d_buttonPress.ino: At global scope:
pb01d_buttonPress:37: error: expected declaration before '}' token
 }
 ^
exit status 1
expected constructor, destructor, or type conversion before '(' token
sendWithDebug(httpHeaders, bodyLength, body_1, objectIDstr, body_2, MACstr, body_3);

That’s not a function prototype.

This is:

template <class T> void sendWithDebug(T dataToSend, WiFiClientSecure& client);

Pieter

Thankyou. Thats solved that. I’m now getting the same issue with checking the response:

/Users/thomascooper/Documents/GitHub/A-Bit-Pushy/pb01_sketch/pb01b_httpPost.ino: In function ‘bool sendRequest(unsigned int)’:
pb01b_httpPost:58: error: ‘checkHTTPstatusOK’ was not declared in this scope
bool HTTPstatusOK = checkHTTPstatusOK(client);
^
/Users/thomascooper/Documents/GitHub/A-Bit-Pushy/pb01_sketch/pb01d_buttonPress.ino: At global scope:
pb01d_buttonPress:37: error: expected declaration before ‘}’ token
}
^
exit status 1
‘checkHTTPstatusOK’ was not declared in this scope

I’ve tried to add the following, but it doesn’t seem to do the job:

template <class T> void checkHTTPstatusOK(client);

As you can probably tell I’m outside of my sphere of competency somewhat!

I’ve tried to add the following, but it doesn’t seem to do the job:

Code: [Select]

template void checkHTTPstatusOK(client);

A function prototype must define the type of all arguments, though the arguments don’t need to be named. client is NOT a type.

I recommend you read the link I posted in reply #8.
Also, checkHTTPstatusOK is not a template function.

Pieter

Hmmm. I'm in a bit over my head here. I need to do some further learning on the new elements that you've presented to me which is going to take some time.

One quick question however:

I understand that going from WiFiClient to WiFiClientSecure is supposed to be relatively straightforward.

I was successfully making REST Posts over http using this: https://github.com/pantri/A-Bit-Pushy/blob/3142657b7cde1693c64169b8fc03054e632ba9c9/pb01_sketch/pb01b_httpPost.ino

Was this & the methods of stringing together the elements to form the POST an incorrect way of doing it in the first place?

Thanks again for your help.

thomascoope:
Hmmm. I'm in a bit over my head here. I need to do some further learning on the new elements that you've presented to me which is going to take some time.

Take a look at the following piece of code:

void setup() {
  Serial.begin(115200);
  sayHello("Thomas"); // function call to sayHello
}
void loop() {}

void sayHello(const char *name) { // function definition for sayHello
  Serial.printf("Hello, %s\r\n", name);
}

The compiler runs through this piece of code from top to bottom. In the setup, it encounters a call to the sayHello function. However, the compiler doesn't know that this function exists, because it is defined further down the page.
It will say something like: error: ‘sayHello’ was not declared in this scope.

To solve this, you add a function prototype for sayHello, before it is called for the first time. A function prototype tells the compiler that the function exists, and that is defined somewhere else. It also needs to tell it what the data types of the parameters are, and what data type it returns. As PaulS mentioned, you don't have to specify the name of the parameters, but it's often done to make it more readable for programmers (just like the compiler, most humans read from top to bottom as well, so encounter the function prototype before the function definition).
A function prototype is basically a declaration for a function.

The following piece of code compiles without problems, because sayHello is declared before it is called, using a prototype.

void sayHello(const char *); // function prototype for sayHello (function declaration)

void setup() {
  Serial.begin(115200);
  sayHello("Thomas"); // function call to sayHello
}
void loop() {}

void sayHello(const char *name) { // function definition for sayHello
  Serial.printf("Hello, %s\r\n", name);
}

Note:

void sayHello(const char *name); // function prototype for sayHello (function declaration)
void sayHello(const char *foobar); // function prototype for sayHello (function declaration)

are both valid prototypes as well.

thomascoope:
I understand that going from WiFiClient to WiFiClientSecure is supposed to be relatively straightforward.

Yes, just remember that HTTPS uses port 443, and HTTP uses port 80. You should also check the fingerprint of the servers TLS/SSL certificate.

thomascoope:
I was successfully making REST Posts over http using this: https://github.com/pantri/A-Bit-Pushy/blob/3142657b7cde1693c64169b8fc03054e632ba9c9/pb01_sketch/pb01b_httpPost.ino

Was this & the methods of stringing together the elements to form the POST an incorrect way of doing it in the first place?

The first method was correct, but inefficient. Strcat has to go to the entire buffer to find the end of the string every time you make a new request, and you create the MAC address string on every request as well. You can just do that once in the setup, the MAC address doesn't change while the program is running.
There's no need to fit everything in a single buffer, since you can use client.print() multiple times with different parts of the request string, no need to concatenate.

Using Strings can cause memory fragmentation in the long run, so are usually not recommended.
There is just no need for Strings here, you can use (static) char arrays combined with a single char buffer for the variable objectID. If you use preprocessor defines for the URI and host, it will be concatenated to the static char arrays at compile time, without causing overhead or memory problems at run time.

Pieter

Hi Peiter

This has been great. Really made sense in the end once I had chance to read through it all & understand it properly. Here is the demo:

https://github.com/thomascoope/httpsTest

However...I've now tried to import it into the main repo & I'm having all sorts of issues...

https://github.com/pantri/A-Bit-Pushy/tree/https_creation

I'm getting the following error output:

pb01b_httpsPost:24: error: 'WiFi' does not name a type
WiFi.macAddress(MAC);
^
pb01b_httpsPost:25: error: expected constructor, destructor, or type conversion before '(' token
sprintf(MACstr, "%02X:%02X:%02X:%02X:%02X:%02X", MAC[5], MAC[4], MAC[3], MAC[2], MAC[1], MAC[0]);
^
pb01b_httpsPost:27: error: 'fixedBodyLength' does not name a type
fixedBodyLength = strlen(body_1) + strlen(body_2) + strlen(body_3) + strlen(MACstr);
^
/Users/thomascooper/Documents/GitHub/A-Bit-Pushy/pb01_sketch/pb01b_httpsPost.ino: In function 'bool sendRequest()':
pb01b_httpsPost:52: error: 'sendWithDebug' was not declared in this scope
sendWithDebug(httpHeaders, client);
^
pb01b_httpsPost:64: error: 'checkHTTPstatusOK' was not declared in this scope
bool HTTPstatusOK = checkHTTPstatusOK(client);
^
Multiple libraries were found for "WiFiClient.h"
Used: /Users/thomascooper/Library/Arduino15/packages/esp8266/hardware/esp8266/2.3.0/libraries/ESP8266WiFi
Not used: /Applications/Arduino.app/Contents/Java/libraries/WiFi
Not used: /Applications/Arduino.app/Contents/Java/libraries/WiFi
Not used: /Applications/Arduino.app/Contents/Java/libraries/WiFi
Not used: /Applications/Arduino.app/Contents/Java/libraries/WiFi
exit status 1
'WiFi' does not name a type

Any idea whats going on as it works fine in the httpsTest repo?!

uint8_t MAC[6];
WiFi.macAddress(MAC);
sprintf(MACstr, "%02X:%02X:%02X:%02X:%02X:%02X", MAC[5], MAC[4], MAC[3], MAC[2], MAC[1], MAC[0]);

fixedBodyLength = strlen(body_1) + strlen(body_2) + strlen(body_3) + strlen(MACstr);

You can’t do this at the global scope. Put this code inside of a function (e.g. httpsSendSetup) and call it inside the setup.

thomascoope:
/Users/thomascooper/Documents/GitHub/A-Bit-Pushy/pb01_sketch/pb01b_httpsPost.ino: In function ‘bool sendRequest()’:
pb01b_httpsPost:52: error: ‘sendWithDebug’ was not declared in this scope
sendWithDebug(httpHeaders, client);
^
pb01b_httpsPost:64: error: ‘checkHTTPstatusOK’ was not declared in this scope
bool HTTPstatusOK = checkHTTPstatusOK(client);

You need function prototypes at the top of the file:

template <class T> void sendWithDebug(T dataToSend, WiFiClientSecure& client); // function prototype
bool checkHTTPstatusOK(WiFiClientSecure& client);

Please refer to reply #13.

Pieter