ESP32 - HTTPS request with API token

Hi,

I am trying to make an HTTPS request using an ESP32 development board.

  • I am using the HTTPClient library to make the request.
  • I have an API token for this request.
  • The documentation of the server I am trying to query states that the API key should be included as a header in all requests in the following format.
auth-token: myapitoken

What code am I using? Full code is uploaded below. I have used the library example "BasicHttpsClient" and added the following line.

https.addHeader("Authorization", "auth_token: uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ");

What error am I getting? The code returns the error "connection refused".

[HTTPS] GET... failed, error: connection refused.

Steps that I have tested:

  1. I have confirmed that my API token is valid by sending the request using curl in my computer shell. I can then access the data as intended. See screenshot below.
  2. I have tested changing the URL for a resource that do not not require authentification: https://api.electricitymap.org/v3/zones. I confirmed that the code then worked as intended.
  3. I don't quite understand how the HTTPClient library work. have tested the following different syntaxes for the addHeader method with no success.
https.addHeader("Authorization", "auth_token: uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ"); //alternative 1
https.addHeader("auth_token", "uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ"); //alternative 2
https.addHeader("Authorization", "uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ"); //alternative 3

Full code below


#include <Arduino.h>

#include <WiFi.h>
#include <WiFiMulti.h>

#include <HTTPClient.h>

#include <WiFiClientSecure.h>


const char* ssid = "My wifi ssid";
const char* password = "My password";

const char*  server = "api.electricitymap.org";  // Server URL

const char* rootCACertificate= \
"-----BEGIN CERTIFICATE-----\n" \
"MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw\n" \
"TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh\n" \
"cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4\n" \
"WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu\n" \
"ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY\n" \
"MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc\n" \
"h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+\n" \
"0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U\n" \
"A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW\n" \
"T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH\n" \
"B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC\n" \
"B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv\n" \
"KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn\n" \
"OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn\n" \
"jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw\n" \
"qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI\n" \
"rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV\n" \
"HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq\n" \
"hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL\n" \
"ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ\n" \
"3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK\n" \
"NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5\n" \
"ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur\n" \
"TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC\n" \
"jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc\n" \
"oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq\n" \
"4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA\n" \
"mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d\n" \
"emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=\n" \
"-----END CERTIFICATE-----\n"; 



void setClock() {
  configTime(0, 0, "pool.ntp.org");

  Serial.print(F("Waiting for NTP time sync: "));
  time_t nowSecs = time(nullptr);
  while (nowSecs < 8 * 3600 * 2) {
    delay(500);
    Serial.print(F("."));
    yield();
    nowSecs = time(nullptr);
  }

  Serial.println();
  struct tm timeinfo;
  gmtime_r(&nowSecs, &timeinfo);
  Serial.print(F("Current time: "));
  Serial.print(asctime(&timeinfo));
}


WiFiMulti WiFiMulti;

void setup() {

  Serial.begin(115200);
  // Serial.setDebugOutput(true);

  Serial.println();
  Serial.println();
  Serial.println();

  WiFi.mode(WIFI_STA);
  WiFiMulti.addAP(ssid, password);

  // wait for WiFi connection
  Serial.print("Waiting for WiFi to connect...");
  while ((WiFiMulti.run() != WL_CONNECTED)) {
    Serial.print(".");
  }
  Serial.println(" connected");

  setClock();  
}

void loop() {
  WiFiClientSecure *client = new WiFiClientSecure;
  if(client) {
    client -> setCACert(rootCACertificate);

    {
      // Add a scoping block for HTTPClient https to make sure it is destroyed before WiFiClientSecure *client is 
      HTTPClient https;
  
      Serial.print("[HTTPS] begin...\n");
      if (https.begin(*client, "https://api-access.electricitymaps.com/2w97h07rvxvuaa1g/carbon-intensity/latest?zone=NL")) {  // HTTPS //https://api.electricitymap.org/v3/zones
        Serial.print("[HTTPS] GET...\n");
        // start connection and send HTTP header
        https.addHeader("Authorization", "auth-token: uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ");
        int httpCode = https.GET();
  
        // httpCode will be negative on error
        if (httpCode > 0) {
          // HTTP header has been send and Server response header has been handled
          Serial.printf("[HTTPS] GET... code: %d\n", httpCode);
  
          // file found at server
          if (httpCode == HTTP_CODE_OK || httpCode == HTTP_CODE_MOVED_PERMANENTLY) {
            String payload = https.getString();
            Serial.println(payload);
          }
        } else {
          Serial.printf("[HTTPS] GET... failed, error: %s\n", https.errorToString(httpCode).c_str());
        }
  
        https.end();
      } else {
        Serial.printf("[HTTPS] Unable to connect\n");
      }

      // End extra scoping block
    }
  
    delete client;
  } else {
    Serial.println("Unable to create client");
  }

  Serial.println();
  Serial.println("Waiting 10s before the next round...");
  delay(10000);
}

Screenshot from request done from shell:

That seems to be wrong. As far as I can see, the header must contain "auth_token" element, so the first one is wrong because you'll get a line like:
Authorization: auth_token: uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ
So the "alternative 2" seems to be the right one:

https.addHeader("auth_token", "uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ");

It adds the same header you send with curl "-H" option.

Even using Chrome I get a response after calling https://api-access.electricitymaps.com/2w97h07rvxvuaa1g/carbon-intensity/latest?zone=NL
I see:
{"requestId":"1701275418941173954-YXdzLWV1LXdlc3QtMQ==-af5e670f6b5b45338a36b17c5d791079b4dfd72b6c2942e38d4b0ca3d7d2bb4d","message":"Forbidden"}
At the moment, I'd say it sounds like something wrong in your request and/or client setup, and not on the authorization (without it, you'll always get a response, even with "forbidden" message like the one I have got before). If the secure channel isn't established, no any subsequent actions (like sending a GET) can be done.

Just as a starting idea, a failed https connection should occour if the secure channel can't be established and most of the times it happens because not completed all the secured requirements and the error could even be thrown by the client itself, and not the remote server.
Just to say, I don't have any ESP32 on my desk to test with, but I make a guess after a few and quick peek inside your code (so I could be wrong...): I see the root CA of the "api-access.electricitymaps.com" server certificate is "Amazon Root CA1" while your rootCACertificate contains the cert by "ISRG Root X1", so I suspect your client doesn't establish the connection due to wrong root certificate.
Double check this and let me know.

1 Like

Thank you for your help! I made some progress thanks to your guidance. I still encounter an error though, I hope you can help me further.

First, I have changed the certificate
You were right in pointing out that I was not using the root certificate. I changed this. I am still getting an error though, but I think there is progress and that it now establishes connection with the server succesfully.

Error when using the previous certificate:

[HTTPS] GET... failed, error: connection refused // (httpCode= -1)

Error when using the new certificate, Amazon Root CA:

[HTTPS] GET... code: 404  // ( httpCode = 404)

I don't have a deep understanding of what could be the cause of the error 404. I made the assumption that this might be due to bad authentification / bad way to add Headers to the request.

Second, I have tried a few alternatives for the authentification method.

  1. Test 1 - without any authentification method. I get the error httpCode = 403
//  https.addHeader("auth-token:", "uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ"); NO AUTHENTIFICATION
  1. Test 2 - using addHeader and the most logical syntax, as you point out. I get the error httpCode = 404
  https.addHeader("auth-token:", "uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ");
  1. Test 3 - using the SetAuthorization method of the class HTTPClient. I get the error httpCode = 404
        //https.addHeader("auth-token:", "uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ");
        https.setAuthorization("auth-token: uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ");
  1. Test 4 - using the addHeader with a different syntax. I get the error httpCode = 404
https.addHeader("Authorization", "auth-token: uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ");

Note about the library HTTPClient:
The reason why I have tried Test 4 above is that I looked into the library and noticed the method addHeader() is built to deal with "Authorization" header in a particular way. I don't fully understand what the method is doing, and in what way "Authorization" headers are treated differently, but sharing below the extract for visibility.

In HTTPClient.cpp:

void HTTPClient::addHeader(const String& name, const String& value, bool first, bool replace)
{
    // not allow set of Header handled by code
    if(!name.equalsIgnoreCase(F("Connection")) &&
       !name.equalsIgnoreCase(F("User-Agent")) &&
       !name.equalsIgnoreCase(F("Host")) &&
       !(name.equalsIgnoreCase(F("Authorization")) && _base64Authorization.length())){

Absolutely not. The http response code 404 means "not found". You made a wrong GET request for a path and/or parameter not recognised by the server, so you need to investigate in this direction.
I don't have neither an ESP32 nor enough time now to make tests, anyway I can suggest you to see THIS page describing https requests in detail (e.g. I don't get why you use pointers like "WiFiClientSecure *client = new WiFiClientSecure;", that page doesn't make use of such pointer...).

Thank you for your reply.

I read through the tutorial and others, but I am still puzzled by the behavior I am seeing. Can you help me make sense of the difference between the two cases below? Some advice on what directions to investigate would be very helpful for me.

  • Error 403 without addHeader(): When I don't include the addHeader() method and do not try to pass any API token, the server returns an HTTP 403 error. My understanding is that the server understands the request but I don't have authorization to access the resource - which is expected.
// output in verbose mode, when commenting out the line https.addHeader("auth-token:", "uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ");
[  1121][D][HTTPClient.cpp:303] beginInternal(): protocol: https, host: api-access.electricitymaps.com port: 443 url: /2w97h07rvxvuaa1g/carbon-intensity/latest?zone=NL
[  2007][D][HTTPClient.cpp:1170] connect():  connected to api-access.electricitymaps.com:443
[  2216][D][HTTPClient.cpp:1321] handleHeaderResponse(): code: 403
  • Error 404 with addHeader() When I try the same code, including the addHeader() line, the server returns an HTTP 404 error. My understanding is that the server did not understand what was requested. I was assuming that this was due to the syntax being wrong / parameters not being passed correctly.
// output Debug level: verbose mode, when including the line https.addHeader("auth-token:", "uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ");
[  1122][D][HTTPClient.cpp:303] beginInternal(): protocol: https, host: api-access.electricitymaps.com port: 443 url: /2w97h07rvxvuaa1g/carbon-intensity/latest?zone=NL
[  1892][D][HTTPClient.cpp:1170] connect():  connected to api-access.electricitymaps.com:443
[  2092][D][HTTPClient.cpp:1321] handleHeaderResponse(): code: 404
  • As shared above, I have confirmed separately that the code can access resources on the server that do not require authentification. And I have confirmed that I can access the resource that require authentification using the API token from my computer shell.

Below the full code, using directly the example from the tutorial you have shared above.

#include <Arduino.h>
#include <WiFi.h>
#include <WiFiClientSecure.h>
#include <HTTPClient.h>


const char* ssid = "wifissid";
const char* password = "wifipassword";

const char* rootCACertificate= \
"-----BEGIN CERTIFICATE-----\n" \
"MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF\n" \
"ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6\n" \
"b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL\n" \
"MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv\n" \
"b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj\n" \
"ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM\n" \
"9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw\n" \
"IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6\n" \
"VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L\n" \
"93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm\n" \
"jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC\n" \
"AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA\n" \
"A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI\n" \
"U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs\n" \
"N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv\n" \
"o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU\n" \
"5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy\n" \
"rqXRfboQnoZsG4q5WTP468SQvvG5\n" \
"-----END CERTIFICATE-----\n";



void setup() {
  Serial.begin(115200);
  Serial.println();
  // Initialize Wi-Fi
  WiFi.mode(WIFI_STA);
  WiFi.begin(ssid, password);
  Serial.print("Connecting to WiFi ..");
  while (WiFi.status() != WL_CONNECTED) {
    Serial.print('.');
    delay(1000);
  }
  Serial.println(WiFi.localIP());
}

void loop() {
  WiFiClientSecure *client = new WiFiClientSecure;
  if(client) {
    // set secure client with certificate
    client->setCACert(rootCACertificate);
    //create an HTTPClient instance
    HTTPClient https;

    //Initializing an HTTPS communication using the secure client
    Serial.print("[HTTPS] begin...\n");
    if (https.begin(*client, "https://api-access.electricitymaps.com/2w97h07rvxvuaa1g/carbon-intensity/latest?zone=NL";)) {  // HTTPS 
      https.addHeader("auth-token:", "uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ"); //this line is commented IN / OUT to test behavior
      Serial.print("[HTTPS] GET...\n");
      // start connection and send HTTP header
      int httpCode = https.GET();
      // httpCode will be negative on error
      if (httpCode > 0) {
      // HTTP header has been send and Server response header has been handled
       Serial.printf("[HTTPS] GET... code: %d\n", httpCode);
      // file found at server
        if (httpCode == HTTP_CODE_OK || httpCode == HTTP_CODE_MOVED_PERMANENTLY) {
          // print server response payload
          String payload = https.getString();
          Serial.println(payload);
        }
      }
      else {
        Serial.printf("[HTTPS] GET... failed, error: %s\n", https.errorToString(httpCode).c_str());
      }
      https.end();
    }
  }
  else {
    Serial.printf("[HTTPS] Unable to connect\n");
  }
  Serial.println();
  Serial.println("Waiting 2min before the next round...");
  delay(120000);
}

If you provide the correct authentication key and the GET is ok, I can't know why you still get a 404 error from that web service.

I don't know if this is related to your issue, but first of all on its API documentation between the examples I read they call "api.electricitymaps.com" while you are requesting "api-access.electricitymaps.com".
Then, I can't see such kind of path you're using here, the latest carbon intensity example has a "/v3/carbon-intensity/latest" path instead.
Lastly, about Arduino code, I start asking myself what is the use of the "const char* server" variable, and if the syntax of your https.begin() call is exact/correct for your specific library: if I' were you, I'd have a look at the library code...

Sorry for not being more helpful than that.

Thanks for your help.

In the end I figured out what was wrong, posting here as it might help others. It turned out this was indeed a syntax issue in the addHeader() method, which seems to add automatically the semi-column.

This is wrong:

https.addHeader("auth-token:", "uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ");

This is correct:

https.addHeader("auth-token", "uJU7AL0FbiGW9TgMt8lHOu8cLs31eFgZ");

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.