Mqtt + ESP32 Wifi

Dear All,
When I connect Wifi and connect a broker with PubSubClient MQTT library, program works properly. I check periodically (2 sec) min heap memory level using ESP.getMinFreeHeap() function. I experienced that heap memory drops down from 250K to 212 K in 12 hours. I can see every 2 sec some times until 1K drop. At the end ESP restarts when heap reaches under 212K. What should be the issue?

Using VScode PlatformIO
platfromio.ini:
[env:esp32dev]
platform = GitHub - platformio/platform-espressif32: Espressif 32: development platform for PlatformIO
board = esp32dev
framework = arduino
monitor_speed = 115200
lib_deps =
knolleary/PubSubClient@^2.8

Are you using any of the memory allocation functions in your code ?

1 Like

This the test program that I get the results as follows. In the main program I don’t use any memory allocation.

Program Code:
main.cpp

#include <Arduino.h>
#include <WiFi.h>
#include <ESP.h>
#include <PubSubClient.h>

const char* ssid = "RajaNet";
const char* password = "rtu1235557x";

/*
const char* broker = "5.189.189.4";
const char* muser = "";
const char* mpass = "";
const char* mid = "esp32user2";
const int port =1883;
*/

const char* broker = "broker.hivemq.com";
const char* muser = "";
const char* mpass = "";
const char* mid = "esp32user2";
const int port =1883;

#define MSG_BUFFER_SIZE (50)
char msg[MSG_BUFFER_SIZE];
int value = 0;

WiFiClient espClient;
PubSubClient client(espClient);

const int Watch_Out = 14;
const int Wifi_Out = 13;

uint32_t heartbeat=0;
uint32_t heartbeat2=0;

unsigned long MqttreconnectTime,WifiPreviousMillis;

void initWifi() {

// We start by connecting to a WiFi network
Serial.println();
Serial.print("Connecting to ");
Serial.println(ssid);

WiFi.begin(ssid, password);

while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}

Serial.println("");
Serial.println("WiFi connected");
Serial.println("IP address: ");
Serial.println(WiFi.localIP());
digitalWrite(Wifi_Out, HIGH);
}

bool reconnectMqtt() {

Serial.println("Attempting MQTT connection...");
Serial.print(muser);
Serial.print(" ");
Serial.print(mpass);
Serial.print(" ");
Serial.println(port);

// Attempt to connect
//client.connect(mid,muser,mpass)
String clientId = mid;
clientId += String(random(0xffff), HEX);
if (client.connect(clientId.c_str())) {
Serial.println("Connected to the Mqtt Broker");
// Subscribe
client.subscribe("esp32/command");
} else {
Serial.print("failed, rc=");
Serial.println(client.state());
return false;
}

return client.connected();

}

void check_heap_mem(){

Serial.printf("Lowest level of free heap since boot: %d\n",ESP.getMinFreeHeap());

}

void callback(char* topic, byte* message, unsigned int length) {

Serial.print("Message arrived on topic: ");
Serial.print(topic);
Serial.print(". Message: ");
String messageTemp;

for (int i = 0; i < length; i++) {
Serial.print((char)message);
messageTemp += (char)message;
}
Serial.println();

// Feel free to add more if statements to control more GPIOs with MQTT

// If a message is received on the topic esp32/output, you check if the message is either "on" or "off".
// Changes the output state according to the message
if (String(topic) == "esp32/output") {
Serial.print("MQTT message: ");

messageTemp.trim();
if (messageTemp.startsWith("q"))
{
Serial.println("q..");
}
if(messageTemp == "mem")
{
check_heap_mem();//
}

else if(messageTemp.startsWith("wr")|messageTemp.startsWith("WR"))
{
Serial.println("write");
}
}

}

void setup() {

Serial.begin(115200);
while (!Serial) continue;

pinMode(Watch_Out, OUTPUT);
pinMode(Wifi_Out, OUTPUT);

initWifi();

client.setServer(broker,1883);
client.setKeepAlive(60);
client.setCallback(callback);
//reconnectMqtt();
//delay(5000);

}

void loop() {

unsigned long currentMillis = millis();
// if WiFi is down, try reconnecting every CHECK_WIFI_TIME seconds
if ((WiFi.status() != WL_CONNECTED) && (currentMillis - WifiPreviousMillis >=10000)) {
Serial.print(millis());
Serial.println("Reconnecting to WiFi...");
WiFi.disconnect();
WiFi.reconnect();
WifiPreviousMillis = currentMillis;
}

unsigned long nowb=millis();
if(nowb - heartbeat>2000)
{
heartbeat = millis();
check_heap_mem();
Serial.println(client.state());

if (client.connected())
{
value=10;
snprintf (msg, MSG_BUFFER_SIZE, "hello world #%ld", value);
Serial.print("Publish message: ");
Serial.println(msg);
client.publish("esp32/output", msg);
}
}

//************** MQTT ****************************

if (!client.connected() && WiFi.status() == WL_CONNECTED) {

long now = millis();
if (now - MqttreconnectTime > 10000) {
MqttreconnectTime = now;
// Attempt to reconnect
if (reconnectMqtt()) {
MqttreconnectTime = 0;
}
}

} else {
// Client connected

client.loop();
}
}

You could try to use c strings instead of the String class to see if String class is causing a problem.
I think the only String your using is messageTemp ?

As you see below I removed Strings and made a simple program to test as you suggested but I still get same result, heap memory drops down...

#include <Arduino.h>
#include <WiFi.h>
#include <ESP.h>
#include <PubSubClient.h>

WiFiClient espClient;
PubSubClient client(espClient);

const char* ssid = "RajaNet";
const char* password = "rxt123456";

const char* broker = "broker.hivemq.com";
const char* muser = "";
const char* mpass = "";
const char* mid = "esp32user2";
const int port =1883;

uint32_t heartbeat=0;
uint32_t heartbeat2=0;

unsigned long MqttreconnectTime,WifiPreviousMillis,MqttLoopTime;

bool reconnectMqtt() {

Serial.println("Attempting MQTT connection...");
Serial.print(muser);
Serial.print(" ");
Serial.print(mpass);
Serial.print(" ");
Serial.println(port);


if (client.connect(mid)) {
  Serial.println("Connected to the Mqtt Broker");
  // Subscribe
  client.subscribe("esp32/command");
} else {
  Serial.print("failed, rc=");
  Serial.println(client.state());
  return false;
}

return client.connected();

}

void initWifi() {

// We start by connecting to a WiFi network
Serial.println();
Serial.print("Connecting to ");
Serial.println(ssid);

WiFi.begin(ssid, password);

while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}

Serial.println("");
Serial.println("WiFi connected");
Serial.println("IP address: ");
Serial.println(WiFi.localIP());
digitalWrite(Wifi_Out, HIGH);
}

void callback(char* topic, byte* message, unsigned int length) {

Serial.print("Message arrived on topic: ");
Serial.print(topic);
Serial.print(". Message: ");  

}

void setup() {

Serial.begin(115200);
while (!Serial);

initWifi();

client.setServer(broker,1883);
client.setKeepAlive(60);
client.setCallback(callback);

reconnectMqtt();
delay(2000);

}

void check_heap_mem(){

value=ESP.getMinFreeHeap();
Serial.printf("Lowest level of free heap since boot: %d\n",value);

}

void loop() {

unsigned long currentMillis = millis();
// if WiFi is down, try reconnecting every CHECK_WIFI_TIME seconds
if ((WiFi.status() != WL_CONNECTED) && (currentMillis - WifiPreviousMillis >=10000)) {
Serial.print(millis());
Serial.println("Reconnecting to WiFi...");
WiFi.disconnect();
WiFi.reconnect();
WifiPreviousMillis = currentMillis;
}

unsigned long nowb=millis();
if(nowb - heartbeat>2000)
{
heartbeat = millis();
check_heap_mem();

// if (client.connected())
//{

 // snprintf (msg, MSG_BUFFER_SIZE, "Minimum Heap Memory:%ld", value);
  //Serial.print("Publish message: ");
  //Serial.println(msg);
 //client.publish("esp32/output", msg);
 
  
//}

}

if (!client.connected()) {

long now = millis();
if (now - MqttreconnectTime > 10000) {
  MqttreconnectTime = now;
  // Attempt to reconnect
  reconnectMqtt(); 
  }
}

else {

client.loop();

}
}

Can we expect to see code tags any time soon?

What you mean code tags? I am sorry I don’t understand

click on the "code/" tag above and place code between the tags. Also auto format your code before
copying and pasting into your forum post.

Do you get any error msgs ? 212K of heap is still a lot. I think there may be another reason for the crash.

I found an explanation about hadware limits in the github of the library as follows,

  • Arduino WiFi Shield - if you want to send packets > 90 bytes with this shield, enable the MQTT_MAX_TRANSFER_SIZE define in PubSubClient.h.

So, I enabled and tried with existing 80. It was better and drop down speed of heap getting slow down with another trial of 120, 256 and 1024 finally. Resut is getting better and seriously slowed down but it wasn’t solved completely. Any suggestion?

By the way, published message buffer size is 50 bytes and user name is max. 10 const char bytes.

I cannot offer a solution to your memory leak problem, however in my ESP32 projects I had to abandon the knolleary library due to severe performance problems. Pushing more than 30 messages per second resulted in freezes and crashes. There is clearly something strange going on with the way this library operates on the ESP32 platform.

Instead I switched to the async-mqtt-client library which is way faster and behaves as you would expect it to given the limitations of the WiFi transport layer.

1 Like

Thank you very much for your support. I also tried the library you suggested. I got same result again, heap memory drops down. I just tried the "examples/FullyFeatured-ESP32/FullyFeatured-ESP32.ino" in the GitHub library.. What you think? Have you got any possibility to try this example with ESP32 with ESP.getMinFreeHeap() function.

Thanks in advance again

Regards,

OK, but what are the test conditions?
The FullyFeatured-ESP32.ino sketch only sends 3 messages, subscribes to 1 topic then does nothing.

I'm currently feeding it about 5 messages per second and the free heap isn't changing.

Update:
After 2 hours it had received 38000 messages.
getMinFreeHeap() started at 242804 and after 2 hours had decreased by 4 bytes to 242800.

Actually I am using VScode-PlatformIO IDE and today I will try with Arduino IDE. My message interval in the trial of FullyFeatured-ESP32 was 2 and 10 sec. Could you please share your test sketch? I will try it in the platformio IDE. By the way I made a some trials in Wokwi.com (online simulator) situation was same. Probably, problem is related to coding or IDE...

The name of the MQTT topic I'm sending from my computer is s_publishMotionDetected. The payload is always a 5 character plain text 'false'.

Therefore in onMqttConnect I changed the subscription to

uint16_t packetIdSub = mqttClient.subscribe("s_publishMotionDetected", 2);

I added the following to the end of onMqttMessage.

  static long count = 0;

  count++;

  Serial.println("message count=");
  Serial.println(count);

  Serial.println("time=");
  Serial.println(millis());

  Serial.println("min free heap=");
  Serial.println(ESP.getMinFreeHeap());

The serial port output in Putty looks like this.

Publish received.
  topic: s_publishMotionDetected
  qos: 0
  dup: 0
  retain: 0
  len: 5
  index: 0
  total: 5
message count=
47445
time=
10134705
min free heap=
242800

Have you ever tried to publish message from ESP with a constant interval?

Look at xTaskCreatePinnedToCore for doing sends at regular intervals. For extra safety you may want to delay sending a message if you are currently in the middle of receiving a message in another task. Research mutexes.

Focus on solving one problem at a time.

  1. Get the receive demo working and prove that it isn't leaking memory.
  2. Do a send demo and prove it also isn't leaking memory.
  3. Combine the two and again check to see that it hasn't created a memory leak.

I made a trial with platformio IDE and Arduino IDE. Result was same on both IDE, heap memory drops down with PubSubClient and other lib. (Platfrom started to drop from 257K, Arduino 217K). Interesting thing is heap memory was constant while connecting WIFI via GSM based router or mobile phone. If home network is used to connect to internet, heap starts to drop down again. I think problem is speed of connection, if speed is not enough to transfer data between ESP and broker, buffer starts to overflow. I don't know it is caused by limitation on broker side socket or router speed... What you think?

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.