Serializing data class

eBuzatti · February 12, 2024, 8:27pm

Hi, I have a custom type that I'd like to serialize as a char array:

typedef struct network {
    char SSID[50];
    char PASS[50];
} network_t, *network_p;

typedef struct nvs {
    network_p* networks;
    uint8_t items;
} nvs_t, *nvs_p;

adding some items:

  nvs_t _1 = nvs_t();
  _1.items = 2;
  _1.networks = (network_p*)malloc( _1.items * sizeof(network_p) );
  _1.networks[0] = (network_p)malloc( sizeof(network_t) );
  _1.networks[1] = (network_p)malloc( sizeof(network_t) );
  strcpy(_1.networks[0]->SSID, "alma");
  strcpy(_1.networks[0]->PASS, "körte");
  strcpy(_1.networks[1]->SSID, "szilva");
  strcpy(_1.networks[1]->PASS, "narancs");

nice until here. My serialization purpose is:

Clipboard01

I tried to copy a whole network_t struct in one pass but it didn't work:

  unsigned char foo[sizeof(uint8_t)+_1.items*sizeof(network_t)] = {0};
  unsigned char *ptr = foo;
  memcpy(foo, &(_1.items), sizeof(uint8_t));
  for(int i=0, offset = sizeof(uint8_t);i<_1.items;i++, offset +=sizeof(network_t)) {
    memcpy(foo + offset, &(_1.networks[i]), sizeof(network_t));
  }

The working version is:

  unsigned char foo[sizeof(uint8_t)+_1.items*sizeof(network_t)] = {0};
  unsigned char *ptr = foo;
  memcpy(foo, &(_1.items), sizeof(uint8_t));
  for(int i=0, offset = sizeof(uint8_t);i<_1.items;i++, offset +=50) {
    strcpy((char *)(foo + offset), (char *)&(_1.networks[i]->SSID));
    offset +=50;
    strcpy((char *)(foo + offset), (char *)&(_1.networks[i]->PASS));
  }

It is actually working but I don't like it. Wiring the offset is ugly, to recover the individual elements in this way is ugly, too.
Is there a nicer solution?

J-M-L · February 13, 2024, 10:49am

do you want the nvs structure to be "flat" ?

eBuzatti · February 13, 2024, 12:34pm

yes, that is my goal! Store the nvs_t structure and belongings in one char array. The purpose is to store credentials under one key in NonVolatileStorage in ESP32.

J-M-L · February 13, 2024, 12:53pm

you are not achieving this if you have pointers to pointers for the networks array

as you won't embed the network_t elements right there but only a pointer to somewhere else in memory

try something like this:

struct Network {
  char SSID[50];
  char PASS[50];
};

Network ntw0 = {"Hello", "World"};
Network ntw1 = {"Bye Bye", "World"};

struct Nvs {
  Network* networks; // pointer to the start of the array of Network items, all contiguous
  uint8_t items; // the number of Network items in the array
};

void print(Nvs& nvs) {
  Serial.print("Networks count : "); Serial.println(nvs.items);
  for (uint8_t i = 0; i < nvs.items; i++ ) {
    Serial.write('\t');
    Serial.print("Network#"); Serial.print(i);
    Serial.write(':'); Serial.print(nvs.networks[i].SSID);
    Serial.write(','); Serial.println(nvs.networks[i].PASS);
  }
}

void setup() {
  Serial.begin(115200);
  Nvs nvsStruct ; 

  nvsStruct.items = 2;
  nvsStruct.networks = (Network*) malloc(nvsStruct.items * sizeof(Network)); // space for 2 Network items

  strcpy(nvsStruct.networks[0].SSID, "alma");
  strcpy(nvsStruct.networks[0].PASS, "körte");
  strcpy(nvsStruct.networks[1].SSID, "szilva");
  strcpy(nvsStruct.networks[1].PASS, "narancs");

  print(nvsStruct);

  nvsStruct.networks[0] = ntw0; // struct copy
  nvsStruct.networks[1] = ntw1; // struct copy
  print(nvsStruct);
}

void loop() {}

but even there the structure is not flat, it's just a pointer and a number. storing the Nvs instance will not save the text content... so you'll have to iterate through multiple saves for each Network

you need to allocate a fixed maximum amount of networks you can save and size the array appropriately

const byte maxNetworks = 5;
struct Nvs {
  Network networks[maxNetworks];
  uint8_t items; // the actual number of active Network items in the array
};

PS/ if you use malloc() don't forget to free() at some point when the space is no longer needed

eBuzatti · February 13, 2024, 4:13pm

Yeah, sort of... But your code isn't dynamic. maxNetworks has to be set, "wired" to a number in the code.
I need dynamic solution as it stands in my sample above: _1.items is the 1st byte in the serialized char array and and when reading, this determines the number of networks to be reserved.

J-M-L · February 13, 2024, 5:41pm

then you can't use a struct as archiving the struct in NVS will just result in archiving the value of the pointers and not the data pointed by the pointers

You need to create a memory buffer of the right size and fill in (serialise) the data in the buffer in the right place.

try this

struct Network {
  char SSID[50];
  char PASS[50];
};

Network theNetworks[] = {
  {"alma", "körte"},
  {"szilva", "narancs"},
};
const  uint8_t networksCount = sizeof theNetworks / sizeof * theNetworks;

void dump(uint8_t * payload) {
  uint8_t cnt = *payload; // The first byte is the number of Network instances that follow
  Serial.print("Networks count : "); Serial.println(cnt);
  for (uint8_t i = 0; i < cnt; i++ ) {
    Network tmpNetwork;
    memcpy(&tmpNetwork, payload + 1 + i * sizeof(Network), sizeof(Network)); // get the i-th Network instance (memcpy will likely not move any memory)
    Serial.write('\t');
    Serial.print("Network#"); Serial.print(i);
    Serial.write(':'); Serial.print(tmpNetwork.SSID);
    Serial.write(','); Serial.println(tmpNetwork.PASS);
  }
}


void setup() {
  Serial.begin(115200);

  uint8_t * buffer =  malloc(1 + networksCount * sizeof(Network)); // space for 2 Network items and the count

  buffer[0] = networksCount;
  for (uint8_t i = 0; i < networksCount; i++ ) {
    memcpy(buffer + 1 + i * sizeof(Network), &(theNetworks[i]), sizeof(Network)); // fill in the buffer 
  }

  dump(buffer);
}

void loop() {}

gfvalvo · February 13, 2024, 9:00pm

I also like the idea of making it self-serializing and self-deserializing :

#include <vector>
using byteVector = std::vector<uint8_t>;

struct Network {
  char ssid[50] = {0};
  char pass[50] = {0};

  Network(const char* s, const char * p) {
    strcpy(ssid, s);
    strcpy(pass, p);
  }
};

class NetworkContainer {
  private:
    using NetworkVector = std::vector<Network>;
    NetworkVector networks;
    static constexpr size_t networkSize = sizeof(Network);

  public:
    NetworkContainer(NetworkVector v) : networks(v) {}
    NetworkContainer() {}

    NetworkContainer(const byteVector &b) {
      size_t numNets = b.size() / networkSize;
      NetworkVector v(numNets, {"", ""});
      memcpy(v.data(), b.data(), numNets * networkSize);
      networks = std::move(v);
    }

    NetworkContainer &operator=(const byteVector &b) {
      size_t numNets = b.size() / networkSize;
      NetworkVector v(numNets, {"", ""});
      memcpy(v.data(), b.data(), numNets * networkSize);
      networks = std::move(v);
      return *this;
    }

    size_t numNetworks() const {
      return networks.size();
    }

    byteVector serialize() {
      size_t numBytes = networkSize * numNetworks();
      byteVector theBytes(numBytes, 0);
      memcpy(theBytes.data(), networks.data(), numBytes);
      return theBytes;
    }
};

NetworkContainer theNetworks1({
  {"alma", "körte"},
  {"szilva", "narancs"}
});

void printBytes(byteVector &bytes);

void setup() {
  NetworkContainer theNetworks2;

  Serial.begin(115200);
  delay(2000);

  // Serialize the global instance
  byteVector bytes1 = theNetworks1.serialize();
  printBytes(bytes1);
  Serial.println();

  // Assign to local instance from bytes;
  theNetworks2 = bytes1;
  byteVector bytes2 = theNetworks2.serialize();
  printBytes(bytes2);
  Serial.println();

  // Construct from bytes
  NetworkContainer theNetworks3(bytes1);
  byteVector bytes3 = theNetworks3.serialize();
  printBytes(bytes3);
  Serial.println();
}

void printBytes(byteVector &bytes) {
  size_t byteCount = 0;
  for (uint8_t b : bytes) {
    Serial.printf("%.2hhX ", b);
    if (++byteCount >= 50) {
      Serial.println();
      byteCount = 0;
    }
  }
}

void loop() {
}

Output:

61 6C 6D 61 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 
6B C3 B6 72 74 65 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 
73 7A 69 6C 76 61 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 
6E 61 72 61 6E 63 73 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 

61 6C 6D 61 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 
6B C3 B6 72 74 65 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 
73 7A 69 6C 76 61 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 
6E 61 72 61 6E 63 73 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 

61 6C 6D 61 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 
6B C3 B6 72 74 65 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 
73 7A 69 6C 76 61 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 
6E 61 72 61 6E 63 73 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00

eBuzatti · February 14, 2024, 6:32am

Thanks, i don't know that much about c++ I need to be prepared to interpret it.

gfvalvo · February 14, 2024, 12:49pm

Other variations are possible such as:

Serializing the class to a caller-supplied byte buffer.
Deserializing data from a caller-supplied byte buffer.
Prepending the count of contained Network structs to the byte stream.

gfvalvo · February 15, 2024, 2:01am

A somewhat better version, C++ style-wise.

#include <vector>
using byteVector = std::vector<uint8_t>;

struct Network {
  char ssid[50] = {0};
  char pass[50] = {0};

  Network(const char* s, const char * p) {
    strcpy(ssid, s);
    strcpy(pass, p);
  }

};

class NetworkContainer {
  private:
    using NetworkVector = std::vector<Network>;
    NetworkVector networks;
    static constexpr size_t networkSize = sizeof(Network);

  public:
    NetworkContainer() {}
    NetworkContainer(NetworkVector v) : networks(v) {}
    NetworkContainer(std::initializer_list<Network> nets) : networks(nets) {}

    NetworkContainer(const byteVector &b) {
      size_t numNets = b.size() / networkSize;
      const Network *networkPtr = reinterpret_cast<const Network *>(b.data());
      networks = NetworkVector {networkPtr, networkPtr + numNets};
    }

    NetworkContainer &operator=(const byteVector &b) {
      size_t numNets = b.size() / networkSize;
      const Network *networkPtr = reinterpret_cast<const Network *>(b.data());
      networks = NetworkVector {networkPtr, networkPtr + numNets};
      return *this;
    }

    size_t numNetworks() const {
      return networks.size();
    }

    byteVector serialize() const {
      size_t numBytes = networkSize * numNetworks();
      const uint8_t *bytePtr = reinterpret_cast<const uint8_t *> (networks.data());
      byteVector theBytes{bytePtr, bytePtr + numBytes};
      return theBytes;
    }
};

NetworkContainer theNetworks1 {
  {"alma", "körte"},
  {"szilva", "narancs"}
};

void printBytes(byteVector &bytes);

void setup() {
  NetworkContainer theNetworks2;

  Serial.begin(115200);
  delay(2000);

  // Serialize the global instance
  byteVector bytes1 = theNetworks1.serialize();
  printBytes(bytes1);
  Serial.println();

  // Assign to local instance from bytes;
  theNetworks2 = bytes1;
  byteVector bytes2 = theNetworks2.serialize();
  printBytes(bytes2);
  Serial.println();

  // Construct from bytes
  NetworkContainer theNetworks3{bytes1};
  byteVector bytes3 = theNetworks3.serialize();
  printBytes(bytes3);
  Serial.println();
}

void printBytes(byteVector &bytes) {
  size_t byteCount = 0;
  for (uint8_t b : bytes) {
    Serial.printf("%.2hhX ", b);
    if (++byteCount >= 50) {
      Serial.println();
      byteCount = 0;
    }
  }
}

void loop() {
}

gfvalvo · February 15, 2024, 2:23pm

Ooops ... looks like my last version may have broken C++'s Strict Aliasing rules. I think this is the way to go:

#include <vector>
using byteVector = std::vector<uint8_t>;

struct Network {
  char ssid[50] = {0};
  char pass[50] = {0};

  Network() {}
  Network(const char* s, const char * p) {
    strcpy(ssid, s);
    strcpy(pass, p);
  }
};

class NetworkContainer {
  private:
    using NetworkVector = std::vector<Network>;
    NetworkVector networks;
    static constexpr size_t networkSize = sizeof(Network);

  public:
    NetworkContainer() {}
    NetworkContainer(NetworkVector v) : networks(v) {}
    NetworkContainer(std::initializer_list<Network> nets) : networks(nets) {}

    NetworkContainer(const byteVector &b) {
      size_t numNets = b.size() / networkSize;
      NetworkVector tempVector {numNets};
      memcpy(tempVector.data(), b.data(), numNets * networkSize);
      networks.swap(tempVector);
    }

    NetworkContainer &operator=(const byteVector &b) {
      NetworkVector{}.swap(networks);
      NetworkContainer(b).networks.swap(networks);
      return *this;
    }

    size_t numNetworks() const {
      return networks.size();
    }

    byteVector serialize() const {
      size_t numBytes = networkSize * numNetworks();
      const uint8_t *bytePtr = reinterpret_cast<const uint8_t *> (networks.data());
      byteVector theBytes{bytePtr, bytePtr + numBytes};
      return theBytes;
    }
};

NetworkContainer theNetworks1 {
  {"alma", "körte"},
  {"szilva", "narancs"}
};

void printBytes(const byteVector &bytes);

void setup() {
  NetworkContainer theNetworks2;

  Serial.begin(115200);
  delay(2000);

  // Serialize the global instance
  byteVector bytes1 = theNetworks1.serialize();
  printBytes(bytes1);
  Serial.println();

  // Assign to local instance from bytes;
  theNetworks2 = bytes1;
  byteVector bytes2 = theNetworks2.serialize();
  printBytes(bytes2);
  Serial.println();

  // Construct from bytes
  NetworkContainer theNetworks3{bytes1};
  byteVector bytes3 = theNetworks3.serialize();
  printBytes(bytes3);
  Serial.println();
}

void printBytes(const byteVector &bytes) {
  size_t byteCount = 0;
  for (uint8_t b : bytes) {
    Serial.printf("%.2hhX ", b);
    if (++byteCount >= 50) {
      Serial.println();
      byteCount = 0;
    }
  }
}

void loop() {
}

system · August 13, 2024, 2:24pm

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.

Topic		Replies	Views
Access multiple structs of differing sizes, like an array? Programming	141	4360	February 17, 2023
Help understanding structure behaviior in esp now example Programming	26	1047	July 23, 2024
How to send and receive struct data packet over serial Programming	99	17107	October 8, 2021
Sending structs through serial. Programming	14	18920	May 5, 2021
Guidance on structures (structs) General Guidance	22	2679	November 22, 2022

Serializing data class

Related topics