Go Down

Topic: Converting byte arrays to strings (Read 385 times) previous topic - next topic

pan19ss

Hello,
my program involves hashing, and the hashing library takes strings as inputs.
However because I need to store many hash values in memory (more than 20, 256-bits each) and these are dynamically changed, I cannot fit them into memory so I thought I could change them into byte arrays. However this involves storing the initial values in flash memory and convert them into Byte arrays, then convert them back to strings so I can hash them.

The problem is with the following snippet, I have an array_to_string() function which works for one bytearray, but if I add a second then the program misbehaves (doesn't output anything).

Anyone has an idea of what could be wrong?

Code: [Select]
const byte MaxByteArraySize = 32;

void hexCharacterStringToBytes(byte *byteArray, const char *hexString)
{
  bool oddLength = strlen(hexString) & 1;

  byte currentByte = 0;
  byte byteIndex = 0;

  for (byte charIndex = 0; charIndex < strlen(hexString); charIndex++)
  {
    bool oddCharIndex = charIndex & 1;

    if (oddLength)
    {
      // If the length is odd
      if (oddCharIndex)
      {
        // odd characters go in high nibble
        currentByte = nibble(hexString[charIndex]) << 4;
      }
      else
      {
        // Even characters go into low nibble
        currentByte |= nibble(hexString[charIndex]);
        byteArray[byteIndex++] = currentByte;
        currentByte = 0;
      }
    }
    else
    {
      // If the length is even
      if (!oddCharIndex)
      {
        // Odd characters go into the high nibble
        currentByte = nibble(hexString[charIndex]) << 4;
      }
      else
      {
        // Odd characters go into low nibble
        currentByte |= nibble(hexString[charIndex]);
        byteArray[byteIndex++] = currentByte;
        currentByte = 0;
      }
    }
  }
}

byte nibble(char c)
{
  if (c >= '0' && c <= '9')
    return c - '0';

  if (c >= 'a' && c <= 'f')
    return c - 'a' + 10;

  if (c >= 'A' && c <= 'F')
    return c - 'A' + 10;

  return 0;  // Not a valid hexadecimal character
}

String array_to_string(byte array[], unsigned int len)
{
    char buffer[32];
    for (unsigned int i = 0; i < len; i++)
    {
        byte nib1 = (array[i] >> 4) & 0x0F;
        byte nib2 = (array[i] >> 0) & 0x0F;
        buffer[i*2+0] = nib1  < 0xA ? '0' + nib1  : 'A' + nib1  - 0xA;
        buffer[i*2+1] = nib2  < 0xA ? '0' + nib2  : 'A' + nib2  - 0xA;
    }
    //buffer[len*2] = '\0';
   
    return (String(buffer));
}

void setup()
{
  Serial.begin(115200);

  byte byteArray[MaxByteArraySize] = {0};
  byte byteArray2[MaxByteArraySize] = {0};

  hexCharacterStringToBytes(byteArray, "173af653133d964edfc16cafe0aba33c8f500a07f3ba3f81943916910c257705");
  Serial.println(array_to_string(byteArray,32));

  hexCharacterStringToBytes(byteArray2, "3a397279c72e019a31729d7536dfd407c37cc63dcde44a0222be2e098d85181a"); //
  Serial.println(array_to_string(byteArray2,32)); //if I comment these 2 lines out then the output for the first byteArray works..

 
}

void loop() {}

jremington

#1
May 23, 2019, 05:47 am Last Edit: May 23, 2019, 05:51 am by jremington
Quote
the hashing library takes strings as inputs.
What library? Post a link to the library.

What do you mean by "strings"?  If you mean String objects, these are to be avoided on Arduino, because they cause memory problems and program crashes. C-strings are zero terminated character arrays, like "12345".

Please try to clearly describe exactly what you are trying to do. Posting snippets is not useful.

20 x 256 bit hash values require 640 bytes of storage. No problem there.

pan19ss

#2
May 23, 2019, 07:05 am Last Edit: May 23, 2019, 07:06 am by pan19ss
The library is this one https://rweather.github.io/arduinolibs/classSHA256.html
(from Tools->Manage libraries)
I also see it here https://github.com/rweather/arduinolibs/tree/master/libraries/Crypto


Here is my complete code. This works fine for up to 6 hash values (SIGMA = 6 in the code). However if I add more the program misbehaves and I also get dynamic memory warnings. Think for 20 values it doesn't work at all.
So my ultimate goal is to minimize memory usage, that's why I thought of converting the hash values to byte arrays...
Code: [Select]
#include <Crypto.h>
#include <SHA256.h>
#include <string.h>

#define HASH_SIZE 32
#define BLOCK_SIZE 64
#define OUTPUT_MAX 32
#include <limits.h> //for infinity
char hex[256];

char *btoh(char *dest, uint8_t *src, int len) {
  char *d = dest;
  while( len-- ) sprintf(d, "%02x", (unsigned char)*src++), d += 2;
  return dest;
}

void PrintHex8(uint8_t *data, uint8_t length) // prints 8-bit data in hex with leading zeroes
{
     char tmp[16];
       for (int i=0; i<length; i++) {
         sprintf(tmp, "0x%.2X",data[i]);
         Serial.print(tmp); Serial.print(" ");
       }
}


char* h(String input) {   
 
SHA256 hash;
uint8_t result[32];
hash.reset();
hash.update(input.c_str(), input.length());
hash.finalize(result, sizeof(result));

  return(btoh(hex, result, 32));
}


//Pebble code
unsigned int SIGMA = 6;

//Stop function
void stop()
{
 while(1);
}

typedef struct Pebble{
  int StartIncr;
  int DestIncr;
  int position;
  int destination;
  String value;
};

String FindValue(struct Pebble pebble_List[])
{
  for (int i = 1; i < SIGMA; i++)
  {
        if (pebble_List[i].position == pebble_List[0].position) {
            return pebble_List[i].value;         
          }
  }
}


int struct_cmp_by_destin(const void *a, const void *b)
{
    struct Pebble *ia = (struct Pebble *)a;
    struct Pebble *ib = (struct Pebble *)b;
    return (ia->destination - ib->destination);
}


unsigned int Pow2(byte ex)
{
   return 1 << ex;
}

void setup()
{
  Serial.begin(115200);
  //boolean DEBUG = 0;
  Pebble pebblelist[SIGMA];
  for (int i = 0; i < SIGMA; i++) {
    pebblelist[i].StartIncr = 3*Pow2(i+1);
    pebblelist[i].DestIncr = Pow2(i+2) ;
    pebblelist[i].position = Pow2(i+1);
    pebblelist[i].destination = Pow2(i+1);
  }
  pebblelist[0].value = "dbd575734d4c87e9287ee5233e9b83611004f9f996befbc9b68d34d4e423bd98";
  pebblelist[1].value = "3a397279c72e019a31729d7536dfd407c37cc63dcde44a0222be2e098d85181a";
  pebblelist[2].value = "701e771b21edc950887818f5c41a5ac3f0e8b471318599aef9c836291987c82f";
  pebblelist[3].value = "c5c6432924112bb666e4eec23d9855c3d817416b36a2a9dfd0acb968a07f203f";
  pebblelist[4].value = "6a85d015df7b6be69c75a9067945052117c4da060d1e95fc3f59079d36216da2";
  pebblelist[5].value = "a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3";

  unsigned int currentposition = 0;

  while(1) {
  //Serial.print("Hash ");
  //Serial.println(currentposition);
  //delay(500);
  //1
  if (currentposition >= Pow2(SIGMA) ) { stop();}
  else {currentposition++ ;};
  //2
  for (int i = 0; i < SIGMA; i++) {
    if (pebblelist[i].position != pebblelist[i].destination) {
      pebblelist[i].position = pebblelist[i].position -2;
      pebblelist[i].value = h(h(pebblelist[i].value));
      };
  }
  String output;
  //3
  if (currentposition % 2 == 1 ) {
    output = h(pebblelist[0].value);
    }
  else {
    output = pebblelist[0].value;
    pebblelist[0].position = pebblelist[0].position + pebblelist[0].StartIncr;
    pebblelist[0].destination = pebblelist[0].destination + pebblelist[0].DestIncr;
    if (pebblelist[0].destination > Pow2(SIGMA) ) {
        pebblelist[0].destination = INT_MAX;
        pebblelist[0].position = INT_MAX;
      }
    else {
        pebblelist[0].value = FindValue(pebblelist);
      };
    qsort(pebblelist, sizeof(pebblelist)/sizeof(pebblelist[0]), sizeof(pebblelist[0]), struct_cmp_by_destin);
    };
  Serial.println(output);

  }
}



void loop()
{



}

PaulS

Quote
So my ultimate goal is to minimize memory usage, that's why I thought of converting the hash values to byte arrays...
Each two characters in the string becomes one byte, so you will save the data in half the space.

What IS the problem? Grabbing two characters at a time from a string is trivial. Converting the 2 characters to a byte is simple. Storing the resulting value in an array is a no-brainer.
The art of getting good answers lies in asking good questions.

GolamMostafa

#4
May 23, 2019, 02:36 pm Last Edit: May 23, 2019, 03:16 pm by GolamMostafa
So my ultimate goal is to minimize memory usage, that's why I thought of converting the hash values to byte arrays...
Each two characters in the string becomes one byte, so you will save the data in half the space.
Example:
Code: [Select]
char srcArray[] = {0x31, 0x32, 0x33, 0x34, 0x35, 0x46};
byte destArray[3];   //12 34 5F

void setup()
{
  Serial.begin(9600);
}

void loop()
{
  for (int i = 0, j = 0; i < sizeof(srcArray) - 2, j < sizeof(srcArray) / 2; i = i + 2, j++)
  {
    byte x = hexDigit(srcArray[i]);    //scArray[0] = 0x31; x = 01
    x = x << 4;                              //x = 10
    // Serial.println(x, HEX);
    byte z = hexDigit(srcArray[i + 1]);  //srcArray[1] = 0x32; z = 02
    z = x | z;                                      // z = 12
    //  Serial.println(z, HEX);
    destArray[j] = z;                            //destArray[0] = 12
  }

  for (int i = 0; i < 3; i++)
  {
    Serial.print(destArray[i], HEX);  //shows: 12 34 5F
    Serial.print(' ');
  }
  Serial.println();
  delay(1000);
}

byte hexDigit(byte y)
{
  if (y <= 0x39)
  {
    y = (y - 0x30);  //01
  }
  else
  {
    y = (y - 0x37);
  }
  return y;
}

PaulS

Code: [Select]
char srcArray[] = {0x31, 0x32, 0x33, 0x34, 0x35, 0x46};
Why do you insist on providing examples that do NOT match what the OP has?

Code: [Select]
char srcArray[] = "12345F";
The art of getting good answers lies in asking good questions.

jremington

#6
May 23, 2019, 05:01 pm Last Edit: May 23, 2019, 05:05 pm by jremington
Quote
So my ultimate goal is to minimize memory usage
This is certainly a problem:
Code: [Select]
String value;
To achieve your goal and vastly increase operational reliability:

1. Don't use Strings
2. As already mentioned, convert hex character data arrays to byte data arrays, and cut space requirements in half

GolamMostafa

#7
May 23, 2019, 06:56 pm Last Edit: May 23, 2019, 06:57 pm by GolamMostafa
Code: [Select]
char srcArray[] = {0x31, 0x32, 0x33, 0x34, 0x35, 0x46};
Why do you insist on providing examples that do NOT match what the OP has?

Code: [Select]
char srcArray[] = "12345F";
Strictly speaking, I should have declared the array in my example as:

Code: [Select]
char srcArray[] = {'1', '2', '3', '4', '5', 'F', '\0'};

in order to  maintain consistency with @PaulS's following statement:
Quote
Each two characters in the string becomes one byte, so you will save the data in half the space.

pan19ss

Ok so yes I have understood that string is "bad" in arduino. However the I am trying to make the hashing function work with char instead of strings but it outputs incorrect values.
Here's a minimal working example of what I mean. h1() which uses string outputs the correct hash of value1 (starts with "6db...") but h2() which uses char outputs wrong hash.
Here's the usage example also from the library.

Code: [Select]
#include <Crypto.h>
#include <SHA256.h>
#include <string.h>

#define HASH_SIZE 32
#define BLOCK_SIZE 64
#define OUTPUT_MAX 32
#include <limits.h> //for infinity
char hex[256];

const char*  value1 = "dbd575734d4c87e9287ee5233e9b83611004f9f996befbc9b68d34d4e423bd98";

char *btoh(char *dest, uint8_t *src, int len) {
  char *d = dest;
  while( len-- ) sprintf(d, "%02x", (unsigned char)*src++), d += 2;
  return dest;
}


char* h1(String input) {   
 
SHA256 hash;
uint8_t result[32];
hash.reset();
hash.update(input.c_str(), input.length());
hash.finalize(result, sizeof(result));

  return(btoh(hex, result, 32));
}


char* h2(char* input) {   
 
SHA256 hash;
uint8_t result[32];
hash.reset();
hash.update(input, sizeof(input));
//hash.update(input.c_str(), input.length());
hash.finalize(result, sizeof(result));

  return(btoh(hex, result, 32));
}


void setup()
{
  Serial.begin(115200);
 
  Serial.println(h1(value1));
  Serial.println(h2(value1));
}



void loop()
{



}

 

jremington

#9
May 25, 2019, 03:41 am Last Edit: May 25, 2019, 03:44 am by jremington
There are no Strings in the linked example code. Start with that and do not use Strings for anything.

pan19ss

There are no Strings in the linked example code. Start with that and do not use Strings for anything.
Ok can you please suggest how to fix my hash function h2() ? I've been struggling with this so long and can't find how to do this right..

jremington

#11
May 25, 2019, 04:26 am Last Edit: May 25, 2019, 04:29 am by jremington
You are confusing the size of a character pointer with the length of a character string. To demonstrate that, I've added a couple of debug prints to the function h2().

The fix should be immediately obvious.

Code: [Select]

char* h2(char* input) {   
 
SHA256 hash;
Serial.print("sizeof(input) ");
Serial.println(sizeof(input));
Serial.print("strlen(input) ");
Serial.println(strlen(input));
uint8_t result[32];
hash.reset();
hash.update(input, sizeof(input));
//hash.update(input.c_str(), input.length());
hash.finalize(result, sizeof(result));

  return(btoh(hex, result, 32));
}


PS: this should not be in the function:

Code: [Select]
SHA256 hash;

pan19ss


Go Up