Reading a 32 byte array from Arduino into python

Hi there,
I am trying to read a 32 byte array into python and not getting it to work properly I feel I am really close but its just that last little bit where its not working

I am receiving a 32 byte message from a lora transmitter on one board and receiving on another then I want to send the received bytes into python the wireless send & recieve works great but I can't get python to see the serial data correctly.

I am using the USB serial to send the binary data and have a USB to serial adaptor hooked up to serial1 for debugging.

arduino code:

// Feather9x_RX
// -*- mode: C++ -*-
// Example sketch showing how to create a simple messaging client (receiver)
// with the RH_RF95 class. RH_RF95 class does not provide for addressing or
// reliability, so you should only use RH_RF95 if you do not need the higher
// level messaging abilities.
// It is designed to work with the other example Feather9x_TX

#include <SPI.h>
#include <RH_RF95.h>
#include <RHEncryptedDriver.h>
#include <Speck.h>

/* for Feather32u4 RFM9x
  #define RFM95_CS 8
  #define RFM95_RST 4
  #define RFM95_INT 7
*/

#define RFM95_CS 8
#define RFM95_RST 4
#define RFM95_INT 3


/* for shield
  #define RFM95_CS 10
  #define RFM95_RST 9
  #define RFM95_INT 7
*/

/* Feather 32u4 w/wing
  #define RFM95_RST     11   // "A"
  #define RFM95_CS      10   // "B"
  #define RFM95_INT     2    // "SDA" (only SDA/SCL/RX/TX have IRQ!)
*/

/* Feather m0 w/wing
  #define RFM95_RST     11   // "A"
  #define RFM95_CS      10   // "B"
  #define RFM95_INT     6    // "D"
*/

#if defined(ESP8266)
/* for ESP w/featherwing */
#define RFM95_CS  2    // "E"
#define RFM95_RST 16   // "D"
#define RFM95_INT 15   // "B"

#elif defined(ESP32)
/* ESP32 feather w/wing */
#define RFM95_RST     27   // "A"
#define RFM95_CS      33   // "B"
#define RFM95_INT     12   //  next to A

#elif defined(NRF52)
/* nRF52832 feather w/wing */
#define RFM95_RST     7   // "A"
#define RFM95_CS      11   // "B"
#define RFM95_INT     31   // "C"

#elif defined(TEENSYDUINO)
/* Teensy 3.x w/wing */
#define RFM95_RST     9   // "A"
#define RFM95_CS      10   // "B"
#define RFM95_INT     4    // "C"
#endif


// Change to 434.0 or other frequency, must match RX's freq!
#define RF95_FREQ 433.0

#define STATIONID_HIGHBYTE 0x00
#define STATIONID_LOWBYTE 0x02

// Singleton instance of the radio driver
RH_RF95 rf95(RFM95_CS, RFM95_INT);


Speck myCipher;   // Instanciate a Speck block ciphering
RHEncryptedDriver myDriver(rf95, myCipher); // Instantiate the driver with those two

unsigned char encryptkey[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; // The very secret key


// Blinky on receipt
#define LED 13


void setup()
{
  pinMode(LED, OUTPUT);
  pinMode(RFM95_RST, OUTPUT);
  digitalWrite(RFM95_RST, HIGH);

  int16_t serialtimeout = 0;
  Serial.begin(115200);
  while (!Serial) {
    delay(1000);
    serialtimeout++;
    if (serialtimeout > 10) {
      break;
    }
  }

  Serial1.begin(115200);


  delay(100);

  //Serial.println("Feather LoRa RX Test!");
  Serial1.println("Feather LoRa RX Test!");
  // manual reset
  digitalWrite(RFM95_RST, LOW);
  delay(10);
  digitalWrite(RFM95_RST, HIGH);
  delay(10);

  while (!rf95.init()) {
    Serial1.println("LoRa radio init failed");
    while (1);
  }
  Serial1.println("LoRa radio init OK!");

  // Defaults after init are 434.0MHz, modulation GFSK_Rb250Fd250, +13dbM
  if (!rf95.setFrequency(RF95_FREQ)) {
    Serial1.println("setFrequency failed");
    while (1);
  }
  Serial1.print("Set Freq to: "); Serial.println(RF95_FREQ);

  // Defaults after init are 434.0MHz, 13dBm, Bw = 125 kHz, Cr = 4/5, Sf = 128chips/symbol, CRC on

  // The default transmitter power is 13dBm, using PA_BOOST.
  // If you are using RFM95/96/97/98 modules which uses the PA_BOOST transmitter pin, then
  // you can set transmitter powers from 5 to 23 dBm:
  rf95.setTxPower(23, false);
  myCipher.setKey(encryptkey, 16);
}



void myprintBuffer(const char* prompt, const uint8_t* buf, uint8_t len)
{
  uint8_t i;
  Serial1.println(prompt);
  for (i = 0; i < len; i++)
  {
    Serial1.print(buf[i], HEX);
    Serial1.print(' ');
  }
  Serial1.println("");
}

void printrecvBuffer(const uint8_t* buf, uint8_t len)
{
  uint8_t i;
  //Serial.write(0x24);
   //Serial.write(buf);
  for (i = 0; i < len; i++)
  {
    Serial.write(buf[i]);

  }
  //Serial.println();
}

void loop()
{
  //if (rf95.available())
  if (myDriver.available())
  {
    // Should be a message for us now
    uint8_t buf[RH_RF95_MAX_MESSAGE_LEN];
    uint8_t len = sizeof(buf);

    //if (rf95.recv(buf, &len))
    if (myDriver.recv(buf, &len))
    {
      digitalWrite(LED, HIGH);
      //myprintBuffer("Received: ", buf, len);
      //Serial1.print("Got: ");
      //Serial1.println((char*)buf);
      //Serial.print("RSSI: ");
      //Serial.println(rf95.lastRssi(), DEC);
      //Serial.println("----------------------------------------------");
      char data[8];
      if (buf[0] == 0x02) { //packet is a data packet do stuff
        //Serial.print("byte 0: ");
        //Serial.println(buf[0]);
        printrecvBuffer(buf, len);
        myprintBuffer("Received: ", buf, len);
        //Serial.write(buf);
        //Serial.println("");
        data[0] = 0x01; //packet type is ack
        data[1] = buf[3];
        data[2] = buf[4];
        data[3] = STATIONID_HIGHBYTE; //Transmitting/Source Station ID highByte
        data[4] = STATIONID_LOWBYTE; //Transmitting/Source Station ID lowByte
        data[5] = buf[5];
        data[6] = buf[6];
        //data[7] = 0x00;
        //Serial.println(data);
        myDriver.send((uint8_t *)data, sizeof(data));
        rf95.waitPacketSent();
      }

      // Send a reply
      //uint8_t data[] = "And hello back to you";
      //rf95.send(data, sizeof(data));
      //rf95.waitPacketSent();
      //Serial.println("Sent a reply");
      digitalWrite(LED, LOW);
    }
    else
    {
      Serial1.println("Receive failed");
    }
  }
}

Python code:

import serial
import time
ser = serial.Serial('COM14', 115200)
ser.flushInput()
buf = bytearray(32)
while (True):
    rx_raw = ser.read(32)
    print("raw:  ")
    print(rx_raw) 
    print("\n")

    mystring = "";
    for c in rx_raw:
        mystring = mystring + " " + hex(c)
    print("hexed: ")
    print(mystring)
    print("\n")

see post below for a break down of my packet structure

I am getting what I expect from my debug output on serial1:

Received: 
2 0 1 0 2 1 69 A 22 1 7 1 35 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 
Received: 
2 0 1 0 2 1 6A A 23 1 7 1 36 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

yes I realise that this is the data converted to ascii by serial.print but I would expect my python code to see the same but as binary data as I am using serial.write

running my python code gives me:

raw:  
b'\x00\x01\x00\x02\x01\xc3\n"\x01\x07\x01;\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00'


hexed: 
 0x0 0x1 0x0 0x2 0x1 0xc3 0xa 0x22 0x1 0x7 0x1 0x3b 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x2 0x0


raw:  
b'\x01\x00\x02\x01\xc4\n\x1e\x01\x07\x01;\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x01'


hexed: 
 0x1 0x0 0x2 0x1 0xc4 0xa 0x1e 0x1 0x7 0x1 0x3b 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x2 0x0 0x1

which doesn't entirely match up with what I am expecting namely the first byte should always be 0x2 and the 14th byte onwards should be always zero so something is happening but I have no clue what.

My 32 byte array is structured as follows:

  mydata[0] = 0x02; //Message Type 01=ack, 02=data (with ack), 03=command (with ack)
  mydata[1] = 0x00; //Receiveing/Destination Station ID highByte set to 00 for bcast
  mydata[2] = 0x01; //Receiveing/Destination Station ID lowByte set to 00 for bcast
  mydata[3] = STATIONID_HIGHBYTE; //Transmitting/Source Station ID highByte
  mydata[4] = STATIONID_LOWBYTE; //Transmitting/Source Station ID lowByte
  mydata[5] = highByte(packetnum); //Message ID highByte
  mydata[6] = lowByte(packetnum); //Message ID lowByte
  mydata[7] = highByte(vbat); //Battery Voltage highbyte
  mydata[8] = lowByte(vbat); //Battery Voltage lowbyte
  mydata[9] = highByte(temp); //temp sensor highbyte
  mydata[10] = lowByte(temp); //temp sensor lowbyte
  mydata[11] = highByte(humi); //humidity sensor highbyte
  mydata[12] = lowByte(humi); //humidity sensor lowbyte
  mydata[13] = 0; //Reserved for future use
  mydata[14] = 0; //Reserved for future use
  mydata[15] = 0; //Reserved for future use
  mydata[16] = 0; //Reserved for future use
  mydata[17] = 0; //Reserved for future use
  mydata[18] = 0; //Reserved for future use
  mydata[19] = 0; //Reserved for future use
  mydata[20] = 0; //Reserved for future use
  mydata[21] = 0; //Reserved for future use
  mydata[22] = 0; //Reserved for future use
  mydata[23] = 0; //Reserved for future use
  mydata[24] = 0; //Reserved for future use
  mydata[25] = 0; //Reserved for future use
  mydata[26] = 0; //Reserved for future use
  mydata[27] = 0; //Reserved for future use
  mydata[28] = 0; //Reserved for future use
  mydata[29] = 0; //Reserved for future use
  mydata[30] = 0; //Reserved for future use
  mydata[31] = 0; //Reserved for future use

This simple Python - Arduino demo may help

It may be useful to read about the Python unpack function

...R

I rewrote it in python 2.7 and added some start and end markers to the arduino code and that let get the data I was expecting now to try to port it back to python 3