Why I am unable to read some bytes in python from arduino?

I am developing a flash memory dumper. In order to do so I need to transfer some bytes to my computer therefore I develop the serial protocol with some emulated data using the following sketch:

#define LASTBLOCK 1023
#define LASTPAGE 63
#define LASTBYTE 2111

#define PAGE_READ 0x13
#define PAGE_READ_CACHE_SEQUENCIAL 0x03
#define READ_FROM_CACHE 0x3B
#define PAGE_READ_CACHE_END 0x3F

#define PAGE_SIZE 2111 
#define BUFFER_LEN 32 

unsigned int page_to_read = 1;
unsigned int block_to_read = 1;
boolean spi_begun = false;
boolean read_start = false;
uint8_t bytes_index = 1;

typedef struct  {

  // Page that is transmitted
  unsigned int page;

  // Block that is transmitted
  unsigned int block;

  // A mumeric representation of the 32 byte segment that I send. 
  // I cannot send the page altogether, therefore a byte intexing is required
  uint8_t bytes_index;
  
  // Despite my buffer is 32 bytes the usable data may be less
  uint8_t length; 

  // Data buffer containing the data to be sent via serial
  byte data[BUFFER_LEN];
} usb_msg;
const size_t message_length = sizeof(usb_msg);


void setup() {
 pinMode(13, OUTPUT);
// SPI.begin();
 Serial.begin(9600);
 digitalWrite(13, LOW);
 delay(1000);
 digitalWrite(13, HIGH);
}

void loop() {
    
  // I need to signal when to start reading
  if(Serial.available() < 0 || !read_start || !Serial.readString().equals("READY")){return;}

  delay(1000);
  read_start=true;
  digitalWrite(13, LOW);
  
  // I need to reset the counts before reading any block 
  if(page_to_read > LASTPAGE){
    page_to_read = 1;
    block_to_read ++;
  }

  if(block_to_read > LASTBLOCK){
    spi_begun=false;
    digitalWrite(13, LOW);
    return ;
  }

  if(!spi_begun){
    digitalWrite(13, LOW);
    delay(600);
    digitalWrite(13, HIGH);
    spi_begun=true;
  }
    
  // Reading page there's no need for seperate function in order to avoid stack usage
  for(int bytes_to_read = PAGE_SIZE; bytes_to_read > 0; bytes_to_read-=BUFFER_LEN){
    
    uint8_t bytes_to_send = bytes_to_read < PAGE_SIZE?bytes_to_read:PAGE_SIZE;
    usb_msg msg;

    msg.page  = page_to_read;
    msg.block = block_to_read;
    msg.bytes_index = bytes_index;
    msg.length = bytes_to_send;

    // Dummy Data
    for(uint8_t i=0 ; i < bytes_to_send; i++){ msg.data[i]= (byte) 1;}

    Serial.write((byte *)&msg,message_length);
    delay(100);
    bytes_index++;
  }
  
  page_to_read++;
  delay(100);
}

In the meantime I made the following python script to read it:

import serial
from time import sleep
import base64

ser = serial.Serial('/dev/ttyACM0', baudrate=9600, timeout=1)

if not ser.isOpen():
    ser.open()

print("OPEN Serial\n")
ser.write(b'READY')

print("READ DATA\n")
bytes = ser.read(80)
while 1:
    print("BYTES "+ bytes.hex()+"\n")
    bytesToRead = ser.inWaiting()
    if bytesToRead == 0: continue;
    print("READ "+ str(bytesToRead)+"\n")
    bytes = ser.read(bytesToRead)

But I am unable to read some bytes. And I have no idea why. Do you have an appropach where the script reads the data?

If this was my project and had your problem, I would Serial.print() the entire Arduino message in hex to the serial monitor and see what was not being shown by the python program.

There is a very simple code you can build from here

More advanced stuff here

And more here

In the end I tried this approach:

Sketch:

#define LASTBLOCK 1023
#define LASTPAGE 63
#define LASTBYTE 2111

#define PAGE_READ 0x13
#define PAGE_READ_CACHE_SEQUENCIAL 0x03
#define READ_FROM_CACHE 0x3B
#define PAGE_READ_CACHE_END 0x3F

#define PAGE_SIZE 2111 
#define BUFFER_LEN 32 

#define ACK 0x06

unsigned int page_to_read = 1;
unsigned int block_to_read = 1;
boolean spi_begun = false;
boolean read_start = false;
uint8_t bytes_index = 1;

typedef struct  {
  // Page that is transmitted
  unsigned int page;

  // Block that is transmitted
  unsigned int block;

  // A mumeric representation of the 32 byte segment that I send. 
  // I cannot send the page altogether, therefore a byte intexing is required
  uint8_t bytes_index;
  
  // Despite my buffer is 32 bytes the usable data may be less
  uint8_t length; 

  // Data buffer containing the data to be sent via serial
  byte data[BUFFER_LEN];
} usb_msg;


void sendMsg(usb_msg msg){
  byte* ptr = (byte*)&msg;
  for (size_t i = 0; i < sizeof(usb_msg); i++){
    Serial.print(*ptr >>4, HEX);
    Serial.print(*ptr & 0x0F, HEX);
    ptr++;
  }
  Serial.println();
}


void setup() {
 Serial.begin(9600);
}

void loop() {
    
  read_start=true;
  
  // I need to reset the counts before reading any block 
  if(page_to_read > LASTPAGE){
    page_to_read = 1;
    block_to_read ++;
  }

  if(block_to_read > LASTBLOCK){
    spi_begun=false;
    return ;
  }

  if(!spi_begun){
    spi_begun=true;
  }
    
  // Reading page there's no need for seperate function in order to avoid stack usage
  for(int bytes_to_read = PAGE_SIZE; bytes_to_read > 0; bytes_to_read-=BUFFER_LEN){
    uint8_t bytes_to_send = bytes_to_read < BUFFER_LEN ? bytes_to_read: BUFFER_LEN;
    usb_msg msg = {page_to_read, block_to_read, bytes_index, bytes_to_send, {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}};
    sendMsg(msg);
    delay(100);
    bytes_index++;
  }
  
  page_to_read++;
  delay(100);
}

And with this simple python script:

import serial
from time import sleep
import base64

ser = serial.Serial('/dev/ttyACM0', baudrate=9600, timeout=1)

if not ser.isOpen():
    ser.open()

print("OPEN Serial\n")
print("READ DATA\n")
while 1:
    line = ser.readline()
    if(line == b""): continue; 
    print(line)

For now, I ditched the command from the python and I actually focus on data encoding/decoding as hex as seen on: How I can translate a struct as hex/base16 - #5 by david_2018

What I actually do is an Telnet-based approach where each messsage is seperated with \r\n a single byte line (using ascii controll characters) would be control indicators.

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.