Information: Source-Level Debugging of Sketch Using OpenOCD and GDB

- Prerequisites & Background Knowledge:
    When you run the 'arduino-cli compile --upload blink/sketch' command,
    a 'sketch.ino_debug.elf' file for debugging is created in the cache
    folder. The 'sketch.ino.elf-zsk.bin' file is uploaded to the MCU address
    0x08100000. It is then relocated to the SRAM 0x20000000 area by the
    'llext_load' function. This relocated information is stored in the SRAM
    area as an 'llext' struct, where 'llext->name' contains the value "sketch".

    To perform source-level debugging, you need to determine the location of
    the 'sketch.ino_debug.elf' file and the '.text', '.rodata', and '.bss'
    section values from the 'llext' struct information.

  . Location of the 'sketch.ino_debug.elf' file:
    1. Generate an MD5 hash of the full path containing the sketch program:
       echo -n "/home/arduino/{SKETCH_FOLEDER} | md5sum | awk '{print toupper($1)}'
       # echo -n "/home/arduino/works/blink/sketch" | md5sum | awk '{print toupper($1)}'
       # EA52F4B169B20C1503833FB344F2E866

    2. Path: /home/arduino/.cache/arduino/sketches/{MD5_HASH}/sketch.ino_debug.elf
       # /home/arduino/.cache/arduino/sketches/EA52F4B169B20C1503833FB344F2E866/sketch.ino_debug.elf


- Setup: Open 3 Terminals
  1) Terminal #1: Used for running OpenOCD and GDB.
  2) Terminal #2: Used for Serial/Monitor via the 'arduino-app-cli monitor'
     command.
  3) Terminal #3: Used for Python to execute 'Bridge.call' and send a
    'Bridge.notify' to trigger the function set as a breakpoint.
    (The 'arduino' package must be installed in the Python .venv).


1. In Terminal #1, run OpenOCD in the background and launch the 
   'arm-zephyr-eabi-gdb' program:
   
   /opt/openocd/bin/openocd -s /opt/openocd -f openocd_gpiod.cfg & [Enter][Enter]

   # To find the gdb path: find ~/.arduino15 -name "arm-zephyr-eabi-gdb"
   # /home/arduino/.arduino15/packages/zephyr/tools/arm-zephyr-eabi/0.16.8/bin/arm-zephyr-eabi-gdb

   /home/arduino/.arduino15/packages/zephyr/tools/arm-zephyr-eabi/0.16.8/bin/arm-zephyr-eabi-gdb

  1.1. Find the 'llext' struct information:

       target extended-remote:3333
       monitor reset halt
       find 0x20002000, 0x20003000, "sketch"
       # print/x $_ + 16
       x/4xw $_ + 16
       #                   .text                          .rodata          .bss
       # 0x20002068:     0x200021b0      0x00000000      0x200087f4      0x20008cc0

 1.2. Remap the symbol tables with the new section information:

      add-symbol-file  /home/arduino/.cache/arduino/sketches/{MD5_HASH}/sketch.ino_debug.elf 0x200021b0 -s .rodata 0x200087f4 -s .bss 0x20008cc0

(gdb) add-symbol-file /home/arduino/.cache/arduino/sketches/EA52F4B169B20C1503833FB344F2E866/sketch.ino_debug.elf 0x200021b0 -s .rodata 0x200087f4 -s .bss 0x20008cc0
add symbol table from file "/home/arduino/.cache/arduino/sketches/EA52F4B169B20C1503833FB344F2E866/sketch.ino_debug.elf" at
        .text_addr = 0x200021b0
        .rodata_addr = 0x200087f4
        .bss_addr = 0x20008cc0
(y or n) y

 1.3. Set a breakpoint and run GDB:

      monitor gdb breakpoint_override hard
      break sketch_info
      # Breakpoint 1 at 0x200022e0: file /home/arduino/works/blink/sketch/sketch.ino, line 69.
      continue


2. Run 'Bridge.call(...)' and 'Bridge.notify('sketch_info')' in Python
   Terminal #3 to hit the breakpoint:

   python
   from arduino.app_utils import Bridge
   Bridge.call('set_led_state', False)
   Bridge.notify('sketch_info')


3. Verify that the execution has stopped at the breakpoint in 
   Terminal #1 (GDB):
   
   Breakpoint 1, sketch_info () at /home/arduino/works/blink/sketch/sketch.ino:69
   69        for (uintptr_t addr = scan_start; addr <= scan_end - sizeof(struct llext); addr +=4) {
  
   Execute the 'next' and 'list' commands. *** Do not use single quotes when
   entering commands in GDB.
   Execute 'until 72'
   Run 'print *candidate'
   Run 'print candidate->name'
   Execute 'continue'

4. Check the output screen on the Serial/Monitor terminal.


This completes the setup for source-level debugging of your sketch program. Happy debugging!!!

sketch.ino

// SPDX-FileCopyrightText: Copyright (C) ARDUINO SRL (http://www.arduino.cc)
//
// SPDX-License-Identifier: MPL-2.0
#include <zephyr/kernel.h>
#include "Arduino_RouterBridge.h"

#if 1
extern "C" {
  #include <zephyr/llext/llext.h>
}
#endif


void setup() {
    pinMode(LED_BUILTIN, OUTPUT);

    Bridge.begin();
    Serial.begin();

    Bridge.provide("reset_led_count", reset_led_count);
    Bridge.provide("set_led_state", set_led_state);
    Bridge.provide("sketch_info", sketch_info);
}

void loop() {
    char buffer[83];

    if (Serial.available() > 0) {
        String s = Serial.readStringUntil('\n');
        s.trim();
        snprintf(buffer, sizeof(buffer), "\"%s\"", s.c_str());
        Serial.print(buffer);
    }

    k_msleep(10);
}

int count = 0;

void reset_led_count() {
    count = 0;
    Serial.println("reset count");
}

String set_led_state(bool state) {
    char buffer[30]; // Buffer for formatted string

    snprintf(buffer, sizeof(buffer), "%4d set_led_state called", ++count); // Format "   1", "  10", etc.
    Serial.println(buffer);

    // LOW state means LED is ON
    digitalWrite(LED_BUILTIN, state ? LOW : HIGH);
    String r = digitalRead(LED_BUILTIN) == LOW ? String("ON") : String("OFF");

    return r;
}

void sketch_info() {
  char buffer[80];

#if 1
  uintptr_t scan_start = 0x20002000;
  uintptr_t scan_end   = 0x20004000;

  const char* name = "sketch";
  size_t name_len = strlen(name);
  struct llext *ext = NULL;

  for (uintptr_t addr = scan_start; addr <= scan_end - sizeof(struct llext); addr +=4) {
    struct llext *candidate = (struct llext *)addr;
    if (strncmp(candidate->name, name, name_len) == 0) {
      uintptr_t next_ptr = (uintptr_t)candidate->llext_list.next;
      if (next_ptr == 0 || (next_ptr >= 0x20000000 && next_ptr <= 0x200BFFFF)) {
        ext = candidate;
        break;
      }
    }
  }

  if (ext != NULL) {
    Serial.println("-------------------------------------------------");
    Serial.print("llext  : 0x");
    Serial.println((uintptr_t)ext, HEX);

    // [1] llext->link
    snprintf(buffer, sizeof(buffer), "  [%p + %d] llext_list :  %p", ext, ((uintptr_t)&(ext->llext_list) - (uintptr_t)ext), ext->llext_list);
    Serial.println(buffer);

    // [2] llext->name
    snprintf(buffer, sizeof(buffer), "  [%p + %d] name       : '%s'", ext, ((uintptr_t)&(ext->name) - (uintptr_t)ext), ext->name);
    Serial.println(buffer);

    // [3] .text (코드 영역)
    uintptr_t text_start = (uintptr_t)ext->mem[LLEXT_MEM_TEXT];
    size_t    text_size  = ext->mem_size[LLEXT_MEM_TEXT];

    snprintf(buffer, sizeof(buffer), "  [%p + %d] .text      : %p %8d", &(ext->mem), LLEXT_MEM_TEXT, text_start, text_size);
    Serial.println(buffer);

    // [4] .rodata (초기화되지 않은 전역변수 영역)
    snprintf(buffer, sizeof(buffer), "  [%p + %d] .rodata    : %p %8d", &(ext->mem), LLEXT_MEM_RODATA, (uintptr_t)ext->mem[LLEXT_MEM_RODATA], (uintptr_t)ext->mem_size[LLEXT_MEM_RODATA]);
    Serial.println(buffer);

    // [5] .bss (초기화되지 않은 전역변수 영역)
    uintptr_t bss_start = (uintptr_t)ext->mem[LLEXT_MEM_BSS];
    size_t    bss_size  = ext->mem_size[LLEXT_MEM_BSS];

    snprintf(buffer, sizeof(buffer), "  [%p + %d] .bss       : %p %8d", &(ext->mem), LLEXT_MEM_BSS, bss_start, bss_size);
    Serial.println(buffer);
    Serial.println("-------------------------------------------------");
  }
#endif

  Serial.println();
  snprintf(buffer, sizeof(buffer), "%-20s: %p","main", ((uintptr_t)&main) & ~1);
  Serial.println(buffer);

  snprintf(buffer, sizeof(buffer), "%-20s: %p","  setup", ((uintptr_t)&setup) & ~1);
  Serial.println(buffer);

  snprintf(buffer, sizeof(buffer), "%-20s: %p","  loop", ((uintptr_t)&loop) & ~1);
  Serial.println(buffer);

  snprintf(buffer, sizeof(buffer), "%-20s: %p","  reset_led_count", ((uintptr_t)&reset_led_count) & ~1);
  Serial.println(buffer);

  snprintf(buffer, sizeof(buffer), "%-20s: %p","  set_led_state", ((uintptr_t)&set_led_state) & ~1);
  Serial.println(buffer);

  snprintf(buffer, sizeof(buffer), "%-20s: %p","  sketch_info", ((uintptr_t)&sketch_info) & ~1);
  Serial.println(buffer);

  Serial.println("  ------------------");
  snprintf(buffer, sizeof(buffer), "%-20s: %p","  count", &count);
  Serial.println(buffer);

}

Thanks for taking the time to share this knowledge @msyang!

An alternative approach would be to use the --export-binaries flag in the arduino-cli compile command:

https://arduino.github.io/arduino-cli/latest/commands/arduino-cli_compile/#options

This will cause the .ino_debug.elf file to be saved to the build/arduino.zephyr.unoq subfolder of the sketch folder.

Another alternative would be to use the --build-path to specify an arbitrary path for Arduino CLI to store the compiled binaries.

Thanks a lot @ptillisch for setting build path. By the way, is there any RPC to Zephyr RTOS to query struct llext *ext for "sketch"?