manual regex check of sketch ?

Hi,

I get the Stackoverflow error on my sketch (see troubleshooting guide)
but it leaves no clue as to where to look.
I run on linux, so maybe someone has the regex checks in a shellscript ?
with that I can check the sketch manually and hopefully get a better idea as to where things go wrong.

any tips as to what regex checks are executed by the preprocessor are also much appreciated.
I lack the skills to delve in the arduino ide source code (and find them).

help ? pretty please ?

Post your code then we might be able to advise.

there we go. I have removed a lot of commentary in the hope it would change something, but alas..
it is my modification of the sketch isp_repair2 from JeeLabs. it is supposed to program attiny85 chips in the end.
other scripts still compile ok, and also changing the target does not help
compiled on ubuntu, arduino 1.0.1

#include <JeeLib.h>
#include <avr/pgmspace.h>
#include <avr/sleep.h>

// pin definitions
#define PIN_SCK 14 // PC0 - AIO1 - serial clock to target avr
#define PIN_MISO 4 // PD4 - DIO1 - input from target avr
#define PIN_MOSI 17 // PC3 - AIO4 - output to target avr
#define RESET 7 // PD7 - DIO4 - reset pin of the target avr
#define DONE_LED 9 // B1 - blue LED on JN USB, blinks on start and when ok

// pins used for the optional config switches
#define CONFIG1 5 // DIO2
#define CONFIG2 15 // AIO2
#define CONFIG3 16 // AIO3
#define CONFIG4 6 // DIO3

// MPU-specific values
#define PAGE_BYTES 128 // ATmega168 and ATmega328, and tiny85 as well

#define LOCK_BITS_AT85 0xFF
#define FUSE_HIGH_AT85 0xDF
#define FUSE_LOW_AT85 0x62
#define FUSE_EXT_AT85 0xFF


// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -


// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -


const char progdata[] PROGMEM = 
":100000000EC015C014C013C012C011C010C00FC064\
:100010000EC00DC00CC00BC00AC009C008C011241E\
:100020001FBECFE5D2E0DEBFCDBF02D02AC0E8CFF1\
:100030000F931F93DF93CF9300D0CDB7DEB787E345\
:1000400090E021E0FC01208388E390E021E0FC01C6\
:10005000208388E390E028E330E0F901308121E05B\
:100060002327FC0120838FEF9FEF9A838983898167\
:100070009A818C01C8010197F1F78C011A830983D9\
:06008000E8CFF894FFCF69\
:00000001FF";

static word singlehex(char c) {
  const String validchars = "0123456789ABCDEF";
  return (word)validchars.indexOf(c);
}

static word hextoval(String heks) {
  word val = 0;
  byte i = 0;
  if (heks.length() < 4) {
    while (i<heks.length()) {
      val = val*16 + singlehex(heks.charAt(i++));
    }
  }
  return val;
}

// ISP Command Words
#define CMD_Program_Enable 0xAC53
#define CMD_Erase_Flash 0xAC80
#define CMD_Poll 0xF000
#define CMD_Read_Flash_Low 0x2000
#define CMD_Read_Flash_High 0x2800
#define CMD_Load_Page_Low 0x4000
#define CMD_Load_Page_High 0x4800
#define CMD_Write_Page 0x4C00
#define CMD_Read_EEPROM 0xA000
#define CMD_Write_EEPROM 0xC000
#define CMD_Read_Lock 0x5800
#define CMD_Write_Lock 0xACE0
#define CMD_Read_Signature 0x3000
#define CMD_Write_Fuse_Low 0xACA0
#define CMD_Write_Fuse_High 0xACA8
#define CMD_Write_Fuse_Extended 0xACA4
#define CMD_Read_Fuse_Low 0x5000
#define CMD_Read_Fuse_High 0x5808
#define CMD_Read_Fuse_Extended 0x5008
#define CMD_Read_Fuse_High 0x5808
#define CMD_Read_Calibration 0x3800


// transfer a byte using software SPI, 
static byte XferByte(byte v) {
    byte result = 0;
    for (byte i = 0; i < 8; ++i) {
       digitalWrite(PIN_MOSI, v & 0x80);
       digitalWrite(PIN_SCK, 0); // slow pulse, max 60KHz
       digitalWrite(PIN_SCK, 1);
       v <<= 1;
       result = (result << 1) | digitalRead(PIN_MISO);
    }
    return result;
}

// send 4 bytes to target microcontroller, returns the fourth MISO byte
static byte Send_ISP (word v01, byte v2 =0, byte v3 =0) {
    XferByte(v01 >> 8);
    XferByte(v01);
    XferByte(v2);
    return XferByte(v3);
}

// send 4 bytes to target microcontroller and wait for completion
static void Send_ISP_wait (word v01, byte v2 =0, byte v3 =0) {
    Send_ISP(v01, v2, v3);
    while (Send_ISP(CMD_Poll) & 1)
        ;
}

// reset the target microcontroller
static void Reset_Target() {
    digitalWrite(PIN_SCK, 0); // has to be set LOW at startup, or PE fails
    digitalWrite(RESET, 1);
    delay(30);
    digitalWrite(RESET, 0);
    delay(50); // minimum delay here is 20ms for the ATmega8
}

// print the 16 signature bytes (device codes)
static void Read_Signature() {
    Serial.print("Signatures:");
    for (byte x = 0 ; x < 8 ; ++x) {
        Serial.print(" ");
        Serial.print(Send_ISP(CMD_Read_Signature, x), HEX);
    }
    Serial.println("");
}

// prints the lock and fuse bits (no leading zeros)
static byte Read_Fuses(byte flo, byte fhi) {
//    Serial.print("Lock Bits required: ");
//    Serial.println(LOCK_BITS_AT85, HEX);
    Serial.print("Lock Bits read: ");
    Serial.println(Send_ISP(CMD_Read_Lock), HEX);
//    Serial.print("Fuse Low required: ");
//    Serial.println(flo, HEX);
    Serial.print("Fuses low read:");
    Serial.println(Send_ISP(CMD_Read_Fuse_Low), HEX);
//    Serial.print("Fuse high required: ");
//    Serial.println(fhi, HEX);
    Serial.print("Fuses high read:");
    Serial.println(Send_ISP(CMD_Read_Fuse_High), HEX);
//    Serial.print("Fuses extended required: ");
//    Serial.println(FUSE_EXT_AT85, HEX);
    Serial.print("Fuses extended read:");
    Serial.println(Send_ISP(CMD_Read_Fuse_Extended), HEX);
    return Send_ISP(CMD_Read_Lock) == LOCK_BITS_AT85 &&
           Send_ISP(CMD_Read_Fuse_Low) == flo &&
           Send_ISP(CMD_Read_Fuse_High) == fhi &&
           Send_ISP(CMD_Read_Fuse_Extended) == FUSE_EXT_AT85;
}

static word addr2page (word addr) {
    return (word)((addr & ~ (PAGE_BYTES-1)) >> 1);
}

static void LoadPageC(word addr, String dta) {
    word cmd = addr & 1 ? CMD_Load_Page_High : CMD_Load_Page_Low;
    Send_ISP(cmd | (addr >> 9), addr >> 1, (byte)hextoval(dta));
}

static void WritePage (word page) {
    Send_ISP_wait(CMD_Write_Page | (page >> 8), page);
}

static void WriteDataS (word start, String data) {
    word page = addr2page(start);
    for (word i = 0; i < data.length(); i += 4) {
        if (page != addr2page(start)) {
            WritePage(page);
            Serial.print('.');
            page = addr2page(start);
        }
        LoadPageC(start++, data.substring(i,i+2));
        LoadPageC(start++, data.substring(i+2,i+4));
    }
    WritePage(page);
    Serial.println(data);
}

static void read_progdata () {
  word oldaddr; 
  word newaddr;
  const char* ptr; 
  String oneline;
  ptr = &progdata[0];
  oldaddr = 0x0000;
  newaddr = 0x0000;
  String oneline = "";
 
  while (pgm_read_byte(ptr) != 0x00) { 
      if (pgm_read_byte(ptr) != ':' ) { 
        oneline += (char)pgm_read_byte(ptr++);
      } 
      else { // current char is colon so next line starts process old one first
        oneline.toUpperCase();
        newaddr = hextoval(oneline.substring(2,6));
        if ((oneline.substring(6,8) == "00") && (oldaddr < newaddr)) {
          oldaddr = newaddr;  
          WriteDataS(oldaddr, oneline.substring(8,oneline.length() - 2)); //leave checksum off
        } 
        else {
          Serial.println("error addresses or rec type ");
          Serial.print(oneline.substring(6,7));
          Serial.print(" : ");
          Serial.println(oneline);
        }
      oneline = "";  
      ptr++; // skip the colon
      }
  }  
}

static byte EnableProgramming () {
    Reset_Target();
    Serial.println(Send_ISP(CMD_Program_Enable, 0x00, 0x00));
    if (Send_ISP(CMD_Program_Enable, 0x00, 0x00) != 0x00) {
        Serial.println("Program Enable Failed");
        return 0;
    }
    return 1;
}

static void blink () {
  pinMode(DONE_LED, OUTPUT);
  digitalWrite(DONE_LED, 0); // inverted logic
  delay(100); // blink briefly
  pinMode(DONE_LED, INPUT);
}

static byte readConfig () {
    static byte pins[] = { CONFIG1, CONFIG2, CONFIG3, CONFIG4 };
    byte switches = 0;
    for (byte i = 0; i < 4; ++i) {
        pinMode(pins[i], INPUT);
        digitalWrite(pins[i], 1); // enable pull-up
        bitWrite(switches, i, digitalRead(pins[i]));
        digitalWrite(pins[i], 0); // disable pull-up
    }
    return switches; // a 4-bit value, i.e. 0..15
}

static byte programSlave () {
    byte f = EnableProgramming();
    if (f) {
        read_progdata();
    }
    return f;
}

void setup () {
    Serial.begin(57600);
    Serial.println("[jeeslaveprog3.01]");
    blink();

    pinMode(PIN_SCK, OUTPUT);
    pinMode(PIN_MOSI, OUTPUT);
    pinMode(RESET, OUTPUT);

    digitalWrite(PIN_SCK, 0);
    digitalWrite(PIN_MOSI, 1);
    digitalWrite(RESET, 0);
  
    byte config = readConfig();
    byte sketch = config & 1; // 0..1 : CONFIG1
    byte xspeed = (config >> 1) & 1; // 0..1 : CONFIG2
    byte bootld = 2 + (config >> 2); // 2..5 : CONFIG4 & CONFIG3
    
    Serial.print("Configuration: ");
    
    if (EnableProgramming()) {
        Serial.println("Erasing Flash");
        Send_ISP_wait(CMD_Erase_Flash, 0x22, 0x22);
        if (EnableProgramming()) {
              if (programSlave()) {
                Read_Signature();
                if (Read_Fuses(FUSE_LOW_AT85, FUSE_HIGH_AT85)) {
                    Serial.println("Done.");
                    blink();
                } else {
                    Serial.println("Fuses NOT OK!");
                }    
              }
        }
    }

    pinMode(PIN_SCK, INPUT);
    pinMode(PIN_MOSI, INPUT);
    pinMode(RESET, INPUT);
    
    digitalWrite(PIN_SCK, 0);
    digitalWrite(PIN_MOSI, 0);
    digitalWrite(RESET, 0);


    Serial.flush();
    delay(10); // let the serial port finish
}

void loop () {
}

it appears to be linked with the definition of the progdata String.
this way of noting it down (see below) made the error disappear.

but the strange thing is that I did not touch this bit of code. It has been
compiled succesfully several times in it's old form (see above) !

it amazes me that where Arduino tries to win over newbies for the embedded computing with
easily accessed and understood concepts, and clear location of errors,
the only available support for this type of
preprocessor error is the advice to try fiddling with backslashes
(at least that is how I read the troubleshooting guide on this).
it's not even reporting the line or string fragment it borks over.

in this form it now compiles

const char progdata[] PROGMEM = 
":100000000EC015C014C013C012C011C010C00FC064"
":100010000EC00DC00CC00BC00AC009C008C011241E"
":100020001FBECFE5D2E0DEBFCDBF02D02AC0E8CFF1"

ps : beware of my published code, it contains errors. it was not finished yet.

Confirmed that this occurs, however on Linux and Windows only. It compiles clean on OS/X.

I should point out that this is really the preferred way of doing long strings:

const char progdata[] PROGMEM = 
":100000000EC015C014C013C012C011C010C00FC064"
":100010000EC00DC00CC00BC00AC009C008C011241E"
":100020001FBECFE5D2E0DEBFCDBF02D02AC0E8CFF1"

You earlier code (with the backslashes) is setting up a very long single string. It would appear that it has confused the preprocessor, as you surmised.

it amazes me that where Arduino tries to win over newbies for the embedded computing with
easily accessed and understood concepts, and clear location of errors,
the only available support for this type of
preprocessor error is the advice to try fiddling with backslashes
(at least that is how I read the troubleshooting guide on this).
it's not even reporting the line or string fragment it borks over.

It's not a compiler error, the regexp has failed. Ah well, I agree the error message is not at all helpful. I'll report the error.

http://code.google.com/p/arduino/issues/detail?id=962

Thanks Nick !

I'm not too familiar with the bug reporting system, so thanks for making the effort to do that for me.

My gripe is just that the preprocessor does not currently report what data it fails on.
If I knew what line or part of my code it generated the error then I could focus
on that part. Now I was lost.

I tried to isolate the error, but as said, it compiles ok (as it did for me earlier) so it must be a combination with something
later on the in program

this compiles OK :

const char progdata[] PROGMEM = 
":100000000EC015C014C013C012C011C010C00FC064\
:100010000EC00DC00CC00BC00AC009C008C011241E\
:100020001FBECFE5D2E0DEBFCDBF02D02AC0E8CFF1\
:100030000F931F93DF93CF9300D0CDB7DEB787E345\
:1000400090E021E0FC01208388E390E021E0FC01C6\
:10005000208388E390E028E330E0F901308121E05B\
:100060002327FC0120838FEF9FEF9A838983898167\
:100070009A818C01C8010197F1F78C011A830983D9\
:06008000E8CFF894FFCF69\
:00000001FF";

void setup () {}

void loop () {
}