thanks i solved that part.
BUT i was a little dumb because the previous ones were not my full of code 
i test it with only that what i posted above and turned out that it works.
so my real question is why doesn't it work, and this time here is the full of code:
(perhaps because the interrupt?)
#define __spi_clock 13 // SCK - hardware SPI
#define __spi_latch 10
#define __spi_data 11 // MOSI - hardware SPI
//#define __spi_data_in 12 // MISO - hardware SPI (unused)
#define __display_enable 9
#define __rows 8
#define __max_row __rows-1
#define __leds_per_row 8
#define __max_led __leds_per_row-1
#define __brightness_levels 32 // 0...15 above 28 is bad for ISR ( move to timer1, lower irq freq ! )
#define __max_brightness __brightness_levels-1
#define __fade_delay 4
#define __TIMER1_MAX 0xFFFF // 16 bit CTR
#define __TIMER1_CNT 0x0130 // 32 levels --> 0x0130; 38 --> 0x0157 (flicker)
#define __TIMER2_MAX 0xFF // 8 bit CTR
#define __TIMER2_CNT 0xFF // max 28 levels !
#include <avr/interrupt.h>
#include <avr/io.h>
#include <stdint.h>
byte input[8]; // = {1,3,5,7,9,11,13,15};
byte sor = 0;
byte val=0;
boolean ok = true;
byte bred[8][8];
byte brightness_red[__leds_per_row][__rows];
byte brightness_green[__leds_per_row][__rows];
byte brightness_blue[__leds_per_row][__rows];
ISR(TIMER1_OVF_vect) {
//TCNT2 = __TIMER2_MAX - __TIMER2_CNT; // precharge TIMER2 to maximize ISR time --> max led brightness
TCNT1 = __TIMER1_MAX - __TIMER1_CNT;
byte cycle;
digitalWrite(__display_enable,LOW); // enable display inside ISR
for(cycle = 0; cycle < __max_brightness; cycle++) {
byte led;
byte row = B00000000; // row: current source. on when (1)
byte red; // current sinker when on (0)
byte green; // current sinker when on (0)
byte blue; // current sinker when on (0)
for(row = 0; row <= __max_row; row++) {
red = B00000000; // off
green = B00000000; // off
blue = B00000000; // off
for(led = 0; led <= __max_led; led++) {
if(cycle < brightness_red[row][led]) {
red |= (1<<led);
}
if(cycle < brightness_green[row][led]) {
green |= (1<<led);
}
if(cycle < brightness_blue[row][led]) {
blue |= (1<<led);
}
}
digitalWrite(__spi_latch,LOW);
spi_transfer(B00000001<<row);
spi_transfer(green);
spi_transfer(blue);
spi_transfer(red);
digitalWrite(__spi_latch,HIGH);
digitalWrite(__spi_latch,LOW);
}
}
digitalWrite(__spi_latch,LOW);
spi_transfer(B00000000); // blue off
spi_transfer(B00000000); // green off
spi_transfer(B00000000); // red off
spi_transfer(B00000000); // rows off
digitalWrite(__spi_latch,HIGH);
digitalWrite(__spi_latch,LOW);
digitalWrite(__display_enable,HIGH); // disable display outside ISR
}
void setup() {
randomSeed(555);
byte ctr1;
byte ctr2;
pinMode(__spi_clock,OUTPUT);
pinMode(__spi_latch,OUTPUT);
pinMode(__spi_data,OUTPUT);
pinMode(__display_enable,OUTPUT);
digitalWrite(__spi_latch,LOW);
digitalWrite(__spi_data,LOW);
digitalWrite(__spi_clock,LOW);
Serial.begin(9600);
setup_hardware_spi();
delay(10);
set_matrix_rgb(0,0,0);
//setup_timer2_ovf();
setup_timer1_ovf();
// display enable/disable is done inside the ISR !
}
void loop(){
for(int i=0; i < 8; i++) {
set_row_rgb(i,0,0,input);
}
}
void serialEvent() {
while(Serial.available()) {
val = Serial.read();
if(val=='>'){
sor=0;
}
else
{
input[sor] = val;
sor++;
}
}
}
////////////////////////////LED SPECIFIC STUFF/////////////////////////////////////
void set_led_red(byte row, byte led, byte red) {
brightness_red[row][led] = red;
}
void set_led_green(byte row, byte led, byte green) {
brightness_green[row][led] = green;
}
void set_led_blue(byte row, byte led, byte blue) {
brightness_blue[row][led] = blue;
}
void set_led_rgb(byte row, byte led, byte red, byte green, byte blue) {
set_led_red(row,led,red);
set_led_green(row,led,green);
set_led_blue(row,led,blue);
}
void set_matrix_rgb(byte red, byte green, byte blue) {
byte ctr1;
byte ctr2;
for(ctr2 = 0; ctr2 <= __max_row; ctr2++) {
for(ctr1 = 0; ctr1 <= __max_led; ctr1++) {
set_led_rgb(ctr2,ctr1,red,green,blue);
}
}
}
void set_row_rgb(byte row, byte red, byte green, byte blue) {
byte ctr1;
for(ctr1 = 0; ctr1 <= __max_led; ctr1++) {
set_led_rgb(row,ctr1,red,green,blue);
}
}
void set_column_rgb(byte column, byte red, byte green, byte blue) {
byte ctr1;
for(ctr1 = 0; ctr1 <= __max_row; ctr1++) {
set_led_rgb(ctr1,column,red,green,blue);
}
}
byte spi_transfer(byte data)
{
SPDR = data; // Start the transmission
while (!(SPSR & (1<<SPIF))) // Wait the end of the transmission
{
};
return SPDR; // return the received byte, we don't need that
}
void setup_hardware_spi(void) {
byte clr;
// spi prescaler:
// SPI2X SPR1 SPR0
// 0 0 0 fosc/4
// 0 0 1 fosc/16
// 0 1 0 fosc/64
// 0 1 1 fosc/128
// 1 0 0 fosc/2
// 1 0 1 fosc/8
// 1 1 0 fosc/32
// 1 1 1 fosc/64
SPCR |= ( (1<<SPE) | (1<<MSTR) ); // enable SPI as master
//SPCR |= ( (1<<SPR1) ); // set prescaler bits
SPCR &= ~ ( (1<<SPR1) | (1<<SPR0) ); // clear prescaler bits
clr=SPSR; // clear SPI status reg
clr=SPDR; // clear SPI data reg
SPSR |= (1<<SPI2X); // set prescaler bits
//SPSR &= ~(1<<SPI2X); // clear prescaler bits
}
void setup_timer1_ovf(void) {
// Arduino runs at 16 Mhz...
// Timer1 (16bit) Settings:
// prescaler (frequency divider) values: CS12 CS11 CS10
// 0 0 0 stopped
// 0 0 1 /1
// 0 1 0 /8
// 0 1 1 /64
// 1 0 0 /256
// 1 0 1 /1024
// 1 1 0 external clock on T1 pin, falling edge
// 1 1 1 external clock on T1 pin, rising edge
//
TCCR1B &= ~ ( (1<<CS11) );
TCCR1B |= ( (1<<CS12) | (1<<CS10) );
//normal mode
TCCR1B &= ~ ( (1<<WGM13) | (1<<WGM12) );
TCCR1A &= ~ ( (1<<WGM11) | (1<<WGM10) );
//Timer1 Overflow Interrupt Enable
TIMSK1 |= (1<<TOIE1);
TCNT1 = __TIMER1_MAX - __TIMER1_CNT;
// enable all interrupts
sei();
}