Working Twitter App to read latest tweets

An application to read a twitter timeline.
With support for Serial and 20x4 lcd output.

I tested it on an ATMega2560.

Individual tweets can contain very long embedded links.
This makes the tweet text limit of ~140 characters a bit of a farce.
So using an Arduino with less memory than the mega would be a challenge.

I have recently updated the software adding #tag and @someone support. (See attachment)
There is limited support for common french accented characters.
The lcd display section tries not to break up words across lines.
I have experienced occasional missing characters in the input section. If these remove part of a "<a href=" reference it can occasionally mess up the tweet parsing.

Constructive suggestions invited.

David

Part one of App:

// YiFi101tweetRead2.ino

// Read Twitter timeline
// tested on ATmega2560
// Will not work on protected  accounts (Nor should it!)
// Serial support

// LCD Support (20 x 4)
  // I used an in-expensive  20x4 LCD "4-Line Module":
    // with Blue Backlight and I2C Interface
    // from http://www.4tronix.co.uk/store/
    // Item #: ARD20X4I2C
    // connected to 0V, 5V, SDA and SCL on the Mega
// lcd backlight toggled with WiFi button

// Select the number of recent tweets to read
// Display tweets on lcd:
  // requires suppression of tweet links
  // Option to suppress images and linked URL's using boolean nolink

// tweet recovery limited to a line length of 998 (+ "\n") characters per tweet
// (unless maxtweets = 0)
//    if maxtweets = 0, all of the response is printed and not processed
//    Some sites have tweets with embedded links that are incredibly long!
//    A tweet should have no more than 140 text characters (ho Ho)

// Reloading un-changing tweets can be prevented with bailout = true

// Led with series 1Kohm resistor between pin 12 and ground- active during reading
// Switch between pins 8 and 9 to toggle lcd scrolling

// D.R.Patterson
// 24/12/2016

#include <SPI.h>
#include <WiFi101.h>

#include <LiquidCrystal_I2C.h>
// set the LCD address to 0x27 for a 20 chars 4 line display
// Set the pins on the I2C chip used for LCD connections:
//                    addr, en,rw,rs,d4,d5,d6,d7,bl,blpol
LiquidCrystal_I2C lcd(0x27, 2, 1, 0, 4, 5, 6, 7, 3, POSITIVE);
#include <Wire.h>
// custom pound using https://omerk.github.io/lcdchargen/

byte customPound[8] = {
  0b01100,
  0b10010,
  0b01000,
  0b11100,
  0b01000,
  0b01000,
  0b11110,
  0b00000
};

const byte wifiSelect = 10; // YiFi101 select (active low)
int mystatus = WL_IDLE_STATUS;
const byte button = 6;      // attached to on-board button

const byte smonitor = 8; // if low no led scrolling
const byte pinlow = 9; // convenient low pin next to monitored pin
const byte led = 12; // led pin

boolean lcdlight = true;

// ######### Modify this section ########################################

#define LIB_DOMAIN "mobile.twitter.com"

const String twittername = "bbcnews"; // cnn, bbc etc

const byte sdSelect = 53;   // SD select (active low)

const char ssid[] = "xxxx"; // your network SSID (name)
                            // add password support/keyIndex if required
                            // and amend line with mystatus = WiFi.begin(ssid);

const unsigned int maxtweets = 10; // 0 get all the available html without selection
                                   // 1 get last tweet, 10 get last 10 tweets etc
                                   // 10 works well without adjustment in main loop timing

const unsigned long reloadTime = 900000;   // time between re-loads 15 minutes
                                           // (at least maxtweets *70000) + 20% if scrolling is used
                                                                       
const unsigned long chardelay = 240; // delay for visible character display during lcd scroll

const unsigned long waitTime = 10000; // wait for wifi101 to settle (mS)

const boolean nolink = true;  // filter links and allow lcd to show tweet
                              // if false, will serial print (if debug true)

const boolean debug = true;   // extended serial output

const boolean bailout = false;   // if true, when identically ordered tweet id's are found,
                                 // data processing ceases for a quick exit.
                                 // if false, reloads the tweets again

boolean scroll = true;            // Scroll tweets on lcd- will be over ruled if using monitored switch
const boolean haveSwitch = true;  // set to true if using a switch beteen pin 9 and 8 to
                                  // control lcd scrolling 

const unsigned int maxSlength = 998; // maximum length of a processed string. 998 for mega

const char pound = 1; // custom char "£" (closest £ on my lcd was 193 and it was poor)

// ######## End of modify ###############################################

int lastTweet = -1;

String content[maxtweets]; // tweet storage

const String clearit = "                    ";
unsigned long tstart;

void setup(){
// setup wifi and sd select
pinMode(sdSelect, OUTPUT); digitalWrite(sdSelect, HIGH);
pinMode(wifiSelect, OUTPUT); digitalWrite(wifiSelect, LOW);
pinMode(button, INPUT_PULLUP);

// setup monitored pin
pinMode(pinlow, OUTPUT); digitalWrite(pinlow, LOW);
pinMode(smonitor, INPUT_PULLUP);
// led pin
pinMode(led, OUTPUT); digitalWrite(led, LOW);

Serial.begin(115200);
  while(!Serial) yield();
lcd.begin(20,4);
delay(250);
lcd.clear();
// create a new custom character
lcd.createChar(1, customPound);
lcd.clear();
lcd.backlight();
lcd.print(F("Twitter Timeline"));
wifiConnect();

// wait for wifi to settle
lcd.setCursor(0,2); Serial.print(F("Settling Wifi"));
lcd.print(F("Settling Wifi"));
delay(waitTime);
lcd.setCursor(0,2); lcd.print(clearit);

String retval = readtwitter(maxtweets);
  if (debug && retval != "0") Serial.println("Connection returned " + retval);
 
tstart = millis();
}

void loop(){
static int dcount = 0;
String message, temp;
int E;

  if(haveSwitch) scroll = digitalRead(smonitor); // use switch to set scroll
                                                // comment out if not using switch

checkbutton(); // look for wifi button press 

unsigned long tnow = millis();
  if ( (tnow - tstart) > reloadTime) { // default = 15 minutes 
                                                      // keep rate down to make twitter happy
                                                      // timing optimized for 10 tweets
  mystatus = WiFi.status();
    if (mystatus != WL_CONNECTED) { // lost the wifi
    lcd.clear(); lcd.print(F("WiFi Signal lost"));
    wifiConnect(); // re-connect
    tnow = millis();
    }
  String retval = readtwitter(maxtweets);
    if (retval != "0" && debug) Serial.println("Connection returned " + retval);
  tstart = tnow;
  dcount = 0; // reset tweet display count for lcd
  }
  
  if(nolink){ // put the tweet on the lcd
    if (lastTweet >= 0){ // require at least 1 tweet (1st tweet = 0)
    message = content[dcount];
    message = String(dcount + 1) +": " + message;
    Serial.print("\n" + message);
    
     // line above line display
    message.replace("\n\n","\n"); // remove any repeated \n to improve lcd layout
    message.replace(char(163),char(pound)); // swap £ for a custom char on the lcd
      while (message.length()) {
      lcd.clear();  
        for(int i = 0; i < 4; i++){ // split message up to display on a 4 line 10x4 lcd
        int S = 0;
        E = message.indexOf("\n");
          if (E > 20) {
          E = 20;
          temp = message.substring(0, E );
          
          // remove leading spaces
          int L = 0;
            while(byte(temp.charAt(L) ) == 32) L++;
            if (L > 0)  temp.remove(0, L);

          // avoid spliiting words
          char N = message.charAt(E);
            if (!(N== 0 || N==10 || N==13 || N == 32  )){
            int B = temp.lastIndexOf(" ");
              if (B>-1){
              E = B;
              temp = temp.substring(0, E );
              S = 1;
              }
            }
            // end of avoid splitting
          }else{
          S = 1;
          temp = message.substring(0, E );
          }
        temp.trim();
        message.remove(0, E+ S);
        lcd.setCursor(0, i);
        lcd.print(temp);
          if (message.length() == 0) break;      
        }
        if (message == "\n") message = "";  // no point in starting a new panel
      mydelay(5000);
      }
    
      if(scroll){
      // scroll the mesage on line 2 of the lcd
      lcd.clear();
      lcd.print( twittername + ", " + String(dcount + 1));
      message = content[dcount];
      message.replace(char(163),char(pound)); // swap £ for a custom char
      lcdscroll(message, 2, false ); // params: (message, line number, clear the line first
      }
    dcount++; // tweet counter
      if(dcount > lastTweet) dcount = 0;
    }
  }
}

YiFi101tweetReadForum.zip (6.9 KB)

Part 2 of App:

String readtwitter(unsigned int maxno){
WiFiSSLClient client;
static String id[maxtweets]; // store tweet id's

char c;
String retval = "0";
boolean firstline = true;
String sentence = "";
int S, E;
unsigned int tweet = 0;

digitalWrite(led, HIGH);
lcd.setCursor(9,3); lcd.print(F(" *Reading* "));
Serial.print(F("\nConnecting to ")); Serial.println(LIB_DOMAIN);
  if (client.connect(LIB_DOMAIN, 443) ) {
  Serial.println(F("Connected\n"));
  // Make a HTTP request:
   //client.println("GET /search?q=" + twittername + "HTTP/1.1");
  client.println("GET /" + twittername + " HTTP/1.1");
  client.println("Host: mobile.twitter.com");
  client.println("Connection: close");
  client.println();
  client.flush();
    while (client.connected()) {
      while (client.available()){
      c = client.read();
      sentence = sentence + c;
        if (c == 10){
          if(firstline){
          S = sentence.indexOf(" ");
          E = sentence.indexOf(" ", S + 1);
          retval = sentence.substring(S + 1, E);
          firstline = false;
          }
          // print everything 
          if (maxno == 0) Serial.print(sentence); else S = sentence.indexOf("\"tweet-text\"");  
         
          if ( (tweet < maxno) && (  S >= 0)  ) {
          S = sentence.indexOf("\"", S + 19);
          E = sentence.indexOf("\"", S+1);
          String sid = sentence.substring(S+1, E); // store tweet id as string
          sentence="";
          
            // test sid against id[tweet] to find out if latest tweet has changed
            if(bailout){ // if the tweet id is the same, stop processing   and get out quick
              if(sid == id[tweet]) {
                while(client.connected()) { // discard rest of html
                  if(client.available())  c =  client.read();
                }
              client.stop();
              Serial.println(F("No change in latest tweet"));
              lcd.setCursor(9,3); lcd.print(F(" *NoChange*"));
               digitalWrite(led, LOW);
              return "0";
              }
            }
          id[tweet] = sid;

            while(client.connected() ) {
              if (client.available() ) {
              c = client.read();

                if (c == 10){
                sentence = sentence + c;
                break;
                }
                // reject rogue characters
                if (sentence.length() < maxSlength){ // avoid sentence length failure!
                byte test = c; // had to do this to test for £ (163) and other characters
                  if ( test == 163 || ( test > 31 && test < 127 ) ) {
                  sentence += c; continue;
                  }

                  // Limited French letter conversion:
                  // lcd does not like french e acute, etc  
                  if  (test == 160 || test == 162){
                  sentence += "a"; continue;
                  }
                  if (test == 167) {
                  sentence += "c"; continue;
                  }
                  if  (test >167 && test < 172) {
                  sentence += "e";
                  }
                  //if (test == 153) sentence += "`";   // ` causes more problems than solutions
                }   
              }
            }
          S = sentence.indexOf(">");
          sentence = sentence.substring(S+3); // miss 2 characters after '>' : +3
                 
            if (nolink){ 
            // replace the links, leaving prompts
            S = sentence.indexOf("<a h"); // find link start
              while(S >= 0){
              E = sentence.indexOf(">", S + 6); // look for link end
                if (E == -1){ // not found so sentence truncated
                sentence.remove(S+1); // rest of line
                sentence = sentence + ">\n";
                } else sentence.remove(S+1, E- S); // delete link leaving "<"
              S = sentence.indexOf("<a href="); // look for the next link
              }
            sentence.replace("</a", ""); // amend end of link leaving ">"
            content[tweet] = sentence; // store the tweet for processing
            }else{ // add /n to print each link on a newline
            sentence.replace("<a h", "\n<a h");
            sentence.replace("</a>", "</a>\n");
            }
            if(debug) {
            Serial.print(tweet + 1); Serial.print(" "); Serial.println(id[tweet]);
            Serial.println( sentence );Serial.flush();
            }
          lastTweet = tweet;
          tweet ++;
          }
        sentence = "";
        }
      }
    }
  client.stop();
  // any sentence with length here, probably came from a remote client shut down
    if(debug && sentence.length()) Serial.println(sentence);
    
    if (nolink){
      for(int i = 0; i <= lastTweet; i++){
      sentence = content[i];
        // check for un-terminated message
        //if (sentence.charAt(sentence.length() - 1)  != 10)  sentence = sentence + "\n";
      // swap encoded characters
      sentence.replace("&#38;#38;#10;", "\n"); // linefeed
      sentence.replace("&#38;#38;#39;", "\'");  // '
      sentence.replace("&amp;", "&"); // &
      sentence.replace("&quot;", "\""); // "
      sentence.replace("  ", " "); // multi spaces
      content[i] = sentence; // store tweets in global array
      }
    }
  Serial.println(F("Finished"));  
  lcd.setCursor(9,3); lcd.print(F(" *Finished*"));  
  } else {
  Serial.println(F("Connection failed"));
  lcd.setCursor(9,3); lcd.print(F("  *Failed* "));
  }
digitalWrite(led, LOW);
return retval;
}

void lcdscroll(String message, byte lineno, boolean clearline){
int L = message.length();
  if (clearline && (L < 20) ){
  lcd.setCursor(0, lineno); lcd.print(clearit);
  }
lcd.setCursor(0, lineno); lcd.print(message.substring(0, 20) );
mydelay(1000);
  for (int i = 0; i <  L - 19; i++) {
  lcd.setCursor(0, lineno); lcd.print(clearit);
  mydelay(65); // dark time
  lcd.setCursor(0, lineno); lcd.print(message.substring(i, i + 20) );
  mydelay(chardelay); // bright time
  }
mydelay(1500);
}

void checkbutton() {
  if (digitalRead(button) == LOW){ // backlight control
  lcdlight = !lcdlight;  
    if(lcdlight) lcd.setBacklight(BACKLIGHT_ON); else lcd.setBacklight(BACKLIGHT_OFF);
    while (digitalRead(button) == LOW){
    delay(1);
    }
  }
}

void mydelay(unsigned long elapsed){
unsigned long tt = millis();
  while ( (millis() - tt) < elapsed) checkbutton();
}

void  wifiConnect(){  // returns if wifi connected, with up to 3 attempts
lcd.setCursor(0,1); lcd.print(F("Connecting"));
// check for the presence of the shield:
mystatus = WiFi.status();
  if (mystatus == WL_NO_SHIELD) {
  Serial.println(F("WiFi shield not present"));
  }else{
  // attempt to connect to Wifi network:
  byte attempts = 0;  
    while ( attempts < 3) {
    attempts++;
    Serial.print(F("Attempting to connect to SSID: ")); Serial.println(ssid);
    mystatus = WiFi.begin(ssid);
    unsigned long tt = millis();
      while ((millis()-tt) < 10000){ // up to 10 second wait
      delay(1000);
      mystatus = WiFi.status();
        if ((mystatus == WL_CONNECTED) || (mystatus == WL_NO_SHIELD)) break;
      }
      if ((mystatus == WL_CONNECTED) || (mystatus == WL_NO_SHIELD)) break;
    }
  }
lcd.setCursor(0,1);
  if (mystatus != WL_CONNECTED){
  Serial.println(F("Wifi not available\n"));
  lcd.print(F("Wifi not available"));
    while(1) yield();
  } else{
  lcd.print(F("WiFi Connected"));
  Serial.println(F("Wifi Connected\n"));
  }
}

cnn feed:

How are you getting around having to use OAuth to connect?

Wow! I posted this over a year ago so I had to test it still works.. and it does.

As the listing says.. " Will not work on protected accounts (Nor should it!)"

It never logs into twitter.

Try typing this into a browser...
www.twitter.com/bbc

This works, so all we have to do is the arduino equivalent
See Part 2 of App for method.

David