Go Down

Topic: Working Twitter App to read latest tweets (Read 1 time) previous topic - next topic

david-

Dec 28, 2016, 09:57 am Last Edit: Jan 08, 2017, 09:28 am by david-
An application to read a twitter timeline.
With support for Serial  and 20x4 lcd output.

I tested it on an ATMega2560.

Individual tweets can contain very long embedded links.
This makes the tweet text limit of ~140 characters a bit of a farce.
So using an Arduino with less memory than the mega would be a challenge.

I have recently updated the software adding #tag and @someone support. (See attachment)
There is limited support for common french accented characters.
The lcd display section tries not to break up words across lines.
I have experienced occasional missing characters in the input section. If these remove part of a "<a href=" reference it can occasionally mess up the tweet parsing.

Constructive suggestions invited.

David

Part one of App:

Code: [Select]

// YiFi101tweetRead2.ino

// Read Twitter timeline
// tested on ATmega2560
// Will not work on protected  accounts (Nor should it!)
// Serial support

// LCD Support (20 x 4)
  // I used an in-expensive  20x4 LCD "4-Line Module":
    // with Blue Backlight and I2C Interface
    // from http://www.4tronix.co.uk/store/
    // Item #: ARD20X4I2C
    // connected to 0V, 5V, SDA and SCL on the Mega
// lcd backlight toggled with WiFi button

// Select the number of recent tweets to read
// Display tweets on lcd:
  // requires suppression of tweet links
  // Option to suppress images and linked URL's using boolean nolink

// tweet recovery limited to a line length of 998 (+ "\n") characters per tweet
// (unless maxtweets = 0)
//    if maxtweets = 0, all of the response is printed and not processed
//    Some sites have tweets with embedded links that are incredibly long!
//    A tweet should have no more than 140 text characters (ho Ho)

// Reloading un-changing tweets can be prevented with bailout = true

// Led with series 1Kohm resistor between pin 12 and ground- active during reading
// Switch between pins 8 and 9 to toggle lcd scrolling

// D.R.Patterson
// 24/12/2016

#include <SPI.h>
#include <WiFi101.h>

#include <LiquidCrystal_I2C.h>
// set the LCD address to 0x27 for a 20 chars 4 line display
// Set the pins on the I2C chip used for LCD connections:
//                    addr, en,rw,rs,d4,d5,d6,d7,bl,blpol
LiquidCrystal_I2C lcd(0x27, 2, 1, 0, 4, 5, 6, 7, 3, POSITIVE);
#include <Wire.h>
// custom pound using https://omerk.github.io/lcdchargen/

byte customPound[8] = {
  0b01100,
  0b10010,
  0b01000,
  0b11100,
  0b01000,
  0b01000,
  0b11110,
  0b00000
};

const byte wifiSelect = 10; // YiFi101 select (active low)
int mystatus = WL_IDLE_STATUS;
const byte button = 6;      // attached to on-board button

const byte smonitor = 8; // if low no led scrolling
const byte pinlow = 9; // convenient low pin next to monitored pin
const byte led = 12; // led pin

boolean lcdlight = true;

// ######### Modify this section ########################################

#define LIB_DOMAIN "mobile.twitter.com"

const String twittername = "bbcnews"; // cnn, bbc etc

const byte sdSelect = 53;   // SD select (active low)

const char ssid[] = "xxxx"; // your network SSID (name)
                            // add password support/keyIndex if required
                            // and amend line with mystatus = WiFi.begin(ssid);

const unsigned int maxtweets = 10; // 0 get all the available html without selection
                                   // 1 get last tweet, 10 get last 10 tweets etc
                                   // 10 works well without adjustment in main loop timing

const unsigned long reloadTime = 900000;   // time between re-loads 15 minutes
                                           // (at least maxtweets *70000) + 20% if scrolling is used
                                                                       
const unsigned long chardelay = 240; // delay for visible character display during lcd scroll

const unsigned long waitTime = 10000; // wait for wifi101 to settle (mS)

const boolean nolink = true;  // filter links and allow lcd to show tweet
                              // if false, will serial print (if debug true)

const boolean debug = true;   // extended serial output

const boolean bailout = false;   // if true, when identically ordered tweet id's are found,
                                 // data processing ceases for a quick exit.
                                 // if false, reloads the tweets again

boolean scroll = true;            // Scroll tweets on lcd- will be over ruled if using monitored switch
const boolean haveSwitch = true;  // set to true if using a switch beteen pin 9 and 8 to
                                  // control lcd scrolling

const unsigned int maxSlength = 998; // maximum length of a processed string. 998 for mega

const char pound = 1; // custom char "£" (closest £ on my lcd was 193 and it was poor)

// ######## End of modify ###############################################

int lastTweet = -1;

String content[maxtweets]; // tweet storage

const String clearit = "                    ";
unsigned long tstart;

void setup(){
// setup wifi and sd select
pinMode(sdSelect, OUTPUT); digitalWrite(sdSelect, HIGH);
pinMode(wifiSelect, OUTPUT); digitalWrite(wifiSelect, LOW);
pinMode(button, INPUT_PULLUP);

// setup monitored pin
pinMode(pinlow, OUTPUT); digitalWrite(pinlow, LOW);
pinMode(smonitor, INPUT_PULLUP);
// led pin
pinMode(led, OUTPUT); digitalWrite(led, LOW);

Serial.begin(115200);
  while(!Serial) yield();
lcd.begin(20,4);
delay(250);
lcd.clear();
// create a new custom character
lcd.createChar(1, customPound);
lcd.clear();
lcd.backlight();
lcd.print(F("Twitter Timeline"));
wifiConnect();

// wait for wifi to settle
lcd.setCursor(0,2); Serial.print(F("Settling Wifi"));
lcd.print(F("Settling Wifi"));
delay(waitTime);
lcd.setCursor(0,2); lcd.print(clearit);

String retval = readtwitter(maxtweets);
  if (debug && retval != "0") Serial.println("Connection returned " + retval);
 
tstart = millis();
}

void loop(){
static int dcount = 0;
String message, temp;
int E;

  if(haveSwitch) scroll = digitalRead(smonitor); // use switch to set scroll
                                                // comment out if not using switch

checkbutton(); // look for wifi button press

unsigned long tnow = millis();
  if ( (tnow - tstart) > reloadTime) { // default = 15 minutes
                                                      // keep rate down to make twitter happy
                                                      // timing optimized for 10 tweets
  mystatus = WiFi.status();
    if (mystatus != WL_CONNECTED) { // lost the wifi
    lcd.clear(); lcd.print(F("WiFi Signal lost"));
    wifiConnect(); // re-connect
    tnow = millis();
    }
  String retval = readtwitter(maxtweets);
    if (retval != "0" && debug) Serial.println("Connection returned " + retval);
  tstart = tnow;
  dcount = 0; // reset tweet display count for lcd
  }
 
  if(nolink){ // put the tweet on the lcd
    if (lastTweet >= 0){ // require at least 1 tweet (1st tweet = 0)
    message = content[dcount];
    message = String(dcount + 1) +": " + message;
    Serial.print("\n" + message);
   
     // line above line display
    message.replace("\n\n","\n"); // remove any repeated \n to improve lcd layout
    message.replace(char(163),char(pound)); // swap £ for a custom char on the lcd
      while (message.length()) {
      lcd.clear(); 
        for(int i = 0; i < 4; i++){ // split message up to display on a 4 line 10x4 lcd
        int S = 0;
        E = message.indexOf("\n");
          if (E > 20) {
          E = 20;
          temp = message.substring(0, E );
         
          // remove leading spaces
          int L = 0;
            while(byte(temp.charAt(L) ) == 32) L++;
            if (L > 0)  temp.remove(0, L);

          // avoid spliiting words
          char N = message.charAt(E);
            if (!(N== 0 || N==10 || N==13 || N == 32  )){
            int B = temp.lastIndexOf(" ");
              if (B>-1){
              E = B;
              temp = temp.substring(0, E );
              S = 1;
              }
            }
            // end of avoid splitting
          }else{
          S = 1;
          temp = message.substring(0, E );
          }
        temp.trim();
        message.remove(0, E+ S);
        lcd.setCursor(0, i);
        lcd.print(temp);
          if (message.length() == 0) break;     
        }
        if (message == "\n") message = "";  // no point in starting a new panel
      mydelay(5000);
      }
   
      if(scroll){
      // scroll the mesage on line 2 of the lcd
      lcd.clear();
      lcd.print( twittername + ", " + String(dcount + 1));
      message = content[dcount];
      message.replace(char(163),char(pound)); // swap £ for a custom char
      lcdscroll(message, 2, false ); // params: (message, line number, clear the line first
      }
    dcount++; // tweet counter
      if(dcount > lastTweet) dcount = 0;
    }
  }
}

david-

#1
Dec 28, 2016, 10:03 am Last Edit: Jan 03, 2017, 11:31 am by david-
Part 2 of App:

Code: [Select]

String readtwitter(unsigned int maxno){
WiFiSSLClient client;
static String id[maxtweets]; // store tweet id's

char c;
String retval = "0";
boolean firstline = true;
String sentence = "";
int S, E;
unsigned int tweet = 0;

digitalWrite(led, HIGH);
lcd.setCursor(9,3); lcd.print(F(" *Reading* "));
Serial.print(F("\nConnecting to ")); Serial.println(LIB_DOMAIN);
  if (client.connect(LIB_DOMAIN, 443) ) {
  Serial.println(F("Connected\n"));
  // Make a HTTP request:
   //client.println("GET /search?q=" + twittername + "HTTP/1.1");
  client.println("GET /" + twittername + " HTTP/1.1");
  client.println("Host: mobile.twitter.com");
  client.println("Connection: close");
  client.println();
  client.flush();
    while (client.connected()) {
      while (client.available()){
      c = client.read();
      sentence = sentence + c;
        if (c == 10){
          if(firstline){
          S = sentence.indexOf(" ");
          E = sentence.indexOf(" ", S + 1);
          retval = sentence.substring(S + 1, E);
          firstline = false;
          }
          // print everything
          if (maxno == 0) Serial.print(sentence); else S = sentence.indexOf("\"tweet-text\""); 
         
          if ( (tweet < maxno) && (  S >= 0)  ) {
          S = sentence.indexOf("\"", S + 19);
          E = sentence.indexOf("\"", S+1);
          String sid = sentence.substring(S+1, E); // store tweet id as string
          sentence="";
         
            // test sid against id[tweet] to find out if latest tweet has changed
            if(bailout){ // if the tweet id is the same, stop processing   and get out quick
              if(sid == id[tweet]) {
                while(client.connected()) { // discard rest of html
                  if(client.available())  c =  client.read();
                }
              client.stop();
              Serial.println(F("No change in latest tweet"));
              lcd.setCursor(9,3); lcd.print(F(" *NoChange*"));
               digitalWrite(led, LOW);
              return "0";
              }
            }
          id[tweet] = sid;

            while(client.connected() ) {
              if (client.available() ) {
              c = client.read();

                if (c == 10){
                sentence = sentence + c;
                break;
                }
                // reject rogue characters
                if (sentence.length() < maxSlength){ // avoid sentence length failure!
                byte test = c; // had to do this to test for £ (163) and other characters
                  if ( test == 163 || ( test > 31 && test < 127 ) ) {
                  sentence += c; continue;
                  }

                  // Limited French letter conversion:
                  // lcd does not like french e acute, etc 
                  if  (test == 160 || test == 162){
                  sentence += "a"; continue;
                  }
                  if (test == 167) {
                  sentence += "c"; continue;
                  }
                  if  (test >167 && test < 172) {
                  sentence += "e";
                  }
                  //if (test == 153) sentence += "`";   // ` causes more problems than solutions
                }   
              }
            }
          S = sentence.indexOf(">");
          sentence = sentence.substring(S+3); // miss 2 characters after '>' : +3
                 
            if (nolink){
            // replace the links, leaving prompts
            S = sentence.indexOf("<a h"); // find link start
              while(S >= 0){
              E = sentence.indexOf(">", S + 6); // look for link end
                if (E == -1){ // not found so sentence truncated
                sentence.remove(S+1); // rest of line
                sentence = sentence + ">\n";
                } else sentence.remove(S+1, E- S); // delete link leaving "<"
              S = sentence.indexOf("<a href="); // look for the next link
              }
            sentence.replace("</a", ""); // amend end of link leaving ">"
            content[tweet] = sentence; // store the tweet for processing
            }else{ // add /n to print each link on a newline
            sentence.replace("<a h", "\n<a h");
            sentence.replace("</a>", "</a>\n");
            }
            if(debug) {
            Serial.print(tweet + 1); Serial.print(" "); Serial.println(id[tweet]);
            Serial.println( sentence );Serial.flush();
            }
          lastTweet = tweet;
          tweet ++;
          }
        sentence = "";
        }
      }
    }
  client.stop();
  // any sentence with length here, probably came from a remote client shut down
    if(debug && sentence.length()) Serial.println(sentence);
   
    if (nolink){
      for(int i = 0; i <= lastTweet; i++){
      sentence = content[i];
        // check for un-terminated message
        //if (sentence.charAt(sentence.length() - 1)  != 10)  sentence = sentence + "\n";
      // swap encoded characters
      sentence.replace("&#38;#38;#10;", "\n"); // linefeed
      sentence.replace("&#38;#38;#39;", "\'");  // '
      sentence.replace("&amp;", "&"); // &
      sentence.replace("&quot;", "\""); // "
      sentence.replace("  ", " "); // multi spaces
      content[i] = sentence; // store tweets in global array
      }
    }
  Serial.println(F("Finished")); 
  lcd.setCursor(9,3); lcd.print(F(" *Finished*")); 
  } else {
  Serial.println(F("Connection failed"));
  lcd.setCursor(9,3); lcd.print(F("  *Failed* "));
  }
digitalWrite(led, LOW);
return retval;
}

void lcdscroll(String message, byte lineno, boolean clearline){
int L = message.length();
  if (clearline && (L < 20) ){
  lcd.setCursor(0, lineno); lcd.print(clearit);
  }
lcd.setCursor(0, lineno); lcd.print(message.substring(0, 20) );
mydelay(1000);
  for (int i = 0; i <  L - 19; i++) {
  lcd.setCursor(0, lineno); lcd.print(clearit);
  mydelay(65); // dark time
  lcd.setCursor(0, lineno); lcd.print(message.substring(i, i + 20) );
  mydelay(chardelay); // bright time
  }
mydelay(1500);
}

void checkbutton() {
  if (digitalRead(button) == LOW){ // backlight control
  lcdlight = !lcdlight; 
    if(lcdlight) lcd.setBacklight(BACKLIGHT_ON); else lcd.setBacklight(BACKLIGHT_OFF);
    while (digitalRead(button) == LOW){
    delay(1);
    }
  }
}

void mydelay(unsigned long elapsed){
unsigned long tt = millis();
  while ( (millis() - tt) < elapsed) checkbutton();
}

void  wifiConnect(){  // returns if wifi connected, with up to 3 attempts
lcd.setCursor(0,1); lcd.print(F("Connecting"));
// check for the presence of the shield:
mystatus = WiFi.status();
  if (mystatus == WL_NO_SHIELD) {
  Serial.println(F("WiFi shield not present"));
  }else{
  // attempt to connect to Wifi network:
  byte attempts = 0; 
    while ( attempts < 3) {
    attempts++;
    Serial.print(F("Attempting to connect to SSID: ")); Serial.println(ssid);
    mystatus = WiFi.begin(ssid);
    unsigned long tt = millis();
      while ((millis()-tt) < 10000){ // up to 10 second wait
      delay(1000);
      mystatus = WiFi.status();
        if ((mystatus == WL_CONNECTED) || (mystatus == WL_NO_SHIELD)) break;
      }
      if ((mystatus == WL_CONNECTED) || (mystatus == WL_NO_SHIELD)) break;
    }
  }
lcd.setCursor(0,1);
  if (mystatus != WL_CONNECTED){
  Serial.println(F("Wifi not available\n"));
  lcd.print(F("Wifi not available"));
    while(1) yield();
  } else{
  lcd.print(F("WiFi Connected"));
  Serial.println(F("Wifi Connected\n"));
  }
}



Go Up
 


Please enter a valid email to subscribe

Confirm your email address

We need to confirm your email address.
To complete the subscription, please click the link in the email we just sent you.

Thank you for subscribing!

Arduino
via Egeo 16
Torino, 10131
Italy