class Twexicon::Scraper

Attributes

tweets[R]

Public Class Methods

new(username) click to toggle source
# File lib/twexicon/scraper.rb, line 4
def initialize(username)
  @tweets = {}
  scrape_tweets(username)
end

Public Instance Methods

refine_tweets() click to toggle source
# File lib/twexicon/scraper.rb, line 13
def refine_tweets
  tweets.each do |num, tweet|
    t = tweet.keys[0].dup
    t.scan(/pic.twitter.com\/\w{10}/){|p| tweet.values[0][:pix] << p.strip}.gsub!(/pic.twitter.com\/\w{10}/, " ")
    t.scan(/https?:\/\/[\w\.\?\=\&\-\/\#\:]+/){|w| tweet.values[0][:links] << w.strip}.gsub!(/https?:\/\/[\w\.\?\=\&\-\/\#\:]+/, " ")
    t.scan(/#\w+/){|h| tweet.values[0][:hashtags] << h.gsub(/\W/, "").prepend("#")}.gsub!(/#\w+/, " ")
    t.scan(/@\w+/){|u| tweet.values[0][:usernames] << u.gsub(/\W/, "").prepend("@")}.gsub!(/@\w+/, " ")
    t.scan(/(\d+[:\.\ ]?\d*)+/){|n| tweet.values[0][:numbers] << n.first.gsub(/(^\W+|\W+$)/, "")}.gsub!(/(\d+[:\.\ ]?\d*)+/, " ")
    t.scan(/(\b[A-Z][\.\ ][A-Z][\.\ ][A-Z][\.\ ]|\b[A-Z][\.\ ][A-Z][\.\ ])/){|a| tweet.values[0][:acronyms] << a.first.strip}.gsub!(/(\b[A-Z][\.\ ][A-Z][\.\ ][A-Z][\.\ ]|\b[A-Z][\.\ ][A-Z][\.\ ])/, " ")
    t.scan(/(([A-Z]+[\s\,\&\:\-]+){2,}|[A-Z]{4,}\W)/){|s| tweet.values[0][:shouts] << s.first.gsub(/\W/, " ").strip}.gsub!(/(([A-Z]+[\s\,\&\:\-]+){2,}|[A-Z]{4,}\W)/, " ")
    t.scan(/\b[A-Z]{2,3}\b/){|a| tweet.values[0][:acronyms] << a.strip}.gsub!(/\b[A-Z]{2,3}\b/, " ")
    t.scan(/\w+['\/]?\w*/){|w| tweet.values[0][:words] << w.strip}.gsub!(/\w+['\/]?\w*/, " ")
  end
end
scrape_tweets(username) click to toggle source
# File lib/twexicon/scraper.rb, line 9
def scrape_tweets(username)
  Nokogiri::HTML(open("https://twitter.com/#{username}")).css(".tweet-text").each{|t| tweets[tweets.length+1] = {t.text => {:pix => [], :links => [], :hashtags => [], :usernames => [], :numbers => [], :acronyms => [], :shouts => [], :words => []}}}
end