module ProxyUtilities::Getter

Public Class Methods

get_all() click to toggle source

Starting all fetching processes

# File lib/proxy_utilities/getter.rb, line 13
def self.get_all()
  result = []
  result += self.get_free_proxy_list_net
  result += self.get_ssl_proxies_org
  result += self.get_gather_proxy_com
  result += self.get_httptunnel_ge
  result += self.get_proxy_list_org
  result += self.get_socks_proxy_list_net
  result += self.get_checker_proxy_net
  return result.uniq
end
get_checker_proxy_net() click to toggle source
# File lib/proxy_utilities/getter.rb, line 175
def self.get_checker_proxy_net
  result = []
  url = "https://checkerproxy.net/api/archive/#{Date.today.strftime('%Y-%m-%d')}"
  page = Nokogiri::HTML(open(url))
  rows = JSON.parse(page)

  rows.each do |row|
    begin
      proxy = {}
      proxy[:ip] = row["ip"]
      proxy[:port] = Integer(row["addr"].split(":")[1].gsub(/^0+/, ''))
      proxy[:country_code] = row["addr_geo_iso"].upcase
      anonymities = {0=>"transparent", 1=> "elite", 2=> "anonymous"}
      proxy[:anonymity] = anonymities[row["kind"]]
      types = {1=>"HTTP", 2=>"HTTPS", 3=>"SOCKS4", 4=>"SOCKS5"}
      proxy[:type] = types[row["type"]]
      result << proxy
    end
  end
  return result
end
get_free_proxy_list_net() click to toggle source
# File lib/proxy_utilities/getter.rb, line 25
def self.get_free_proxy_list_net
  result = []
  url = "https://free-proxy-list.net/"
  page = Nokogiri::HTML(open(url))
  rows = page.xpath('//table[@id="proxylisttable"]/tbody/tr')

  rows.each do |row|
    begin
      proxy = {}
      proxy[:ip] = row.xpath('td[1]').text
      proxy[:port] = Integer(row.xpath('td[2]').text.gsub(/^0+/, ''))
      proxy[:country_code] = row.xpath('td[3]').text
      proxy[:anonymity] = row.xpath('td[5]').text.gsub(" proxy", "")
      if row.xpath('td[7]').text == 'yes'
        proxy[:type] = 'HTTPS'
      else
        proxy[:type] = 'HTTP'
      end
      result << proxy
    end
  end
  return result
end
get_gather_proxy_com() click to toggle source
# File lib/proxy_utilities/getter.rb, line 73
def self.get_gather_proxy_com
  result = []
  url = "http://www.gatherproxy.com/"
  page = Nokogiri::HTML(open(url))
  rows = page.xpath('//div[@class="proxy-list"]/table/script')
  javascript = rows.text[/{.+}/im].gsub("gp.insertPrx(", "").gsub(");", "")
  json = JSON.parse("[" + javascript.split("\r\n").join(",").gsub("               ,                    ", "") + "]")
  json.each do |row|
    begin
      proxy = {}
      proxy[:ip] = row["PROXY_IP"]
      proxy[:port] = row['PROXY_PORT'].to_i(16)
      begin
        proxy[:country_code] = IsoCountryCodes.search_by_name(row["PROXY_COUNTRY"]).first.alpha2
      rescue
        proxy[:country_code] = nil
      end
      proxy[:anonymity] = row['PROXY_TYPE'].downcase
      proxy[:type] = 'HTTP'
      result << proxy
    end
  end
  return result
end
get_httptunnel_ge() click to toggle source
# File lib/proxy_utilities/getter.rb, line 98
def self.get_httptunnel_ge
  result = []
  url = "http://www.httptunnel.ge/ProxyListForFree.aspx"
  page = Nokogiri::HTML(open(url))
  rows = page.xpath('//table[contains(@id, "GridView")]/tr[(count(td)>2)]')

  rows.each do |row|
    begin
      proxy = {}
      proxy[:ip] = row.xpath('td[1]').text.split(":")[0].strip
      proxy[:port] = Integer(row.xpath('td[1]').text.split(":")[1].strip)
      proxy[:country_code] = row.xpath('td[8]/img/@title').text
      transparency = row.xpath('td[5]').text.to_sym
      proxy[:anonymity] = { A: 'anonymous',  E: 'elite', T: 'transparent', U: 'unknown'}.fetch(transparency, 'Unknown')
      proxy[:type] = 'HTTP'
      result << proxy
    end
  end
  return result
end
get_proxy_list_org() click to toggle source
# File lib/proxy_utilities/getter.rb, line 119
def self.get_proxy_list_org
  result = []
  (1..10).to_a.each do |page_num|
    begin
      url = "https://proxy-list.org/english/index.php?p=#{page_num}"
      page = Nokogiri::HTML(open(url))
      rows = page.css('.table-wrap .table ul')

      rows.each do |row|
        begin
          proxy = {}
          proxy[:ip] = ::Base64.decode64(rows.first.at_css('li script').text.match(/'(.+)'/)[1]).split(":")[0].strip
          proxy[:port] = Integer(::Base64.decode64(rows.first.at_css('li script').text.match(/'(.+)'/)[1]).split(":")[1].strip)
          country_code = row.xpath('li[5]/div/span/span/span[2]').text.scan(/([A-Z]{2}) /)
          unless country_code.nil?
            begin
              proxy[:country_code] = country_code.first.first.strip
            rescue
              next
            end
          else
            next
          end
          proxy[:anonymity] = row.xpath('li[4]').text.strip.downcase
          proxy[:type] = row.xpath('li[2]').text.strip.upcase
          if proxy[:type] == "-"
            proxy[:type] = nil
          end
          result << proxy
        end
      end
    end
  end
  return result
end
get_socks_proxy_list_net() click to toggle source
# File lib/proxy_utilities/getter.rb, line 155
def self.get_socks_proxy_list_net
  result = []
  url = "https://www.socks-proxy.net/"
  page = Nokogiri::HTML(open(url))
  rows = page.xpath('//table[@id="proxylisttable"]/tbody/tr')

  rows.each do |row|
    begin
      proxy = {}
      proxy[:ip] = row.xpath('td[1]').text
      proxy[:port] = Integer(row.xpath('td[2]').text.gsub(/^0+/, ''))
      proxy[:country_code] = row.xpath('td[3]').text
      proxy[:anonymity] = row.xpath('td[6]').text.downcase
      proxy[:type] = row.xpath('td[5]').text.upcase
      result << proxy
    end
  end
  return result
end
get_ssl_proxies_org() click to toggle source
# File lib/proxy_utilities/getter.rb, line 49
def self.get_ssl_proxies_org
  result = []
  url = "https://www.sslproxies.org/"
  page = Nokogiri::HTML(open(url))
  rows = page.xpath('//table[@id="proxylisttable"]/tbody/tr')

  rows.each do |row|
    begin
      proxy = {}
      proxy[:ip] = row.xpath('td[1]').text
      proxy[:port] = Integer(row.xpath('td[2]').text.gsub(/^0+/, ''))
      proxy[:country_code] = row.xpath('td[3]').text
      proxy[:anonymity] = row.xpath('td[5]').text.gsub(" proxy", "")
      if row.xpath('td[7]').text == 'yes'
        proxy[:type] = 'HTTPS'
      else
        proxy[:type] = 'HTTP'
      end
      result << proxy
    end
  end
  return result
end