module ProxyUtilities::Getter
Public Class Methods
get_all()
click to toggle source
Starting all fetching processes
# File lib/proxy_utilities/getter.rb, line 13 def self.get_all() result = [] result += self.get_free_proxy_list_net result += self.get_ssl_proxies_org result += self.get_gather_proxy_com result += self.get_httptunnel_ge result += self.get_proxy_list_org result += self.get_socks_proxy_list_net result += self.get_checker_proxy_net return result.uniq end
get_checker_proxy_net()
click to toggle source
# File lib/proxy_utilities/getter.rb, line 175 def self.get_checker_proxy_net result = [] url = "https://checkerproxy.net/api/archive/#{Date.today.strftime('%Y-%m-%d')}" page = Nokogiri::HTML(open(url)) rows = JSON.parse(page) rows.each do |row| begin proxy = {} proxy[:ip] = row["ip"] proxy[:port] = Integer(row["addr"].split(":")[1].gsub(/^0+/, '')) proxy[:country_code] = row["addr_geo_iso"].upcase anonymities = {0=>"transparent", 1=> "elite", 2=> "anonymous"} proxy[:anonymity] = anonymities[row["kind"]] types = {1=>"HTTP", 2=>"HTTPS", 3=>"SOCKS4", 4=>"SOCKS5"} proxy[:type] = types[row["type"]] result << proxy end end return result end
get_free_proxy_list_net()
click to toggle source
# File lib/proxy_utilities/getter.rb, line 25 def self.get_free_proxy_list_net result = [] url = "https://free-proxy-list.net/" page = Nokogiri::HTML(open(url)) rows = page.xpath('//table[@id="proxylisttable"]/tbody/tr') rows.each do |row| begin proxy = {} proxy[:ip] = row.xpath('td[1]').text proxy[:port] = Integer(row.xpath('td[2]').text.gsub(/^0+/, '')) proxy[:country_code] = row.xpath('td[3]').text proxy[:anonymity] = row.xpath('td[5]').text.gsub(" proxy", "") if row.xpath('td[7]').text == 'yes' proxy[:type] = 'HTTPS' else proxy[:type] = 'HTTP' end result << proxy end end return result end
get_gather_proxy_com()
click to toggle source
# File lib/proxy_utilities/getter.rb, line 73 def self.get_gather_proxy_com result = [] url = "http://www.gatherproxy.com/" page = Nokogiri::HTML(open(url)) rows = page.xpath('//div[@class="proxy-list"]/table/script') javascript = rows.text[/{.+}/im].gsub("gp.insertPrx(", "").gsub(");", "") json = JSON.parse("[" + javascript.split("\r\n").join(",").gsub(" , ", "") + "]") json.each do |row| begin proxy = {} proxy[:ip] = row["PROXY_IP"] proxy[:port] = row['PROXY_PORT'].to_i(16) begin proxy[:country_code] = IsoCountryCodes.search_by_name(row["PROXY_COUNTRY"]).first.alpha2 rescue proxy[:country_code] = nil end proxy[:anonymity] = row['PROXY_TYPE'].downcase proxy[:type] = 'HTTP' result << proxy end end return result end
get_httptunnel_ge()
click to toggle source
# File lib/proxy_utilities/getter.rb, line 98 def self.get_httptunnel_ge result = [] url = "http://www.httptunnel.ge/ProxyListForFree.aspx" page = Nokogiri::HTML(open(url)) rows = page.xpath('//table[contains(@id, "GridView")]/tr[(count(td)>2)]') rows.each do |row| begin proxy = {} proxy[:ip] = row.xpath('td[1]').text.split(":")[0].strip proxy[:port] = Integer(row.xpath('td[1]').text.split(":")[1].strip) proxy[:country_code] = row.xpath('td[8]/img/@title').text transparency = row.xpath('td[5]').text.to_sym proxy[:anonymity] = { A: 'anonymous', E: 'elite', T: 'transparent', U: 'unknown'}.fetch(transparency, 'Unknown') proxy[:type] = 'HTTP' result << proxy end end return result end
get_proxy_list_org()
click to toggle source
# File lib/proxy_utilities/getter.rb, line 119 def self.get_proxy_list_org result = [] (1..10).to_a.each do |page_num| begin url = "https://proxy-list.org/english/index.php?p=#{page_num}" page = Nokogiri::HTML(open(url)) rows = page.css('.table-wrap .table ul') rows.each do |row| begin proxy = {} proxy[:ip] = ::Base64.decode64(rows.first.at_css('li script').text.match(/'(.+)'/)[1]).split(":")[0].strip proxy[:port] = Integer(::Base64.decode64(rows.first.at_css('li script').text.match(/'(.+)'/)[1]).split(":")[1].strip) country_code = row.xpath('li[5]/div/span/span/span[2]').text.scan(/([A-Z]{2}) /) unless country_code.nil? begin proxy[:country_code] = country_code.first.first.strip rescue next end else next end proxy[:anonymity] = row.xpath('li[4]').text.strip.downcase proxy[:type] = row.xpath('li[2]').text.strip.upcase if proxy[:type] == "-" proxy[:type] = nil end result << proxy end end end end return result end
get_socks_proxy_list_net()
click to toggle source
# File lib/proxy_utilities/getter.rb, line 155 def self.get_socks_proxy_list_net result = [] url = "https://www.socks-proxy.net/" page = Nokogiri::HTML(open(url)) rows = page.xpath('//table[@id="proxylisttable"]/tbody/tr') rows.each do |row| begin proxy = {} proxy[:ip] = row.xpath('td[1]').text proxy[:port] = Integer(row.xpath('td[2]').text.gsub(/^0+/, '')) proxy[:country_code] = row.xpath('td[3]').text proxy[:anonymity] = row.xpath('td[6]').text.downcase proxy[:type] = row.xpath('td[5]').text.upcase result << proxy end end return result end
get_ssl_proxies_org()
click to toggle source
# File lib/proxy_utilities/getter.rb, line 49 def self.get_ssl_proxies_org result = [] url = "https://www.sslproxies.org/" page = Nokogiri::HTML(open(url)) rows = page.xpath('//table[@id="proxylisttable"]/tbody/tr') rows.each do |row| begin proxy = {} proxy[:ip] = row.xpath('td[1]').text proxy[:port] = Integer(row.xpath('td[2]').text.gsub(/^0+/, '')) proxy[:country_code] = row.xpath('td[3]').text proxy[:anonymity] = row.xpath('td[5]').text.gsub(" proxy", "") if row.xpath('td[7]').text == 'yes' proxy[:type] = 'HTTPS' else proxy[:type] = 'HTTP' end result << proxy end end return result end