module GetProxyList
Public Class Methods
get_list(limit,page)
click to toggle source
# File lib/get_proxy_list.rb, line 45 def self.get_list(limit,page) fromProxyCn = FromProxyCn.new() base = Base.new() proxylist = fromProxyCn.get_proxylist(page) proxylist = base.get_proxy_in_time_limit(limit,proxylist) return proxylist end
get_proxy_in_time_limt(limit,proxylist)
click to toggle source
# File lib/get_proxy_list.rb, line 53 def self.get_proxy_in_time_limt(limit,proxylist) limit_list = [] proxylist.each do |proxy| url = proxy time_start = Time.now.to_i begin timeout(limit+1) do doc = Nokogiri::HTML(open("http://http://www.ikanke.cn:8080/api/herf_links/test_link",:proxy=> url)).css("body")[0].content end time_end = Time.now.to_i time_use = time_end - time_start p "url use_time:#{time_use}" rescue Exception =>e case e when Errno::ETIMEDOUT p "Use #{url} timeout" when Timeout::Error p "Use #{url} timeout" when Errno::ECONNREFUSED p "Use #{url} Error connection" else p "Use #{url} Error:#{e.to_s}" end time_use = -1 end if(time_use > 0 && time_use < limit&& doc == "{\"ResultMessage\":true}") limit_list << url end end limit_list end
proxy_list(limit,page,proxy_path)
click to toggle source
# File lib/get_proxy_list.rb, line 7 def self.proxy_list(limit,page,proxy_path) proxy_list = [] proxy_path = File.dirname(proxy_path) crt_date = DateTime.now.strftime('%F') proxy_path = "#{proxy_path}/%s.txt" % [crt_date] p "Proxy_Path: #{proxy_path}" if File.exist?(proxy_path) file_proxy = File.open(proxy_path,"r") p "Write in file" file_proxy.each_line {|line| proxy_list << line.chomp.to_s } p "Get Proxy_list:#{proxy_list}" file_proxy.flush file_proxy.close else proxy_list=get_list(limit,page) dirpath = "#{File.dirname(proxy_path)}/" p "DirPath=#{dirpath}" Dir.open(dirpath) {|fna| fna.each do |fn| if(fn.to_s != ".." && fn.to_s != ".") File.delete("#{dirpath + fn.to_s}") end end } file_proxy = File.new(proxy_path,"a") proxy_list.each do |proxy| p "Proxy:#{proxy}" file_proxy.puts proxy end file_proxy.flush file_proxy.close end return proxy_list end