module GetProxyList

Public Class Methods

get_list(limit,page) click to toggle source
# File lib/get_proxy_list.rb, line 45
def self.get_list(limit,page)
  fromProxyCn = FromProxyCn.new()
  base = Base.new()
  proxylist = fromProxyCn.get_proxylist(page)
  proxylist = base.get_proxy_in_time_limit(limit,proxylist)
  return proxylist
end
get_proxy_in_time_limt(limit,proxylist) click to toggle source
# File lib/get_proxy_list.rb, line 53
def self.get_proxy_in_time_limt(limit,proxylist)
  limit_list = []
  proxylist.each do |proxy|
    url = proxy
    time_start = Time.now.to_i
    begin
      timeout(limit+1) do
        doc = Nokogiri::HTML(open("http://http://www.ikanke.cn:8080/api/herf_links/test_link",:proxy=> url)).css("body")[0].content
      end
      time_end = Time.now.to_i
      time_use = time_end - time_start
      p  "url   use_time:#{time_use}"
    rescue Exception =>e
      case e
        when Errno::ETIMEDOUT
          p "Use #{url} timeout"
        when Timeout::Error
          p "Use #{url} timeout"
        when Errno::ECONNREFUSED
          p "Use #{url} Error connection"
        else
          p "Use #{url} Error:#{e.to_s}"
      end
      time_use = -1
    end
    if(time_use > 0 && time_use < limit&& doc == "{\"ResultMessage\":true}")
      limit_list << url
    end
  end
  limit_list
end
proxy_list(limit,page,proxy_path) click to toggle source
# File lib/get_proxy_list.rb, line 7
def self.proxy_list(limit,page,proxy_path)
  proxy_list = []
  proxy_path = File.dirname(proxy_path)
  crt_date = DateTime.now.strftime('%F')
  proxy_path = "#{proxy_path}/%s.txt" % [crt_date]
  p "Proxy_Path: #{proxy_path}"
  if File.exist?(proxy_path)
    file_proxy = File.open(proxy_path,"r")
    p "Write in file"
    file_proxy.each_line {|line|
      proxy_list << line.chomp.to_s
    }
    p "Get Proxy_list:#{proxy_list}"
    file_proxy.flush
    file_proxy.close
  else
    proxy_list=get_list(limit,page)
    dirpath = "#{File.dirname(proxy_path)}/"
    p "DirPath=#{dirpath}"
    Dir.open(dirpath)  {|fna|
      fna.each do |fn|
        if(fn.to_s != ".." && fn.to_s != ".")
          File.delete("#{dirpath + fn.to_s}")
        end
      end
    }
    file_proxy = File.new(proxy_path,"a")
    proxy_list.each do |proxy|
      p "Proxy:#{proxy}"
      file_proxy.puts proxy
    end
    file_proxy.flush
    file_proxy.close
  end
  return proxy_list
end