class Npm::Mirror::Mirror

Attributes

from[R]
parallelism[R]
server[R]
to[R]

Public Class Methods

new(from = DEFAULT_FROM, to = DEFAULT_TO, server = DEFAULT_SERVER, parallelism = 10, recheck = false) click to toggle source
# File lib/npm/mirror.rb, line 16
def initialize(from = DEFAULT_FROM, to = DEFAULT_TO,
               server = DEFAULT_SERVER, parallelism = 10,
               recheck = false)
  @from, @to, @server = from, to, server
  parallelism ||= 10
  recheck ||= false
  @pool = Pool.new parallelism
  @http = Net::HTTP::Persistent.new self.class.name
  @recheck = recheck

  puts "Mirroring  : #{from}"
  puts "Datadir    : #{to}"
  puts "Server     : #{server}"
  puts "Parallelism: #{parallelism}"
  puts "Recheck    : #{recheck}"
end

Public Instance Methods

correct_tarball_url() click to toggle source
# File lib/npm/mirror.rb, line 206
def correct_tarball_url
  Dir.glob("#{to}/*/index.json").each do |filename|
    json = JSON.load(File.open(filename, 'rb').read)
    next unless json['versions']
    versions = json['versions'].keys
    versions.each do |version|
      next unless json['versions'][version]['dist']
      tarball = URI json['versions'][version]['dist']['tarball']
      tarball = link tarball.path
      json['versions'][version]['dist']['tarball'] = tarball
    end
    File.open(filename, 'wb') { |f| f << json.to_json }
    write_package_versions json['name'], json
    puts filename
  end
end
etag_for(path) click to toggle source
# File lib/npm/mirror.rb, line 195
def etag_for(path)
  etag_path = path_for_etag path
  if !etag_path
    nil
  elsif File.file?(etag_path) && File.readable?(etag_path)
    File.open(etag_path, 'rb') { |f| f.readline.strip }
  else
    nil
  end
end
fetch(url, path = nil) click to toggle source
# File lib/npm/mirror.rb, line 54
def fetch(url, path = nil)
  uri = URI url
  etag = etag_for path

  req = Net::HTTP::Get.new uri.path
  req.add_field 'If-None-Match', etag if etag

  begin
    resp = @http.request uri, req
    fail Error, '503' if resp.code == '503'  # 503 backend read error
  rescue => e
    puts "[#{id}] Error fetching #{uri.path}: #{e.inspect}"
    sleep 10
    retry
  end

  puts "[#{id}] Fetched(#{resp.code}): #{uri}"
  case resp.code.to_i
  when 301  # Moved
    return fetch resp['location'], path
  when 302  # Found
    return fetch resp['location'], path
  when 200
    return resp
  when 304  # Not modified
    return resp if @recheck
    return nil
  when 403
    return nil
  when 404  # couchdb returns json even it's 404
    return resp
  else
    fail Error, "unexpected response #{resp.inspect}"
  end
end
fetch_index() click to toggle source
# File lib/npm/mirror.rb, line 90
def fetch_index
  url = from '-/all'
  path = to '-/all/index.json'
  resp = fetch url, path
  fail Error, "Failed to fetch #{url}" if resp.nil?

  if resp.code == '304'
    if File.exist? path
      json = JSON.load(File.open(path, 'rb').read)
    else
      json = {}
    end
  else
    json = JSON.load resp.body
  end

  @pool.run

  packages = json.keys
  packages.each_slice(@pool.size * 10) do |slice|
    slice.each do |package|
      next if package.start_with? '_'
      @pool.enqueue_job(package, &method(:fetch_package))
    end
    sleep 0.5
  end

  @pool.enqueue_job do
    write_file path, resp.body, resp['etag']
  end unless resp.code == '304'
end
fetch_package(package) click to toggle source
# File lib/npm/mirror.rb, line 122
def fetch_package(package)
  url = from package
  path = to package, 'index.json'
  resp = fetch url, path
  return if resp.nil?

  if resp.code == '304'
    json = JSON.load(File.open(path, 'rb').read)
    tarball_links json
  else
    json = JSON.load resp.body
    tarball_links json
    write_file path, json.to_json, resp['etag']
    write_package_versions(package, json)
  end
end
fetch_tarball(tarball_uri) click to toggle source
# File lib/npm/mirror.rb, line 147
def fetch_tarball(tarball_uri)
  url = from tarball_uri
  path = to tarball_uri
  resp = fetch url, path
  return if resp.nil? || resp.code == '304' || resp.body.size.zero?
  write_file path, resp.body, resp['etag']
end
id() click to toggle source
# File lib/npm/mirror.rb, line 45
def id
  l = @pool.size.to_s.size
  if Thread.current[:id]
    "Thread-#{Thread.current[:id].to_s.rjust(l, '0')}"
  else
    'Master'.ljust(%w(Thread- Master).map(&:size).max + l)
  end
end
path_for_etag(path) click to toggle source
# File lib/npm/mirror.rb, line 184
def path_for_etag(path)
  if path.nil?
    nil
  elsif File.directory? path
    File.join path, '.etag'
  else
    dirname, basename = File.split path
    File.join dirname, ".#{basename}.etag"
  end
end
run() click to toggle source
# File lib/npm/mirror.rb, line 179
def run
  fetch_index
  @pool.run_til_done
end
write_file(path, bytes, etag = nil) click to toggle source
# File lib/npm/mirror.rb, line 155
def write_file(path, bytes, etag = nil)
  FileUtils.mkdir_p File.dirname(path)
  File.open(path, 'wb') { |f| f << bytes }
  File.open(path_for_etag(path), 'wb') { |f| f << etag } if etag
end
write_package_versions(package, json) click to toggle source
# File lib/npm/mirror.rb, line 139
def write_package_versions(package, json)
  return unless json['versions']
  json['versions'].each do |k, v|
    path = to package, k, 'index.json'
    write_file path, v.to_json, nil
  end
end