module NHKore::CLI::GetCmd
@author Jonathan Bradley Whited @since 0.2.0
Constants
- DEFAULT_GET_CHUNK_SIZE
- DEFAULT_GET_URL_LENGTH
- GET_URL
- GET_URL_FILENAME
Public Instance Methods
build_get_cmd()
click to toggle source
# File lib/nhkore/cli/get_cmd.rb, line 27 def build_get_cmd app = self @get_cmd = @app_cmd.define_command do name 'get' usage 'get [OPTIONS] [COMMAND]...' aliases :g summary "Download NHKore's pre-scraped files from the latest release" \ " (aliases: #{app.color_alias('g')})" description(<<-DESC) Download NHKore's pre-scraped files from the latest release & save to folder: #{Util::CORE_DIR} Note: the latest NHK articles may not have been scraped yet. DESC option :o,:out,'directory to save downloaded files to',argument: :required,default: Util::CORE_DIR, transform: lambda { |value| app.check_empty_opt(:out,value) } flag nil,:'show-url','show download URL and exit (for downloading manually)' do |value,cmd| puts GET_URL exit end run do |opts,args,cmd| app.refresh_cmd(opts,args,cmd) app.run_get_cmd end end end
run_get_cmd()
click to toggle source
# File lib/nhkore/cli/get_cmd.rb, line 60 def run_get_cmd require 'down/net_http' require 'tempfile' require 'zip' build_out_dir(:out,default_dir: Util::CORE_DIR) return unless check_out_dir(:out) chunk_size = DEFAULT_GET_CHUNK_SIZE down = nil dry_run = @cmd_opts[:dry_run] force = @cmd_opts[:force] max_retries = @scraper_kargs[:max_retries] max_retries = 3 if max_retries.nil? out_dir = @cmd_opts[:out] begin start_spin('Opening URL') begin down = Down::NetHttp.open(GET_URL,rewindable: false,**@scraper_kargs) rescue Down::ConnectionError raise if (max_retries -= 1) < 0 retry end stop_spin return if dry_run Tempfile.create(["#{App::NAME}_",'.zip'],binmode: true) do |file| puts puts "Downloading #{GET_URL_FILENAME} to temp file:" puts "> #{file.path}" len = down.size len = DEFAULT_GET_LENGTH if len.nil? || len < 1 bar = build_progress_bar('> Downloading',download: true,total: len) bar.start while !down.eof? file.write(down.read(chunk_size)) bar.advance(chunk_size) end down.close file.close bar.finish puts puts "Extracting #{GET_URL_FILENAME}..." # We manually ask the user whether to overwrite each file, so set this to # true so that Zip extract() will force overwrites and not raise an error. Zip.on_exists_proc = true Zip::File.open(file) do |zip_file| zip_file.each do |entry| if !entry.name_safe? raise ZipError,"unsafe entry name[#{entry.name}] in Zip file" end name = Util.strip_web_str(File.basename(entry.name)) next if name.empty? out_file = File.join(out_dir,name) puts "> #{name}" if !force && File.exist?(out_file) puts puts 'Warning: output file already exists!' puts "> '#{out_file}'" overwrite = @high.agree('Overwrite this file (yes/no)? ') puts next unless overwrite end entry.extract(out_file) end end puts puts "Extracted #{GET_URL_FILENAME} to directory:" puts "> #{out_dir}" end ensure down.close if !down.nil? && !down.closed? end end