class BilibiliSunday::Downloader
Public Class Methods
new(work_path = nil, downloader = nil, logger = nil)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 18 def initialize(work_path = nil, downloader = nil, logger = nil) @work_path = File.expand_path(work_path || '~/.bilibili_sunday') @downloader = downloader || Aria2::Downloader.new @logger = logger || Logger.new($stdout) @cacher = BilibiliSunday::Cacher.new(cacher_store_path) @mutex = Mutex.new FileUtils.mkdir_p(@work_path) end
Public Instance Methods
active_videos()
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 99 def active_videos @mutex.synchronize do all_videos.select { |i| !concat_completed?(i)} end end
all_videos()
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 71 def all_videos @mutex.synchronize do get_all_videos end end
cid_for_video_url(url)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 77 def cid_for_video_url(url) doc = Nokogiri::HTML(gzip_inflate(@cacher.read_url(url))) res = /secure,cid=([0-9]*)/.match(doc.to_s) if res && res[1] return res[1].to_i else raise "Not a valid Bilibili video page URL. " end end
query_status(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 42 def query_status(cid) @mutex.synchronize do status = :unknown status = :caching if cache_in_progress?(cid) status = :concatenating if concat_in_progress?(cid) status = :complete if concat_completed?(cid) { cid: cid, status: status, downloads: load_yaml(status_yaml_path(cid)) || [], path: concat_completed?(cid) ? concat_output_file_path(cid) : nil, comments_path: comments_path(cid) } end end
remove_cache(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 65 def remove_cache(cid) @mutex.synchronize do remove(cid) end end
request_cache(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 59 def request_cache(cid) @mutex.synchronize do cache(cid) end end
routine_work()
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 28 def routine_work @mutex.synchronize do @logger.info 'Carrying out routine work. ' videos = get_all_videos videos.each do |cid| update_status(cid) cleanup(cid) concat(cid) if (cache_completed?(cid) && (!concat_started?(cid))) end end end
title_for_video_url(url)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 89 def title_for_video_url(url) doc = Nokogiri::HTML(gzip_inflate(@cacher.read_url(url))) doc.css('meta').each do |i| return i['content'] if i['name'] == 'title' end raise "Not a valid Bilibili video page URL. " end
Private Instance Methods
cache(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 280 def cache(cid) return false if cache_started?(cid) FileUtils.mkdir_p(video_path(cid)) filelist = fetch_filelist(cid) downloads = [] filelist.each_with_index do |url, order| to_name = "#{order}#{fix_extname(File.extname(URI.parse(url).path))}" to_path = File.join(video_path(cid), to_name) downloads << { order: order, url: url, download_id: @downloader.download(url, to_path), path: to_path } end write_yaml(downloads_yaml_path(cid), downloads) true end
cache_completed?(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 214 def cache_completed?(cid) yaml_exists?(downloaded_yaml_path(cid)) end
cache_in_progress?(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 218 def cache_in_progress?(cid) cache_started?(cid) && (!cache_completed?(cid)) end
cache_started?(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 210 def cache_started?(cid) yaml_exists?(downloads_yaml_path(cid)) end
cacher_store_path()
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 246 def cacher_store_path File.join(@work_path, 'cache') end
cleanup(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 107 def cleanup(cid) if concat_completed?(cid) # Deletes the partial files downloads = load_yaml(downloads_yaml_path(cid)) downloads.each do |download| path = download[:path] FileUtils.rm(path) if File.exists?(path) end end end
comments_path(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 250 def comments_path(cid) "http://comment.bilibili.tv/#{cid}.xml" end
concat(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 321 def concat(cid) return false if concat_started?(cid) unless cache_completed?(cid) raise "Cannot concat clips. Downloads not completed. " end write_ffmpeg_concat_input_file(cid) Thread.start(cid) do |cid| ffmpeg = "ffmpeg" command = "\"#{ffmpeg}\" -f concat -i \"#{ffmpeg_concat_input_path(cid)}\" -c copy \"#{concat_output_file_path(cid)}\"" system(command) if $? == 0 mark_concat_complete(cid) else remove_ffmpeg_concat_input_file(cid) end end end
concat_completed?(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 226 def concat_completed?(cid) yaml_exists?(concatenated_yaml_path(cid)) end
concat_in_progress?(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 230 def concat_in_progress?(cid) concat_started?(cid) && (!concat_completed?(cid)) end
concat_output_file_path(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 268 def concat_output_file_path(cid) File.join(video_path(cid), "entirety#{video_ext(cid)}") end
concat_started?(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 222 def concat_started?(cid) yaml_exists?(ffmpeg_concat_input_path(cid)) end
concatenated_yaml_path(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 242 def concatenated_yaml_path(cid) File.join(video_path(cid), 'concatenated.yaml') end
downloaded_yaml_path(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 238 def downloaded_yaml_path(cid) File.join(video_path(cid), 'downloaded.yml') end
downloads_yaml_path(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 234 def downloads_yaml_path(cid) File.join(video_path(cid), 'downloads.yml') end
fetch_filelist(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 179 def fetch_filelist(cid) url = "http://interface.bilibili.tv/v_cdn_play?cid=#{cid}" xml = XmlSimple::xml_in(@cacher.read_url(url)) filelist = [''] * xml['durl'].length xml['durl'].each do |file| order = file['order'][0].to_i url = file['url'][0] filelist[order - 1] = url end filelist end
ffmpeg_concat_input_path(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 276 def ffmpeg_concat_input_path(cid) File.join(video_path(cid), 'concat_list') end
fix_extname(extname)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 263 def fix_extname(extname) return '.flv' if extname == '.hlv' extname end
get_all_videos()
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 118 def get_all_videos Dir.glob(File.join(video_store_path, '*')).select {|f| File.directory? f}.map { |f| File.basename(f).to_i } end
gzip_inflate(string)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 122 def gzip_inflate(string) # TODO Ugly workaround... begin Zlib::GzipReader.new(StringIO.new(string)).read rescue string end end
load_yaml(path)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 164 def load_yaml(path) File.exists?(path) ? YAML.load(File.open(path) { |f| f.read }) : nil end
mark_cache_complete(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 194 def mark_cache_complete(cid) write_yaml(downloaded_yaml_path(cid), true) end
mark_concat_complete(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 198 def mark_concat_complete(cid) write_yaml(concatenated_yaml_path(cid), true) end
remove(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 345 def remove(cid) return true unless cache_started?(cid) return false if concat_in_progress?(cid) downloads = load_yaml(downloads_yaml_path(cid)) status = load_yaml(status_yaml_path(cid)) downloads.each_with_index do |download, order| if !status || status[order][:status]['status'] == 'active' @downloader.remove(download[:download_id]) end end FileUtils.rm_rf(video_path(cid)) true end
remove_ffmpeg_concat_input_file(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 317 def remove_ffmpeg_concat_input_file(cid) FileUtils.rm(ffmpeg_concat_input_path(cid)) end
status_yaml_path(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 272 def status_yaml_path(cid) File.join(video_path(cid), 'status.yaml') end
update_status(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 131 def update_status(cid) # If download has not started yet, skip updating status. return unless cache_started?(cid) # If download already completed, skip updating status. return if cache_completed?(cid) downloads = load_yaml(downloads_yaml_path(cid)) old_status = load_yaml(status_yaml_path(cid)) status = [] incomplete = false downloads.each_with_index do |download, order| # If already completed, stop updating status. if old_status last_status = old_status[order][:status] if last_status['status'] == 'complete' download[:status] = last_status status << download next end end download[:status] = @downloader.query_status(download[:download_id]) incomplete = true unless download[:status]['status'] == 'complete' status << download end write_yaml(status_yaml_path(cid), status) mark_cache_complete(cid) unless incomplete end
video_ext(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 254 def video_ext(cid) # TODO better way of identifying file types downloads = load_yaml(downloads_yaml_path(cid)) ext = File.extname(downloads[0][:path]) ext && !ext.empty? ? ext : '.flv' end
video_path(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 206 def video_path(cid) File.join(video_store_path, "#{cid}") end
video_store_path()
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 202 def video_store_path File.join(@work_path, "store") end
write_ffmpeg_concat_input_file(cid)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 305 def write_ffmpeg_concat_input_file(cid) downloads = load_yaml(downloads_yaml_path(cid)) File.open(ffmpeg_concat_input_path(cid), 'w') do |f| downloads.sort_by! { |f| f[:order] } downloads.each do |v| f.puts "file '#{v[:path]}'" end end end
write_yaml(path, obj)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 170 def write_yaml(path, obj) FileUtils.mkdir_p(File.dirname(path)) File.open(path, 'w') { |f| f.write(YAML.dump(obj))} end
yaml_exists?(rel_path)
click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 175 def yaml_exists?(rel_path) File.exists?(rel_path) end