class BilibiliSunday::Downloader

Public Class Methods

new(work_path = nil, downloader = nil, logger = nil) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 18
def initialize(work_path = nil, downloader = nil, logger = nil)
        @work_path = File.expand_path(work_path || '~/.bilibili_sunday')
        @downloader = downloader || Aria2::Downloader.new
        @logger = logger || Logger.new($stdout)
        @cacher = BilibiliSunday::Cacher.new(cacher_store_path)
        @mutex = Mutex.new

        FileUtils.mkdir_p(@work_path)
end

Public Instance Methods

active_videos() click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 99
def active_videos
        @mutex.synchronize do
                all_videos.select { |i| !concat_completed?(i)}
        end
end
all_videos() click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 71
def all_videos
        @mutex.synchronize do
                get_all_videos
        end
end
cid_for_video_url(url) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 77
def cid_for_video_url(url)
        doc = Nokogiri::HTML(gzip_inflate(@cacher.read_url(url)))

        res = /secure,cid=([0-9]*)/.match(doc.to_s)

        if res && res[1]
                return res[1].to_i
        else
                raise "Not a valid Bilibili video page URL. "
        end
end
query_status(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 42
def query_status(cid)
        @mutex.synchronize do
                status = :unknown
                status = :caching if cache_in_progress?(cid)
                status = :concatenating if concat_in_progress?(cid)
                status = :complete if concat_completed?(cid)

                {
                        cid: cid,
                        status: status, 
                        downloads: load_yaml(status_yaml_path(cid)) || [], 
                        path: concat_completed?(cid) ? concat_output_file_path(cid) : nil,
                        comments_path: comments_path(cid)
                }
        end
end
remove_cache(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 65
def remove_cache(cid)
        @mutex.synchronize do
                remove(cid)
        end
end
request_cache(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 59
def request_cache(cid)
        @mutex.synchronize do
                cache(cid)
        end
end
routine_work() click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 28
def routine_work
        @mutex.synchronize do
                @logger.info 'Carrying out routine work. '

                videos = get_all_videos

                videos.each do |cid|
                        update_status(cid)
                        cleanup(cid)
                        concat(cid) if (cache_completed?(cid) && (!concat_started?(cid)))
                end
        end
end
title_for_video_url(url) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 89
def title_for_video_url(url)
        doc = Nokogiri::HTML(gzip_inflate(@cacher.read_url(url)))

        doc.css('meta').each do |i|
                return i['content'] if i['name'] == 'title'
        end

        raise "Not a valid Bilibili video page URL. "
end

Private Instance Methods

cache(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 280
def cache(cid)
        return false if cache_started?(cid)

        FileUtils.mkdir_p(video_path(cid))

        filelist = fetch_filelist(cid)
        downloads = []

        filelist.each_with_index do |url, order|
                to_name = "#{order}#{fix_extname(File.extname(URI.parse(url).path))}"
                to_path = File.join(video_path(cid), to_name)

                downloads << {
                        order: order, 
                        url: url, 
                        download_id: @downloader.download(url, to_path), 
                        path: to_path
                }
        end

        write_yaml(downloads_yaml_path(cid), downloads)

        true
end
cache_completed?(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 214
def cache_completed?(cid)
        yaml_exists?(downloaded_yaml_path(cid))
end
cache_in_progress?(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 218
def cache_in_progress?(cid)
        cache_started?(cid) && (!cache_completed?(cid))
end
cache_started?(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 210
def cache_started?(cid)
        yaml_exists?(downloads_yaml_path(cid))
end
cacher_store_path() click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 246
def cacher_store_path
        File.join(@work_path, 'cache')
end
cleanup(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 107
def cleanup(cid)
        if concat_completed?(cid)
                # Deletes the partial files
                downloads = load_yaml(downloads_yaml_path(cid))
                downloads.each do |download|
                        path = download[:path]
                        FileUtils.rm(path) if File.exists?(path)
                end
        end
end
comments_path(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 250
def comments_path(cid)
        "http://comment.bilibili.tv/#{cid}.xml"
end
concat(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 321
def concat(cid)
        return false if concat_started?(cid)

        unless cache_completed?(cid) 
                raise "Cannot concat clips. Downloads not completed. "
        end

        write_ffmpeg_concat_input_file(cid)

        Thread.start(cid) do |cid|
                ffmpeg = "ffmpeg"
                command = "\"#{ffmpeg}\" -f concat -i \"#{ffmpeg_concat_input_path(cid)}\" -c copy \"#{concat_output_file_path(cid)}\""

                system(command)

                if $? == 0
                        mark_concat_complete(cid)
                else
                        remove_ffmpeg_concat_input_file(cid)
                end
        end

end
concat_completed?(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 226
def concat_completed?(cid)
        yaml_exists?(concatenated_yaml_path(cid))
end
concat_in_progress?(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 230
def concat_in_progress?(cid)
        concat_started?(cid) && (!concat_completed?(cid))
end
concat_output_file_path(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 268
def concat_output_file_path(cid)
        File.join(video_path(cid), "entirety#{video_ext(cid)}")
end
concat_started?(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 222
def concat_started?(cid)
        yaml_exists?(ffmpeg_concat_input_path(cid))
end
concatenated_yaml_path(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 242
def concatenated_yaml_path(cid)
        File.join(video_path(cid), 'concatenated.yaml') 
end
downloaded_yaml_path(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 238
def downloaded_yaml_path(cid)
        File.join(video_path(cid), 'downloaded.yml')
end
downloads_yaml_path(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 234
def downloads_yaml_path(cid)
        File.join(video_path(cid), 'downloads.yml')
end
fetch_filelist(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 179
def fetch_filelist(cid)
        url = "http://interface.bilibili.tv/v_cdn_play?cid=#{cid}"
        xml = XmlSimple::xml_in(@cacher.read_url(url))

        filelist = [''] * xml['durl'].length

        xml['durl'].each do |file|
                order = file['order'][0].to_i
                url = file['url'][0]
                filelist[order - 1] = url
        end

        filelist
end
ffmpeg_concat_input_path(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 276
def ffmpeg_concat_input_path(cid)
        File.join(video_path(cid), 'concat_list')
end
fix_extname(extname) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 263
def fix_extname(extname)
        return '.flv' if extname == '.hlv'
        extname
end
get_all_videos() click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 118
def get_all_videos
        Dir.glob(File.join(video_store_path, '*')).select {|f| File.directory? f}.map { |f| File.basename(f).to_i }
end
gzip_inflate(string) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 122
def gzip_inflate(string)
        # TODO Ugly workaround...
    begin
      Zlib::GzipReader.new(StringIO.new(string)).read
    rescue
      string
    end
end
load_yaml(path) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 164
def load_yaml(path)
        File.exists?(path) ?
                YAML.load(File.open(path) { |f| f.read }) :
                nil
end
mark_cache_complete(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 194
def mark_cache_complete(cid)
        write_yaml(downloaded_yaml_path(cid), true)
end
mark_concat_complete(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 198
def mark_concat_complete(cid)
        write_yaml(concatenated_yaml_path(cid), true)
end
remove(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 345
def remove(cid)
        return true unless cache_started?(cid)
        return false if concat_in_progress?(cid)

        downloads = load_yaml(downloads_yaml_path(cid))
        status = load_yaml(status_yaml_path(cid))

        downloads.each_with_index do |download, order|
                if !status || status[order][:status]['status'] == 'active'
                        @downloader.remove(download[:download_id])
                end
        end         

        FileUtils.rm_rf(video_path(cid))

        true
end
remove_ffmpeg_concat_input_file(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 317
def remove_ffmpeg_concat_input_file(cid)
        FileUtils.rm(ffmpeg_concat_input_path(cid))
end
status_yaml_path(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 272
def status_yaml_path(cid)
        File.join(video_path(cid), 'status.yaml')
end
update_status(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 131
def update_status(cid)
        # If download has not started yet, skip updating status.
        return unless cache_started?(cid)

        # If download already completed, skip updating status.
        return if cache_completed?(cid)

        downloads = load_yaml(downloads_yaml_path(cid))
        old_status = load_yaml(status_yaml_path(cid))

        status = []
        incomplete = false

        downloads.each_with_index do |download, order|
                # If already completed, stop updating status.
                if old_status
                        last_status = old_status[order][:status]
                        if last_status['status'] == 'complete'
                                download[:status] = last_status
                                status << download
                                next
                        end
                end

                download[:status] = @downloader.query_status(download[:download_id])
                incomplete = true unless download[:status]['status'] == 'complete'
                status << download
        end

        write_yaml(status_yaml_path(cid), status)
        mark_cache_complete(cid) unless incomplete
end
video_ext(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 254
def video_ext(cid)
        # TODO better way of identifying file types
        downloads = load_yaml(downloads_yaml_path(cid))
        ext = File.extname(downloads[0][:path])
        ext && !ext.empty? ?
                ext :
                '.flv'
end
video_path(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 206
def video_path(cid)
        File.join(video_store_path, "#{cid}")
end
video_store_path() click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 202
def video_store_path
        File.join(@work_path, "store")
end
write_ffmpeg_concat_input_file(cid) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 305
def write_ffmpeg_concat_input_file(cid)
        downloads = load_yaml(downloads_yaml_path(cid))

        File.open(ffmpeg_concat_input_path(cid), 'w') do |f|
                downloads.sort_by! { |f| f[:order] }

                downloads.each do |v|
                        f.puts "file '#{v[:path]}'"
                end
        end
end
write_yaml(path, obj) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 170
def write_yaml(path, obj)
        FileUtils.mkdir_p(File.dirname(path))
        File.open(path, 'w') { |f| f.write(YAML.dump(obj))}
end
yaml_exists?(rel_path) click to toggle source
# File lib/bilibili_sunday/downloader.rb, line 175
def yaml_exists?(rel_path)
        File.exists?(rel_path)
end