class Asperalm::Cli::Plugins::Preview

Constants

ACTIONS
AK_MARKER_FILE
DEFAULT_PREVIEWS_FOLDER
LOCAL_STORAGE_PCVL
PREVIEW_BASENAME

basename of preview files

PREVIEW_FOLDER_SUFFIX

defined by node API: suffix for folder containing previews

PREV_GEN_TAG

special tag to identify transfers related to generator

TMP_DIR_PREFIX

subfolder in system tmp folder

Attributes

option_file_access[RW]
option_folder_reset_cache[RW]
option_overwrite[RW]
option_previews_folder[RW]

option_skip_format has special accessors

option_skip_folders[RW]

Public Class Methods

new(env) click to toggle source
Calls superclass method Asperalm::Cli::BasicAuthPlugin::new
# File lib/asperalm/cli/plugins/preview.rb, line 34
def initialize(env)
  super(env)
  @skip_types=[]
  @default_transfer_spec=nil
  # by default generate all supported formats
  @preview_formats_to_generate=Asperalm::Preview::Generator::PREVIEW_FORMATS.clone
  # options for generation
  @gen_options=Asperalm::Preview::Options.new
  # link CLI options to gen_info attributes
  self.options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
  self.options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
  self.options.set_obj_attr(:skip_types,self,:option_skip_types)
  self.options.set_obj_attr(:previews_folder,self,:option_previews_folder,DEFAULT_PREVIEWS_FOLDER)
  self.options.set_obj_attr(:skip_folders,self,:option_skip_folders,[]) # no skip
  self.options.set_obj_attr(:overwrite,self,:option_overwrite,:mtime)
  self.options.set_obj_attr(:file_access,self,:option_file_access,:local)
  self.options.add_opt_list(:skip_format,Asperalm::Preview::Generator::PREVIEW_FORMATS,'skip this preview format (multiple possible)')
  self.options.add_opt_list(:folder_reset_cache,[:no,:header,:read],'force detection of generated preview by refresh cache')
  self.options.add_opt_simple(:skip_types,'skip types in comma separated list')
  self.options.add_opt_simple(:previews_folder,'preview folder in storage root')
  self.options.add_opt_simple(:temp_folder,'path to temp folder')
  self.options.add_opt_simple(:skip_folders,'list of folder to skip')
  self.options.add_opt_simple(:case,'test case name')
  self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
  self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
  self.options.set_option(:temp_folder,Dir.tmpdir)

  # add other options for generator (and set default values)
  Asperalm::Preview::Options::DESCRIPTIONS.each do |opt|
    self.options.set_obj_attr(opt[:name],@gen_options,opt[:name],opt[:default])
    if opt.has_key?(:values)
      self.options.add_opt_list(opt[:name],opt[:values],opt[:description])
    elsif [:yes,:no].include?(opt[:default])
      self.options.add_opt_boolean(opt[:name],opt[:description])
    else
      self.options.add_opt_simple(opt[:name],opt[:description])
    end
  end

  self.options.parse_options!
  raise 'skip_folder shall be an Array, use @json:[...]' unless @option_skip_folders.is_a?(Array)
  @tmp_folder=File.join(self.options.get_option(:temp_folder,:mandatory),"#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}")
  FileUtils.mkdir_p(@tmp_folder)
  Log.log.debug("tmpdir: #{@tmp_folder}")
end

Public Instance Methods

do_transfer(direction,folder_id,source_filename,destination='/') click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 166
def do_transfer(direction,folder_id,source_filename,destination='/')
  raise "error" if destination.nil? and direction.eql?('receive')
  if @default_transfer_spec.nil?
    # make a dummy call to get some default transfer parameters
    res=@api_node.create('files/upload_setup',{'transfer_requests'=>[{'transfer_request'=>{'paths'=>[{}],'destination_root'=>'/'}}]})
    sample_transfer_spec=res[:data]['transfer_specs'].first['transfer_spec']
    # get ports, anyway that should be 33001 for both. add remote_user ?
    @default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=sample_transfer_spec[e];h}
    # note: we use the same address for ascp than for node api instead of the one from upload_setup
    @default_transfer_spec.merge!({
      'token'            => "Basic #{Base64.strict_encode64("#{@access_key_self['id']}:#{self.options.get_option(:password,:mandatory)}")}",
      'remote_host'      => @transfer_server_address,
      'remote_user'      => Fasp::ACCESS_KEY_TRANSFER_USER
    })
  end
  tspec=@default_transfer_spec.merge({
    'direction'  => direction,
    'paths'      => [{'source'=>source_filename}],
    'tags'       => { 'aspera' => {
    PREV_GEN_TAG   => true,
    'node'         => {
    'access_key'     => @access_key_self['id'],
    'file_id'        => folder_id }}}
  })
  # force destination
  # tspec['destination_root']=destination
  self.transfer.option_transfer_spec_deep_merge({'destination_root'=>destination})
  Main.result_transfer(self.transfer.start(tspec,{:src=>:node_gen4}))
end
entry_preview_folder_name(entry) click to toggle source

defined by node api

# File lib/asperalm/cli/plugins/preview.rb, line 231
def entry_preview_folder_name(entry)
  "#{entry['id']}#{PREVIEW_FOLDER_SUFFIX}"
end
execute_action() click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 348
def execute_action
  command=self.options.get_next_command(ACTIONS)
  unless [:check,:test].include?(command)
    # this will use node api
    @api_node=basic_auth_api
    @transfer_server_address=URI.parse(@api_node.params[:base_url]).host
    # get current access key
    @access_key_self=@api_node.read('access_keys/self')[:data]
    # TODO: check events is activated here:
    # note that docroot is good to look at as well
    node_info=@api_node.read('info')[:data]
    Log.log.debug("root: #{node_info['docroot']}")
    @access_remote=@option_file_access.eql?(:remote)
    Log.log.debug("remote: #{@access_remote}")
    Log.log.debug("access key info: #{@access_key_self}")
    #TODO: can the previews folder parameter be read from node api ?
    @option_skip_folders.push('/'+@option_previews_folder)
    if @access_remote
      # note the filter "name", it's why we take the first one
      @previews_folder_entry=get_folder_entries(@access_key_self['root_file_id'],{:name=>@option_previews_folder}).first
      raise CliError,"Folder #{@option_previews_folder} does not exist on node. Please create it in the storage root, or specify an alternate name." if @previews_folder_entry.nil?
    else
      raise "only local storage allowed in this mode" unless @access_key_self['storage']['type'].eql?('local')
      @local_storage_root=@access_key_self['storage']['path']
      #TODO: option to override @local_storage_root='xxx'
      @local_storage_root=@local_storage_root[LOCAL_STORAGE_PCVL.length..-1] if @local_storage_root.start_with?(LOCAL_STORAGE_PCVL)
      #TODO: windows could have "C:" ?
      raise "not local storage: #{@local_storage_root}" unless @local_storage_root.start_with?('/')
      raise CliError,"Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
      @local_preview_folder=File.join(@local_storage_root,@option_previews_folder)
      raise CliError,"Folder #{@local_preview_folder} does not exist locally. Please create it, or specify an alternate name." unless File.directory?(@local_preview_folder)
      # protection to avoid clash of file id for two different access keys
      marker_file=File.join(@local_preview_folder,AK_MARKER_FILE)
      Log.log.debug("marker file: #{marker_file}")
      if File.exist?(marker_file)
        ak=File.read(marker_file)
        raise "mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}" unless @access_key_self['id'].eql?(ak)
      else
        File.write(marker_file,@access_key_self['id'])
      end
    end
  end
  case command
  when :scan
    scan_folder_files({
      'id'   => @access_key_self['root_file_id'],
      'name' => '/',
      'type' => 'folder',
      'path' => '/' })
    return Main.result_status('scan finished')
  when :events
    iteration_data=[]
    iteration_persistency=nil
    if self.options.get_option(:once_only,:mandatory)
      iteration_persistency=PersistencyFile.new(
      data: iteration_data,
      ids:  ['preview_iteration_events',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
    end
    iteration_data[0]=process_file_events(iteration_data[0])
    iteration_persistency.save unless iteration_persistency.nil?
    return Main.result_status('events finished')
  when :trevents
    iteration_data=[]
    iteration_persistency=nil
    if self.options.get_option(:once_only,:mandatory)
      iteration_persistency=PersistencyFile.new(
      data: iteration_data,
      ids:  ['preview_iteration_transfer',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
    end
    iteration_data[0]=process_transfer_events(iteration_data[0])
    iteration_persistency.save unless iteration_persistency.nil?
    return Main.result_status('trevents finished')
  when :folder
    file_id=self.options.get_next_argument('file id')
    file_info=@api_node.read("files/#{file_id}")[:data]
    scan_folder_files(file_info)
    return Main.result_status('file finished')
  when :check
    Asperalm::Preview::Utils.check_tools(@skip_types)
    return Main.result_status('tools validated')
  when :test
    format = self.options.get_next_argument('format',Asperalm::Preview::Generator::PREVIEW_FORMATS)
    source = self.options.get_next_argument('source file')
    dest=preview_filename(format,self.options.get_option(:case,:optional))
    g=Asperalm::Preview::Generator.new(@gen_options,source,dest,@tmp_folder)
    raise "format not supported: #{format}" unless g.supported?
    g.generate
    return Main.result_status("generated: #{dest}")
  end
ensure
  FileUtils.rm_rf(@tmp_folder)
end
generate_preview(entry) click to toggle source

generate preview files for one folder entry (file) if necessary entry must contain “parent_file_id” if remote.

# File lib/asperalm/cli/plugins/preview.rb, line 241
def generate_preview(entry)
  #Log.log.debug(">>>> #{entry}".red)
  # folder where previews will be generated for this particular entry
  local_entry_preview_dir=String.new
  # prepare generic information
  gen_infos=@preview_formats_to_generate.map do |preview_format|
    {
      :preview_format => preview_format,
      :base_dest      => preview_filename(preview_format)
    }
  end
  # lets gather some infos on possibly existing previews
  # it depends if files access locally or remotely
  if @access_remote
    get_infos_remote(gen_infos,entry,local_entry_preview_dir)
  else # direct local file system access
    get_infos_local(gen_infos,entry,local_entry_preview_dir)
  end
  # here we have the status on preview files
  # let's find if they need generation
  gen_infos.select! do |gen_info|
    # if it exists, what about overwrite policy ?
    if gen_info[:preview_exist]
      case @option_overwrite
      when :always
        # continue: generate
      when :never
        # never overwrite
        next false
      when :mtime
        # skip if preview is newer than original
        next false if gen_info[:preview_newer_than_original]
      end
    end
    # need generator for further checks
    gen_info[:generator]=Asperalm::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'],false)
    # get conversion_type (if known) and check if supported
    next false unless gen_info[:generator].supported?
    # shall we skip it ?
    next false if @skip_types.include?(gen_info[:generator].conversion_type)
    # ok we need to generate
    true
  end
  return if gen_infos.empty?
  # create folder if needed
  FileUtils.mkdir_p(local_entry_preview_dir)
  if @access_remote
    raise 'missing parent_file_id in entry' if entry['parent_file_id'].nil?
    #  download original file to temp folder
    do_transfer('receive',entry['parent_file_id'],entry['name'],@tmp_folder)
  end
  Log.log.info("source: #{entry['id']}: #{entry['path']})")
  gen_infos.each do |gen_info|
    gen_info[:generator].generate rescue nil
  end
  if @access_remote
    # upload
    do_transfer('send',@previews_folder_entry['id'],local_entry_preview_dir)
    # cleanup after upload
    FileUtils.rm_rf(local_entry_preview_dir)
    File.delete(File.join(@tmp_folder,entry['name']))
  end
  # force read file updated previews
  if @option_folder_reset_cache.eql?(:read)
    @api_node.read("files/#{entry['id']}")
  end
rescue => e
  Log.log.error("#{e.message}")
  Log.log.debug(e.backtrace.join("\n").red)
end
get_folder_entries(file_id,request_args=nil) click to toggle source

/files/id/files is normally cached in redis, but we can discard the cache but /files/id is not cached

# File lib/asperalm/cli/plugins/preview.rb, line 105
def get_folder_entries(file_id,request_args=nil)
  headers={'Accept'=>'application/json'}
  headers.merge!({'X-Aspera-Cache-Control'=>'no-cache'}) if @option_folder_reset_cache.eql?(:header)
  return @api_node.call({:operation=>'GET',:subpath=>"files/#{file_id}/files",:headers=>headers,:url_params=>request_args})[:data]
  #return @api_node.read("files/#{file_id}/files",request_args)[:data]
end
get_infos_local(gen_infos,entry,local_entry_preview_dir) click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 196
def get_infos_local(gen_infos,entry,local_entry_preview_dir)
  local_original_filepath=File.join(@local_storage_root,entry['path'])
  original_mtime=File.mtime(local_original_filepath)
  # out
  local_entry_preview_dir.replace(File.join(@local_preview_folder, entry_preview_folder_name(entry)))
  gen_infos.each do |gen_info|
    gen_info[:src]=local_original_filepath
    gen_info[:dst]=File.join(local_entry_preview_dir, gen_info[:base_dest])
    gen_info[:preview_exist]=File.exist?(gen_info[:dst])
    gen_info[:preview_newer_than_original] = (gen_info[:preview_exist] and (File.mtime(gen_info[:dst])>original_mtime))
  end
end
get_infos_remote(gen_infos,entry,local_entry_preview_dir) click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 209
def get_infos_remote(gen_infos,entry,local_entry_preview_dir)
  #Log.log.debug(">>>> get_infos_remote #{entry}".red)
  # store source directly here
  local_original_filepath=File.join(@tmp_folder,entry['name'])
  #original_mtime=DateTime.parse(entry['modified_time'])
  # out: where previews are generated
  local_entry_preview_dir.replace(File.join(@tmp_folder,entry_preview_folder_name(entry)))
  file_info=@api_node.read("files/#{entry['id']}")[:data]
  #TODO: this does not work because previews is hidden in api (gen4)
  #this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{:name=>@entry_preview_folder_name})
  # TODO: use gen3 api to list files and get date
  gen_infos.each do |gen_info|
    gen_info[:src]=local_original_filepath
    gen_info[:dst]=File.join(local_entry_preview_dir, gen_info[:base_dest])
    # TODO: use this_preview_folder_entries (but it's hidden)
    gen_info[:preview_exist]=file_info.has_key?('preview')
    # TODO: get change time and compare, useful ?
    gen_info[:preview_newer_than_original] = gen_info[:preview_exist]
  end
end
option_skip_format() click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 97
def option_skip_format
  return @preview_formats_to_generate.map{|i|i.to_s}.join(',')
end
option_skip_format=(value) click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 93
def option_skip_format=(value)
  @preview_formats_to_generate.delete(value)
end
option_skip_types() click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 89
def option_skip_types
  return @skip_types.map{|i|i.to_s}.join(',')
end
option_skip_types=(value) click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 80
def option_skip_types=(value)
  @skip_types=[]
  value.split(',').each do |v|
    s=v.to_sym
    raise "not supported: #{v}" unless Asperalm::Preview::FileTypes::CONVERSION_TYPES.include?(s)
    @skip_types.push(s)
  end
end
preview_filename(preview_format,filename=PREVIEW_BASENAME) click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 235
def preview_filename(preview_format,filename=PREVIEW_BASENAME)
  return "#{filename}.#{preview_format.to_s}"
end
process_file_events(iteration_token) click to toggle source

requests recent events on node api and process newly modified folders

# File lib/asperalm/cli/plugins/preview.rb, line 138
def process_file_events(iteration_token)
  # get new file creation by access key (TODO: what if file already existed?)
  events_filter={
    'access_key'=>@access_key_self['id'],
    'type'=>'file.*'
  }
  # and optionally by iteration token
  events_filter['iteration_token']=iteration_token unless iteration_token.nil?
  events=@api_node.read('events',events_filter)[:data]
  return if events.empty?
  events.each do |event|
    # process only files
    next unless event.dig('data','type').eql?('file')
    file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
    next if file_entry.nil?
    next unless @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
    file_entry['parent_file_id']=event['data']['parent_file_id']
    if event['types'].include?('file.deleted')
      Log.log.error('TODO'.red)
    end
    if event['types'].include?('file.deleted')
      generate_preview(file_entry)
    end
  end
  # write new iteration file
  return events.last['id'].to_s
end
process_transfer_events(iteration_token) click to toggle source

old version based on folders

# File lib/asperalm/cli/plugins/preview.rb, line 113
def process_transfer_events(iteration_token)
  events_filter={
    'access_key'=>@access_key_self['id'],
    'type'=>'download.ended'
  }
  # optionally by iteration token
  events_filter['iteration_token']=iteration_token unless iteration_token.nil?
  events=@api_node.read('events',events_filter)[:data]
  return if events.empty?
  events.each do |event|
    next unless event['data']['direction'].eql?('receive')
    next unless event['data']['status'].eql?('completed')
    next unless event['data']['error_code'].eql?(0)
    next unless event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
    folder_id=event.dig('data','tags','aspera','node','file_id')
    folder_id||=event.dig('data','file_id')
    next if folder_id.nil?
    folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil
    next if folder_entry.nil?
    scan_folder_files(folder_entry)
  end
  return events.last['id'].to_s
end
scan_folder_files(top_entry) click to toggle source

scan all files in provided folder entry

# File lib/asperalm/cli/plugins/preview.rb, line 313
def scan_folder_files(top_entry)
  Log.log.debug("scan: #{top_entry}")
  # don't use recursive call, use list instead
  items_to_process=[top_entry]
  while !items_to_process.empty?
    entry=items_to_process.shift
    Log.log.debug("item:#{entry}")
    case entry['type']
    when 'file'
      generate_preview(entry)
    when 'link'
      Log.log.debug('Ignoring link.')
    when 'folder'
      if @option_skip_folders.include?(entry['path'])
        Log.log.debug("#{entry['path']} folder (skip)".bg_red)
      else
        Log.log.debug("#{entry['path']} folder")
        # get folder content
        folder_entries=get_folder_entries(entry['id'])
        # process all items in current folder
        folder_entries.each do |folder_entry|
          # add path for older versions of ES
          if !folder_entry.has_key?('path')
            folder_entry['path']=(entry['path'].eql?('/')?'':entry['path'])+'/'+folder_entry['name']
          end
          folder_entry['parent_file_id']=entry['id']
          items_to_process.push(folder_entry)
        end
      end
    else
      Log.log.warn("unknown entry type: #{entry['type']}")
    end
  end
end