class Envoi::Mam::Cantemo::Agent

Constants

DEFAULT_DESTINATION_PATH
DEFAULT_PRESERVE_FILE_PATH
DEFAULT_SHAPE_TAG
WATCH_FOLDER_MANAGER_CLASS

Attributes

default_aspera_ascp_args[RW]
default_aspera_ascp_path[RW]
default_preserve_file_path[RW]
default_vidispine_shape_tag[RW]

Public Instance Methods

after_initialize() click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 22
def after_initialize
  args                        = initial_args
  @default_aspera_ascp_path   = args[:default_aspera_ascp_path]
  @default_aspera_args        = args.fetch(:default_ascp_args, Envoi::Mam::Agent::TransferClient::Aspera::DEFAULT_ASCP_ARGS)
  @default_preserve_file_path = args.fetch(:default_preserve_file_path, DEFAULT_PRESERVE_FILE_PATH)
end
agent_config() click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 33
def agent_config
  @agent_config ||= config['cantemo'] || config || {}
end
agent_config_storages() click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 37
def agent_config_storages
  agent_config['storages']
end
download(args = {}) click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 66
def download(args = {})

  item_id  = args[:item_id]
  shape_id = args[:shape_id]
  unless shape_id && !shape_id.empty?
    shape_tag = args[:shape_tag] || default_vidispine_shape_tag
    shape_id  = item_get_shape_by_tag(item_id, shape_tag)
  end

  logger.info { "Getting file path for Item ID: #{item_id} Shape ID: #{shape_id}" }
  item_shape_get_response = api_client.item_shape_get(:item_id => item_id, :shape_id => shape_id)

  files = item_shape_get_response['containerComponent']['file']
  logger.debug { "Files: #{files}" }

  # file = files.first
  files = [files.first] # just do the first file for now
  files.each do |file|
    begin
      download_file(args, file)
    rescue => e
      logger.warn { "Exception: #{$!}" }
    end
  end
  logger.info { 'DONE' }
end
download_file(args, file) click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 93
def download_file(args, file)
  logger.debug { "File: #{file}" }
  transfer_type = args[:transfer_type]

  file_storage_id = file['storage']
  file_path       = file['path']

  file_storage_config = agent_config_storages[file_storage_id]

  unless file_storage_config && !file_storage_config.empty?
    raise Exception, "No configuration found for storage '#{file_storage_id}'"
  end

  logger.info { "Transferring File Path: '#{file_path}'" }
  preserve_path = args.fetch(:preserve_path, file_storage_config.fetch('preserve_path', default_preserve_file_path))

  destination_path = args[:destination_path] || file_storage_config['destination_path'] || DEFAULT_DESTINATION_PATH
  relative_path    = preserve_path ? File.dirname(file_path) : nil
  relative_path    = nil if relative_path == '.'

  target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
  target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')

  aspera_config = file_storage_config['aspera']
  if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
    client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
    return client.download(aspera_config, file_path, target_path)
    # download_using_aspera(aspera_config, file_path, target_path)
  end

  s3_config = file_storage_config['s3']
  if (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
    target_path = File.expand_path(target_path) if target_path == '.'
    target_path = File.join(target_path, File.basename(file_path))
    client      = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
    return client.download(s3_config, file_path, target_path)
  end

  logger.warn { "No Supported TransferClient Configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}Found in Storage Configuration." }
end
dry_run?() click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 29
def dry_run?;
  @dry_run
end
import_file(args = {}) click to toggle source

@param [Hash] args @option args [String] :file_path @option args [Hash] :import_args ({}) @option args [Hash] :import_options ({}) @option args [String] :item_id @option args [Hash] :response_object ({}) @option args [String] :shape_tag (@default_vidispine_shape_tag) @option args [String] :storage_id @option args [String] :target_path

@return [Hash{ :success->Boolean, :file_create_response->{}, :import_response->{} }]

# File lib/envoi/mam/cantemo/agent.rb, line 256
def import_file(args = {})
  logger.debug { "Agent - Import File - Args: #{args}"}
  _response = args[:response_object] || {}

  file_path  = args[:file_path]
  storage_id = args[:storage_id]

  target_path = args[:target_path]

  item_id   = args[:item_id]
  shape_tag = args[:shape_tag] || default_vidispine_shape_tag

  item_add_args_in = args[:item_add_args] || { }
  item_add_options_in = args[:item_add_options] || { }

  # attach file to item as shape
  path_on_storage = File.join(target_path, File.basename(file_path))
  path_on_storage = path_on_storage[1..-1] if path_on_storage.start_with?('/')
  # file_create_response = api_client.storage_file_create(storage_id: storage_id,
  #                                                       path: path_on_storage, state: 'CLOSED')
  file_create_response             = api_client.storage_file_get_or_create(storage_id, path_on_storage, { :extended_response => true })
  _response[:file_create_response] = file_create_response
  file                             = file_create_response[:file]

  file_id = file['id']

  unless file_id
    _file = file.dup
    _file.keep_if { |k, v| v }
    logger.error { "Failed to create file. #{_file}" }
    _response[:success] = false
    return _response
  end

  item = (file['item'] || []).first

  if item
    shape = (item['shape'] || []).first
    msg   = "File already exist and is associated to item #{item['id']} as shape #{shape['id']}."
    logger.warn { "#{msg} #{file}" }
    _response[:error]   = { :message => msg }
    _response[:success] = false
    return _response
  end

  if item_id
    item_shape_import_response = api_client.item_shape_import(item_id: item_id,
                                                              tag:     shape_tag, file_id: file_id)
  else
    item_add_args = {
      storage_id:       storage_id,
      file_path:        path_on_storage,
      file_id:           file_id,
      storage_path_map: { '/' => storage_id }
    }
    item_add_args.merge!(item_add_args_in) if item_add_args_in.is_a?(Hash)

    item_add_options = { }
    item_add_options.merge!(item_add_options_in) if item_add_options_in.is_a?(Hash)

    logger.debug { "Executing Item Add/Import. ARGS: #{item_add_args} OPTS: #{item_add_options}" }
    item_shape_import_response = api_client.item_add_using_file_path(item_add_args, item_add_options)
  end
  _response[:import_response] = item_shape_import_response

  _response
end
ingest_file() click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 324
def ingest_file
  # 1) export ASPERA_SCP_TOKEN="ATB3_AEA9dUI5dd2u1k8XBMVD0qInR-Lyylkz8AylTXLVt5_SMVGR8SO7Sdc0lj6RkoHK_DvAAEWac8bllF_Yan1NbbDTyPj_3BTA"
  # 2) ascp -i asperaweb_id_dsa.openssh -P 33001 -l 20m /Users/nicholasstokes/Desktop/CantemoAgent.mov xfer@ats-aws-us-west-2.aspera.io:
  #
  # single line equivalent
  # ascp -i asperaweb_id_dsa.openssh -W 'ATB3_AEAiBdrf-7sMQu782sPoQ4Q7Yh4LadgvYK9BP4Kt1hcVlZGR8SO7Sdc0lj6RkoHK_DvAAG5XlBBIZGEINicYOxMRE7g_3BTA' -P 33001 -l 20m /Users/nicholasstokes/Desktop/CantemoAgent.mov xfer@ats-aws-us-west-2.aspera.io:

  # Aspera:
  # `GET /aspera/ping/` - to check if Aspera upload is available
  # `POST /vs/item/new/` to create a placeholder to upload to
  # `POST /aspera/upload_setup/` - Aspera token
  # ... upload the file with Aspera ...
  # `POST /API/v1/placeholder/VX-3012/associatefile/?filepath=Screen%20Shot%202018-08-07%20at%2013.06.58.png&storageid=VX-11&platform=mac`  - associate new file to placeholder

  # HTTP upload:
  # `POST /vs/item/new/` - create a placeholder
  # then streamind POST (?) requests to `VSAPI/item/VX-3014/shape/essence/raw?transferId=2D30030A-6EC3-4364-9598-5BF20972A218&throttle=false&filename=Screen%20Shot%202018-08-07%20at%2014.34.15.png&jobmetadata=portal_groups:StringArray%3dAdmin`  to upload the data.
  # that end-point for HTTP is documented in http://apidoc.vidispine.com/latest/ref/item/shape.html#import-a-shape-using-the-request-body
  #
end
initialize_api_client(args = {}) click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 41
def initialize_api_client(args = {})
  api_config  = agent_config
  @api_client = args[:api_client] || begin
    client_args = Hash[api_config.map { |k,v| [ k.respond_to?(:to_sym) ? k.to_sym.downcase : k, v ]}]
    client_args[:logger] = logger
    _client = ::Cantemo::Portal::API::Client.new(client_args)

    begin
      _client.version
    rescue => e
      raise "Error connecting to Portal: #{e.message}"
    end

    _client
  end

  @default_vidispine_shape_tag = args[:default_shape_tag] || api_config['default_shape_tag'] || api_config['shape_tag'] || DEFAULT_SHAPE_TAG

end
item_get_shape_by_tag(item_id, shape_tag) click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 61
def item_get_shape_by_tag(item_id, shape_tag)
  item_shapes_get_response = api_client.item_shapes_get(:item_id => item_id, :tag => shape_tag)
  shape_id                 = item_shapes_get_response['uri'].first
end
upload(args = {}) click to toggle source

@param [Hash] args @option args [String] :file_path @option args [String] :storage_id @option args [String] :transfer_type ('') @option args [Boolean] :import_file (true) @option args [Boolean] :preserve_path (storage_config || default_preserve_path)

@return [Hash]

# File lib/envoi/mam/cantemo/agent.rb, line 142
def upload(args = {})
  _response = {}

  file_path = args[:file_path]
  raise ArgumentError, "Path not found: '#{file_path}'" unless File.exists?(file_path)

  if File.directory?(file_path)
    # Non-recursive directory upload
    file_paths = Dir.glob(File.join(file_path, '*.*'))
    logger.debug { "File Paths: #{file_paths}" }
    file_paths.map { |fp| upload(args.merge(file_path: fp)) }
    return file_paths
  end
  logger.debug { "Preparing to upload '#{file_path}'" }

  transfer_type            = args[:transfer_type] || ''
  storage_id               = args[:storage_id]
  storage_config = agent_config_storages[storage_id]

  unless storage_config && !storage_config.empty?
    raise "No configuration found for storage '#{storage_id}'"
  end

  should_import_file = args.fetch(:import_file, storage_config.fetch('import', true))

  should_preserve_path = args.fetch(:preserve_path,
                                    storage_config.fetch('preserve_path', default_preserve_file_path))

  destination_path = args[:destination_path] || storage_config['destination_path'] || '/'
  relative_path    = should_preserve_path ? File.dirname(file_path) : nil
  relative_path    = File.expand_path(relative_path) if relative_path == '.'

  target_path = relative_path ? File.join(destination_path, relative_path) : destination_path
  target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/')


  # upload file

  transfer_response = begin
    response      = nil
    aspera_config = storage_config['aspera']
    begin
      if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?)
        client   = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self)
        response = client.upload(aspera_config, file_path, target_path)
      end
    rescue => e
      logger.error { "Aspera Transfer Failed. '#{e.message}'\n#{e.backtrace.first}" }
    end

    s3_config = storage_config['s3']
    begin
      if !response && (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?)
        _target_path = target_path
        _target_path = File.expand_path(_target_path) if target_path == '.'
        _target_path = File.join(_target_path, File.basename(file_path))
        client       = Envoi::Mam::Agent::TransferClient::S3.new(agent: self)
        response     = client.upload(s3_config, file_path, _target_path)
      end
    rescue => e
      logger.error { "S3 Transfer Failed. '#{e.message}'" }
    end

    response
  end
  transfer_response = { success: transfer_response } if transfer_response == true || transfer_response == false

  logger.debug { "Transfer Response: #{transfer_response}" }
  _response[:transfer_response] = transfer_response

  if transfer_response.nil?
    logger.warn { "No supported TransferClient configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}'" : ''} found in storage configuration." }
    _response[:success] = false
    return _response
  end

  unless transfer_response[:success]
    logger.error { "Error transferring file." }
    _response[:success] = false
    return _response
  end

  unless should_import_file
    _response[:success] = transfer_response[:success]
    return _response
  end


  ### IMPORT - START
  import_file_args                   = args.dup
  import_file_args[:response_object] = _response
  import_file_args[:target_path]     = target_path
  import_file(import_file_args)
  _response[:success] = true unless _response[:success] === false
  ### IMPORT - END

  _response
rescue => e
  logger.error { "Exception: #{e.message}" }
  _response[:exception] = e
  return _response
end