class Envoi::Mam::Cantemo::Agent
Constants
- DEFAULT_DESTINATION_PATH
- DEFAULT_PRESERVE_FILE_PATH
- DEFAULT_SHAPE_TAG
- WATCH_FOLDER_MANAGER_CLASS
Attributes
default_aspera_ascp_args[RW]
default_aspera_ascp_path[RW]
default_preserve_file_path[RW]
default_vidispine_shape_tag[RW]
Public Instance Methods
after_initialize()
click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 22 def after_initialize args = initial_args @default_aspera_ascp_path = args[:default_aspera_ascp_path] @default_aspera_args = args.fetch(:default_ascp_args, Envoi::Mam::Agent::TransferClient::Aspera::DEFAULT_ASCP_ARGS) @default_preserve_file_path = args.fetch(:default_preserve_file_path, DEFAULT_PRESERVE_FILE_PATH) end
agent_config()
click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 33 def agent_config @agent_config ||= config['cantemo'] || config || {} end
agent_config_storages()
click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 37 def agent_config_storages agent_config['storages'] end
download(args = {})
click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 66 def download(args = {}) item_id = args[:item_id] shape_id = args[:shape_id] unless shape_id && !shape_id.empty? shape_tag = args[:shape_tag] || default_vidispine_shape_tag shape_id = item_get_shape_by_tag(item_id, shape_tag) end logger.info { "Getting file path for Item ID: #{item_id} Shape ID: #{shape_id}" } item_shape_get_response = api_client.item_shape_get(:item_id => item_id, :shape_id => shape_id) files = item_shape_get_response['containerComponent']['file'] logger.debug { "Files: #{files}" } # file = files.first files = [files.first] # just do the first file for now files.each do |file| begin download_file(args, file) rescue => e logger.warn { "Exception: #{$!}" } end end logger.info { 'DONE' } end
download_file(args, file)
click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 93 def download_file(args, file) logger.debug { "File: #{file}" } transfer_type = args[:transfer_type] file_storage_id = file['storage'] file_path = file['path'] file_storage_config = agent_config_storages[file_storage_id] unless file_storage_config && !file_storage_config.empty? raise Exception, "No configuration found for storage '#{file_storage_id}'" end logger.info { "Transferring File Path: '#{file_path}'" } preserve_path = args.fetch(:preserve_path, file_storage_config.fetch('preserve_path', default_preserve_file_path)) destination_path = args[:destination_path] || file_storage_config['destination_path'] || DEFAULT_DESTINATION_PATH relative_path = preserve_path ? File.dirname(file_path) : nil relative_path = nil if relative_path == '.' target_path = relative_path ? File.join(destination_path, relative_path) : destination_path target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/') aspera_config = file_storage_config['aspera'] if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?) client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self) return client.download(aspera_config, file_path, target_path) # download_using_aspera(aspera_config, file_path, target_path) end s3_config = file_storage_config['s3'] if (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?) target_path = File.expand_path(target_path) if target_path == '.' target_path = File.join(target_path, File.basename(file_path)) client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self) return client.download(s3_config, file_path, target_path) end logger.warn { "No Supported TransferClient Configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}' " : ''}Found in Storage Configuration." } end
dry_run?()
click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 29 def dry_run?; @dry_run end
import_file(args = {})
click to toggle source
@param [Hash] args @option args [String] :file_path @option args [Hash] :import_args ({}) @option args [Hash] :import_options ({}) @option args [String] :item_id @option args [Hash] :response_object ({}) @option args [String] :shape_tag (@default_vidispine_shape_tag) @option args [String] :storage_id @option args [String] :target_path
@return [Hash{ :success->Boolean, :file_create_response->{}, :import_response->{} }]
# File lib/envoi/mam/cantemo/agent.rb, line 256 def import_file(args = {}) logger.debug { "Agent - Import File - Args: #{args}"} _response = args[:response_object] || {} file_path = args[:file_path] storage_id = args[:storage_id] target_path = args[:target_path] item_id = args[:item_id] shape_tag = args[:shape_tag] || default_vidispine_shape_tag item_add_args_in = args[:item_add_args] || { } item_add_options_in = args[:item_add_options] || { } # attach file to item as shape path_on_storage = File.join(target_path, File.basename(file_path)) path_on_storage = path_on_storage[1..-1] if path_on_storage.start_with?('/') # file_create_response = api_client.storage_file_create(storage_id: storage_id, # path: path_on_storage, state: 'CLOSED') file_create_response = api_client.storage_file_get_or_create(storage_id, path_on_storage, { :extended_response => true }) _response[:file_create_response] = file_create_response file = file_create_response[:file] file_id = file['id'] unless file_id _file = file.dup _file.keep_if { |k, v| v } logger.error { "Failed to create file. #{_file}" } _response[:success] = false return _response end item = (file['item'] || []).first if item shape = (item['shape'] || []).first msg = "File already exist and is associated to item #{item['id']} as shape #{shape['id']}." logger.warn { "#{msg} #{file}" } _response[:error] = { :message => msg } _response[:success] = false return _response end if item_id item_shape_import_response = api_client.item_shape_import(item_id: item_id, tag: shape_tag, file_id: file_id) else item_add_args = { storage_id: storage_id, file_path: path_on_storage, file_id: file_id, storage_path_map: { '/' => storage_id } } item_add_args.merge!(item_add_args_in) if item_add_args_in.is_a?(Hash) item_add_options = { } item_add_options.merge!(item_add_options_in) if item_add_options_in.is_a?(Hash) logger.debug { "Executing Item Add/Import. ARGS: #{item_add_args} OPTS: #{item_add_options}" } item_shape_import_response = api_client.item_add_using_file_path(item_add_args, item_add_options) end _response[:import_response] = item_shape_import_response _response end
ingest_file()
click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 324 def ingest_file # 1) export ASPERA_SCP_TOKEN="ATB3_AEA9dUI5dd2u1k8XBMVD0qInR-Lyylkz8AylTXLVt5_SMVGR8SO7Sdc0lj6RkoHK_DvAAEWac8bllF_Yan1NbbDTyPj_3BTA" # 2) ascp -i asperaweb_id_dsa.openssh -P 33001 -l 20m /Users/nicholasstokes/Desktop/CantemoAgent.mov xfer@ats-aws-us-west-2.aspera.io: # # single line equivalent # ascp -i asperaweb_id_dsa.openssh -W 'ATB3_AEAiBdrf-7sMQu782sPoQ4Q7Yh4LadgvYK9BP4Kt1hcVlZGR8SO7Sdc0lj6RkoHK_DvAAG5XlBBIZGEINicYOxMRE7g_3BTA' -P 33001 -l 20m /Users/nicholasstokes/Desktop/CantemoAgent.mov xfer@ats-aws-us-west-2.aspera.io: # Aspera: # `GET /aspera/ping/` - to check if Aspera upload is available # `POST /vs/item/new/` to create a placeholder to upload to # `POST /aspera/upload_setup/` - Aspera token # ... upload the file with Aspera ... # `POST /API/v1/placeholder/VX-3012/associatefile/?filepath=Screen%20Shot%202018-08-07%20at%2013.06.58.png&storageid=VX-11&platform=mac` - associate new file to placeholder # HTTP upload: # `POST /vs/item/new/` - create a placeholder # then streamind POST (?) requests to `VSAPI/item/VX-3014/shape/essence/raw?transferId=2D30030A-6EC3-4364-9598-5BF20972A218&throttle=false&filename=Screen%20Shot%202018-08-07%20at%2014.34.15.png&jobmetadata=portal_groups:StringArray%3dAdmin` to upload the data. # that end-point for HTTP is documented in http://apidoc.vidispine.com/latest/ref/item/shape.html#import-a-shape-using-the-request-body # end
initialize_api_client(args = {})
click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 41 def initialize_api_client(args = {}) api_config = agent_config @api_client = args[:api_client] || begin client_args = Hash[api_config.map { |k,v| [ k.respond_to?(:to_sym) ? k.to_sym.downcase : k, v ]}] client_args[:logger] = logger _client = ::Cantemo::Portal::API::Client.new(client_args) begin _client.version rescue => e raise "Error connecting to Portal: #{e.message}" end _client end @default_vidispine_shape_tag = args[:default_shape_tag] || api_config['default_shape_tag'] || api_config['shape_tag'] || DEFAULT_SHAPE_TAG end
item_get_shape_by_tag(item_id, shape_tag)
click to toggle source
# File lib/envoi/mam/cantemo/agent.rb, line 61 def item_get_shape_by_tag(item_id, shape_tag) item_shapes_get_response = api_client.item_shapes_get(:item_id => item_id, :tag => shape_tag) shape_id = item_shapes_get_response['uri'].first end
upload(args = {})
click to toggle source
@param [Hash] args @option args [String] :file_path @option args [String] :storage_id @option args [String] :transfer_type ('') @option args [Boolean] :import_file (true) @option args [Boolean] :preserve_path (storage_config || default_preserve_path)
@return [Hash]
# File lib/envoi/mam/cantemo/agent.rb, line 142 def upload(args = {}) _response = {} file_path = args[:file_path] raise ArgumentError, "Path not found: '#{file_path}'" unless File.exists?(file_path) if File.directory?(file_path) # Non-recursive directory upload file_paths = Dir.glob(File.join(file_path, '*.*')) logger.debug { "File Paths: #{file_paths}" } file_paths.map { |fp| upload(args.merge(file_path: fp)) } return file_paths end logger.debug { "Preparing to upload '#{file_path}'" } transfer_type = args[:transfer_type] || '' storage_id = args[:storage_id] storage_config = agent_config_storages[storage_id] unless storage_config && !storage_config.empty? raise "No configuration found for storage '#{storage_id}'" end should_import_file = args.fetch(:import_file, storage_config.fetch('import', true)) should_preserve_path = args.fetch(:preserve_path, storage_config.fetch('preserve_path', default_preserve_file_path)) destination_path = args[:destination_path] || storage_config['destination_path'] || '/' relative_path = should_preserve_path ? File.dirname(file_path) : nil relative_path = File.expand_path(relative_path) if relative_path == '.' target_path = relative_path ? File.join(destination_path, relative_path) : destination_path target_path = target_path[0..-1] if target_path.start_with?('/') && !destination_path.start_with?('/') # upload file transfer_response = begin response = nil aspera_config = storage_config['aspera'] begin if (transfer_type.empty? || transfer_type == :aspera) && (aspera_config && !aspera_config.empty?) client = Envoi::Mam::Agent::TransferClient::Aspera.new(agent: self) response = client.upload(aspera_config, file_path, target_path) end rescue => e logger.error { "Aspera Transfer Failed. '#{e.message}'\n#{e.backtrace.first}" } end s3_config = storage_config['s3'] begin if !response && (transfer_type.empty? || transfer_type == :s3) && (s3_config && !s3_config.empty?) _target_path = target_path _target_path = File.expand_path(_target_path) if target_path == '.' _target_path = File.join(_target_path, File.basename(file_path)) client = Envoi::Mam::Agent::TransferClient::S3.new(agent: self) response = client.upload(s3_config, file_path, _target_path) end rescue => e logger.error { "S3 Transfer Failed. '#{e.message}'" } end response end transfer_response = { success: transfer_response } if transfer_response == true || transfer_response == false logger.debug { "Transfer Response: #{transfer_response}" } _response[:transfer_response] = transfer_response if transfer_response.nil? logger.warn { "No supported TransferClient configuration#{transfer_type && !transfer_type.empty? ? " for transfer type '#{transfer_type}'" : ''} found in storage configuration." } _response[:success] = false return _response end unless transfer_response[:success] logger.error { "Error transferring file." } _response[:success] = false return _response end unless should_import_file _response[:success] = transfer_response[:success] return _response end ### IMPORT - START import_file_args = args.dup import_file_args[:response_object] = _response import_file_args[:target_path] = target_path import_file(import_file_args) _response[:success] = true unless _response[:success] === false ### IMPORT - END _response rescue => e logger.error { "Exception: #{e.message}" } _response[:exception] = e return _response end