class Libis::Ingester::ManifestationBuilder
Protected Instance Methods
process(item)
click to toggle source
noinspection RubyResolve
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 34 def process(item) item.status_progress(self.namepath, 0, item.get_run.ingest_model.manifestations.count) # Build all manifestations item.get_run.ingest_model.manifestations.each do |manifestation| debug 'Building manifestation %s', item, manifestation.representation_info.name rep = item.representation(manifestation.name) unless rep rep = Libis::Ingester::Representation.new rep.representation_info = manifestation.representation_info rep.access_right = manifestation.access_right rep.name = manifestation.name rep.label = manifestation.label rep.parent = item end set_status(rep, :STARTED) build_manifestation(rep, manifestation) if rep.items.size == 0 if manifestation.optional warn "Manifestation %s '%s' is marked optional and no items were found. Representation will not be created.", item, manifestation.name, manifestation.label set_status(rep, :DONE) rep.destroy! else error "Representation %s is empty.", item, rep.name set_status(rep, :FAILED) raise Libis::WorkflowError, 'Could not find content for representation %s.' % [rep.name] end else set_status(rep, :DONE) end item.status_progress(self.namepath) end stop_processing_subitems end
Private Instance Methods
add_file_to_registry(name)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 364 def add_file_to_registry(name) @file_registry ||= {} return @file_registry[name] if @file_registry.has_key?(name) @file_registry[name] = @file_registry.count + 1 end
assemble(items, representation, formats, name, convert_id) { |source_files, new_file| ... }
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 204 def assemble(items, representation, formats, name, convert_id) return if representation.get_items.where(:'properties.convert_info' => convert_id).count > 0 source_files = items.to_a.map do |item| # Collect all files from the list of items case item when Libis::Ingester::FileItem item when Libis::Ingester::Division item.all_files else nil end end.select do |file| match_file(file, formats) end return if source_files.empty? new_file = File.join( representation.get_run.work_dir, representation.parent.id.to_s, name ) FileUtils.mkpath(File.dirname(new_file)) debug 'Building %s for %s from %d source files', representation, new_file, representation.name, source_files.count yield source_files, new_file FileUtils.chmod('a+rw', new_file) assembly = Libis::Ingester::FileItem.new assembly.filename = new_file assembly.parent = representation assembly.properties['convert_info'] = convert_id format_identifier(assembly) register_file(assembly) assembly.save! assembly end
assemble_images(items, representation, convert_hash)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 165 def assemble_images(items, representation, convert_hash) target_format = convert_hash[:target_format].to_sym assemble( items, representation, convert_hash[:source_formats], "#{representation.parent.name}_#{convert_hash[:name]}." + "#{Libis::Format::TypeDatabase.type_extentions(target_format).first}", convert_hash[:id] ) do |sources, new_file| options = nil options = convert_hash[:options] if convert_hash[:options] && convert_hash[:options].is_a?(Hash) options = convert_hash[:options].first if convert_hash[:options] && convert_hash[:options].is_a?(Array) sources = options ? sources.map do |source| source_file = source.fullpath source_format = Libis::Format::TypeDatabase.mime_types(source.properties[:mimetype]).first convert_file(source_file, nil, source_format, source_format, options)[0] end : sources.map {|source| source.fullpath} Libis::Format::Converter::ImageConverter.new.assemble_and_convert(sources, new_file, target_format) sources.each {|f| FileUtils.rm_f f} if options options = convert_hash[:options][1] rescue nil convert_file(new_file, new_file, target_format, target_format, options) if options end end
assemble_pdf(items, representation, convert_hash)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 190 def assemble_pdf(items, representation, convert_hash) file_name = "#{convert_hash[:generated_file] ? Kernel.eval(convert_hash[:generated_file]) : "#{representation.parent.name}_#{convert_hash[:name]}" }.pdf" assemble(items, representation, [:PDF], file_name, convert_hash[:id]) do |sources, new_file| source_files = sources.map {|file| file.fullpath} Libis::Format::Tool::PdfMerge.run(source_files, new_file) unless convert_hash[:options].blank? convert_file(new_file, new_file, :PDF, :PDF, convert_hash[:options]) end end end
build_manifestation(representation, manifestation)
click to toggle source
noinspection RubyResolve
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 76 def build_manifestation(representation, manifestation) # special case: no conversion info means to move the original files. This is typically the preservation master. if manifestation.convert_infos.empty? representation.parent.originals.each do |original| move_file(original, representation) end return end # Perform each conversion manifestation.convert_infos.each do |convert_info| debug 'Processing convert_info [%s]', representation, convert_info.to_hash.to_s # Get the source files # - either from the given representation # - or the originals from_manifestation = convert_info.from_manifestation && manifestation.ingest_model.manifestations.find_by(name: convert_info.from_manifestation) source_rep = from_manifestation && representation.parent.representation(from_manifestation.name) # noinspection ConvertOneChainedExprToSafeNavigation source_items = source_rep&.get_items || representation.parent.originals convert_hash = convert_info.to_hash convert_hash[:name] = representation.name convert_hash[:id] = convert_info.id # Check if a generator is given case convert_info.generator when 'assemble_images' # The image assembly generator assemble_images source_items, representation, convert_hash when 'assemble_pdf' # The PDF assembly generator assemble_pdf source_items, representation, convert_hash when 'thumbnail' generate_thumbnail source_items, representation, convert_hash when 'from_ie' generate_from_ie(representation, convert_hash) else # No generator - convert each source file according to the specifications representation.status_progress(self.namepath, 0, source_items.count) source_items.each do |item| convert item, representation, convert_hash if convert_hash[:source_files].blank? || Regexp.new(convert_hash[:source_files]).match(item.filename) representation.status_progress(self.namepath) end end end end
convert(item, new_parent, convert_hash)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 246 def convert(item, new_parent, convert_hash) case item when Libis::Ingester::Division div = item unless convert_hash[:replace] div = item.dup div.parent = new_parent div.save! div end item.get_items.each {|child| convert(child, div, convert_hash)} when Libis::Ingester::FileItem begin return if new_parent.get_items.where(:'properties.converted_from' => item.id).count > 0 mimetype = item.properties['mimetype'] raise Libis::WorkflowError, 'File item %s format not identified.' % [item] unless mimetype type_id = Libis::Format::TypeDatabase.mime_types(mimetype).first unless convert_hash[:source_formats].blank? unless type_id warn 'Ignoring file item (%s) with unsupported file format (%s) in format conversion.', item, mimetype return end group = Libis::Format::TypeDatabase.type_group(type_id) check_list = [type_id, group].compact.map {|v| [v.to_s, v.to_sym]}.flatten if (convert_hash[:source_formats] & check_list).empty? debug 'File item format (%s) does not match conversion criteria (%s)', item, check_list.to_s, convert_hash[:source_formats].to_s return end end (convert_hash[:properties] || {}).each do |key, value| return nil unless item.properties[key] == value end if convert_hash[:target_format].blank? return copy_file(item, new_parent) if convert_hash[:copy_file] return move_file(item, new_parent) end src_file = item.fullpath new_file = File.join( item.get_run.work_dir, new_parent.id.to_s, item.id.to_s, [item.name, convert_hash[:name], extname(convert_hash[:target_format]) ].join('.') ) raise Libis::WorkflowError, 'File item %s format (%s) is not supported.' % [item, mimetype] unless type_id new_file_name, converter = convert_file(src_file, new_file, type_id, convert_hash[:target_format].to_sym, convert_hash[:options]) new_file = new_file_name return nil unless new_file FileUtils.chmod('a+rw', new_file) new_item = Libis::Ingester::FileItem.new new_item.name = item.name new_item.label = item.label new_item.parent = new_parent register_file(new_item) new_item.options = item.options new_item.properties['converter'] = converter new_item.properties['converted_from'] = item.id new_item.properties['convert_info'] = convert_hash[:id] new_item.filename = new_file format_identifier(new_item) new_item.properties['original_path'] = File.join( File.dirname(item.properties['original_path']), "%s.%s.%s" % [File.basename(item.properties['original_path']), convert_hash[:name], extname(convert_hash[:target_format])] ) if item.properties['original_path'] new_item.save! new_item rescue ::RuntimeError => e case parameter(:on_convert_error) when 'COPY' copy_file(item, new_parent) when 'DROP' warn "Ignoring file '%s' because of error: '%s' @ '%s'", item.filepath, e.message, e.backtrace.first return else raise end end else # no action end end
convert_file(source_file, target_file, source_format, target_format, options = [{}])
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 370 def convert_file(source_file, target_file, source_format, target_format, options = [{}]) src_file = source_file src_format = source_format converterlist = [] temp_files = [] case options when Hash [options] when Array options else [{}] end.each do |opts| opts = opts.dup tgt_format = opts.delete(:target_format) || target_format tgt_file = tempname(src_file, tgt_format) temp_files << tgt_file begin src_file, converter = convert_one_file(src_file, tgt_file, src_format, tgt_format, opts) rescue Exception => e raise Libis::WorkflowError, "File conversion of '%s' from '%s' to '%s' failed: %s @ %s" % [src_file, src_format, tgt_format, e.message, e.backtrace.first] end src_format = tgt_format converterlist << converter end converter = converterlist.join(' + ') if target_file FileUtils.mkpath(File.dirname(target_file)) FileUtils.copy(src_file, target_file) else target_file = temp_files.pop end temp_files.each {|tmp_file| FileUtils.rm_f tmp_file} [target_file, converter] end
convert_one_file(source_file, target_file, source_format, target_format, options)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 407 def convert_one_file(source_file, target_file, source_format, target_format, options) converter = Libis::Format::Converter::Repository.get_converter_chain( source_format, target_format, options ) unless converter raise Libis::WorkflowError, "Could not find converter for #{source_format} -> #{target_format} with #{options}" end converter_name = converter.to_s debug 'Converting file %s to %s with %s ', source_file, target_file, converter_name converted = converter.convert(source_file, target_file) unless converted && converted == target_file error 'File conversion failed (%s).', converter_name return [nil, converter_name] end [target_file, converter_name] end
copy_file(file, to_parent)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 129 def copy_file(file, to_parent) debug "Copying '%s' to '%s' in object tree.", to_parent, file.name, to_parent.name file = to_parent.copy_item(file) process_files(file) end
extname(format)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 443 def extname(format) Libis::Format::TypeDatabase.type_extentions(format).first end
format_identifier(item)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 429 def format_identifier(item) result = Libis::Format::Identifier.get(item.fullpath, tool: :fido) || {} process_messages(result, item) apply_formats(item, result[:formats]) end
generate_from_ie(representation, convert_hash)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 156 def generate_from_ie(representation, convert_hash) ie = representation.parent ie.extend Libis::Workflow::Base::FileItem file = Libis::Ingester::FileItem.new file.filename = ie.fullpath format_identifier(file) convert(file, representation, convert_hash) end
generate_thumbnail(items, representation, convert_hash)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 149 def generate_thumbnail(items, representation, convert_hash) source_id = representation.parent.properties['thumbnail_source'] source_item = items.find_by('options.use_as_thumbnail' => true) source_item ||= source_id ? FileItem.find_by(id: source_id) : items.first convert(source_item, representation, convert_hash) end
match_file(file, formats)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 348 def match_file(file, formats) return true if formats.blank? mimetype = file.properties['mimetype'] type_id = Libis::Format::TypeDatabase.mime_types(mimetype).first group = Libis::Format::TypeDatabase.type_group(type_id.to_s) check_list = [type_id, group].compact.map {|v| [v.to_s, v.to_sym]}.flatten return false if (formats & check_list).empty? true end
move_file(file, to_parent)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 135 def move_file(file, to_parent) debug "Moving '%s' to '%s' in object tree.", to_parent, file.name, to_parent.name file = to_parent.move_item(file) process_files(file) end
process_files(file_or_div)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 141 def process_files(file_or_div) if file_or_div.is_a?(Libis::Ingester::FileItem) register_file(file_or_div) else file_or_div.get_items.each {|file| process_files(file)} end end
register_file(new_file)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 360 def register_file(new_file) new_file.properties['group_id'] = add_file_to_registry(new_file.label) end
tempname(source_file, target_format)
click to toggle source
# File lib/libis/ingester/tasks/manifestation_builder.rb, line 435 def tempname(source_file, target_format) # noinspection RubyResolve Dir::Tmpname.create( [File.basename(source_file, '.*'), ".#{extname(target_format)}"], Libis::Ingester::Config.tempdir ) {} end