class Contentful::Importer::ParallelImporter
Attributes
config[R]
content_type[RW]
data_organizer[R]
logger[R]
space[R]
Public Class Methods
new(settings)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 17 def initialize(settings) @config = settings @logger = Logger.new(STDOUT) @data_organizer = DataOrganizer.new(@config) Contentful::Management::Client.new(config.config['access_token'], default_locale: config.config['default_locale'] || 'en-US') end
Public Instance Methods
asset_not_imported_yet?(asset_attributes, assets_ids)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 100 def asset_not_imported_yet?(asset_attributes, assets_ids) !assets_ids.to_a.flatten.include?(asset_attributes['id']) end
asset_status(asset, asset_attributes)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 112 def asset_status(asset, asset_attributes) if asset.is_a?(Contentful::Management::Asset) logger.info "Process asset - #{asset.id} " asset.process_file CSV.open("#{config.log_files_dir}/success_assets.csv", 'a') { |csv| csv << [asset.id] } else logger.info "Error - #{asset.message} - #{asset.response.raw} " CSV.open("#{config.log_files_dir}/failure_assets.csv", 'a') { |csv| csv << [asset_attributes['id'], asset.message, asset.response.raw] } end end
asset_url_param_start_with_http?(asset_attributes)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 96 def asset_url_param_start_with_http?(asset_attributes) asset_attributes['url'] && asset_attributes['url'].start_with?('http') end
create_asset_file(asset_title, params)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 104 def create_asset_file(asset_title, params) Contentful::Management::File.new.tap do |file| file.properties[:contentType] = file_content_type(params) file.properties[:fileName] = asset_title file.properties[:upload] = params['url'] end end
create_contentful_model(space)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 24 def create_contentful_model(space) initialize_space(space) import_content_types end
import_asset(asset_attributes)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 86 def import_asset(asset_attributes) logger.info "Import asset - #{asset_attributes['id']} " asset_title = asset_attributes['name'].present? ? asset_attributes['name'] : asset_attributes['id'] asset_description = asset_attributes['description'].present? ? asset_attributes['description'] : '' asset_file = create_asset_file(asset_title, asset_attributes) space = Contentful::Management::Space.find(config.config['space_id']) asset = space.assets.create(id: "#{asset_attributes['id']}", title: "#{asset_title}", description: asset_description, file: asset_file) asset_status(asset, asset_attributes) end
import_data(threads)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 29 def import_data(threads) clean_threads_dir_before_import(threads) data_organizer.execute(threads) import_in_threads end
import_entries(path, space_id)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 64 def import_entries(path, space_id) log_file_name = "success_thread_#{File.basename(path)}" create_log_file(log_file_name) load_log_files Dir.glob("#{path}/*.json") do |entry_path| content_type_id = File.basename(entry_path).match(/(.+)_\d+/)[1] entry_file_name = File.basename(entry_path) import_entry(entry_path, space_id, content_type_id, log_file_name) unless config.imported_entries.flatten.include?(entry_file_name) end end
import_in_threads()
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 52 def import_in_threads threads = [] number_of_threads.times do |thread_id| threads << Thread.new do self.class.new(config).import_entries("#{config.threads_dir}/#{thread_id}", config.space_id) end end threads.each do |thread| thread.join end end
import_only_assets()
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 75 def import_only_assets create_log_file('success_assets') assets_ids = Set.new(CSV.read("#{config.data_dir}/logs/success_assets.csv", 'r')) Dir.glob("#{config.assets_dir}/**/*json") do |file_path| asset_attributes = JSON.parse(File.read(file_path)) if asset_url_param_start_with_http?(asset_attributes) && asset_not_imported_yet?(asset_attributes, assets_ids) import_asset(asset_attributes) end end end
number_of_threads()
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 44 def number_of_threads number_of_threads = 0 Dir.glob("#{config.threads_dir}/*") do |thread| number_of_threads += 1 if File.basename(thread).size == 1 end number_of_threads end
publish_all_entries(thread_dir)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 164 def publish_all_entries(thread_dir) create_log_file('success_published_entries') config.published_entries << CSV.read("#{config.log_files_dir}/success_published_entries.csv", 'r').flatten Dir.glob("#{thread_dir}/*json") do |entry_file| entry_id = JSON.parse(File.read(entry_file))['id'] publish_entry(entry_id) unless config.published_entries.flatten.include?(entry_id) end end
publish_asset(asset_id)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 158 def publish_asset(asset_id) logger.info "Publish an Asset - ID: #{asset_id}" asset = Contentful::Management::Asset.find(config.config['space_id'], asset_id).publish publish_status(asset, asset_id, 'published_assets') end
publish_assets(thread_dir)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 149 def publish_assets(thread_dir) create_log_file('success_published_assets') config.published_assets << CSV.read("#{config.log_files_dir}/success_published_assets.csv", 'r').flatten Dir.glob("#{thread_dir}/*json") do |asset_file| asset_id = JSON.parse(File.read(asset_file))['id'] publish_asset(asset_id) unless config.published_assets.flatten.include?(asset_id) end end
publish_assets_in_threads(number_of_threads)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 135 def publish_assets_in_threads(number_of_threads) clean_assets_threads_dir_before_publish(number_of_threads) data_organizer.split_assets_to_threads(number_of_threads) threads =[] number_of_threads.times do |thread_id| threads << Thread.new do self.class.new(config).publish_assets("#{config.threads_dir}/assets/#{thread_id}") end end threads.each do |thread| thread.join end end
publish_entries_in_threads()
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 123 def publish_entries_in_threads threads =[] number_of_threads.times do |thread_id| threads << Thread.new do self.class.new(config).publish_all_entries("#{config.threads_dir}/#{thread_id}") end end threads.each do |thread| thread.join end end
publish_entry(entry_id)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 173 def publish_entry(entry_id) logger.info "Publish entries for #{entry_id}." entry = Contentful::Management::Entry.find(config.config['space_id'], entry_id).publish publish_status(entry, entry_id, 'published_entries') end
test_credentials()
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 35 def test_credentials spaces = Contentful::Management::Space.all if spaces.is_a? Contentful::Management::Array logger.info 'Contentful Management API credentials: OK' end rescue NoMethodError => _error logger.info 'Contentful Management API credentials: INVALID (check README)' end
Private Instance Methods
active_status(ct_object)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 358 def active_status(ct_object) if ct_object.is_a? Contentful::Management::Error logger.info "### Failure! - #{ct_object.message} - #{ct_object.response.raw} ###" else logger.info 'Successfully activated!' end end
additional_field_params(field)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 376 def additional_field_params(field) field_type = field['type'] if field_type == 'Entry' || field_type == 'Asset' {type: 'Link', link_type: field_type} elsif field_type == 'Array' {type: 'Array', items: create_array_field(field)} else {type: field_type} end end
clean_assets_threads_dir_before_publish(threads)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 427 def clean_assets_threads_dir_before_publish(threads) threads.times do |thread| if File.directory?("#{config.threads_dir}/assets/#{thread}") logger.info "Remove directory threads/#{thread} from #{config.threads_dir}/assets path." FileUtils.rm_r("#{config.threads_dir}/assets/#{thread}") end end end
clean_threads_dir_before_import(threads)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 418 def clean_threads_dir_before_import(threads) threads.times do |thread| if File.directory?("#{config.threads_dir}/#{thread}") logger.info "Remove directory threads/#{thread} from #{config.threads_dir} path." FileUtils.rm_r("#{config.threads_dir}/#{thread}") end end end
create_array_field(params)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 411 def create_array_field(params) Contentful::Management::Field.new.tap do |field| field.type = params['link'] || 'Link' field.link_type = params['link_type'] end end
create_asset(space_id, params)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 294 def create_asset(space_id, params) if params['id'] space = Contentful::Management::Space.find(space_id) found_asset = space.assets.find(params['id']) asset = found_asset.is_a?(Contentful::Management::Asset) ? found_asset : initialize_asset_file(params) asset end end
create_content_type_field(field_params)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 327 def create_content_type_field(field_params) Contentful::Management::Field.new.tap do |field| field.id = field_params[:id] field.name = field_params[:name] field.type = field_params[:type] field.link_type = field_params[:link_type] field.required = field_params[:required] field.items = field_params[:items] field.validations = field_params[:validations] if field_params[:validations] end end
create_content_type_fields(collection_attributes, content_type)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 208 def create_content_type_fields(collection_attributes, content_type) fields = collection_attributes['fields'].each_with_object([]) do |field, fields| fields << create_field(field) end content_type.fields = fields content_type.save end
create_directory(path)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 436 def create_directory(path) FileUtils.mkdir_p(path) unless File.directory?(path) end
create_entry(params, space_id, content_type_id)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 286 def create_entry(params, space_id, content_type_id) entry_id = get_id(params) content_type = content_type(content_type_id, space_id) content_type.entries.new.tap do |entry| entry.id = entry_id end end
create_entry_parameters(content_type_id, entry_attributes, space_id)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 225 def create_entry_parameters(content_type_id, entry_attributes, space_id) entry_attributes.each_with_object({}) do |(attr, value), entry_params| next if attr.start_with?('@') entry_param = if value.is_a? Hash parse_attributes_from_hash(value, space_id, content_type_id) elsif value.is_a? Array parse_attributes_from_array(value, space_id, content_type_id) else value end entry_params[attr.to_sym] = entry_param unless validate_param(entry_param) end end
create_field(field)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 317 def create_field(field) field_params = {id: field['id'], name: field['name'], required: field['required']} field_params.merge!(additional_field_params(field)) if field['validations'] field_params[:validations] = create_validations(field['validations']) end logger.info "Creating field: #{field_params[:type]}" create_content_type_field(field_params) end
create_location_file(params)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 310 def create_location_file(params) Contentful::Management::Location.new.tap do |file| file.lat = params['lat'] file.lon = params['lng'] end end
create_log_file(path)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 440 def create_log_file(path) create_directory("#{config.data_dir}/logs") File.open("#{config.data_dir}/logs/#{path}.csv", 'a') { |file| file.write('') } end
create_new_content_type(space, collection_attributes)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 395 def create_new_content_type(space, collection_attributes) space.content_types.new.tap do |content_type| content_type.id = collection_attributes['id'] content_type.name = collection_attributes['name'] content_type.description = collection_attributes['description'] end end
create_space(name_space)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 186 def create_space(name_space) logger.info "Creating a space with name: #{name_space}" new_space = Contentful::Management::Space.create(name: name_space, organization_id: config.config['organization_id']) logger.info "Space was created successfully! Space id: #{new_space.id}" new_space end
create_validation(validation_params)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 346 def create_validation(validation_params) validation = Contentful::Management::Validation.new mappings = {'linkContentType' => 'link_content_type'} type = validation_params['type'] type = mappings[type] if mappings[type] params = validation_params['params'] validation.send("#{type}=", params) return validation end
create_validations(validations_params)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 339 def create_validations(validations_params) validations = validations_params.each_with_object([]) do |validation_params, validations| validations << create_validation(validation_params) end return validations end
file_content_type(params)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 403 def file_content_type(params) params['contentType'].present? ? params['contentType'] : MimeContentType::EXTENSION_LIST[File.extname(params['url'])] end
format_json(item)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 407 def format_json(item) JSON.pretty_generate(JSON.parse(item.to_json)) end
get_id(params)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 204 def get_id(params) File.basename(params['id'] || params['url']) end
import_content_types()
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 193 def import_content_types Dir.glob("#{config.collections_dir}/*json") do |file_path| collection_attributes = JSON.parse(File.read(file_path)) content_type = create_new_content_type(space, collection_attributes) logger.info "Importing content_type: #{content_type.name}" content_type.properties[:displayField] = collection_attributes['displayField'] or "" create_content_type_fields(collection_attributes, content_type) active_status(content_type.activate) end end
import_entry(file_path, space_id, content_type_id, log_file)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 216 def import_entry(file_path, space_id, content_type_id, log_file) entry_attributes = JSON.parse(File.read(file_path)) logger.info "Creating entry: #{entry_attributes['id']}." entry_params = create_entry_parameters(content_type_id, entry_attributes, space_id) content_type = content_type(content_type_id, space_id) entry = content_type.entries.create(entry_params) import_status(entry, file_path, log_file) end
import_status(entry, file_path, log_file)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 268 def import_status(entry, file_path, log_file) if entry.is_a? Contentful::Management::Entry entry_file_name = File.basename(file_path) logger.info 'Imported successfully!' CSV.open("#{config.log_files_dir}/#{log_file}.csv", 'a') { |csv| csv << [entry_file_name] } else logger.info "### Failure! - #{entry.message} - #{entry.response.raw}###" failure_filename = log_file.match(/(thread_\d)/)[1] CSV.open("#{config.log_files_dir}/failure_#{failure_filename}.csv", 'a') { |csv| csv << [file_path, entry.message, entry.response.raw] } end end
initialize_asset_file(params)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 303 def initialize_asset_file(params) Contentful::Management::Asset.new.tap do |asset| asset.id = params['id'] asset.link_type = 'Asset' end end
initialize_space(space)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 181 def initialize_space(space) fail 'You need to specify \'--space_id\' argument to find an existing Space or \'--space_name\' to create a new Space.' if space[:space_id].nil? && [:space_name].nil? @space = space[:space_id].present? ? Contentful::Management::Space.find(space[:space_id]) : create_space(space[:space_name]) end
load_log_files()
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 445 def load_log_files Dir.glob("#{config.log_files_dir}/*.csv") do |log_files| file_name = File.basename(log_files) imported_ids = CSV.read(log_files, 'r').flatten config.imported_entries << imported_ids if file_name.start_with?('success_thread') && !config.imported_entries.include?(imported_ids) end end
parse_attributes_from_array(params, space_id, content_type_id)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 255 def parse_attributes_from_array(params, space_id, content_type_id) params.each_with_object([]) do |attr, array_attributes| value = if attr['type'].present? && attr['type'] != 'File' create_entry(attr, space_id, content_type_id) elsif attr['type'] == 'File' create_asset(space_id, attr) else attr end array_attributes << value unless value.nil? end end
parse_attributes_from_hash(params, space_id, content_type_id)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 239 def parse_attributes_from_hash(params, space_id, content_type_id) type = params['type'] if type case type when 'Location' create_location_file(params) when 'File' create_asset(space_id, params) else create_entry(params, space_id, content_type_id) end else params end end
publish_status(ct_object, object_id, log_file_name)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 366 def publish_status(ct_object, object_id, log_file_name) if ct_object.is_a? Contentful::Management::Error logger.info "### Failure! - #{ct_object.message} - #{ct_object.response.raw} ###" CSV.open("#{config.log_files_dir}/failure_#{log_file_name}.csv", 'a') { |csv| csv << [object_id, ct_object.message, ct_object.response.raw] } else logger.info 'Successfully activated!' CSV.open("#{config.log_files_dir}/success_#{log_file_name}.csv", 'a') { |csv| csv << [ct_object.id] } end end
validate_param(param)
click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 387 def validate_param(param) if param.is_a? Array param.empty? else param.nil? end end