class Contentful::Importer::ParallelImporter

Attributes

config[R]
content_type[RW]
data_organizer[R]
logger[R]
space[R]

Public Class Methods

new(settings) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 17
def initialize(settings)
  @config = settings
  @logger = Logger.new(STDOUT)
  @data_organizer = DataOrganizer.new(@config)
  Contentful::Management::Client.new(config.config['access_token'], default_locale: config.config['default_locale'] || 'en-US')
end

Public Instance Methods

asset_not_imported_yet?(asset_attributes, assets_ids) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 100
def asset_not_imported_yet?(asset_attributes, assets_ids)
  !assets_ids.to_a.flatten.include?(asset_attributes['id'])
end
asset_status(asset, asset_attributes) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 112
def asset_status(asset, asset_attributes)
  if asset.is_a?(Contentful::Management::Asset)
    logger.info "Process asset - #{asset.id} "
    asset.process_file
    CSV.open("#{config.log_files_dir}/success_assets.csv", 'a') { |csv| csv << [asset.id] }
  else
    logger.info "Error - #{asset.message} - #{asset.response.raw} "
    CSV.open("#{config.log_files_dir}/failure_assets.csv", 'a') { |csv| csv << [asset_attributes['id'], asset.message, asset.response.raw] }
  end
end
asset_url_param_start_with_http?(asset_attributes) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 96
def asset_url_param_start_with_http?(asset_attributes)
  asset_attributes['url'] && asset_attributes['url'].start_with?('http')
end
create_asset_file(asset_title, params) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 104
def create_asset_file(asset_title, params)
  Contentful::Management::File.new.tap do |file|
    file.properties[:contentType] = file_content_type(params)
    file.properties[:fileName] = asset_title
    file.properties[:upload] = params['url']
  end
end
create_contentful_model(space) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 24
def create_contentful_model(space)
  initialize_space(space)
  import_content_types
end
import_asset(asset_attributes) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 86
def import_asset(asset_attributes)
  logger.info "Import asset - #{asset_attributes['id']} "
  asset_title = asset_attributes['name'].present? ? asset_attributes['name'] : asset_attributes['id']
  asset_description = asset_attributes['description'].present? ? asset_attributes['description'] : ''
  asset_file = create_asset_file(asset_title, asset_attributes)
  space = Contentful::Management::Space.find(config.config['space_id'])
  asset = space.assets.create(id: "#{asset_attributes['id']}", title: "#{asset_title}", description: asset_description, file: asset_file)
  asset_status(asset, asset_attributes)
end
import_data(threads) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 29
def import_data(threads)
  clean_threads_dir_before_import(threads)
  data_organizer.execute(threads)
  import_in_threads
end
import_entries(path, space_id) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 64
def import_entries(path, space_id)
  log_file_name = "success_thread_#{File.basename(path)}"
  create_log_file(log_file_name)
  load_log_files
  Dir.glob("#{path}/*.json") do |entry_path|
    content_type_id = File.basename(entry_path).match(/(.+)_\d+/)[1]
    entry_file_name = File.basename(entry_path)
    import_entry(entry_path, space_id, content_type_id, log_file_name) unless config.imported_entries.flatten.include?(entry_file_name)
  end
end
import_in_threads() click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 52
def import_in_threads
  threads = []
  number_of_threads.times do |thread_id|
    threads << Thread.new do
      self.class.new(config).import_entries("#{config.threads_dir}/#{thread_id}", config.space_id)
    end
  end
  threads.each do |thread|
    thread.join
  end
end
import_only_assets() click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 75
def import_only_assets
  create_log_file('success_assets')
  assets_ids = Set.new(CSV.read("#{config.data_dir}/logs/success_assets.csv", 'r'))
  Dir.glob("#{config.assets_dir}/**/*json") do |file_path|
    asset_attributes = JSON.parse(File.read(file_path))
    if asset_url_param_start_with_http?(asset_attributes) && asset_not_imported_yet?(asset_attributes, assets_ids)
      import_asset(asset_attributes)
    end
  end
end
number_of_threads() click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 44
def number_of_threads
  number_of_threads = 0
  Dir.glob("#{config.threads_dir}/*") do |thread|
    number_of_threads += 1 if File.basename(thread).size == 1
  end
  number_of_threads
end
publish_all_entries(thread_dir) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 164
def publish_all_entries(thread_dir)
  create_log_file('success_published_entries')
  config.published_entries << CSV.read("#{config.log_files_dir}/success_published_entries.csv", 'r').flatten
  Dir.glob("#{thread_dir}/*json") do |entry_file|
    entry_id = JSON.parse(File.read(entry_file))['id']
    publish_entry(entry_id) unless config.published_entries.flatten.include?(entry_id)
  end
end
publish_asset(asset_id) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 158
def publish_asset(asset_id)
  logger.info "Publish an Asset - ID: #{asset_id}"
  asset = Contentful::Management::Asset.find(config.config['space_id'], asset_id).publish
  publish_status(asset, asset_id, 'published_assets')
end
publish_assets(thread_dir) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 149
def publish_assets(thread_dir)
  create_log_file('success_published_assets')
  config.published_assets << CSV.read("#{config.log_files_dir}/success_published_assets.csv", 'r').flatten
  Dir.glob("#{thread_dir}/*json") do |asset_file|
    asset_id = JSON.parse(File.read(asset_file))['id']
    publish_asset(asset_id) unless config.published_assets.flatten.include?(asset_id)
  end
end
publish_assets_in_threads(number_of_threads) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 135
def publish_assets_in_threads(number_of_threads)
  clean_assets_threads_dir_before_publish(number_of_threads)
  data_organizer.split_assets_to_threads(number_of_threads)
  threads =[]
  number_of_threads.times do |thread_id|
    threads << Thread.new do
      self.class.new(config).publish_assets("#{config.threads_dir}/assets/#{thread_id}")
    end
  end
  threads.each do |thread|
    thread.join
  end
end
publish_entries_in_threads() click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 123
def publish_entries_in_threads
  threads =[]
  number_of_threads.times do |thread_id|
    threads << Thread.new do
      self.class.new(config).publish_all_entries("#{config.threads_dir}/#{thread_id}")
    end
  end
  threads.each do |thread|
    thread.join
  end
end
publish_entry(entry_id) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 173
def publish_entry(entry_id)
  logger.info "Publish entries for #{entry_id}."
  entry = Contentful::Management::Entry.find(config.config['space_id'], entry_id).publish
  publish_status(entry, entry_id, 'published_entries')
end
test_credentials() click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 35
def test_credentials
  spaces = Contentful::Management::Space.all
  if spaces.is_a? Contentful::Management::Array
    logger.info 'Contentful Management API credentials: OK'
  end
rescue NoMethodError => _error
  logger.info 'Contentful Management API credentials: INVALID (check README)'
end

Private Instance Methods

active_status(ct_object) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 358
def active_status(ct_object)
  if ct_object.is_a? Contentful::Management::Error
    logger.info "### Failure! - #{ct_object.message} - #{ct_object.response.raw} ###"
  else
    logger.info 'Successfully activated!'
  end
end
additional_field_params(field) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 376
def additional_field_params(field)
  field_type = field['type']
  if field_type == 'Entry' || field_type == 'Asset'
    {type: 'Link', link_type: field_type}
  elsif field_type == 'Array'
    {type: 'Array', items: create_array_field(field)}
  else
    {type: field_type}
  end
end
clean_assets_threads_dir_before_publish(threads) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 427
def clean_assets_threads_dir_before_publish(threads)
  threads.times do |thread|
    if File.directory?("#{config.threads_dir}/assets/#{thread}")
      logger.info "Remove directory threads/#{thread} from #{config.threads_dir}/assets path."
      FileUtils.rm_r("#{config.threads_dir}/assets/#{thread}")
    end
  end
end
clean_threads_dir_before_import(threads) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 418
def clean_threads_dir_before_import(threads)
  threads.times do |thread|
    if File.directory?("#{config.threads_dir}/#{thread}")
      logger.info "Remove directory threads/#{thread} from #{config.threads_dir} path."
      FileUtils.rm_r("#{config.threads_dir}/#{thread}")
    end
  end
end
create_array_field(params) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 411
def create_array_field(params)
  Contentful::Management::Field.new.tap do |field|
    field.type = params['link'] || 'Link'
    field.link_type = params['link_type']
  end
end
create_asset(space_id, params) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 294
def create_asset(space_id, params)
  if params['id']
    space = Contentful::Management::Space.find(space_id)
    found_asset = space.assets.find(params['id'])
    asset = found_asset.is_a?(Contentful::Management::Asset) ? found_asset : initialize_asset_file(params)
    asset
  end
end
create_content_type_field(field_params) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 327
def create_content_type_field(field_params)
  Contentful::Management::Field.new.tap do |field|
    field.id = field_params[:id]
    field.name = field_params[:name]
    field.type = field_params[:type]
    field.link_type = field_params[:link_type]
    field.required = field_params[:required]
    field.items = field_params[:items]
    field.validations = field_params[:validations] if field_params[:validations]
  end
end
create_content_type_fields(collection_attributes, content_type) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 208
def create_content_type_fields(collection_attributes, content_type)
  fields = collection_attributes['fields'].each_with_object([]) do |field, fields|
    fields << create_field(field)
  end
  content_type.fields = fields
  content_type.save
end
create_directory(path) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 436
def create_directory(path)
  FileUtils.mkdir_p(path) unless File.directory?(path)
end
create_entry(params, space_id, content_type_id) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 286
def create_entry(params, space_id, content_type_id)
  entry_id = get_id(params)
  content_type = content_type(content_type_id, space_id)
  content_type.entries.new.tap do |entry|
    entry.id = entry_id
  end
end
create_entry_parameters(content_type_id, entry_attributes, space_id) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 225
def create_entry_parameters(content_type_id, entry_attributes, space_id)
  entry_attributes.each_with_object({}) do |(attr, value), entry_params|
    next if attr.start_with?('@')
    entry_param = if value.is_a? Hash
                    parse_attributes_from_hash(value, space_id, content_type_id)
                  elsif value.is_a? Array
                    parse_attributes_from_array(value, space_id, content_type_id)
                  else
                    value
                  end
    entry_params[attr.to_sym] = entry_param unless validate_param(entry_param)
  end
end
create_field(field) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 317
def create_field(field)
  field_params = {id: field['id'], name: field['name'], required: field['required']}
  field_params.merge!(additional_field_params(field))
  if field['validations']
    field_params[:validations] = create_validations(field['validations'])
  end
  logger.info "Creating field: #{field_params[:type]}"
  create_content_type_field(field_params)
end
create_location_file(params) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 310
def create_location_file(params)
  Contentful::Management::Location.new.tap do |file|
    file.lat = params['lat']
    file.lon = params['lng']
  end
end
create_log_file(path) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 440
def create_log_file(path)
  create_directory("#{config.data_dir}/logs")
  File.open("#{config.data_dir}/logs/#{path}.csv", 'a') { |file| file.write('') }
end
create_new_content_type(space, collection_attributes) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 395
def create_new_content_type(space, collection_attributes)
  space.content_types.new.tap do |content_type|
    content_type.id = collection_attributes['id']
    content_type.name = collection_attributes['name']
    content_type.description = collection_attributes['description']
  end
end
create_space(name_space) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 186
def create_space(name_space)
  logger.info "Creating a space with name: #{name_space}"
  new_space = Contentful::Management::Space.create(name: name_space, organization_id: config.config['organization_id'])
  logger.info "Space was created successfully! Space id: #{new_space.id}"
  new_space
end
create_validation(validation_params) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 346
def create_validation(validation_params)
  validation = Contentful::Management::Validation.new

  mappings = {'linkContentType' => 'link_content_type'}
  type = validation_params['type']
  type = mappings[type] if mappings[type]

  params = validation_params['params']
  validation.send("#{type}=", params)
  return validation
end
create_validations(validations_params) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 339
def create_validations(validations_params)
  validations = validations_params.each_with_object([]) do |validation_params, validations|
    validations << create_validation(validation_params)
  end
  return validations
end
file_content_type(params) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 403
def file_content_type(params)
  params['contentType'].present? ? params['contentType'] : MimeContentType::EXTENSION_LIST[File.extname(params['url'])]
end
format_json(item) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 407
def format_json(item)
  JSON.pretty_generate(JSON.parse(item.to_json))
end
get_id(params) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 204
def get_id(params)
  File.basename(params['id'] || params['url'])
end
import_content_types() click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 193
def import_content_types
  Dir.glob("#{config.collections_dir}/*json") do |file_path|
    collection_attributes = JSON.parse(File.read(file_path))
    content_type = create_new_content_type(space, collection_attributes)
    logger.info "Importing content_type: #{content_type.name}"
    content_type.properties[:displayField] = collection_attributes['displayField'] or ""
    create_content_type_fields(collection_attributes, content_type)
    active_status(content_type.activate)
  end
end
import_entry(file_path, space_id, content_type_id, log_file) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 216
def import_entry(file_path, space_id, content_type_id, log_file)
  entry_attributes = JSON.parse(File.read(file_path))
  logger.info "Creating entry: #{entry_attributes['id']}."
  entry_params = create_entry_parameters(content_type_id, entry_attributes, space_id)
  content_type = content_type(content_type_id, space_id)
  entry = content_type.entries.create(entry_params)
  import_status(entry, file_path, log_file)
end
import_status(entry, file_path, log_file) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 268
def import_status(entry, file_path, log_file)
  if entry.is_a? Contentful::Management::Entry
    entry_file_name = File.basename(file_path)
    logger.info 'Imported successfully!'
    CSV.open("#{config.log_files_dir}/#{log_file}.csv", 'a') { |csv| csv << [entry_file_name] }
  else
    logger.info "### Failure! - #{entry.message}  - #{entry.response.raw}###"
    failure_filename = log_file.match(/(thread_\d)/)[1]
    CSV.open("#{config.log_files_dir}/failure_#{failure_filename}.csv", 'a') { |csv| csv << [file_path, entry.message, entry.response.raw] }
  end
end
initialize_asset_file(params) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 303
def initialize_asset_file(params)
  Contentful::Management::Asset.new.tap do |asset|
    asset.id = params['id']
    asset.link_type = 'Asset'
  end
end
initialize_space(space) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 181
def initialize_space(space)
  fail 'You need to specify \'--space_id\' argument to find an existing Space or \'--space_name\' to create a new Space.' if space[:space_id].nil? && [:space_name].nil?
  @space = space[:space_id].present? ? Contentful::Management::Space.find(space[:space_id]) : create_space(space[:space_name])
end
load_log_files() click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 445
def load_log_files
  Dir.glob("#{config.log_files_dir}/*.csv") do |log_files|
    file_name = File.basename(log_files)
    imported_ids = CSV.read(log_files, 'r').flatten
    config.imported_entries << imported_ids if file_name.start_with?('success_thread') && !config.imported_entries.include?(imported_ids)
  end
end
parse_attributes_from_array(params, space_id, content_type_id) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 255
def parse_attributes_from_array(params, space_id, content_type_id)
  params.each_with_object([]) do |attr, array_attributes|
    value = if attr['type'].present? && attr['type'] != 'File'
              create_entry(attr, space_id, content_type_id)
            elsif attr['type'] == 'File'
              create_asset(space_id, attr)
            else
              attr
            end
    array_attributes << value unless value.nil?
  end
end
parse_attributes_from_hash(params, space_id, content_type_id) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 239
def parse_attributes_from_hash(params, space_id, content_type_id)
  type = params['type']
  if type
    case type
      when 'Location'
        create_location_file(params)
      when 'File'
        create_asset(space_id, params)
      else
        create_entry(params, space_id, content_type_id)
    end
  else
    params
  end
end
publish_status(ct_object, object_id, log_file_name) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 366
def publish_status(ct_object, object_id, log_file_name)
  if ct_object.is_a? Contentful::Management::Error
    logger.info "### Failure! - #{ct_object.message} - #{ct_object.response.raw} ###"
    CSV.open("#{config.log_files_dir}/failure_#{log_file_name}.csv", 'a') { |csv| csv << [object_id, ct_object.message, ct_object.response.raw] }
  else
    logger.info 'Successfully activated!'
    CSV.open("#{config.log_files_dir}/success_#{log_file_name}.csv", 'a') { |csv| csv << [ct_object.id] }
  end
end
validate_param(param) click to toggle source
# File lib/contentful/importer/parallel_importer.rb, line 387
def validate_param(param)
  if param.is_a? Array
    param.empty?
  else
    param.nil?
  end
end