class Contentful::Importer::DataOrganizer

Attributes

config[R]
logger[R]
split_params[R]

Public Class Methods

new(settings) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 11
def initialize(settings)
  @config = settings
  @split_params = {object_index: 0, current_thread: 0}
  @logger = Logger.new(STDOUT)
end

Public Instance Methods

content_type_id_from_file(collection_file) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 72
def content_type_id_from_file(collection_file)
  JSON.parse(File.read("#{config.collections_dir}/#{collection_file}.json"))['id']
end
copy_asset(asset_path) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 84
def copy_asset(asset_path)
  FileUtils.cp asset_path, "#{config.threads_dir}/assets/#{split_params[:current_thread]}/#{File.basename(asset_path)}"
end
copy_entry(entry_path, current_thread, content_type_id) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 80
def copy_entry(entry_path, current_thread, content_type_id)
  FileUtils.cp entry_path, "#{config.threads_dir}/#{current_thread}/#{new_entry_name(content_type_id, entry_path)}"
end
count_index_files(objects_per_thread_count, threads_count) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 56
def count_index_files(objects_per_thread_count, threads_count)
  if split_params[:object_index] == objects_per_thread_count
    split_params[:object_index] = 0
    set_current_thread(threads_count)
  end
end
create_directory(path) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 96
def create_directory(path)
  FileUtils.mkdir_p(path) unless File.directory?(path)
end
create_threads_subdirectories(threads_count, validate, dir = {}) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 88
def create_threads_subdirectories(threads_count, validate, dir = {})
  validate_collections_files if validate
  create_directory(config.threads_dir)
  threads_count.times do |thread_id|
    create_directory("#{config.threads_dir}/#{dir[:assets]}#{thread_id}")
  end
end
execute(threads_count) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 17
def execute(threads_count)
  create_threads_subdirectories(threads_count, true)
  split_entries(threads_count)
end
has_contentful_structure?(collection_file) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 68
def has_contentful_structure?(collection_file)
  File.exist?("#{config.collections_dir}/#{collection_file}.json")
end
new_entry_name(content_type_id, entry_path) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 76
def new_entry_name(content_type_id, entry_path)
  "#{content_type_id}_#{File.basename(entry_path, '.*').match(/(\d+)/)[0]}.json"
end
process_collection_files(content_type_id, dir_path, entries_per_thread_count, threads_count) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 47
def process_collection_files(content_type_id, dir_path, entries_per_thread_count, threads_count)
  logger.info "Processing collection: #{content_type_id}"
  Dir.glob("#{dir_path}/*.json") do |entry_path|
    copy_entry(entry_path, split_params[:current_thread], content_type_id)
    split_params[:object_index] += 1
    count_index_files(entries_per_thread_count, threads_count)
  end
end
set_current_thread(threads_count) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 63
def set_current_thread(threads_count)
  split_params[:current_thread] += 1
  split_params[:current_thread] = 0 if split_params[:current_thread] == threads_count
end
split_assets(threads_count) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 38
def split_assets(threads_count)
  asset_per_thread = total_assets_count / threads_count
  Dir.glob("#{config.assets_dir}/**/*json") do |asset_path|
    copy_asset(asset_path)
    split_params[:object_index] += 1
    count_index_files(asset_per_thread, threads_count)
  end
end
split_assets_to_threads(threads_count) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 22
def split_assets_to_threads(threads_count)
  create_threads_subdirectories(threads_count, false, {assets: 'assets/'})
  split_assets(threads_count)
end
split_entries(threads_count) click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 27
def split_entries(threads_count)
  entries_per_thread_count = total_entries_count / threads_count
  Dir.glob("#{config.entries_dir}/*") do |dir_path|
    collection_name = File.basename(dir_path)
    if has_contentful_structure?(collection_name)
      content_type_id = content_type_id_from_file(collection_name)
      process_collection_files(content_type_id, dir_path, entries_per_thread_count, threads_count)
    end
  end
end
total_assets_count() click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 109
def total_assets_count
  Dir.glob("#{config.assets_dir}/**/*json").count
end
total_entries_count() click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 100
def total_entries_count
  total_number = 0
  Dir.glob("#{config.entries_dir}/*") do |dir_path|
    collection_name = File.basename(dir_path)
    total_number += Dir.glob("#{config.entries_dir}/#{collection_name}/*").count if has_contentful_structure?(collection_name)
  end
  total_number
end
validate_collections_files() click to toggle source
# File lib/contentful/importer/data_organizer.rb, line 113
def validate_collections_files
  fail ArgumentError, "Make sure the #{config.collections_dir} directory exists and the content structure resides within it. View README" unless Dir.exist?(config.collections_dir)
  fail ArgumentError, 'Collections directory is empty! Create content types JSON files. View README' if Dir.glob("#{config.collections_dir}/*").empty?
end