class JekyllOpenSdgPlugins::FetchRemoteData

Public Instance Methods

fetch_build(path) click to toggle source

Get a build from a local folder on disk or a remote URL on the Internet.

# File lib/jekyll-open-sdg-plugins/fetch_remote_data.rb, line 34
def fetch_build(path)

  is_remote = opensdg_is_path_remote(path)
  build = {}
  get_endpoints().each do |key, value|
    endpoint = is_remote ? path + '/' + value : File.join(path, fix_path(value))

    begin
      json_file = is_remote ? open(endpoint) : File.open(endpoint)
      build[key] = JSON.load(json_file)
    rescue StandardError => e
      # For backwards compatibility, forego the exception in some cases.
      abort_build = true
      if ['translations', 'indicator_downloads', 'disaggregation', 'data_packages'].include? key
        abort_build = false
      elsif endpoint.include? '/untranslated/'
        abort_build = false
      end
      if abort_build
        puts e.message
        abort 'Unable to read data from: ' + endpoint
      end
    end
  end

  return build
end
fix_path(path) click to toggle source

Fix a Unix path in case we are on Windows.

# File lib/jekyll-open-sdg-plugins/fetch_remote_data.rb, line 13
def fix_path(path)
  path_parts = path.split('/')
  return path_parts.join(File::SEPARATOR)
end
generate(site) click to toggle source
# File lib/jekyll-open-sdg-plugins/fetch_remote_data.rb, line 83
def generate(site)

  # For below, make sure there is at least an empty hash at
  # site.data.translations.
  if !site.data.has_key?('translations')
    site.data['translations'] = {}
  end

  remote = site.config['remote_data_prefix']
  local = site.config['local_data_folder']

  if !remote && !local
    abort 'Site config must include "remote_data_prefix".'
  end

  build_location = remote ? remote : local
  is_remote = opensdg_is_path_remote(build_location)

  build_location = is_remote ? build_location : File.join(Dir.pwd, build_location)
  translated_builds = site_uses_translated_builds(build_location)

  if translated_builds
    # For translated builds, we get a build for each language, and
    # place them in "subfolders" (so to speak) of site.data.
    subfolders = site.config['languages'].clone
    subfolders.append('untranslated')
    subfolders.each do |language|
      data_target = site.data[language]
      translated_build = is_remote ? build_location + '/' + language : File.join(build_location, language)
      data_source = fetch_build(translated_build)
      if !data_source.empty?
        if data_target
          data_target.deep_merge(data_source)
        else
          site.data[language] = data_source
        end
      end
    end
    # We move the language-specific translations to the
    # site.data.translations location, where all translations are kept.
    site.config['languages'].each do |language|
      translation_target = site.data['translations'][language]
      translation_source = site.data[language]['translations']
      if translation_target
        translation_target.deep_merge(translation_source)
      else
        site.data['translations'][language] = translation_source
      end
    end
    # And there are some parts of the build that don't need to be translated
    # and should be moved to the top level.
    first_language = site.config['languages'][0]
    site.data['reporting'] = site.data[first_language]['reporting']
    site.data['schema'] = site.data[first_language]['schema']
    site.data['zip'] = site.data[first_language]['zip']
  else
    # For untranslated builds, we download one build only, and place it
    # in the "root" (so to speak) of site.data. Nothing else is needed.
    target = site.data
    source = fetch_build(build_location)
    if !source.empty?
      target.deep_merge(source)
    end
  end

  # Finally support the deprecated 'remote_translations' option.
  # This is deprecated because translations should now be in the
  # data repository, where they will be fetched in fetch_build().
  if site.config['remote_translations']
    key = 'translations'
    target = site.data[key]
    site.config['remote_translations'].each do |endpoint|
      begin
        source = JSON.load(open(endpoint))
        if target
          target.deep_merge(source)
        else
          site.data[key] = source
        end
      rescue StandardError => e
        puts e.message
        abort 'Unable to fetch remote translation from: ' + endpoint
      end
    end
  end
end
get_endpoints() click to toggle source

Our hardcoded list of pieces of the build that we expect.

# File lib/jekyll-open-sdg-plugins/fetch_remote_data.rb, line 19
def get_endpoints()
  return {
    'meta' => 'meta/all.json',
    'headlines' => 'headline/all.json',
    'schema' => 'meta/schema.json',
    'reporting' => 'stats/reporting.json',
    'disaggregation' => 'stats/disaggregation.json',
    'translations' => 'translations/translations.json',
    'zip' => 'zip/all_indicators.json',
    'indicator_downloads' => 'downloads/indicator-downloads.json',
    'data_packages' => 'data-packages/all.json',
  }
end
site_uses_translated_builds(path) click to toggle source

Predict (before data has been fetched) whether the site is using translated builds or not.

# File lib/jekyll-open-sdg-plugins/fetch_remote_data.rb, line 64
def site_uses_translated_builds(path)

  is_remote = opensdg_is_path_remote(path)
  endpoints = get_endpoints()
  # For a quick test, we just use 'meta'.
  meta = endpoints['meta']
  endpoint = is_remote ? path + '/' + meta : File.join(path, fix_path(meta))

  begin
    json_file = is_remote ? open(endpoint) : File.open(endpoint)
  rescue StandardError => e
    # If we didn't find an untranslated 'meta', we assume translated builds.
    return true
  end

  # Other wise assume untranslated builds.
  return false
end