class Tumugi::Plugin::BigqueryLoadTask

Public Instance Methods

output() click to toggle source
# File lib/tumugi/plugin/task/bigquery_load.rb, line 27
def output
  return @output if @output

  opts = { dataset_id: dataset_id, table_id: table_id }
  opts[:project_id] = project_id if project_id
  @output = Tumugi::Plugin::BigqueryTableTarget.new(opts)
end
run() click to toggle source
# File lib/tumugi/plugin/task/bigquery_load.rb, line 35
def run
  if mode != 'append'
    raise Tumugi::ParameterError.new("Parameter 'schema' is required when 'mode' is 'truncate' or 'empty'") if schema.nil?
  end

  src_uri = "gs://#{bucket}#{normalize_path(key)}"
  log "Source: #{src_uri}"
  log "Destination: #{output}"

  bq_client = output.client
  opts = {
    schema: schema,
    field_delimiter: field_delimiter,
    mode: mode.to_sym,
    allow_jagged_rows: allow_jagged_rows,
    max_bad_records: max_bad_records,
    ignore_unknown_values: ignore_unknown_values,
    allow_quoted_newlines: allow_quoted_newlines,
    quote: quote,
    skip_leading_rows: skip_leading_rows,
    source_format: source_format,
    project_id: output.project_id,
    wait: wait
  }
  bq_client.load(output.dataset_id, output.table_id, src_uri, opts)
end

Private Instance Methods

normalize_path(path) click to toggle source
# File lib/tumugi/plugin/task/bigquery_load.rb, line 64
def normalize_path(path)
  unless path.start_with?('/')
    "/#{path}"
  else
    path
  end
end