class Tumugi::Plugin::BigqueryLoadTask
Public Instance Methods
output()
click to toggle source
# File lib/tumugi/plugin/task/bigquery_load.rb, line 27 def output return @output if @output opts = { dataset_id: dataset_id, table_id: table_id } opts[:project_id] = project_id if project_id @output = Tumugi::Plugin::BigqueryTableTarget.new(opts) end
run()
click to toggle source
# File lib/tumugi/plugin/task/bigquery_load.rb, line 35 def run if mode != 'append' raise Tumugi::ParameterError.new("Parameter 'schema' is required when 'mode' is 'truncate' or 'empty'") if schema.nil? end src_uri = "gs://#{bucket}#{normalize_path(key)}" log "Source: #{src_uri}" log "Destination: #{output}" bq_client = output.client opts = { schema: schema, field_delimiter: field_delimiter, mode: mode.to_sym, allow_jagged_rows: allow_jagged_rows, max_bad_records: max_bad_records, ignore_unknown_values: ignore_unknown_values, allow_quoted_newlines: allow_quoted_newlines, quote: quote, skip_leading_rows: skip_leading_rows, source_format: source_format, project_id: output.project_id, wait: wait } bq_client.load(output.dataset_id, output.table_id, src_uri, opts) end
Private Instance Methods
normalize_path(path)
click to toggle source
# File lib/tumugi/plugin/task/bigquery_load.rb, line 64 def normalize_path(path) unless path.start_with?('/') "/#{path}" else path end end