class Google::Cloud::Bigquery::Service

@private Represents the Bigquery service and API calls.

Constants

API

Alias to the Google Client API module

Attributes

credentials[RW]

@private

host[R]

@private

mocked_service[RW]
project[RW]

@private

retries[R]

@private

timeout[R]

@private

Public Class Methods

get_table_ref(table, default_ref: nil) click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 459
def self.get_table_ref table, default_ref: nil
  if table.respond_to? :table_ref
    table.table_ref
  else
    table_ref_from_s table, default_ref: default_ref
  end
end
new(project, credentials, retries: nil, timeout: nil, host: nil, quota_project: nil) click to toggle source

Creates a new Service instance.

# File lib/google/cloud/bigquery/service.rb, line 46
def initialize project, credentials, retries: nil, timeout: nil, host: nil, quota_project: nil
  @project = project
  @credentials = credentials
  @retries = retries
  @timeout = timeout
  @host = host
  @quota_project = quota_project
end
table_ref_from_s(str, default_ref: {}) click to toggle source

Extracts at least `tbl` group, and possibly `dts` and `prj` groups, from strings in the formats: “my_table”, “my_dataset.my_table”, or “my-project:my_dataset.my_table”. Then merges project_id and dataset_id from the default table ref if they are missing.

The regex matches both Standard SQL (“bigquery-public-data.samples.shakespeare”) and Legacy SQL (“bigquery-public-data:samples.shakespeare”).

# File lib/google/cloud/bigquery/service.rb, line 476
def self.table_ref_from_s str, default_ref: {}
  str = str.to_s
  m = /\A(((?<prj>\S*)(:|\.))?(?<dts>\S*)\.)?(?<tbl>\S*)\z/.match str
  raise ArgumentError, "unable to identify table from #{str.inspect}" unless m
  str_table_ref_hash = {
    project_id: m["prj"],
    dataset_id: m["dts"],
    table_id:   m["tbl"]
  }.delete_if { |_, v| v.nil? }
  str_table_ref_hash = default_ref.to_h.merge str_table_ref_hash
  ref = Google::Apis::BigqueryV2::TableReference.new(**str_table_ref_hash)
  validate_table_ref ref
  ref
end
validate_table_ref(table_ref) click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 491
def self.validate_table_ref table_ref
  [:project_id, :dataset_id, :table_id].each do |f|
    raise ArgumentError, "TableReference is missing #{f}" if table_ref.send(f).nil?
  end
end

Public Instance Methods

cancel_job(job_id, location: nil) click to toggle source

Cancel the job specified by jobId.

# File lib/google/cloud/bigquery/service.rb, line 391
def cancel_job job_id, location: nil
  # The BigQuery team has told us cancelling is considered idempotent
  execute backoff: true do
    service.cancel_job @project, job_id, location: location
  end
end
copy_table(copy_job_gapi) click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 435
def copy_table copy_job_gapi
  execute backoff: true do
    service.insert_job @project, copy_job_gapi
  end
end
dataset_ref_from(dts, pjt = nil) click to toggle source

API object for dataset.

# File lib/google/cloud/bigquery/service.rb, line 518
def dataset_ref_from dts, pjt = nil
  return nil if dts.nil?
  if dts.respond_to? :dataset_id
    Google::Apis::BigqueryV2::DatasetReference.new(
      project_id: (pjt || dts.project_id || @project),
      dataset_id: dts.dataset_id
    )
  else
    Google::Apis::BigqueryV2::DatasetReference.new(
      project_id: (pjt || @project),
      dataset_id: dts
    )
  end
end
delete_dataset(dataset_id, force = nil) click to toggle source

Deletes the dataset specified by the datasetId value. Before you can delete a dataset, you must delete all its tables, either manually or by specifying force: true in options. Immediately after deletion, you can create another dataset with the same name.

# File lib/google/cloud/bigquery/service.rb, line 129
def delete_dataset dataset_id, force = nil
  execute do
    service.delete_dataset @project, dataset_id, delete_contents: force
  end
end
delete_model(dataset_id, model_id) click to toggle source

Deletes the model specified by modelId from the dataset. If the model contains data, all the data will be deleted.

# File lib/google/cloud/bigquery/service.rb, line 319
def delete_model dataset_id, model_id
  execute { service.delete_model @project, dataset_id, model_id }
end
delete_routine(dataset_id, routine_id) click to toggle source

Deletes the routine specified by routine_id from the dataset.

# File lib/google/cloud/bigquery/service.rb, line 369
def delete_routine dataset_id, routine_id
  execute { service.delete_routine @project, dataset_id, routine_id }
end
delete_table(dataset_id, table_id) click to toggle source

Deletes the table specified by tableId from the dataset. If the table contains data, all the data will be deleted.

# File lib/google/cloud/bigquery/service.rb, line 216
def delete_table dataset_id, table_id
  execute { service.delete_table @project, dataset_id, table_id }
end
extract_table(extract_job_gapi) click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 441
def extract_table extract_job_gapi
  execute backoff: true do
    service.insert_job @project, extract_job_gapi
  end
end
get_dataset(dataset_id) click to toggle source

Returns the dataset specified by datasetID.

# File lib/google/cloud/bigquery/service.rb, line 94
def get_dataset dataset_id
  # The get operation is considered idempotent
  execute backoff: true do
    service.get_dataset @project, dataset_id
  end
end
get_job(job_id, location: nil) click to toggle source

Returns the job specified by jobID.

# File lib/google/cloud/bigquery/service.rb, line 400
def get_job job_id, location: nil
  # The get operation is considered idempotent
  execute backoff: true do
    service.get_job @project, job_id, location: location
  end
end
get_model(dataset_id, model_id) click to toggle source

Gets the specified model resource by model ID. This method does not return the data in the model, it only returns the model resource, which describes the structure of this model.

# File lib/google/cloud/bigquery/service.rb, line 295
def get_model dataset_id, model_id
  get_project_model @project, dataset_id, model_id
end
get_project_model(project_id, dataset_id, model_id) click to toggle source

Gets the specified model resource by full model reference.

# File lib/google/cloud/bigquery/service.rb, line 285
def get_project_model project_id, dataset_id, model_id
  # The get operation is considered idempotent
  execute backoff: true do
    json_txt = service.get_model project_id, dataset_id, model_id, options: { skip_deserialization: true }
    JSON.parse json_txt, symbolize_names: true
  end
end
get_project_table(project_id, dataset_id, table_id) click to toggle source

Gets the specified table resource by full table reference.

# File lib/google/cloud/bigquery/service.rb, line 147
def get_project_table project_id, dataset_id, table_id
  # The get operation is considered idempotent
  execute backoff: true do
    service.get_table project_id, dataset_id, table_id
  end
end
get_routine(dataset_id, routine_id) click to toggle source

Gets the specified routine resource by routine ID.

# File lib/google/cloud/bigquery/service.rb, line 345
def get_routine dataset_id, routine_id
  # The get operation is considered idempotent
  execute backoff: true do
    service.get_routine @project, dataset_id, routine_id
  end
end
get_table(dataset_id, table_id) click to toggle source

Gets the specified table resource by table ID. This method does not return the data in the table, it only returns the table resource, which describes the structure of this table.

# File lib/google/cloud/bigquery/service.rb, line 159
def get_table dataset_id, table_id
  get_project_table @project, dataset_id, table_id
end
get_table_policy(dataset_id, table_id) click to toggle source

Returns Google::Apis::BigqueryV2::Policy

# File lib/google/cloud/bigquery/service.rb, line 187
def get_table_policy dataset_id, table_id
  policy_options = API::GetPolicyOptions.new requested_policy_version: 1
  execute do
    service.get_table_iam_policy table_path(dataset_id, table_id),
                                 API::GetIamPolicyRequest.new(options: policy_options)
  end
end
insert_dataset(new_dataset_gapi) click to toggle source

Creates a new empty dataset.

# File lib/google/cloud/bigquery/service.rb, line 103
def insert_dataset new_dataset_gapi
  execute { service.insert_dataset @project, new_dataset_gapi }
end
insert_job(config, location: nil) click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 407
def insert_job config, location: nil
  job_object = API::Job.new job_reference: job_ref_from(nil, nil, location: location), configuration: config
  # Jobs have generated id, so this operation is considered idempotent
  execute backoff: true do
    service.insert_job @project, job_object
  end
end
insert_routine(dataset_id, new_routine_gapi) click to toggle source

Creates a new routine in the dataset.

# File lib/google/cloud/bigquery/service.rb, line 325
def insert_routine dataset_id, new_routine_gapi
  execute { service.insert_routine @project, dataset_id, new_routine_gapi }
end
insert_table(dataset_id, new_table_gapi) click to toggle source

Creates a new, empty table in the dataset.

# File lib/google/cloud/bigquery/service.rb, line 165
def insert_table dataset_id, new_table_gapi
  execute { service.insert_table @project, dataset_id, new_table_gapi }
end
insert_tabledata(dataset_id, table_id, rows, insert_ids: nil, ignore_unknown: nil, skip_invalid: nil) click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 235
def insert_tabledata dataset_id, table_id, rows, insert_ids: nil, ignore_unknown: nil, skip_invalid: nil
  json_rows = Array(rows).map { |row| Convert.to_json_row row }
  insert_tabledata_json_rows dataset_id, table_id, json_rows, insert_ids:     insert_ids,
                                                              ignore_unknown: ignore_unknown,
                                                              skip_invalid:   skip_invalid
end
insert_tabledata_json_rows(dataset_id, table_id, json_rows, insert_ids: nil, ignore_unknown: nil, skip_invalid: nil) click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 242
def insert_tabledata_json_rows dataset_id, table_id, json_rows, insert_ids: nil, ignore_unknown: nil,
                               skip_invalid: nil
  rows_and_ids = Array(json_rows).zip Array(insert_ids)
  insert_rows = rows_and_ids.map do |json_row, insert_id|
    if insert_id == :skip
      { json: json_row }
    else
      insert_id ||= SecureRandom.uuid
      {
        insertId: insert_id,
        json:     json_row
      }
    end
  end

  insert_req = {
    rows:                insert_rows,
    ignoreUnknownValues: ignore_unknown,
    skipInvalidRows:     skip_invalid
  }.to_json

  # The insertAll with insertId operation is considered idempotent
  execute backoff: true do
    service.insert_all_table_data(
      @project, dataset_id, table_id, insert_req,
      options: { skip_serialization: true }
    )
  end
end
inspect() click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 533
def inspect
  "#{self.class}(#{@project})"
end
job_query_results(job_id, location: nil, max: nil, token: nil, start: nil, timeout: nil) click to toggle source

Returns the query data for the job

# File lib/google/cloud/bigquery/service.rb, line 423
def job_query_results job_id, location: nil, max: nil, token: nil, start: nil, timeout: nil
  # The get operation is considered idempotent
  execute backoff: true do
    service.get_job_query_results @project, job_id,
                                  location:    location,
                                  max_results: max,
                                  page_token:  token,
                                  start_index: start,
                                  timeout_ms:  timeout
  end
end
job_ref_from(job_id, prefix, location: nil) click to toggle source

If no job_id or prefix is given, always generate a client-side job ID anyway, for idempotent retry in the google-api-client layer. See cloud.google.com/bigquery/docs/managing-jobs#generate-jobid

# File lib/google/cloud/bigquery/service.rb, line 508
def job_ref_from job_id, prefix, location: nil
  prefix ||= "job_"
  job_id ||= "#{prefix}#{generate_id}"
  job_ref = API::JobReference.new project_id: @project, job_id: job_id
  # BigQuery does not allow nil location, but missing is ok.
  job_ref.location = location if location
  job_ref
end
list_datasets(all: nil, filter: nil, max: nil, token: nil) click to toggle source

Lists all datasets in the specified project to which you have been granted the READER dataset role.

# File lib/google/cloud/bigquery/service.rb, line 85
def list_datasets all: nil, filter: nil, max: nil, token: nil
  # The list operation is considered idempotent
  execute backoff: true do
    service.list_datasets @project, all: all, filter: filter, max_results: max, page_token: token
  end
end
list_jobs(all: nil, token: nil, max: nil, filter: nil, min_created_at: nil, max_created_at: nil, parent_job_id: nil) click to toggle source

Lists all jobs in the specified project to which you have been granted the READER job role.

# File lib/google/cloud/bigquery/service.rb, line 376
def list_jobs all: nil, token: nil, max: nil, filter: nil, min_created_at: nil, max_created_at: nil,
              parent_job_id: nil
  # The list operation is considered idempotent
  min_creation_time = Convert.time_to_millis min_created_at
  max_creation_time = Convert.time_to_millis max_created_at
  execute backoff: true do
    service.list_jobs @project, all_users: all, max_results: max,
                                page_token: token, projection: "full", state_filter: filter,
                                min_creation_time: min_creation_time, max_creation_time: max_creation_time,
                                parent_job_id: parent_job_id
  end
end
list_models(dataset_id, max: nil, token: nil) click to toggle source

Lists all models in the specified dataset. Requires the READER dataset role.

# File lib/google/cloud/bigquery/service.rb, line 275
def list_models dataset_id, max: nil, token: nil
  options = { skip_deserialization: true }
  # The list operation is considered idempotent
  execute backoff: true do
    json_txt = service.list_models @project, dataset_id, max_results: max, page_token: token, options: options
    JSON.parse json_txt, symbolize_names: true
  end
end
list_projects(max: nil, token: nil) click to toggle source

Lists all projects to which you have been granted any project role.

# File lib/google/cloud/bigquery/service.rb, line 499
def list_projects max: nil, token: nil
  execute backoff: true do
    service.list_projects max_results: max, page_token: token
  end
end
list_routines(dataset_id, max: nil, token: nil, filter: nil) click to toggle source

Lists all routines in the specified dataset. Requires the READER dataset role. Unless readMask is set in the request, only the following fields are populated:

etag, projectId, datasetId, routineId, routineType, creationTime, lastModifiedTime, and language.
# File lib/google/cloud/bigquery/service.rb, line 334
def list_routines dataset_id, max: nil, token: nil, filter: nil
  # The list operation is considered idempotent
  execute backoff: true do
    service.list_routines @project, dataset_id, max_results: max,
                                                page_token:  token,
                                                filter:      filter
  end
end
list_tabledata(dataset_id, table_id, max: nil, token: nil, start: nil) click to toggle source

Retrieves data from the table.

# File lib/google/cloud/bigquery/service.rb, line 222
def list_tabledata dataset_id, table_id, max: nil, token: nil, start: nil
  # The list operation is considered idempotent
  execute backoff: true do
    json_txt = service.list_table_data \
      @project, dataset_id, table_id,
      max_results: max,
      page_token:  token,
      start_index: start,
      options:     { skip_deserialization: true }
    JSON.parse json_txt, symbolize_names: true
  end
end
list_tables(dataset_id, max: nil, token: nil) click to toggle source

Lists all tables in the specified dataset. Requires the READER dataset role.

# File lib/google/cloud/bigquery/service.rb, line 138
def list_tables dataset_id, max: nil, token: nil
  # The list operation is considered idempotent
  execute backoff: true do
    service.list_tables @project, dataset_id, max_results: max, page_token: token
  end
end
load_table_file(file, load_job_gapi) click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 453
def load_table_file file, load_job_gapi
  execute backoff: true do
    service.insert_job @project, load_job_gapi, upload_source: file, content_type: mime_type_for(file)
  end
end
load_table_gs_url(load_job_gapi) click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 447
def load_table_gs_url load_job_gapi
  execute backoff: true do
    service.insert_job @project, load_job_gapi
  end
end
patch_dataset(dataset_id, patched_dataset_gapi) click to toggle source

Updates information in an existing dataset, only replacing fields that are provided in the submitted dataset resource.

# File lib/google/cloud/bigquery/service.rb, line 110
def patch_dataset dataset_id, patched_dataset_gapi
  patch_with_backoff = false
  options = {}
  if patched_dataset_gapi.etag
    options[:header] = { "If-Match" => patched_dataset_gapi.etag }
    # The patch with etag operation is considered idempotent
    patch_with_backoff = true
  end
  execute backoff: patch_with_backoff do
    service.patch_dataset @project, dataset_id, patched_dataset_gapi, options: options
  end
end
patch_model(dataset_id, model_id, patched_model_gapi, etag = nil) click to toggle source

Updates information in an existing model, replacing fields that are provided in the submitted model resource.

# File lib/google/cloud/bigquery/service.rb, line 302
def patch_model dataset_id, model_id, patched_model_gapi, etag = nil
  patch_with_backoff = false
  options = { skip_deserialization: true }
  if etag
    options[:header] = { "If-Match" => etag }
    # The patch with etag operation is considered idempotent
    patch_with_backoff = true
  end
  execute backoff: patch_with_backoff do
    json_txt = service.patch_model @project, dataset_id, model_id, patched_model_gapi, options: options
    JSON.parse json_txt, symbolize_names: true
  end
end
patch_table(dataset_id, table_id, patched_table_gapi) click to toggle source

Updates information in an existing table, replacing fields that are provided in the submitted table resource.

# File lib/google/cloud/bigquery/service.rb, line 172
def patch_table dataset_id, table_id, patched_table_gapi
  patch_with_backoff = false
  options = {}
  if patched_table_gapi.etag
    options[:header] = { "If-Match" => patched_table_gapi.etag }
    # The patch with etag operation is considered idempotent
    patch_with_backoff = true
  end
  execute backoff: patch_with_backoff do
    service.patch_table @project, dataset_id, table_id, patched_table_gapi, options: options
  end
end
project_service_account() click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 78
def project_service_account
  service.get_project_service_account project
end
query_job(query_job_gapi) click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 415
def query_job query_job_gapi
  execute backoff: true do
    service.insert_job @project, query_job_gapi
  end
end
service() click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 55
def service
  return mocked_service if mocked_service
  @service ||= begin
    service = API::BigqueryService.new
    service.client_options.application_name    = "gcloud-ruby"
    service.client_options.application_version = Google::Cloud::Bigquery::VERSION
    service.client_options.open_timeout_sec = timeout
    service.client_options.read_timeout_sec = timeout
    service.client_options.send_timeout_sec = timeout
    service.request_options.retries = 0 # handle retries in #execute
    service.request_options.header ||= {}
    service.request_options.header["x-goog-api-client"] = \
      "gl-ruby/#{RUBY_VERSION} gccl/#{Google::Cloud::Bigquery::VERSION}"
    service.request_options.query ||= {}
    service.request_options.query["prettyPrint"] = false
    service.request_options.quota_project = @quota_project if @quota_project
    service.authorization = @credentials.client
    service.root_url = host if host
    service
  end
end
set_table_policy(dataset_id, table_id, new_policy) click to toggle source

@param [Google::Apis::BigqueryV2::Policy] new_policy

# File lib/google/cloud/bigquery/service.rb, line 197
def set_table_policy dataset_id, table_id, new_policy
  execute do
    service.set_table_iam_policy table_path(dataset_id, table_id),
                                 API::SetIamPolicyRequest.new(policy: new_policy)
  end
end
test_table_permissions(dataset_id, table_id, permissions) click to toggle source

Returns Google::Apis::BigqueryV2::TestIamPermissionsResponse

# File lib/google/cloud/bigquery/service.rb, line 206
def test_table_permissions dataset_id, table_id, permissions
  execute do
    service.test_table_iam_permissions table_path(dataset_id, table_id),
                                       API::TestIamPermissionsRequest.new(permissions: permissions)
  end
end
update_routine(dataset_id, routine_id, new_routine_gapi) click to toggle source

Updates information in an existing routine, replacing the entire routine resource.

# File lib/google/cloud/bigquery/service.rb, line 354
def update_routine dataset_id, routine_id, new_routine_gapi
  update_with_backoff = false
  options = {}
  if new_routine_gapi.etag
    options[:header] = { "If-Match" => new_routine_gapi.etag }
    # The update with etag operation is considered idempotent
    update_with_backoff = true
  end
  execute backoff: update_with_backoff do
    service.update_routine @project, dataset_id, routine_id, new_routine_gapi, options: options
  end
end

Protected Instance Methods

execute(backoff: nil) { || ... } click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 557
def execute backoff: nil, &block
  if backoff
    Backoff.new(retries: retries).execute(&block)
  else
    yield
  end
rescue Google::Apis::Error => e
  raise Google::Cloud::Error.from_error e
end
generate_id() click to toggle source

Generate a random string similar to the BigQuery service job IDs.

# File lib/google/cloud/bigquery/service.rb, line 545
def generate_id
  SecureRandom.urlsafe_base64 21
end
mime_type_for(file) click to toggle source
# File lib/google/cloud/bigquery/service.rb, line 549
def mime_type_for file
  mime_type = MiniMime.lookup_by_filename Pathname(file).to_path
  return nil if mime_type.nil?
  mime_type.content_type
rescue StandardError
  nil
end
table_path(dataset_id, table_id) click to toggle source

Creates a formatted table path.

# File lib/google/cloud/bigquery/service.rb, line 540
def table_path dataset_id, table_id
  "projects/#{@project}/datasets/#{dataset_id}/tables/#{table_id}"
end