将 Cloud Storage 中的 ORC 文件加载到新表中。
深入探索
如需查看包含此代码示例的详细文档,请参阅以下内容:
代码示例
C#
试用此示例之前,请按照 BigQuery 快速入门:使用客户端库中的 C# 设置说明进行操作。 如需了解详情,请参阅 BigQuery C# API 参考文档。
如需向 BigQuery 进行身份验证,请设置应用默认凭据。 如需了解详情,请参阅为客户端库设置身份验证。
using Google.Apis.Bigquery.v2.Data;
using Google.Cloud.BigQuery.V2;
using System;
public class BigQueryLoadTableGcsOrc
{
public void LoadTableGcsOrc(
string projectId = "your-project-id",
string datasetId = "your_dataset_id"
)
{
BigQueryClient client = BigQueryClient.Create(projectId);
var gcsURI = "gs://cloud-samples-data/bigquery/us-states/us-states.orc";
var dataset = client.GetDataset(datasetId);
TableReference destinationTableRef = dataset.GetTableReference(
tableId: "us_states");
// Create job configuration
var jobOptions = new CreateLoadJobOptions()
{
SourceFormat = FileFormat.Orc
};
// Create and run job
var loadJob = client.CreateLoadJob(
sourceUri: gcsURI,
destination: destinationTableRef,
// Pass null as the schema because the schema is inferred when
// loading Orc data
schema: null,
options: jobOptions
);
loadJob = loadJob.PollUntilCompleted().ThrowOnAnyError(); // Waits for the job to complete.
// Display the number of rows uploaded
BigQueryTable table = client.GetTable(destinationTableRef);
Console.WriteLine(
$Loa"ded {table.Resource.NumRows} rows to {table.FullyQualifiedId});
" }
}
Go
试用此示例之前,请按照 BigQuery 快速入门:使用客户端库中的 Go 设置说明进行操作。 如需了解详情,请参阅 BigQuery Go API 参考文档。
如需向 BigQuery 进行身份验证,请设置应用默认凭据。 如需了解详情,请参阅为客户端库设置身份验证。
import (
"context"
"fmt"
"cloud.google.com/go/bigquery"
)
// importORC demonstrates loading Apache ORC data from Cloud Storage into a table.
func importORC(projectID, datasetID, tableID string) error {
// projectID := "my-project-id"
// datasetID := "mydataset"
// tableID := "mytable"
ctx := context.Background()
client, err := bigquery.NewClient(ctx, projectID)
if err != nil {
return fmt.Errorf("bigquery.NewClient: %w", err)
}
defer client.Close()
gcsRef := bigquery.NewGCSReference("gs://cloud-samples-data/bigquery/us-states/us-states.orc")
gcsRef.SourceFormat = bigquery.ORC
loader := client.Dataset(datasetID).Table(tableID).LoaderFrom(gcsRef)
job, err := loader.Run(ctx)
if err != nil {
return err
}
status, err := job.Wait(ctx)
if err != nil {
return err
}
if status.Err() != nil {
return fmt.Errorf("job completed with error: %w", status.Err())
}
return nil
}
Java
试用此示例之前,请按照 BigQuery 快速入门:使用客户端库中的 Java 设置说明进行操作。 如需了解详情,请参阅 BigQuery Java API 参考文档。
如需向 BigQuery 进行身份验证,请设置应用默认凭据。 如需了解详情,请参阅为客户端库设置身份验证。
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.BigQueryOptions;
import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.FormatOptions;
import com.google.cloud.bigquery.Job;
import com.google.cloud.bigquery.JobInfo;
import com.google.cloud.bigquery.LoadJobConfiguration;
import com.google.cloud.bigquery.Schema;
import com.google.cloud.bigquery.StandardSQLTypeName;
import com.google.cloud.bigquery.TableId;
// Sample to load ORC data from Cloud Storage into a new BigQuery table
public class LoadOrcFromGcs {
public static void main(String[] args) {
// TODO(developer): Replace these variables before running the sample.
String datasetName = "MY_DATASET_NAME";
String tableName = "MY_TABLE_NAME";
String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.orc";
Schema schema =
Schema.of(
Field.of("name", StandardSQLTypeName.STRING),
Field.of("post_abbr", StandardSQLTypeName.STRING));
loadOrcFromGcs(datasetName, tableName, sourceUri, schema);
}
public static void loadOrcFromGcs(
String datasetName, String tableName, String sourceUri, Schema schema) {
try {
// Initialize client that will be used to send requests. This client only needs to be created
// once, and can be reused for multiple requests.
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
TableId tableId = TableId.of(datasetName, tableName);
LoadJobConfiguration loadConfig =
LoadJobConfiguration.newBuilder(tableId, sourceUri, FormatOptions.orc())
.setSchema(schema)
.build();
// Load data from a GCS ORC file into the table
Job job = bigquery.create(JobInfo.of(loadConfig));
// Blocks until this load table job completes its execution, either failing or succeeding.
job = job.waitFor();
if (job.isDone() && job.getStatus().getError() == null) {
System.out.println("ORC from GCS successfully added during load append job");
} else {
System.out.println(
"BigQuery was unable to load into the table due to an error:"
job.getStatus().getError());
}
} catch (BigQueryException | InterruptedException e) {
System.out.println("Column not added during load append \n" e.toString());
}
}
}
Node.js
试用此示例之前,请按照 BigQuery 快速入门:使用客户端库中的 Node.js 设置说明进行操作。 如需了解详情,请参阅 BigQuery Node.js API 参考文档。
如需向 BigQuery 进行身份验证,请设置应用默认凭据。 如需了解详情,请参阅为客户端库设置身份验证。
// Import the Google Cloud client libraries
const {BigQuery} = require('@google-cloud/bigquery');
const {Storage} = require('@google-cloud/storage');
// Instantiate clients
const bigquery = new BigQuery();
const storage = new Storage();
/**
* This sample loads the ORC file at
* https://storage.googleapis.com/cloud-samples-data/bigquery/us-states/us-states.orc
*
* TODO(developer): Replace the following lines with the path to your file.
*/
const bucketName = 'cloud-samples-data';
const filename = 'bigquery/us-states/us-states.orc';
async function loadTableGCSORC() {
// Imports a GCS file into a table with ORC source format.
/**
* TODO(developer): Uncomment the following line before running the sample.
*/
// const datasetId = 'my_dataset';
// const tableId = 'my_table'
// Configure the load job. For full list of options, see:
// https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad
const metadata = {
sourceFormat: 'ORC',
location: 'US',
};
// Load data from a Google Cloud Storage file into the table
const [job] = await bigquery
.dataset(datasetId)
.table(tableId)
.load(storage.bucket(bucketName).file(filename), metadata);
// load() waits for the job to finish
console.log(`Job ${job.id} completed.`);
}
PHP
试用此示例之前,请按照 BigQuery 快速入门:使用客户端库中的 PHP 设置说明进行操作。 如需了解详情,请参阅 BigQuery PHP API 参考文档。
如需向 BigQuery 进行身份验证,请设置应用默认凭据。 如需了解详情,请参阅为客户端库设置身份验证。
use Google\Cloud\BigQuery\BigQueryClient;
/**
* Import data from storage orc.
*
* @param string $projectId The project Id of your Google Cloud Project.
* @param string $datasetId The BigQuery dataset ID.
* @param string $tableId The BigQuery table ID.
*/
function import_from_storage_orc(
string $projectId,
string $datasetId,
string $tableId = 'us_states'
): void {
// instantiate the bigquery table service
$bigQuery = new BigQueryClient([
'projectId' => $projectId,
]);
$dataset = $bigQuery->dataset($datasetId);
$table = $dataset->table($tableId);
// create the import job
$gcsUri = 'gs://cloud-samples-data/bigquery/us-states/us-states.orc';
$loadConfig = $table->loadFromStorage($gcsUri)->sourceFormat('ORC');
$job = $table->runJob($loadConfig);
// check if the job is complete
$job->reload();
if (!$job->isComplete()) {
throw new \Exception('Job has not yet completed', 500);
}
// check if the job has errors
if (isset($job->info()['status']['errorResult'])) {
$error = $job->info()['status']['errorResult']['message'];
printf('Error running job: %s' . PHP_EOL, $error);
} else {
print('Data imported successfully' . PHP_EOL);
}
}
Python
试用此示例之前,请按照 BigQuery 快速入门:使用客户端库中的 Python 设置说明进行操作。 如需了解详情,请参阅 BigQuery Python API 参考文档。
如需向 BigQuery 进行身份验证,请设置应用默认凭据。 如需了解详情,请参阅为客户端库设置身份验证。
from google.cloud import bigquery
# Construct a BigQuery client object.
client = bigquery.Client()
# TODO(developer): Set table_id to the ID of the table to create.
# table_id = "your-project.your_dataset.your_table_name
job_config = bigquery.LoadJobConfig(source_format=bigquery.SourceFormat.ORC)
uri = "gs://cloud-samples-data/bigquery/us-states/us-states.orc"
load_job = client.load_table_from_uri(
uri, table_id, job_config=job_config
) # Make an API request.
load_job.result() # Waits for the job to complete.
destination_table = client.get_table(table_id)
print("Loaded {} rows.".format(destination_table.num_rows))
Ruby
试用此示例之前,请按照 BigQuery 快速入门:使用客户端库中的 Ruby 设置说明进行操作。 如需了解详情,请参阅 BigQuery Ruby API 参考文档。
如需向 BigQuery 进行身份验证,请设置应用默认凭据。 如需了解详情,请参阅为客户端库设置身份验证。
require "google/cloud/bigquery"
def load_table_gcs_orc dataset_id = "your_dataset_id"
bigquery = Google::Cloud::Bigquery.new
dataset = bigquery.dataset dataset_id
gcs_uri = "gs://cloud-samples-data/bigquery/us-states/us-states.orc"
table_id = "us_states"
load_job = dataset.load_job table_id, gcs_uri, format: "orc"
puts "Starting job #{load_job.job_id}"
load_job.wait_until_done! # Waits for table load to complete.
puts "Job finished."
table = dataset.table table_id
puts "Loaded #{table.rows_count} rows to table #{table.id}"
end
后续步骤
如需搜索和过滤其他 Google Cloud 产品的代码示例,请参阅 Google Cloud 示例浏览器。