Cloud Storage からの JSON データの読み込み

 using Google.Apis.Bigquery.v2.Data; using Google.Cloud.BigQuery.V2; using System;  public class BigQueryLoadTableGcsJson {     public void LoadTableGcsJson(         string projectId = "your-project-id",         string datasetId = "your_dataset_id"     )     {         BigQueryClient client = BigQueryClient.Create(projectId);         var gcsURI = "gs://cloud-samples-data/bigquery/us-states/us-states.json";         var dataset = client.GetDataset(datasetId);         var schema = new TableSchemaBuilder {             { "name", BigQueryDbType.String },             { "post_abbr", BigQueryDbType.String }         }.Build();         TableReference destinationTableRef = dataset.GetTableReference(             tableId: "us_states");         // Create job configuration         var jobOptions = new CreateLoadJobOptions()         {             SourceFormat = FileFormat.NewlineDelimitedJson         };         // Create and run job         BigQueryJob loadJob = client.CreateLoadJob(             sourceUri: gcsURI, destination: destinationTableRef,             schema: schema, options: jobOptions);         loadJob = loadJob.PollUntilCompleted().ThrowOnAnyError();  // Waits for the job to complete.         // Display the number of rows uploaded         BigQueryTable table = client.GetTable(destinationTableRef);         Console.WriteLine(             $"Loaded {table.Resource.NumRows} rows to {table.FullyQualifiedId}");     } }

import ( 	"context" 	"fmt"  	"cloud.google.com/go/bigquery" )  // importJSONExplicitSchema demonstrates loading newline-delimited JSON data from Cloud Storage // into a BigQuery table and providing an explicit schema for the data. func importJSONExplicitSchema(projectID, datasetID, tableID string) error { 	// projectID := "my-project-id" 	// datasetID := "mydataset" 	// tableID := "mytable" 	ctx := context.Background() 	client, err := bigquery.NewClient(ctx, projectID) 	if err != nil { 		return fmt.Errorf("bigquery.NewClient: %v", err) 	} 	defer client.Close()  	gcsRef := bigquery.NewGCSReference("gs://cloud-samples-data/bigquery/us-states/us-states.json") 	gcsRef.SourceFormat = bigquery.JSON 	gcsRef.Schema = bigquery.Schema{ 		{Name: "name", Type: bigquery.StringFieldType}, 		{Name: "post_abbr", Type: bigquery.StringFieldType}, 	} 	loader := client.Dataset(datasetID).Table(tableID).LoaderFrom(gcsRef) 	loader.WriteDisposition = bigquery.WriteEmpty  	job, err := loader.Run(ctx) 	if err != nil { 		return err 	} 	status, err := job.Wait(ctx) 	if err != nil { 		return err 	}  	if status.Err() != nil { 		return fmt.Errorf("job completed with error: %v", status.Err()) 	} 	return nil } 

import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FormatOptions; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobInfo; import com.google.cloud.bigquery.LoadJobConfiguration; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.TableId;  // Sample to load JSON data from Cloud Storage into a new BigQuery table public class LoadJsonFromGCS {    public static void runLoadJsonFromGCS() {     // TODO(developer): Replace these variables before running the sample.     String datasetName = "MY_DATASET_NAME";     String tableName = "MY_TABLE_NAME";     String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.json";     Schema schema =         Schema.of(             Field.of("name", StandardSQLTypeName.STRING),             Field.of("post_abbr", StandardSQLTypeName.STRING));     loadJsonFromGCS(datasetName, tableName, sourceUri, schema);   }    public static void loadJsonFromGCS(       String datasetName, String tableName, String sourceUri, Schema schema) {     try {       // Initialize client that will be used to send requests. This client only needs to be created       // once, and can be reused for multiple requests.       BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();        TableId tableId = TableId.of(datasetName, tableName);       LoadJobConfiguration loadConfig =           LoadJobConfiguration.newBuilder(tableId, sourceUri)               .setFormatOptions(FormatOptions.json())               .setSchema(schema)               .build();        // Load data from a GCS JSON file into the table       Job job = bigquery.create(JobInfo.of(loadConfig));       // Blocks until this load table job completes its execution, either failing or succeeding.       job = job.waitFor();       if (job.isDone()) {         System.out.println("Json from GCS successfully loaded in a table");       } else {         System.out.println(             "BigQuery was unable to load into the table due to an error:"                 + job.getStatus().getError());       }     } catch (BigQueryException | InterruptedException e) {       System.out.println("Column not added during load append \n" + e.toString());     }   } }

// Import the Google Cloud client libraries const {BigQuery} = require('@google-cloud/bigquery'); const {Storage} = require('@google-cloud/storage');  // Instantiate clients const bigquery = new BigQuery(); const storage = new Storage();  /**  * This sample loads the json file at  * https://storage.googleapis.com/cloud-samples-data/bigquery/us-states/us-states.json  *  * TODO(developer): Replace the following lines with the path to your file.  */ const bucketName = 'cloud-samples-data'; const filename = 'bigquery/us-states/us-states.json';  async function loadJSONFromGCS() {   // Imports a GCS file into a table with manually defined schema.    /**    * TODO(developer): Uncomment the following lines before running the sample.    */   // const datasetId = "my_dataset";   // const tableId = "my_table";    // Configure the load job. For full list of options, see:   // https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad   const metadata = {     sourceFormat: 'NEWLINE_DELIMITED_JSON',     schema: {       fields: [         {name: 'name', type: 'STRING'},         {name: 'post_abbr', type: 'STRING'},       ],     },     location: 'US',   };    // Load data from a Google Cloud Storage file into the table   const [job] = await bigquery     .dataset(datasetId)     .table(tableId)     .load(storage.bucket(bucketName).file(filename), metadata);   // load() waits for the job to finish   console.log(`Job ${job.id} completed.`);    // Check the job's status for errors   const errors = job.status.errors;   if (errors && errors.length > 0) {     throw errors;   } }

use Google\Cloud\BigQuery\BigQueryClient; use Google\Cloud\Core\ExponentialBackoff;  /** Uncomment and populate these variables in your code */ // $projectId  = 'The Google project ID'; // $datasetId  = 'The BigQuery dataset ID';  // instantiate the bigquery table service $bigQuery = new BigQueryClient([     'projectId' => $projectId, ]); $dataset = $bigQuery->dataset($datasetId); $table = $dataset->table('us_states');  // create the import job $gcsUri = 'gs://cloud-samples-data/bigquery/us-states/us-states.json'; $schema = [     'fields' => [         ['name' => 'name', 'type' => 'string'],         ['name' => 'post_abbr', 'type' => 'string']     ] ]; $loadConfig = $table->loadFromStorage($gcsUri)->schema($schema)->sourceFormat('NEWLINE_DELIMITED_JSON'); $job = $table->runJob($loadConfig); // poll the job until it is complete $backoff = new ExponentialBackoff(10); $backoff->execute(function () use ($job) {     print('Waiting for job to complete' . PHP_EOL);     $job->reload();     if (!$job->isComplete()) {         throw new Exception('Job has not yet completed', 500);     } }); // check if the job has errors if (isset($job->info()['status']['errorResult'])) {     $error = $job->info()['status']['errorResult']['message'];     printf('Error running job: %s' . PHP_EOL, $error); } else {     print('Data imported successfully' . PHP_EOL); }

from google.cloud import bigquery  # Construct a BigQuery client object. client = bigquery.Client()  # TODO(developer): Set table_id to the ID of the table to create. # table_id = "your-project.your_dataset.your_table_name"  job_config = bigquery.LoadJobConfig(     schema=[         bigquery.SchemaField("name", "STRING"),         bigquery.SchemaField("post_abbr", "STRING"),     ],     source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON, ) uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json"  load_job = client.load_table_from_uri(     uri,     table_id,     location="US",  # Must match the destination dataset location.     job_config=job_config, )  # Make an API request.  load_job.result()  # Waits for the job to complete.  destination_table = client.get_table(table_id) print("Loaded {} rows.".format(destination_table.num_rows))

require "google/cloud/bigquery"  def load_table_gcs_json dataset_id = "your_dataset_id"   bigquery = Google::Cloud::Bigquery.new   dataset  = bigquery.dataset dataset_id   gcs_uri  = "gs://cloud-samples-data/bigquery/us-states/us-states.json"   table_id = "us_states"    load_job = dataset.load_job table_id, gcs_uri, format: "json" do |schema|     schema.string "name"     schema.string "post_abbr"   end   puts "Starting job #{load_job.job_id}"    load_job.wait_until_done! # Waits for table load to complete.   puts "Job finished."    table = dataset.table table_id   puts "Loaded #{table.rows_count} rows to table #{table.id}" end

import ( 	"context" 	"fmt"  	"cloud.google.com/go/bigquery" )  // importJSONTruncate demonstrates loading data from newline-delimeted JSON data in Cloud Storage // and overwriting/truncating data in the existing table. func importJSONTruncate(projectID, datasetID, tableID string) error { 	// projectID := "my-project-id" 	// datasetID := "mydataset" 	// tableID := "mytable" 	ctx := context.Background() 	client, err := bigquery.NewClient(ctx, projectID) 	if err != nil { 		return fmt.Errorf("bigquery.NewClient: %v", err) 	} 	defer client.Close()  	gcsRef := bigquery.NewGCSReference("gs://cloud-samples-data/bigquery/us-states/us-states.json") 	gcsRef.SourceFormat = bigquery.JSON 	gcsRef.AutoDetect = true 	loader := client.Dataset(datasetID).Table(tableID).LoaderFrom(gcsRef) 	loader.WriteDisposition = bigquery.WriteTruncate  	job, err := loader.Run(ctx) 	if err != nil { 		return err 	} 	status, err := job.Wait(ctx) 	if err != nil { 		return err 	}  	if status.Err() != nil { 		return fmt.Errorf("job completed with error: %v", status.Err()) 	}  	return nil } 

import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FormatOptions; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobInfo; import com.google.cloud.bigquery.LoadJobConfiguration; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.TableId;  // Sample to overwrite the BigQuery table data by loading a JSON file from GCS public class LoadJsonFromGCSTruncate {    public static void runLoadJsonFromGCSTruncate() {     // TODO(developer): Replace these variables before running the sample.     String datasetName = "MY_DATASET_NAME";     String tableName = "MY_TABLE_NAME";     String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.json";     Schema schema =         Schema.of(             Field.of("name", StandardSQLTypeName.STRING),             Field.of("post_abbr", StandardSQLTypeName.STRING));     loadJsonFromGCSTruncate(datasetName, tableName, sourceUri, schema);   }    public static void loadJsonFromGCSTruncate(       String datasetName, String tableName, String sourceUri, Schema schema) {     try {       // Initialize client that will be used to send requests. This client only needs to be created       // once, and can be reused for multiple requests.       BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();        TableId tableId = TableId.of(datasetName, tableName);       LoadJobConfiguration loadConfig =           LoadJobConfiguration.newBuilder(tableId, sourceUri)               .setFormatOptions(FormatOptions.json())               // Set the write disposition to overwrite existing table data               .setWriteDisposition(JobInfo.WriteDisposition.WRITE_TRUNCATE)               .setSchema(schema)               .build();        // Load data from a GCS JSON file into the table       Job job = bigquery.create(JobInfo.of(loadConfig));       // Blocks until this load table job completes its execution, either failing or succeeding.       job = job.waitFor();       if (job.isDone()) {         System.out.println("Table is successfully overwritten by JSON file loaded from GCS");       } else {         System.out.println(             "BigQuery was unable to load into the table due to an error:"                 + job.getStatus().getError());       }     } catch (BigQueryException | InterruptedException e) {       System.out.println("Column not added during load append \n" + e.toString());     }   } }

// Import the Google Cloud client libraries const {BigQuery} = require('@google-cloud/bigquery'); const {Storage} = require('@google-cloud/storage');  // Instantiate clients const bigquery = new BigQuery(); const storage = new Storage();  /**  * This sample loads the JSON file at  * https://storage.googleapis.com/cloud-samples-data/bigquery/us-states/us-states.json  *  * TODO(developer): Replace the following lines with the path to your file.  */ const bucketName = 'cloud-samples-data'; const filename = 'bigquery/us-states/us-states.json';  async function loadJSONFromGCSTruncate() {   /**    * Imports a GCS file into a table and overwrites    * table data if table already exists.    */    /**    * TODO(developer): Uncomment the following lines before running the sample.    */   // const datasetId = "my_dataset";   // const tableId = "my_table";    // Configure the load job. For full list of options, see:   // https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad   const metadata = {     sourceFormat: 'NEWLINE_DELIMITED_JSON',     schema: {       fields: [         {name: 'name', type: 'STRING'},         {name: 'post_abbr', type: 'STRING'},       ],     },     // Set the write disposition to overwrite existing table data.     writeDisposition: 'WRITE_TRUNCATE',   };    // Load data from a Google Cloud Storage file into the table   const [job] = await bigquery     .dataset(datasetId)     .table(tableId)     .load(storage.bucket(bucketName).file(filename), metadata);   // load() waits for the job to finish   console.log(`Job ${job.id} completed.`);    // Check the job's status for errors   const errors = job.status.errors;   if (errors && errors.length > 0) {     throw errors;   } }

use Google\Cloud\BigQuery\BigQueryClient; use Google\Cloud\Core\ExponentialBackoff;  /** Uncomment and populate these variables in your code */ // $projectId = 'The Google project ID'; // $datasetId = 'The BigQuery dataset ID'; // $tableID = 'The BigQuery table ID';  // instantiate the bigquery table service $bigQuery = new BigQueryClient([     'projectId' => $projectId, ]); $table = $bigQuery->dataset($datasetId)->table($tableId);  // create the import job $gcsUri = 'gs://cloud-samples-data/bigquery/us-states/us-states.json'; $loadConfig = $table->loadFromStorage($gcsUri)->sourceFormat('NEWLINE_DELIMITED_JSON')->writeDisposition('WRITE_TRUNCATE'); $job = $table->runJob($loadConfig);  // poll the job until it is complete $backoff = new ExponentialBackoff(10); $backoff->execute(function () use ($job) {     print('Waiting for job to complete' . PHP_EOL);     $job->reload();     if (!$job->isComplete()) {         throw new Exception('Job has not yet completed', 500);     } });  // check if the job has errors if (isset($job->info()['status']['errorResult'])) {     $error = $job->info()['status']['errorResult']['message'];     printf('Error running job: %s' . PHP_EOL, $error); } else {     print('Data imported successfully' . PHP_EOL); }

import io  from google.cloud import bigquery  # Construct a BigQuery client object. client = bigquery.Client()  # TODO(developer): Set table_id to the ID of the table to create. # table_id = "your-project.your_dataset.your_table_name  job_config = bigquery.LoadJobConfig(     schema=[         bigquery.SchemaField("name", "STRING"),         bigquery.SchemaField("post_abbr", "STRING"),     ], )  body = io.BytesIO(b"Washington,WA") client.load_table_from_file(body, table_id, job_config=job_config).result() previous_rows = client.get_table(table_id).num_rows assert previous_rows > 0  job_config = bigquery.LoadJobConfig(     write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,     source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON, )  uri = "gs://cloud-samples-data/bigquery/us-states/us-states.json" load_job = client.load_table_from_uri(     uri, table_id, job_config=job_config )  # Make an API request.  load_job.result()  # Waits for the job to complete.  destination_table = client.get_table(table_id) print("Loaded {} rows.".format(destination_table.num_rows))

require "google/cloud/bigquery"  def load_table_gcs_json_truncate dataset_id = "your_dataset_id",                                  table_id   = "your_table_id"   bigquery = Google::Cloud::Bigquery.new   dataset  = bigquery.dataset dataset_id   gcs_uri  = "gs://cloud-samples-data/bigquery/us-states/us-states.json"    load_job = dataset.load_job table_id,                               gcs_uri,                               format: "json",                               write:  "truncate"   puts "Starting job #{load_job.job_id}"    load_job.wait_until_done! # Waits for table load to complete.   puts "Job finished."    table = dataset.table table_id   puts "Loaded #{table.rows_count} rows to table #{table.id}" end