Skip to content

Commit 3810366

Browse files
docs(samples): add load CSV from GCS sample (#426)
1 parent 342573a commit 3810366

File tree

2 files changed

+156
-0
lines changed

2 files changed

+156
-0
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquery;
18+
19+
// [START bigquery_load_table_gcs_csv]
20+
import com.google.cloud.bigquery.BigQuery;
21+
import com.google.cloud.bigquery.BigQueryException;
22+
import com.google.cloud.bigquery.BigQueryOptions;
23+
import com.google.cloud.bigquery.FormatOptions;
24+
import com.google.cloud.bigquery.Job;
25+
import com.google.cloud.bigquery.Table;
26+
27+
// Sample to load CSV data from Cloud Storage into a new BigQuery table
28+
public class LoadCsvFromGcs {
29+
30+
public static void runLoadCsvFromGcs() throws Exception {
31+
// TODO(developer): Replace these variables before running the sample.
32+
String datasetName = "MY_DATASET_NAME";
33+
String tableName = "MY_TABLE_NAME";
34+
String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv";
35+
loadCsvFromGcs(datasetName, tableName, sourceUri);
36+
}
37+
38+
public static void loadCsvFromGcs(String datasetName, String tableName, String sourceUri)
39+
throws Exception {
40+
try {
41+
// Initialize client that will be used to send requests. This client only needs to be created
42+
// once, and can be reused for multiple requests.
43+
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
44+
45+
Table table = bigquery.getTable(datasetName, tableName);
46+
Job loadJob = table.load(FormatOptions.csv(), sourceUri);
47+
48+
// Load data from a GCS parquet file into the table
49+
// Blocks until this load table job completes its execution, either failing or succeeding.
50+
Job completedJob = loadJob.waitFor();
51+
52+
// Check for errors
53+
if (completedJob == null) {
54+
throw new Exception("Job not executed since it no longer exists.");
55+
} else if (completedJob.getStatus().getError() != null) {
56+
// You can also look at queryJob.getStatus().getExecutionErrors() for all
57+
// errors, not just the latest one.
58+
throw new Exception(
59+
"BigQuery was unable to load into the table due to an error: \n"
60+
+ loadJob.getStatus().getError());
61+
}
62+
System.out.println("CSV from GCS successfully added during load append job");
63+
} catch (BigQueryException | InterruptedException e) {
64+
System.out.println("Column not added during load append \n" + e.toString());
65+
}
66+
}
67+
}
68+
// [END bigquery_load_table_gcs_csv]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquery;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
import static junit.framework.TestCase.assertNotNull;
21+
22+
import com.google.cloud.bigquery.Field;
23+
import com.google.cloud.bigquery.LegacySQLTypeName;
24+
import com.google.cloud.bigquery.Schema;
25+
import java.io.ByteArrayOutputStream;
26+
import java.io.PrintStream;
27+
import java.util.UUID;
28+
import org.junit.After;
29+
import org.junit.Before;
30+
import org.junit.BeforeClass;
31+
import org.junit.Test;
32+
33+
public class LoadCsvFromGcsIT {
34+
35+
private String tableName;
36+
private ByteArrayOutputStream bout;
37+
private PrintStream out;
38+
39+
private static final String BIGQUERY_DATASET_NAME = requireEnvVar("BIGQUERY_DATASET_NAME");
40+
41+
private static String requireEnvVar(String varName) {
42+
String value = System.getenv(varName);
43+
assertNotNull(
44+
"Environment variable " + varName + " is required to perform these tests.",
45+
System.getenv(varName));
46+
return value;
47+
}
48+
49+
@BeforeClass
50+
public static void checkRequirements() {
51+
requireEnvVar("BIGQUERY_DATASET_NAME");
52+
}
53+
54+
@Before
55+
public void setUp() {
56+
bout = new ByteArrayOutputStream();
57+
out = new PrintStream(bout);
58+
System.setOut(out);
59+
60+
// Create a test table
61+
tableName = "loadCsvFromGcs_TEST_" + UUID.randomUUID().toString().replace('-', '_');
62+
63+
Schema schema =
64+
Schema.of(
65+
Field.of("name", LegacySQLTypeName.STRING),
66+
Field.of("post_abbr", LegacySQLTypeName.STRING));
67+
68+
CreateTable.createTable(BIGQUERY_DATASET_NAME, tableName, schema);
69+
70+
bout = new ByteArrayOutputStream();
71+
out = new PrintStream(bout);
72+
System.setOut(out);
73+
}
74+
75+
@After
76+
public void tearDown() {
77+
// Clean up
78+
DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName);
79+
System.setOut(null);
80+
}
81+
82+
@Test
83+
public void loadCsvFromGcs() throws Exception {
84+
String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.csv";
85+
LoadCsvFromGcs.loadCsvFromGcs(BIGQUERY_DATASET_NAME, tableName, sourceUri);
86+
assertThat(bout.toString()).contains("CSV from GCS successfully added during load append job");
87+
}
88+
}

0 commit comments

Comments
 (0)