Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
ab0513f
Add Iceberg Catalog for Google BigLake Metastore
Apr 22, 2023
4f1bce1
fix test errors in style check
Apr 25, 2023
3a9948b
removed hadoop conf dependency, fix styles and tests
Apr 28, 2023
c53b9b8
update biglake config names
Apr 28, 2023
4936b16
Merge branch 'apache:master' into biglake
May 11, 2023
2d8dd2c
fix review comments
May 11, 2023
5ddbb14
minor fix BigLake client error message
May 15, 2023
4fcc500
Merge branch 'apache:master' into biglake
May 30, 2023
0b1c479
Merge branch 'apache:master' into biglake
Jun 6, 2023
3fa9e9e
add whitespace after control blocks
Jun 6, 2023
b38a82f
add negative tests of testNamespaceWithSlash and testTableNameWithSlash
Jun 6, 2023
1be35ff
use mocked BigLake gRPC service
Jun 6, 2023
836d8f5
runs a grpc server per test to avoid interference
Jun 7, 2023
ea5a55a
Merge branch 'apache:master' into biglake
Jun 9, 2023
592eb63
fix :iceberg-gcp:checkstyleMain failure
Jun 9, 2023
54e33a4
Merge branch 'apache:master' into biglake
Jun 15, 2023
44e8656
Merge branch 'apache:master' into biglake
Jun 19, 2023
ac57328
Merge branch 'apache:master' into biglake
Jun 23, 2023
1b25736
Merge branch 'apache:master' into biglake
Jul 11, 2023
c875ed3
fix style issues from review
Jul 11, 2023
425d9a8
fix more review comments
Jul 12, 2023
c8b6726
Merge branch 'apache:master' into biglake
Jul 12, 2023
2a3878f
fix more comments on tests
Jul 13, 2023
7274769
Merge branch 'apache:master' into biglake
Jul 15, 2023
79ddc3d
fix more review comments
Jul 16, 2023
467b3bb
fix more review comments.
Jul 16, 2023
22d6d0a
Merge branch 'apache:master' into biglake
Jul 16, 2023
a45a256
Merge branch 'apache:master' into biglake
Jul 18, 2023
59d1280
fix more review comments
Jul 18, 2023
abc420a
fix more review comments
Jul 18, 2023
93e6bd2
Merge branch 'apache:master' into biglake
Jul 18, 2023
28d9dfc
fix more review comment
Jul 19, 2023
060a331
Merge branch 'apache:master' into biglake
Jul 19, 2023
ce48844
Merge branch 'master' into biglake
Jul 21, 2023
c17d205
Merge branch 'apache:master' into biglake
Jul 23, 2023
2193f2f
Merge branch 'apache:master' into biglake
Jul 25, 2023
23c1dc2
fix review comments
Jul 25, 2023
a63ebda
Merge branch 'apache:master' into biglake
Jul 27, 2023
ebfd0ea
fix merge conflict
Aug 5, 2023
19128c5
Merge branch 'master' into biglake
Fokko Aug 30, 2023
4606845
Merge branch 'apache:master' into biglake
Oct 3, 2023
afcdd6e
fix review comment
Oct 3, 2023
469b09e
fix an extra space
Oct 3, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -624,20 +624,30 @@ project(':iceberg-gcp') {
implementation project(':iceberg-common')
implementation project(':iceberg-core')

compileOnly platform(libs.google.libraries.bom)
compileOnly "com.google.cloud:google-cloud-storage"

testImplementation "com.google.cloud:google-cloud-nio"
implementation platform(libs.google.libraries.bom)
implementation 'com.google.cloud:google-cloud-storage'
implementation libs.google.cloud.biglake
implementation 'com.google.cloud:google-cloud-nio'
compileOnly(libs.hadoop2.common) {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
exclude group: 'javax.servlet', module: 'servlet-api'
exclude group: 'com.google.code.gson', module: 'gson'
}

testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')

testImplementation libs.esotericsoftware.kryo
testImplementation libs.google.cloud.biglake.grpc
testImplementation libs.google.cloud.nio
testImplementation 'com.google.api:gax-grpc:2.28.1:testlib'
testImplementation(libs.hadoop2.common) {
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
exclude group: 'javax.servlet', module: 'servlet-api'
exclude group: 'com.google.code.gson', module: 'gson'
}
testImplementation libs.esotericsoftware.kryo
}
}

Expand Down
12 changes: 12 additions & 0 deletions gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.iceberg.util.PropertyUtil;

public class GCPProperties implements Serializable {
// For Google Cloud Storage (GCS).
// Service Options
public static final String GCS_PROJECT_ID = "gcs.project-id";
public static final String GCS_CLIENT_LIB_TOKEN = "gcs.client-lib-token";
Expand All @@ -49,6 +50,17 @@ public class GCPProperties implements Serializable {
*/
public static final int GCS_DELETE_BATCH_SIZE_DEFAULT = 50;

// The GCP project ID. Required.
public static final String PROJECT_ID = "project-id";

// The GCP region (https://cloud.google.com/bigquery/docs/locations). Required.
public static final String REGION = "region";

// The BigLake Metastore catalog ID. It is the container resource of databases and tables.
// It links a BLMS catalog with this Iceberg catalog.
// Optional, default to the Spark catalog plugin name.
public static final String BIGLAKE_CATALOG_ID = "catalog-id";

private String projectId;
private String clientLibToken;
private String serviceHost;
Expand Down
Loading