Skip to content

Commit

Permalink
feat: Add APIs to enable hot backups (#2313)
Browse files Browse the repository at this point in the history
Co-authored-by: Derek Lee <[email protected]>
  • Loading branch information
DerekLeeCS and Derek Lee authored Sep 12, 2024
1 parent c618969 commit 6d004cd
Show file tree
Hide file tree
Showing 8 changed files with 395 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,56 @@ public com.google.bigtable.admin.v2.Backup.State toProto() {
}
}

public enum BackupType {
/** Not specified. */
BACKUP_TYPE_UNSPECIFIED(com.google.bigtable.admin.v2.Backup.BackupType.BACKUP_TYPE_UNSPECIFIED),

/**
* The default type for Cloud Bigtable managed backups. Supported for backups created in both
* HDD and SSD instances. Requires optimization when restored to a table in an SSD instance.
*/
STANDARD(com.google.bigtable.admin.v2.Backup.BackupType.STANDARD),
/**
* A backup type with faster restore to SSD performance. Only supported for backups created in
* SSD instances. A new SSD table restored from a hot backup reaches production performance more
* quickly than a standard backup.
*/
HOT(com.google.bigtable.admin.v2.Backup.BackupType.HOT),

/** The backup type of the backup is not known by this client. Please upgrade your client. */
UNRECOGNIZED(com.google.bigtable.admin.v2.Backup.BackupType.UNRECOGNIZED);

private final com.google.bigtable.admin.v2.Backup.BackupType proto;

BackupType(com.google.bigtable.admin.v2.Backup.BackupType proto) {
this.proto = proto;
}

/**
* Wraps the protobuf. This method is considered an internal implementation detail and not meant
* to be used by applications.
*/
@InternalApi
public static Backup.BackupType fromProto(
com.google.bigtable.admin.v2.Backup.BackupType proto) {
for (Backup.BackupType backupType : values()) {
if (backupType.proto.equals(proto)) {
return backupType;
}
}
return BACKUP_TYPE_UNSPECIFIED;
}

/**
* Creates the request protobuf. This method is considered an internal implementation detail and
* not meant to be used by applications.
*/
@InternalApi
public com.google.bigtable.admin.v2.Backup.BackupType toProto() {
return proto;
}
}

@Nonnull private final com.google.bigtable.admin.v2.Backup proto;
@Nonnull private final String id;
@Nonnull private final String instanceId;
Expand Down Expand Up @@ -147,6 +197,20 @@ public State getState() {
return State.fromProto(proto.getState());
}

/** Get the backup type of this backup. */
public BackupType getBackupType() {
return BackupType.fromProto(proto.getBackupType());
}

/** Get the time at which this backup will be converted from a hot backup to a standard backup. */
@Nullable
public Instant getHotToStandardTime() {
if (proto.hasHotToStandardTime()) {
return Instant.ofEpochMilli(Timestamps.toMillis(proto.getHotToStandardTime()));
}
return null;
}

/**
* Get the encryption information for the backup.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,23 @@ public CreateBackupRequest setExpireTime(Instant expireTime) {
return this;
}

public CreateBackupRequest setBackupType(Backup.BackupType backupType) {
Preconditions.checkNotNull(backupType);
requestBuilder.getBackupBuilder().setBackupType(backupType.toProto());
return this;
}

// The time at which this backup will be converted from a hot backup to a standard backup. Only
// applicable for hot backups. If not set, the backup will remain as a hot backup until it is
// deleted.
public CreateBackupRequest setHotToStandardTime(Instant hotToStandardTime) {
Preconditions.checkNotNull(hotToStandardTime);
requestBuilder
.getBackupBuilder()
.setHotToStandardTime(Timestamps.fromMillis(hotToStandardTime.toEpochMilli()));
return this;
}

@Override
public boolean equals(Object o) {
if (this == o) {
Expand All @@ -69,12 +86,23 @@ public boolean equals(Object o) {
CreateBackupRequest that = (CreateBackupRequest) o;
return Objects.equal(requestBuilder.getBackupId(), that.requestBuilder.getBackupId())
&& Objects.equal(clusterId, that.clusterId)
&& Objects.equal(sourceTableId, that.sourceTableId);
&& Objects.equal(sourceTableId, that.sourceTableId)
&& Objects.equal(
requestBuilder.getBackup().getBackupType(),
that.requestBuilder.getBackup().getBackupType())
&& Objects.equal(
requestBuilder.getBackup().getHotToStandardTime(),
that.requestBuilder.getBackup().getHotToStandardTime());
}

@Override
public int hashCode() {
return Objects.hashCode(requestBuilder.getBackupId(), clusterId, sourceTableId);
return Objects.hashCode(
requestBuilder.getBackupId(),
clusterId,
sourceTableId,
requestBuilder.getBackup().getBackupType(),
requestBuilder.getBackup().getHotToStandardTime());
}

@InternalApi
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,12 @@
package com.google.cloud.bigtable.admin.v2.models;

import com.google.api.core.InternalApi;
import com.google.bigtable.admin.v2.Backup;
import com.google.cloud.bigtable.admin.v2.internal.NameUtil;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.protobuf.FieldMask;
import com.google.protobuf.util.FieldMaskUtil;
import com.google.protobuf.util.Timestamps;
import javax.annotation.Nonnull;
import org.threeten.bp.Instant;
Expand All @@ -43,12 +45,35 @@ private UpdateBackupRequest(String clusterId, String backupId) {
this.clusterId = clusterId;
}

private void updateFieldMask(int fieldNumber) {
FieldMask newMask = FieldMaskUtil.fromFieldNumbers(Backup.class, fieldNumber);
requestBuilder.setUpdateMask(FieldMaskUtil.union(requestBuilder.getUpdateMask(), newMask));
}

public UpdateBackupRequest setExpireTime(Instant expireTime) {
Preconditions.checkNotNull(expireTime);
requestBuilder
.getBackupBuilder()
.setExpireTime(Timestamps.fromMillis(expireTime.toEpochMilli()));
requestBuilder.setUpdateMask(FieldMask.newBuilder().addPaths("expire_time"));
updateFieldMask(Backup.EXPIRE_TIME_FIELD_NUMBER);
return this;
}

// The time at which this backup will be converted from a hot backup to a standard backup. Only
// applicable for hot backups. If not set, the backup will remain as a hot backup until it is
// deleted.
public UpdateBackupRequest setHotToStandardTime(Instant hotToStandardTime) {
Preconditions.checkNotNull(hotToStandardTime);
requestBuilder
.getBackupBuilder()
.setHotToStandardTime(Timestamps.fromMillis(hotToStandardTime.toEpochMilli()));
updateFieldMask(Backup.HOT_TO_STANDARD_TIME_FIELD_NUMBER);
return this;
}

public UpdateBackupRequest clearHotToStandardTime() {
requestBuilder.getBackupBuilder().clearHotToStandardTime();
updateFieldMask(Backup.HOT_TO_STANDARD_TIME_FIELD_NUMBER);
return this;
}

Expand All @@ -64,6 +89,9 @@ public boolean equals(Object o) {
return Objects.equal(
requestBuilder.getBackupBuilder().getExpireTime(),
that.requestBuilder.getBackupBuilder().getExpireTime())
&& Objects.equal(
requestBuilder.getBackupBuilder().getHotToStandardTime(),
that.requestBuilder.getBackupBuilder().getHotToStandardTime())
&& Objects.equal(requestBuilder.getUpdateMask(), that.requestBuilder.getUpdateMask())
&& Objects.equal(clusterId, that.clusterId)
&& Objects.equal(backupId, that.backupId);
Expand All @@ -73,6 +101,7 @@ public boolean equals(Object o) {
public int hashCode() {
return Objects.hashCode(
requestBuilder.getBackupBuilder().getExpireTime(),
requestBuilder.getBackupBuilder().getHotToStandardTime(),
requestBuilder.getUpdateMask(),
backupId);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -459,7 +459,7 @@ public void testGetEncryptionInfos() {
Map<String, List<com.google.cloud.bigtable.admin.v2.models.EncryptionInfo>> actualResult =
adminClient.getEncryptionInfo(TABLE_ID);

// Verify that the encryption info is transfered from the proto to the model.
// Verify that the encryption info is transferred from the proto to the model.
assertThat(actualResult)
.containsExactly(
"cluster1", ImmutableList.of(EncryptionInfo.fromProto(expectedEncryptionInfo)));
Expand Down Expand Up @@ -615,7 +615,9 @@ public void testCreateBackup() {
Timestamp expireTime = Timestamp.newBuilder().setSeconds(789).build();
long sizeBytes = 123456789;
CreateBackupRequest req =
CreateBackupRequest.of(CLUSTER_ID, BACKUP_ID).setSourceTableId(TABLE_ID);
CreateBackupRequest.of(CLUSTER_ID, BACKUP_ID)
.setSourceTableId(TABLE_ID)
.setExpireTime(Instant.ofEpochMilli(Timestamps.toMillis(expireTime)));
mockOperationResult(
mockCreateBackupOperationCallable,
req.toProto(PROJECT_ID, INSTANCE_ID),
Expand Down Expand Up @@ -648,6 +650,61 @@ public void testCreateBackup() {
assertThat(actualResult.getSizeBytes()).isEqualTo(sizeBytes);
}

@Test
public void testCreateHotBackup() {
// Setup
Mockito.when(mockStub.createBackupOperationCallable())
.thenReturn(mockCreateBackupOperationCallable);

String backupName = NameUtil.formatBackupName(PROJECT_ID, INSTANCE_ID, CLUSTER_ID, BACKUP_ID);
Timestamp startTime = Timestamp.newBuilder().setSeconds(123).build();
Timestamp endTime = Timestamp.newBuilder().setSeconds(456).build();
Timestamp expireTime = Timestamp.newBuilder().setSeconds(789).build();
Timestamp hotToStandardTime = Timestamp.newBuilder().setSeconds(500).build();
long sizeBytes = 123456789;
CreateBackupRequest req =
CreateBackupRequest.of(CLUSTER_ID, BACKUP_ID)
.setSourceTableId(TABLE_ID)
.setExpireTime(Instant.ofEpochMilli(Timestamps.toMillis(expireTime)))
.setBackupType(Backup.BackupType.HOT)
.setHotToStandardTime(Instant.ofEpochMilli(Timestamps.toMillis(hotToStandardTime)));
mockOperationResult(
mockCreateBackupOperationCallable,
req.toProto(PROJECT_ID, INSTANCE_ID),
com.google.bigtable.admin.v2.Backup.newBuilder()
.setName(backupName)
.setSourceTable(TABLE_NAME)
.setStartTime(startTime)
.setEndTime(endTime)
.setExpireTime(expireTime)
.setSizeBytes(sizeBytes)
.setBackupType(com.google.bigtable.admin.v2.Backup.BackupType.HOT)
.setHotToStandardTime(hotToStandardTime)
.build(),
CreateBackupMetadata.newBuilder()
.setName(backupName)
.setStartTime(startTime)
.setEndTime(endTime)
.setSourceTable(TABLE_NAME)
.build());
// Execute
Backup actualResult = adminClient.createBackup(req);

// Verify
assertThat(actualResult.getId()).isEqualTo(BACKUP_ID);
assertThat(actualResult.getSourceTableId()).isEqualTo(TABLE_ID);
assertThat(actualResult.getStartTime())
.isEqualTo(Instant.ofEpochMilli(Timestamps.toMillis(startTime)));
assertThat(actualResult.getEndTime())
.isEqualTo(Instant.ofEpochMilli(Timestamps.toMillis(endTime)));
assertThat(actualResult.getExpireTime())
.isEqualTo(Instant.ofEpochMilli(Timestamps.toMillis(expireTime)));
assertThat(actualResult.getBackupType()).isEqualTo(Backup.BackupType.HOT);
assertThat(actualResult.getHotToStandardTime())
.isEqualTo(Instant.ofEpochMilli(Timestamps.toMillis(hotToStandardTime)));
assertThat(actualResult.getSizeBytes()).isEqualTo(sizeBytes);
}

@Test
public void testGetBackup() {
// Setup
Expand All @@ -674,6 +731,7 @@ public void testGetBackup() {
.setEndTime(endTime)
.setSizeBytes(sizeBytes)
.setState(state)
.setBackupType(com.google.bigtable.admin.v2.Backup.BackupType.STANDARD)
.build()));

// Execute
Expand All @@ -690,6 +748,7 @@ public void testGetBackup() {
.isEqualTo(Instant.ofEpochMilli(Timestamps.toMillis(endTime)));
assertThat(actualResult.getSizeBytes()).isEqualTo(sizeBytes);
assertThat(actualResult.getState()).isEqualTo(Backup.State.fromProto(state));
assertThat(actualResult.getBackupType()).isEqualTo(Backup.BackupType.STANDARD);
}

@Test
Expand All @@ -698,6 +757,7 @@ public void testUpdateBackup() {
Mockito.when(mockStub.updateBackupCallable()).thenReturn(mockUpdateBackupCallable);

Timestamp expireTime = Timestamp.newBuilder().setSeconds(123456789).build();
Timestamp hotToStandardTime = Timestamp.newBuilder().setSeconds(123456789).build();
long sizeBytes = 12345L;
UpdateBackupRequest req = UpdateBackupRequest.of(CLUSTER_ID, BACKUP_ID);
Mockito.when(mockUpdateBackupCallable.futureCall(req.toProto(PROJECT_ID, INSTANCE_ID)))
Expand All @@ -709,6 +769,7 @@ public void testUpdateBackup() {
.setSourceTable(NameUtil.formatTableName(PROJECT_ID, INSTANCE_ID, TABLE_ID))
.setExpireTime(expireTime)
.setSizeBytes(sizeBytes)
.setHotToStandardTime(hotToStandardTime)
.build()));

// Execute
Expand All @@ -719,6 +780,8 @@ public void testUpdateBackup() {
assertThat(actualResult.getSourceTableId()).isEqualTo(TABLE_ID);
assertThat(actualResult.getExpireTime())
.isEqualTo(Instant.ofEpochMilli(Timestamps.toMillis(expireTime)));
assertThat(actualResult.getHotToStandardTime())
.isEqualTo(Instant.ofEpochMilli(Timestamps.toMillis(hotToStandardTime)));
assertThat(actualResult.getSizeBytes()).isEqualTo(sizeBytes);
}

Expand Down
Loading

0 comments on commit 6d004cd

Please sign in to comment.