Skip to content

Commit 50ec61f

Browse files
KodaiDKodaiD
committed
Add support for Cloud Storage (#3179)
1 parent 359a6af commit 50ec61f

26 files changed

+1041
-45
lines changed

‎.github/workflows/object-storage-adapter-check.yaml‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ env:
4444
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
4545
S3_REGION: ap-northeast-1
4646
S3_BUCKET_NAME: scalardb-test-bucket
47+
CLOUD_STORAGE_PROJECT_ID: ${{ secrets.CLOUD_STORAGE_PROJECT_ID }}
48+
CLOUD_STORAGE_SERVICE_ACCOUNT_KEY: ${{ secrets.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }}
49+
CLOUD_STORAGE_BUCKET_NAME: scalardb-test-bucket
4750

4851
jobs:
4952
integration-test-s3:
@@ -98,5 +101,59 @@ jobs:
98101
if: always()
99102
uses: actions/upload-artifact@v5
100103
with:
101-
name: cassandra_3.0_integration_test_reports_${{ matrix.mode.label }}
104+
name: s3_integration_test_reports_${{ matrix.mode.label }}
105+
path: core/build/reports/tests/integrationTestObjectStorage
106+
integration-test-cloud-storage:
107+
name: Cloud Storage integration test (${{ matrix.mode.label }})
108+
runs-on: ubuntu-latest
109+
110+
strategy:
111+
fail-fast: false
112+
matrix:
113+
mode:
114+
- label: default
115+
group_commit_enabled: false
116+
- label: with_group_commit
117+
group_commit_enabled: true
118+
119+
steps:
120+
- uses: actions/checkout@v5
121+
122+
- name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
123+
uses: actions/setup-java@v5
124+
with:
125+
java-version: ${{ env.JAVA_VERSION }}
126+
distribution: ${{ env.JAVA_VENDOR }}
127+
128+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
129+
uses: actions/setup-java@v5
130+
if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
131+
with:
132+
java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
133+
distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
134+
135+
- name: Login to Oracle container registry
136+
uses: docker/login-action@v3
137+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
138+
with:
139+
registry: container-registry.oracle.com
140+
username: ${{ secrets.OCR_USERNAME }}
141+
password: ${{ secrets.OCR_TOKEN }}
142+
143+
- name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
144+
if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
145+
run: |
146+
container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
147+
docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
148+
- name: Setup Gradle
149+
uses: gradle/actions/setup-gradle@v5
150+
151+
- name: Execute Gradle 'integrationTestObjectStorage' task
152+
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=scalardb-test-bucket -Dscalardb.object_storage.username=${{ env.CLOUD_STORAGE_PROJECT_ID }} -Dscalardb.object_storage.password=${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
153+
154+
- name: Upload Gradle test reports
155+
if: always()
156+
uses: actions/upload-artifact@v5
157+
with:
158+
name: cloud_storage_integration_test_reports_${{ matrix.mode.label }}
102159
path: core/build/reports/tests/integrationTestObjectStorage

‎build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ subprojects {
4141
db2DriverVersion = '12.1.3.0'
4242
mariadDbDriverVersion = '3.5.6'
4343
alloyDbJdbcConnectorVersion = '1.2.8'
44+
googleCloudStorageVersion = '2.60.0'
4445
picocliVersion = '4.7.7'
4546
commonsTextVersion = '1.14.0'
4647
junitVersion = '5.14.1'

‎core/build.gradle‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,9 @@ dependencies {
193193
implementation("com.google.cloud:alloydb-jdbc-connector:${alloyDbJdbcConnectorVersion}") {
194194
exclude group: 'org.slf4j', module: 'slf4j-api'
195195
}
196+
implementation("com.google.cloud:google-cloud-storage:${googleCloudStorageVersion}") {
197+
exclude group: 'org.slf4j', module: 'slf4j-api'
198+
}
196199
implementation "org.apache.commons:commons-text:${commonsTextVersion}"
197200
testImplementation platform("org.junit:junit-bom:${junitVersion}")
198201
testImplementation 'org.junit.jupiter:junit-jupiter'
@@ -226,7 +229,7 @@ task integrationTestCassandra(type: Test) {
226229
classpath = sourceSets.integrationTestCassandra.runtimeClasspath
227230
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
228231
options {
229-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
232+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
230233
}
231234
}
232235

@@ -237,7 +240,7 @@ task integrationTestCosmos(type: Test) {
237240
classpath = sourceSets.integrationTestCosmos.runtimeClasspath
238241
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
239242
options {
240-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
243+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
241244
}
242245
jvmArgs '-XX:MaxDirectMemorySize=4g', '-Xmx6g',
243246
// INFO com.azure.cosmos.implementation.RxDocumentClientImpl - Initializing DocumentClient [3] with serviceEndpoint [https://localhost:8081/], ...
@@ -255,7 +258,7 @@ task integrationTestDynamo(type: Test) {
255258
classpath = sourceSets.integrationTestDynamo.runtimeClasspath
256259
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
257260
options {
258-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
261+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
259262
}
260263
maxParallelForks = 10
261264
}
@@ -267,7 +270,7 @@ task integrationTestJdbc(type: Test) {
267270
classpath = sourceSets.integrationTestJdbc.runtimeClasspath
268271
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
269272
options {
270-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
273+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
271274
}
272275
maxHeapSize = "4g"
273276
}
@@ -279,7 +282,7 @@ task integrationTestObjectStorage(type: Test) {
279282
classpath = sourceSets.integrationTestObjectStorage.runtimeClasspath
280283
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
281284
options {
282-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
285+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
283286
}
284287
}
285288

@@ -290,7 +293,7 @@ task integrationTestMultiStorage(type: Test) {
290293
classpath = sourceSets.integrationTestMultiStorage.runtimeClasspath
291294
outputs.upToDateWhen { false } // ensures integration tests are run every time when called
292295
options {
293-
systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
296+
systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") })
294297
}
295298
}
296299

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageEnv.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.scalar.db.config.DatabaseConfig;
44
import com.scalar.db.storage.objectstorage.blobstorage.BlobStorageConfig;
5+
import com.scalar.db.storage.objectstorage.cloudstorage.CloudStorageConfig;
56
import com.scalar.db.storage.objectstorage.s3.S3Config;
67
import java.util.Collections;
78
import java.util.Map;
@@ -76,6 +77,11 @@ public static boolean isBlobStorage() {
7677
.equals(BlobStorageConfig.STORAGE_NAME);
7778
}
7879

80+
public static boolean isCloudStorage() {
81+
return System.getProperty(PROP_OBJECT_STORAGE_STORAGE, DEFAULT_OBJECT_STORAGE_STORAGE)
82+
.equals(CloudStorageConfig.STORAGE_NAME);
83+
}
84+
7985
public static boolean isS3() {
8086
return System.getProperty(PROP_OBJECT_STORAGE_STORAGE, DEFAULT_OBJECT_STORAGE_STORAGE)
8187
.equals(S3Config.STORAGE_NAME);

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageWrapperIntegrationTest.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,11 @@ public class ObjectStorageWrapperIntegrationTest {
2828
private static final String TEST_OBJECT2 = "test-object2";
2929
private static final String TEST_OBJECT3 = "test-object3";
3030
private static final int BLOB_STORAGE_LIST_MAX_KEYS = 5000;
31+
private static final int CLOUD_STORAGE_LIST_MAX_KEYS = 1000;
32+
private static final int S3_LIST_MAX_KEYS = 1000;
3133

3234
private ObjectStorageWrapper wrapper;
35+
private int listMaxKeys;
3336

3437
@BeforeAll
3538
public void beforeAll() throws ObjectStorageWrapperException {
@@ -38,6 +41,16 @@ public void beforeAll() throws ObjectStorageWrapperException {
3841
ObjectStorageUtils.getObjectStorageConfig(new DatabaseConfig(properties));
3942
wrapper = ObjectStorageWrapperFactory.create(objectStorageConfig);
4043
createObjects();
44+
45+
if (ObjectStorageEnv.isBlobStorage()) {
46+
listMaxKeys = BLOB_STORAGE_LIST_MAX_KEYS;
47+
} else if (ObjectStorageEnv.isCloudStorage()) {
48+
listMaxKeys = CLOUD_STORAGE_LIST_MAX_KEYS;
49+
} else if (ObjectStorageEnv.isS3()) {
50+
listMaxKeys = S3_LIST_MAX_KEYS;
51+
} else {
52+
throw new AssertionError();
53+
}
4154
}
4255

4356
@AfterAll
@@ -152,14 +165,14 @@ public void update_NonExistingObjectKeyGiven_ShouldThrowPreconditionFailedExcept
152165
String objectKey = "non-existing-key";
153166

154167
// Act Assert
155-
assertThatCode(() -> wrapper.update(objectKey, "some-object", "some-version"))
168+
assertThatCode(() -> wrapper.update(objectKey, "some-object", "123456789"))
156169
.isInstanceOf(PreconditionFailedException.class);
157170
}
158171

159172
@Test
160173
public void update_WrongVersionGiven_ShouldThrowPreconditionFailedException() {
161174
// Arrange
162-
String wrongVersion = "wrong-version";
175+
String wrongVersion = "123456789";
163176

164177
// Act Assert
165178
assertThatCode(() -> wrapper.update(TEST_KEY2, "another-object", wrongVersion))
@@ -219,7 +232,7 @@ public void delete_ExistingObjectKeyWithWrongVersionGiven_ShouldThrowPreconditio
219232
// Arrange
220233
Optional<ObjectStorageWrapperResponse> response1 = wrapper.get(TEST_KEY1);
221234
assertThat(response1.isPresent()).isTrue();
222-
String wrongVersion = "wrong-version";
235+
String wrongVersion = "123456789";
223236

224237
// Act Assert
225238
assertThatCode(() -> wrapper.delete(TEST_KEY1, wrongVersion))
@@ -253,7 +266,7 @@ public void getKeys_WithNonExistingPrefix_ShouldReturnEmptySet() throws Exceptio
253266
public void getKeys_WithPrefixForTheNumberOfObjectsExceedingTheListLimit_ShouldReturnAllKeys()
254267
throws Exception {
255268
String prefix = "prefix-";
256-
int numberOfObjects = BLOB_STORAGE_LIST_MAX_KEYS + 1;
269+
int numberOfObjects = listMaxKeys + 1;
257270
try {
258271
// Arrange
259272
for (int i = 0; i < numberOfObjects; i++) {
@@ -313,7 +326,7 @@ public void deleteByPrefix_WithNonExistingPrefix_ShouldDoNothing() throws Except
313326
deleteByPrefix_WithPrefixForTheNumberOfObjectsExceedingTheListLimit_ShouldDeleteAllObjects()
314327
throws Exception {
315328
String prefix = "prefix-";
316-
int numberOfObjects = BLOB_STORAGE_LIST_MAX_KEYS + 1;
329+
int numberOfObjects = listMaxKeys + 1;
317330
try {
318331
// Arrange
319332
for (int i = 0; i < numberOfObjects; i++) {

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageWrapperLargeObjectWriteIntegrationTest.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import com.scalar.db.config.DatabaseConfig;
77
import com.scalar.db.storage.objectstorage.blobstorage.BlobStorageConfig;
8+
import com.scalar.db.storage.objectstorage.cloudstorage.CloudStorageConfig;
89
import com.scalar.db.storage.objectstorage.s3.S3Config;
910
import java.util.Arrays;
1011
import java.util.Optional;
@@ -47,6 +48,13 @@ public void beforeAll() throws ObjectStorageWrapperException {
4748
BlobStorageConfig.PARALLEL_UPLOAD_THRESHOLD_IN_BYTES,
4849
String.valueOf(parallelUploadUnit * 2));
4950
parallelUploadThresholdInBytes = parallelUploadUnit * 2;
51+
} else if (ObjectStorageEnv.isCloudStorage()) {
52+
// Minimum block size must be greater than or equal to 256KB for Cloud Storage
53+
Long parallelUploadUnit = 256 * 1024L; // 256KB
54+
properties.setProperty(
55+
CloudStorageConfig.PARALLEL_UPLOAD_BLOCK_SIZE_IN_BYTES,
56+
String.valueOf(parallelUploadUnit));
57+
parallelUploadThresholdInBytes = parallelUploadUnit * 2;
5058
} else if (ObjectStorageEnv.isS3()) {
5159
// Minimum part size must be greater than or equal to 5MB for S3
5260
Long parallelUploadUnit = 5 * 1024 * 1024L; // 5MB
@@ -59,7 +67,7 @@ public void beforeAll() throws ObjectStorageWrapperException {
5967
throw new AssertionError();
6068
}
6169

62-
char[] charArray = new char[(int) parallelUploadThresholdInBytes];
70+
char[] charArray = new char[(int) parallelUploadThresholdInBytes + 1];
6371
Arrays.fill(charArray, 'a');
6472
testObject1 = new String(charArray);
6573
Arrays.fill(charArray, 'b');
@@ -163,14 +171,14 @@ public void update_NonExistingObjectKeyGiven_ShouldThrowPreconditionFailedExcept
163171
String objectKey = "non-existing-key";
164172

165173
// Act Assert
166-
assertThatCode(() -> wrapper.update(objectKey, "some-object", "some-version"))
174+
assertThatCode(() -> wrapper.update(objectKey, "some-object", "123456789"))
167175
.isInstanceOf(PreconditionFailedException.class);
168176
}
169177

170178
@Test
171179
public void update_WrongVersionGiven_ShouldThrowPreconditionFailedException() {
172180
// Arrange
173-
String wrongVersion = "wrong-version";
181+
String wrongVersion = "123456789";
174182

175183
// Act Assert
176184
assertThatCode(() -> wrapper.update(TEST_KEY2, "another-object", wrongVersion))

‎core/src/main/java/com/scalar/db/common/CoreError.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -931,6 +931,18 @@ public enum CoreError implements ScalarDbError {
931931
"Conditions on indexed columns in cross-partition scan operations are not allowed in the SERIALIZABLE isolation level",
932932
"",
933933
""),
934+
OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_NOT_FOUND(
935+
Category.USER_ERROR,
936+
"0263",
937+
"The service account key for Cloud Storage was not found.",
938+
"",
939+
""),
940+
OBJECT_STORAGE_CLOUD_STORAGE_SERVICE_ACCOUNT_KEY_LOAD_FAILED(
941+
Category.USER_ERROR,
942+
"0264",
943+
"Failed to load the service account key for Cloud Storage.",
944+
"",
945+
""),
934946

935947
//
936948
// Errors for the concurrency error category

‎core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorage.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,10 @@ public void mutate(List<? extends Mutation> mutations) throws ExecutionException
154154

155155
@Override
156156
public void close() {
157-
wrapper.close();
157+
try {
158+
wrapper.close();
159+
} catch (ObjectStorageWrapperException e) {
160+
logger.warn("Failed to close the ObjectStorageWrapper", e);
161+
}
158162
}
159163
}

‎core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageAdmin.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,15 @@
2222
import java.util.stream.Collectors;
2323
import javax.annotation.Nullable;
2424
import javax.annotation.concurrent.ThreadSafe;
25+
import org.slf4j.Logger;
26+
import org.slf4j.LoggerFactory;
2527

2628
@ThreadSafe
2729
public class ObjectStorageAdmin implements DistributedStorageAdmin {
2830
public static final String NAMESPACE_METADATA_TABLE = "namespaces";
2931
public static final String TABLE_METADATA_TABLE = "metadata";
3032

33+
private static final Logger logger = LoggerFactory.getLogger(ObjectStorageAdmin.class);
3134
private static final StorageInfo STORAGE_INFO =
3235
new StorageInfoImpl(
3336
"object_storage",
@@ -81,7 +84,11 @@ public StorageInfo getStorageInfo(String namespace) throws ExecutionException {
8184

8285
@Override
8386
public void close() {
84-
wrapper.close();
87+
try {
88+
wrapper.close();
89+
} catch (ObjectStorageWrapperException e) {
90+
logger.warn("Failed to close the ObjectStorageWrapper", e);
91+
}
8592
}
8693

8794
@Override

‎core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageConfig.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,6 @@ public interface ObjectStorageConfig {
99
*/
1010
String getStorageName();
1111

12-
/**
13-
* Returns the username for authentication.
14-
*
15-
* @return the username
16-
*/
17-
String getUsername();
18-
1912
/**
2013
* Returns the password for authentication.
2114
*

0 commit comments

Comments
 (0)