Skip to content

Add validation for secondary indexes in table metadata with Object Storage #3127

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 10, 2025
Merged

Add validation for secondary indexes in table metadata with Object Storage #3127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5c0a768
Add validation for secondary indexes in table metadata
KodaiD Nov 7, 2025
3c55b65
Fix
KodaiD Nov 7, 2025
ff0ac32
Fix
KodaiD Nov 7, 2025
73164df
Apply suggestions
KodaiD Nov 7, 2025
4d87cb4
Merge branch 'master' into validate-secondary-index-with-object-storage
feeblefakie Nov 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions ...scalar/db/storage/objectstorage/ConsensusCommitAdminIntegrationTestWithObjectStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.scalar.db.storage.objectstorage;

import com.scalar.db.api.TableMetadata;
import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
import com.scalar.db.util.AdminTestUtils;
import java.util.Properties;
Expand All @@ -13,6 +14,14 @@ protected Properties getProps(String testName) {
return ObjectStorageEnv.getProperties(testName);
}

@Override
protected TableMetadata getTableMetadata() {
return TableMetadata.newBuilder(TABLE_METADATA)
.removeSecondaryIndex(COL_NAME5)
.removeSecondaryIndex(COL_NAME6)
.build();
}

@Override
protected AdminTestUtils getAdminTestUtils(String testName) {
return new ObjectStorageAdminTestUtils(getProperties(testName));
Expand Down
28 changes: 28 additions & 0 deletions .../db/storage/objectstorage/ConsensusCommitAdminRepairIntegrationTestWithObjectStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,39 @@
package com.scalar.db.storage.objectstorage;

import com.scalar.db.api.Scan;
import com.scalar.db.api.TableMetadata;
import com.scalar.db.io.DataType;
import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminRepairIntegrationTestBase;
import java.util.Properties;

public class ConsensusCommitAdminRepairIntegrationTestWithObjectStorage
extends ConsensusCommitAdminRepairIntegrationTestBase {

@Override
protected TableMetadata getTableMetadata() {
return TableMetadata.newBuilder()
.addColumn(COL_NAME1, DataType.INT)
.addColumn(COL_NAME2, DataType.TEXT)
.addColumn(COL_NAME3, DataType.TEXT)
.addColumn(COL_NAME4, DataType.INT)
.addColumn(COL_NAME5, DataType.INT)
.addColumn(COL_NAME6, DataType.TEXT)
.addColumn(COL_NAME7, DataType.BIGINT)
.addColumn(COL_NAME8, DataType.FLOAT)
.addColumn(COL_NAME9, DataType.DOUBLE)
.addColumn(COL_NAME10, DataType.BOOLEAN)
.addColumn(COL_NAME11, DataType.BLOB)
.addColumn(COL_NAME12, DataType.DATE)
.addColumn(COL_NAME13, DataType.TIME)
.addColumn(COL_NAME14, DataType.TIMESTAMPTZ)
.addColumn(COL_NAME15, DataType.TIMESTAMP)
.addPartitionKey(COL_NAME2)
.addPartitionKey(COL_NAME1)
.addClusteringKey(COL_NAME4, Scan.Ordering.Order.ASC)
.addClusteringKey(COL_NAME3, Scan.Ordering.Order.DESC)
.build();
}

@Override
protected Properties getProps(String testName) {
return ObjectStorageEnv.getProperties(testName);
Expand Down
27 changes: 27 additions & 0 deletions ...com/scalar/db/storage/objectstorage/ObjectStorageAdminCaseSensitivityIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package com.scalar.db.storage.objectstorage;

import com.scalar.db.api.DistributedStorageAdminCaseSensitivityIntegrationTestBase;
import com.scalar.db.api.Scan;
import com.scalar.db.api.TableMetadata;
import com.scalar.db.io.DataType;
import com.scalar.db.util.AdminTestUtils;
import java.util.Map;
import java.util.Properties;
Expand All @@ -14,6 +17,30 @@ protected Properties getProperties(String testName) {
return ObjectStorageEnv.getProperties(testName);
}

@Override
protected TableMetadata getTableMetadata() {
return TableMetadata.newBuilder()
.addColumn(getColumnName1(), DataType.INT)
.addColumn(getColumnName2(), DataType.TEXT)
.addColumn(getColumnName3(), DataType.TEXT)
.addColumn(getColumnName4(), DataType.INT)
.addColumn(getColumnName5(), DataType.INT)
.addColumn(getColumnName6(), DataType.TEXT)
.addColumn(getColumnName7(), DataType.BIGINT)
.addColumn(getColumnName8(), DataType.FLOAT)
.addColumn(getColumnName9(), DataType.DOUBLE)
.addColumn(getColumnName10(), DataType.BOOLEAN)
.addColumn(getColumnName11(), DataType.BLOB)
.addColumn(getColumnName12(), DataType.DATE)
.addColumn(getColumnName13(), DataType.TIME)
.addColumn(getColumnName14(), DataType.TIMESTAMPTZ)
.addPartitionKey(getColumnName2())
.addPartitionKey(getColumnName1())
.addClusteringKey(getColumnName4(), Scan.Ordering.Order.ASC)
.addClusteringKey(getColumnName3(), Scan.Ordering.Order.DESC)
.build();
}

@Override
protected Map<String, String> getCreationOptions() {
return ObjectStorageEnv.getCreationOptions();
Expand Down
27 changes: 27 additions & 0 deletions ...tion-test/java/com/scalar/db/storage/objectstorage/ObjectStorageAdminIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package com.scalar.db.storage.objectstorage;

import com.scalar.db.api.DistributedStorageAdminIntegrationTestBase;
import com.scalar.db.api.Scan;
import com.scalar.db.api.TableMetadata;
import com.scalar.db.io.DataType;
import com.scalar.db.util.AdminTestUtils;
import java.util.Properties;
import org.junit.jupiter.api.Disabled;
Expand All @@ -12,6 +15,30 @@ protected Properties getProperties(String testName) {
return ObjectStorageEnv.getProperties(testName);
}

@Override
protected TableMetadata getTableMetadata() {
return TableMetadata.newBuilder()
.addColumn(getColumnName1(), DataType.INT)
.addColumn(getColumnName2(), DataType.TEXT)
.addColumn(getColumnName3(), DataType.TEXT)
.addColumn(getColumnName4(), DataType.INT)
.addColumn(getColumnName5(), DataType.INT)
.addColumn(getColumnName6(), DataType.TEXT)
.addColumn(getColumnName7(), DataType.BIGINT)
.addColumn(getColumnName8(), DataType.FLOAT)
.addColumn(getColumnName9(), DataType.DOUBLE)
.addColumn(getColumnName10(), DataType.BOOLEAN)
.addColumn(getColumnName11(), DataType.BLOB)
.addColumn(getColumnName12(), DataType.DATE)
.addColumn(getColumnName13(), DataType.TIME)
.addColumn(getColumnName14(), DataType.TIMESTAMPTZ)
.addPartitionKey(getColumnName2())
.addPartitionKey(getColumnName1())
.addClusteringKey(getColumnName4(), Scan.Ordering.Order.ASC)
.addClusteringKey(getColumnName3(), Scan.Ordering.Order.DESC)
.build();
}

@Override
protected boolean isIndexOnBooleanColumnSupported() {
return false;
Expand Down
28 changes: 28 additions & 0 deletions ...est/java/com/scalar/db/storage/objectstorage/ObjectStorageAdminRepairIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,40 @@
package com.scalar.db.storage.objectstorage;

import com.scalar.db.api.DistributedStorageAdminRepairIntegrationTestBase;
import com.scalar.db.api.Scan;
import com.scalar.db.api.TableMetadata;
import com.scalar.db.io.DataType;
import java.util.Properties;
import org.junit.jupiter.api.Disabled;

public class ObjectStorageAdminRepairIntegrationTest
extends DistributedStorageAdminRepairIntegrationTestBase {

@Override
protected TableMetadata getTableMetadata() {
return TableMetadata.newBuilder()
.addColumn(COL_NAME1, DataType.INT)
.addColumn(COL_NAME2, DataType.TEXT)
.addColumn(COL_NAME3, DataType.TEXT)
.addColumn(COL_NAME4, DataType.INT)
.addColumn(COL_NAME5, DataType.INT)
.addColumn(COL_NAME6, DataType.TEXT)
.addColumn(COL_NAME7, DataType.BIGINT)
.addColumn(COL_NAME8, DataType.FLOAT)
.addColumn(COL_NAME9, DataType.DOUBLE)
.addColumn(COL_NAME10, DataType.BOOLEAN)
.addColumn(COL_NAME11, DataType.BLOB)
.addColumn(COL_NAME12, DataType.DATE)
.addColumn(COL_NAME13, DataType.TIME)
.addColumn(COL_NAME14, DataType.TIMESTAMPTZ)
.addColumn(COL_NAME15, DataType.TIMESTAMP)
.addPartitionKey(COL_NAME2)
.addPartitionKey(COL_NAME1)
.addClusteringKey(COL_NAME4, Scan.Ordering.Order.ASC)
.addClusteringKey(COL_NAME3, Scan.Ordering.Order.DESC)
.build();
}

@Override
protected Properties getProperties(String testName) {
return ObjectStorageEnv.getProperties(testName);
Expand Down
9 changes: 9 additions & 0 deletions ...ge/objectstorage/SingleCrudOperationTransactionAdminIntegrationTestWithObjectStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.scalar.db.storage.objectstorage;

import com.scalar.db.api.TableMetadata;
import com.scalar.db.transaction.singlecrudoperation.SingleCrudOperationTransactionAdminIntegrationTestBase;
import java.util.Properties;
import org.junit.jupiter.api.Disabled;
Expand All @@ -12,6 +13,14 @@ protected Properties getProps(String testName) {
return ObjectStorageEnv.getProperties(testName);
}

@Override
protected TableMetadata getTableMetadata() {
return TableMetadata.newBuilder(TABLE_METADATA)
.removeSecondaryIndex(COL_NAME5)
.removeSecondaryIndex(COL_NAME6)
.build();
}

@Override
@Disabled("Temporarily disabled because it includes DML operations")
public void truncateTable_ShouldTruncateProperly() {}
Expand Down
10 changes: 10 additions & 0 deletions core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ public void createTable(
String namespace, String table, TableMetadata metadata, Map<String, String> options)
throws ExecutionException {
try {
checkTableMetadata(metadata);
// Insert the table metadata
String tableMetadataKey = getTableMetadataKey(namespace, table);
Map<String, String> readVersionMap = new HashMap<>();
Expand Down Expand Up @@ -268,6 +269,7 @@ public void repairTable(
String namespace, String table, TableMetadata metadata, Map<String, String> options)
throws ExecutionException {
try {
checkTableMetadata(metadata);
// Upsert the table metadata
String tableMetadataKey = getTableMetadataKey(namespace, table);
Map<String, String> readVersionMap = new HashMap<>();
Expand Down Expand Up @@ -470,4 +472,12 @@ private static String getTableNameFromTableMetadataKey(String tableMetadataKey)
}
return parts.get(1);
}

private void checkTableMetadata(TableMetadata metadata) {
Set<String> secondaryIndexNames = metadata.getSecondaryIndexNames();
if (!secondaryIndexNames.isEmpty()) {
throw new IllegalArgumentException(
CoreError.OBJECT_STORAGE_INDEX_NOT_SUPPORTED.buildMessage());
}
}
}
6 changes: 3 additions & 3 deletions ...tion-test/src/main/java/com/scalar/db/api/DistributedStorageAdminIntegrationTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,9 +330,9 @@ public void getTableMetadata_CorrectTableGiven_ShouldReturnCorrectMetadata()
assertThat(tableMetadata.getClusteringOrder(getColumnName14())).isNull();
assertThat(tableMetadata.getClusteringOrder(getColumnName15())).isNull();

assertThat(tableMetadata.getSecondaryIndexNames().size()).isEqualTo(2);
assertThat(tableMetadata.getSecondaryIndexNames().contains(getColumnName5())).isTrue();
assertThat(tableMetadata.getSecondaryIndexNames().contains(getColumnName6())).isTrue();
Set<String> expectedSecondaryIndexNames = getTableMetadata().getSecondaryIndexNames();
assertThat(tableMetadata.getSecondaryIndexNames())
.containsExactlyInAnyOrderElementsOf(expectedSecondaryIndexNames);
}

@Test
Expand Down
30 changes: 15 additions & 15 deletions ...est/src/main/java/com/scalar/db/api/DistributedStorageAdminRepairIntegrationTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,21 @@ public abstract class DistributedStorageAdminRepairIntegrationTestBase {
private static final String TEST_NAME = "storage_admin_repair";
private static final String NAMESPACE = "int_test_" + TEST_NAME;
private static final String TABLE = "test_table";
private static final String COL_NAME1 = "c1";
private static final String COL_NAME2 = "c2";
private static final String COL_NAME3 = "c3";
private static final String COL_NAME4 = "c4";
private static final String COL_NAME5 = "c5";
private static final String COL_NAME6 = "c6";
private static final String COL_NAME7 = "c7";
private static final String COL_NAME8 = "c8";
private static final String COL_NAME9 = "c9";
private static final String COL_NAME10 = "c10";
private static final String COL_NAME11 = "c11";
private static final String COL_NAME12 = "c12";
private static final String COL_NAME13 = "c13";
private static final String COL_NAME14 = "c14";
private static final String COL_NAME15 = "c15";
protected static final String COL_NAME1 = "c1";
protected static final String COL_NAME2 = "c2";
protected static final String COL_NAME3 = "c3";
protected static final String COL_NAME4 = "c4";
protected static final String COL_NAME5 = "c5";
protected static final String COL_NAME6 = "c6";
protected static final String COL_NAME7 = "c7";
protected static final String COL_NAME8 = "c8";
protected static final String COL_NAME9 = "c9";
protected static final String COL_NAME10 = "c10";
protected static final String COL_NAME11 = "c11";
protected static final String COL_NAME12 = "c12";
protected static final String COL_NAME13 = "c13";
protected static final String COL_NAME14 = "c14";
protected static final String COL_NAME15 = "c15";

protected DistributedStorageAdmin admin;

Expand Down
28 changes: 17 additions & 11 deletions ...-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminIntegrationTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,10 @@ protected void initialize(String testName) throws Exception {}

protected abstract Properties getProperties(String testName);

protected TableMetadata getTableMetadata() {
return TABLE_METADATA;
}

protected String getNamespaceBaseName() {
return NAMESPACE_BASE_NAME;
}
Expand All @@ -120,7 +124,7 @@ private void createTables() throws ExecutionException {
for (String namespace : Arrays.asList(namespace1, namespace2)) {
admin.createNamespace(namespace, true, options);
for (String table : Arrays.asList(TABLE1, TABLE2, TABLE3)) {
admin.createTable(namespace, table, TABLE_METADATA, true, options);
admin.createTable(namespace, table, getTableMetadata(), true, options);
}
}
admin.createCoordinatorTables(true, options);
Expand Down Expand Up @@ -240,9 +244,9 @@ public void getTableMetadata_CorrectTableGiven_ShouldReturnCorrectMetadata()
assertThat(tableMetadata.getClusteringOrder(COL_NAME14)).isNull();
assertThat(tableMetadata.getClusteringOrder(COL_NAME15)).isNull();

assertThat(tableMetadata.getSecondaryIndexNames().size()).isEqualTo(2);
assertThat(tableMetadata.getSecondaryIndexNames().contains(COL_NAME5)).isTrue();
assertThat(tableMetadata.getSecondaryIndexNames().contains(COL_NAME6)).isTrue();
Set<String> expectedSecondaryIndexNames = getTableMetadata().getSecondaryIndexNames();
assertThat(tableMetadata.getSecondaryIndexNames())
.containsExactlyInAnyOrderElementsOf(expectedSecondaryIndexNames);
}

@Test
Expand Down Expand Up @@ -322,7 +326,7 @@ public void dropNamespace_ForNonEmptyNamespace_ShouldThrowIllegalArgumentExcepti
try {
// Arrange
admin.createNamespace(namespace3, getCreationOptions());
admin.createTable(namespace3, TABLE1, TABLE_METADATA, getCreationOptions());
admin.createTable(namespace3, TABLE1, getTableMetadata(), getCreationOptions());

// Act Assert
assertThatThrownBy(() -> admin.dropNamespace(namespace3))
Expand All @@ -349,7 +353,7 @@ public void createTable_ForNonExistingTable_ShouldCreateTableProperly()
Map<String, String> options = getCreationOptions();

// Act
admin.createTable(namespace1, TABLE4, TABLE_METADATA, options);
admin.createTable(namespace1, TABLE4, getTableMetadata(), options);

// Assert
assertThat(admin.tableExists(namespace1, TABLE4)).isTrue();
Expand All @@ -364,7 +368,7 @@ public void createTable_ForExistingTable_ShouldThrowIllegalArgumentException() {

// Act Assert
assertThatThrownBy(
() -> admin.createTable(namespace1, TABLE1, TABLE_METADATA, getCreationOptions()))
() -> admin.createTable(namespace1, TABLE1, getTableMetadata(), getCreationOptions()))
.isInstanceOf(IllegalArgumentException.class);
}

Expand All @@ -374,7 +378,7 @@ public void createTable_ForNonExistingNamespace_ShouldThrowIllegalArgumentExcept

// Act Assert
assertThatThrownBy(
() -> admin.createTable(namespace3, TABLE1, TABLE_METADATA, getCreationOptions()))
() -> admin.createTable(namespace3, TABLE1, getTableMetadata(), getCreationOptions()))
.isInstanceOf(IllegalArgumentException.class);
}

Expand All @@ -384,7 +388,9 @@ public void createTable_IfNotExists_ForExistingNamespace_ShouldNotThrowAnyExcept

// Act Assert
assertThatCode(
() -> admin.createTable(namespace1, TABLE1, TABLE_METADATA, true, getCreationOptions()))
() ->
admin.createTable(
namespace1, TABLE1, getTableMetadata(), true, getCreationOptions()))
.doesNotThrowAnyException();
}

Expand All @@ -393,7 +399,7 @@ public void dropTable_ForExistingTable_ShouldDropTableProperly() throws Executio
try {
// Arrange
Map<String, String> options = getCreationOptions();
admin.createTable(namespace1, TABLE4, TABLE_METADATA, options);
admin.createTable(namespace1, TABLE4, getTableMetadata(), options);

// Act
admin.dropTable(namespace1, TABLE4);
Expand Down Expand Up @@ -432,7 +438,7 @@ public void truncateTable_ShouldTruncateProperly()
try {
// Arrange
Map<String, String> options = getCreationOptions();
admin.createTable(namespace1, table, TABLE_METADATA, true, options);
admin.createTable(namespace1, table, getTableMetadata(), true, options);
Key partitionKey = Key.of(COL_NAME2, "aaa", COL_NAME1, 1);
Key clusteringKey = Key.of(COL_NAME4, 2, COL_NAME3, "bbb");
transactionalInsert(
Expand Down
8 changes: 4 additions & 4 deletions ...src/main/java/com/scalar/db/api/DistributedTransactionAdminRepairIntegrationTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,10 @@ public abstract class DistributedTransactionAdminRepairIntegrationTestBase {
protected static final String COL_NAME9 = "c9";
protected static final String COL_NAME10 = "c10";
protected static final String COL_NAME11 = "c11";
private static final String COL_NAME12 = "c12";
private static final String COL_NAME13 = "c13";
private static final String COL_NAME14 = "c14";
private static final String COL_NAME15 = "c15";
protected static final String COL_NAME12 = "c12";
protected static final String COL_NAME13 = "c13";
protected static final String COL_NAME14 = "c14";
protected static final String COL_NAME15 = "c15";

protected DistributedTransactionAdmin admin;
protected DistributedStorageAdmin storageAdmin;
Expand Down
Loading