Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix custom catalog store plugin loading (v2) #22886

Merged
merged 4 commits into from
Aug 27, 2024
Merged

Fix custom catalog store plugin loading (v2) #22886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0fd22cb
Fix Guice bindings for CatalogStoreManager
hashhar Aug 12, 2024
9bbcd7c
Use catalog.store to choose CatalogStoreFactory
hashhar Aug 12, 2024
4dfc8ac
Rename test class to match convention
hashhar Jul 24, 2024
2735a15
Allow using custom catalog store with TestingTrinoServer
hashhar Jul 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 10 additions & 13 deletions core/trino-main/src/main/java/io/trino/connector/CatalogStoreManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,7 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicReference;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
import static java.util.Objects.requireNonNull;

Expand All @@ -45,15 +43,16 @@ public class CatalogStoreManager
{
private static final Logger log = Logger.get(CatalogStoreManager.class);
private static final File CATALOG_STORE_CONFIGURATION = new File("etc/catalog-store.properties");
private static final String CATALOG_STORE_PROPERTY_NAME = "catalog-store.name";
private final Map<String, CatalogStoreFactory> catalogStoreFactories = new ConcurrentHashMap<>();
private final AtomicReference<Optional<CatalogStore>> configuredCatalogStore = new AtomicReference<>(Optional.empty());
private final SecretsResolver secretsResolver;
private final String catalogStoreKind;

@Inject
public CatalogStoreManager(SecretsResolver secretsResolver)
public CatalogStoreManager(SecretsResolver secretsResolver, CatalogStoreConfig catalogStoreConfig)
{
this.secretsResolver = requireNonNull(secretsResolver, "secretsResolver is null");
this.catalogStoreKind = requireNonNull(catalogStoreConfig.getCatalogStoreKind(), "catalogStoreKind is null");
}

public void addCatalogStoreFactory(CatalogStoreFactory catalogStoreFactory)
Expand All @@ -68,22 +67,20 @@ public void addCatalogStoreFactory(CatalogStoreFactory catalogStoreFactory)
public void loadConfiguredCatalogStore()
throws IOException
{
loadConfiguredCatalogStore(CATALOG_STORE_CONFIGURATION);
loadConfiguredCatalogStore(catalogStoreKind, CATALOG_STORE_CONFIGURATION);
}

@VisibleForTesting
void loadConfiguredCatalogStore(File catalogStoreFile)
void loadConfiguredCatalogStore(String catalogStoreName, File catalogStoreFile)
throws IOException
{
if (configuredCatalogStore.get().isPresent() || !catalogStoreFile.exists()) {
if (configuredCatalogStore.get().isPresent()) {
return;
}
Map<String, String> properties = new HashMap<>(loadPropertiesFrom(catalogStoreFile.getPath()));

String catalogStoreName = properties.remove(CATALOG_STORE_PROPERTY_NAME);
checkArgument(!isNullOrEmpty(catalogStoreName),
"Catalog store configuration %s does not contain %s", catalogStoreFile.getAbsoluteFile(), CATALOG_STORE_PROPERTY_NAME);

Map<String, String> properties = new HashMap<>();
if (catalogStoreFile.exists()) {
properties = new HashMap<>(loadPropertiesFrom(catalogStoreFile.getPath()));
}
setConfiguredCatalogStore(catalogStoreName, properties);
}

Expand Down
2 changes: 0 additions & 2 deletions core/trino-main/src/main/java/io/trino/connector/DynamicCatalogManagerModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@

import java.util.Locale;

import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder;
import static io.airlift.configuration.ConfigBinder.configBinder;

public class DynamicCatalogManagerModule
Expand All @@ -44,7 +43,6 @@ protected void setup(Binder binder)
}
default -> {
binder.bind(CatalogStoreManager.class).in(Scopes.SINGLETON);
newOptionalBinder(binder, CatalogStoreManager.class).setBinding().to(CatalogStoreManager.class).in(Scopes.SINGLETON);
binder.bind(CatalogStore.class).to(CatalogStoreManager.class).in(Scopes.SINGLETON);
}
}
Expand Down
5 changes: 4 additions & 1 deletion core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,10 @@ private TestingTrinoServer(

if (coordinator) {
if (catalogMangerKind == CatalogMangerKind.DYNAMIC) {
serverProperties.put("catalog.store", "memory");
Optional<String> catalogStore = Optional.ofNullable(properties.get("catalog.store"));
if (catalogStore.isEmpty()) {
serverProperties.put("catalog.store", "memory");
}
}
serverProperties.put("failure-detector.enabled", "false");

Expand Down
2 changes: 1 addition & 1 deletion .../connector/TestingCatalogStoreConfig.java → ...ino/connector/TestCatalogStoreConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults;

public class TestingCatalogStoreConfig
public class TestCatalogStoreConfig
{
@Test
public void testDefaults()
Expand Down
17 changes: 14 additions & 3 deletions core/trino-main/src/test/java/io/trino/connector/TestCatalogStoreManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,25 @@ void testCatalogStoreIsLoaded()
throws IOException
{
try (TempFile tempFile = new TempFile()) {
Files.writeString(tempFile.path(), "catalog-store.name=test");
CatalogStoreManager catalogStoreManager = new CatalogStoreManager(new SecretsResolver(ImmutableMap.of()));
Files.writeString(tempFile.path(), "some-property=some-value");
CatalogStoreConfig catalogStoreConfig = new CatalogStoreConfig().setCatalogStoreKind("test");
CatalogStoreManager catalogStoreManager = new CatalogStoreManager(new SecretsResolver(ImmutableMap.of()), catalogStoreConfig);
catalogStoreManager.addCatalogStoreFactory(new TestingCatalogStoreFactory());
catalogStoreManager.loadConfiguredCatalogStore(tempFile.file());
catalogStoreManager.loadConfiguredCatalogStore(catalogStoreConfig.getCatalogStoreKind(), tempFile.file());
assertThat(catalogStoreManager.getCatalogs()).containsExactly(TestingCatalogStore.STORED_CATALOG);
}
}

@Test
void testCatalogStoreIsLoadedWithoutConfiguration()
throws IOException
{
CatalogStoreManager catalogStoreManager = new CatalogStoreManager(new SecretsResolver(ImmutableMap.of()), new CatalogStoreConfig().setCatalogStoreKind("test"));
catalogStoreManager.addCatalogStoreFactory(new TestingCatalogStoreFactory());
catalogStoreManager.loadConfiguredCatalogStore();
assertThat(catalogStoreManager.getCatalogs()).containsExactly(TestingCatalogStore.STORED_CATALOG);
}

private static class TestingCatalogStoreFactory
implements CatalogStoreFactory
{
Expand Down
23 changes: 23 additions & 0 deletions core/trino-main/src/test/java/io/trino/server/testing/TestTestingTrinoServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import com.google.inject.Key;
import io.trino.connector.ConnectorServicesProvider;
import io.trino.connector.CoordinatorDynamicCatalogManager;
import io.trino.connector.FileCatalogStore;
import io.trino.connector.InMemoryCatalogStore;
import io.trino.connector.StaticCatalogManager;
import io.trino.connector.WorkerDynamicCatalogManager;
Expand Down Expand Up @@ -63,4 +64,26 @@ void testSetCatalogManagementToStatic()
.isInstanceOf(StaticCatalogManager.class);
}
}

@Test
void testDefaultCatalogStore()
throws IOException
{
try (TestingTrinoServer server = TestingTrinoServer.builder().build()) {
assertThat(server.getInstance(Key.get(CatalogStore.class)))
.isInstanceOf(InMemoryCatalogStore.class);
}
}

@Test
void testExplicitCatalogStore()
throws IOException
{
try (TestingTrinoServer server = TestingTrinoServer.builder()
.addProperty("catalog.store", "file")
.build()) {
assertThat(server.getInstance(Key.get(CatalogStore.class)))
.isInstanceOf(FileCatalogStore.class);
}
}
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You should test the Dynamic custom store.
In invoking a New plugin based on catalogstore catalogstorefactory and plugin interface that contains the getcatalogstorefactoriesn method.

My last test in your code configuration let me confuse. I m not sure that it works well on a custom plugin for catalog store.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've tested this with a custom catalog store (that always returns single catalog). I am submitting a follow up PR as mentioned in above comment which would allow me to add a test.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks

}