Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,29 @@ public class MightyEtlConfigAliasEnvironmentPostProcessor implements Environment
"etl.max-batch-records",
"etl.jobs.intake-enabled",
"connectors.databricks.enabled",
"connectors.databricks.host",
"connectors.databricks.http-path",
"connectors.databricks.token",
"connectors.databricks.catalog",
"connectors.databricks.schema",
"connectors.databricks.table",
"connectors.databricks.write-mode",
"connectors.snowflake.enabled",
"connectors.qlik-sense.enabled"
"connectors.snowflake.account",
"connectors.snowflake.warehouse",
"connectors.snowflake.database",
"connectors.snowflake.schema",
"connectors.snowflake.user",
"connectors.snowflake.password",
"connectors.snowflake.private-key",
"connectors.snowflake.role",
"connectors.snowflake.table",
"connectors.snowflake.merge-keys",
"connectors.qlik-sense.enabled",
"connectors.qlik-sense.tenant-url",
"connectors.qlik-sense.api-key",
"connectors.qlik-sense.app-id",
"connectors.qlik-sense.mode"
);

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.xtrmetl.etl.config;

import com.xtrmetl.etl.connector.ConnectorProperties;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.boot.Banner;
Expand All @@ -10,9 +11,13 @@
import org.springframework.mock.env.MockEnvironment;

import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
Expand All @@ -22,6 +27,74 @@
*/
class MightyEtlConfigAliasEnvironmentPostProcessorTest {

private static final Map<String, String> CONNECTOR_SETTINGS = Map.ofEntries(
Map.entry("connectors.databricks.enabled", "true"),
Map.entry("connectors.databricks.host", "workspace.example"),
Map.entry("connectors.databricks.http-path", "/sql/1.0/warehouses/demo"),
Map.entry("connectors.databricks.token", "test-databricks-token"),
Map.entry("connectors.databricks.catalog", "main"),
Map.entry("connectors.databricks.schema", "analytics"),
Map.entry("connectors.databricks.table", "events"),
Map.entry("connectors.databricks.write-mode", "append"),
Map.entry("connectors.snowflake.enabled", "true"),
Map.entry("connectors.snowflake.account", "test-account"),
Map.entry("connectors.snowflake.warehouse", "COMPUTE_WH"),
Map.entry("connectors.snowflake.database", "ANALYTICS"),
Map.entry("connectors.snowflake.schema", "PUBLIC"),
Map.entry("connectors.snowflake.user", "etl_user"),
Map.entry("connectors.snowflake.password", "test-snowflake-password"),
Map.entry("connectors.snowflake.private-key", "test-private-key"),
Map.entry("connectors.snowflake.role", "ETL_ROLE"),
Map.entry("connectors.snowflake.table", "EVENTS"),
Map.entry("connectors.snowflake.merge-keys", "id"),
Map.entry("connectors.qlik-sense.enabled", "true"),
Map.entry("connectors.qlik-sense.tenant-url", "https://tenant.example"),
Map.entry("connectors.qlik-sense.api-key", "test-qlik-api-key"),
Map.entry("connectors.qlik-sense.app-id", "test-app-id"),
Map.entry("connectors.qlik-sense.mode", "reload-only")
);

@Test
void aliasAllowListMatchesCompleteConnectorBindingSurface() {
Set<String> expected = new TreeSet<>();
expected.addAll(writableConnectorKeys("databricks", ConnectorProperties.DatabricksProps.class));
expected.addAll(writableConnectorKeys("snowflake", ConnectorProperties.SnowflakeProps.class));
expected.addAll(writableConnectorKeys("qlik-sense", ConnectorProperties.QlikSenseProps.class));

Set<String> actual = new TreeSet<>();
MightyEtlConfigAliasEnvironmentPostProcessor.RELATIVE_KEYS.stream()
.filter(key -> key.startsWith("connectors."))
.forEach(actual::add);

assertEquals(
expected,
actual,
"Connector aliases must stay machine-bound to every writable ConnectorProperties setting"
);
}

@Test
void mirrorsEveryModernConnectorSettingToLegacyConsumers() {
MockEnvironment env = new MockEnvironment();
CONNECTOR_SETTINGS.forEach((relative, value) -> env.setProperty("mightyetl." + relative, value));

Map<String, Object> aliases = MightyEtlConfigAliasEnvironmentPostProcessor.buildAliases(env);

CONNECTOR_SETTINGS.forEach((relative, value) ->
assertEquals(value, aliases.get("xtrmetl." + relative), relative));
}

@Test
void mirrorsEveryLegacyConnectorSettingForModernTooling() {
MockEnvironment env = new MockEnvironment();
CONNECTOR_SETTINGS.forEach((relative, value) -> env.setProperty("xtrmetl." + relative, value));

Map<String, Object> aliases = MightyEtlConfigAliasEnvironmentPostProcessor.buildAliases(env);

CONNECTOR_SETTINGS.forEach((relative, value) ->
assertEquals(value, aliases.get("mightyetl." + relative), relative));
}

@Test
void mirrorsModernConnectorFlagsToLegacyKeys() {
MockEnvironment env = new MockEnvironment();
Expand All @@ -32,6 +105,17 @@ void mirrorsModernConnectorFlagsToLegacyKeys() {
assertEquals("true", aliases.get("xtrmetl.connectors.databricks.enabled"));
}

@Test
void modernConnectorValueWinsWhenBothNamespacesAreSet() {
MockEnvironment env = new MockEnvironment();
env.setProperty("mightyetl.connectors.snowflake.password", "modern-test-secret");
env.setProperty("xtrmetl.connectors.snowflake.password", "legacy-test-secret");

Map<String, Object> aliases = MightyEtlConfigAliasEnvironmentPostProcessor.buildAliases(env);

assertEquals("modern-test-secret", aliases.get("xtrmetl.connectors.snowflake.password"));
}

@Test
void mirrorsModernBatchLimitsToLegacyKeys() {
MockEnvironment env = new MockEnvironment();
Expand Down Expand Up @@ -109,6 +193,23 @@ void emptyWhenUnset() {
assertTrue(MightyEtlConfigAliasEnvironmentPostProcessor.buildAliases(new MockEnvironment()).isEmpty());
}

/** Returns the environment-property keys represented by writable JavaBean-style setters. */
private static Set<String> writableConnectorKeys(String connectorId, Class<?> propertiesType) {
Set<String> keys = new TreeSet<>();
for (Method method : propertiesType.getMethods()) {
if (!method.getName().startsWith("set") || method.getName().length() <= 3
|| method.getParameterCount() != 1 || method.getReturnType() != void.class) {
continue;
}
String property = method.getName().substring(3);
property = Character.toLowerCase(property.charAt(0)) + property.substring(1);
String kebab = property.replaceAll("([a-z0-9])([A-Z])", "$1-$2")
.toLowerCase(Locale.ROOT);
keys.add("connectors." + connectorId + "." + kebab);
}
return keys;
}

@Configuration(proxyBeanMethods = false)
static class AliasOrderProbeConfiguration {
}
Expand Down
Loading