diff --git a/build.gradle b/build.gradle index c039194bbd..172ddbeb48 100644 --- a/build.gradle +++ b/build.gradle @@ -71,6 +71,10 @@ subprojects { apply plugin: "java-library" } + project(":polaris-eclipselink") { + apply plugin: "java-library" + } + dependencies { implementation(platform("com.fasterxml.jackson:jackson-bom:${jacksonVersion}")) implementation("com.fasterxml.jackson.core:jackson-annotations") diff --git a/extension/persistence/eclipselink/build.gradle b/extension/persistence/eclipselink/build.gradle index f872994358..ab4593150c 100644 --- a/extension/persistence/eclipselink/build.gradle +++ b/extension/persistence/eclipselink/build.gradle @@ -19,6 +19,7 @@ dependencies { implementation(project(":polaris-service")) implementation("org.eclipse.persistence:eclipselink:4.0.3") implementation("io.dropwizard:dropwizard-jackson:${dropwizardVersion}") + implementation("com.h2database:h2:2.2.224") testImplementation("com.h2database:h2:2.2.224") testImplementation(testFixtures(project(":polaris-core"))) diff --git a/polaris-server.yml b/polaris-server.yml index 886221937e..aa75e2fc8a 100644 --- a/polaris-server.yml +++ b/polaris-server.yml @@ -74,7 +74,6 @@ featureConfiguration: - AZURE - FILE - # Whether we want to enable Snowflake OAuth locally. Setting this to true requires # that you go through the setup outlined in the `README.md` file, specifically the # `OAuth + Snowflake: Local Testing And Then Some` section @@ -88,7 +87,7 @@ defaultRealms: - default-realm metaStoreManager: - type: in-memory + type: eclipse-link # TODO - avoid duplicating token broker config oauth2: @@ -126,7 +125,6 @@ cors: # Logging settings. logging: - # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. level: INFO @@ -136,7 +134,6 @@ logging: io.polaris: DEBUG appenders: - - type: console # If true, write log statements to stdout. # enabled: true diff --git a/polaris-service/build.gradle b/polaris-service/build.gradle index fb99ef01ef..ed43ecd261 100644 --- a/polaris-service/build.gradle +++ b/polaris-service/build.gradle @@ -21,6 +21,7 @@ plugins { dependencies { implementation(project(":polaris-core")) + runtimeOnly(project(":polaris-eclipselink")) implementation(platform("org.apache.iceberg:iceberg-bom:${icebergVersion}")) implementation("org.apache.iceberg:iceberg-api") @@ -193,6 +194,15 @@ task runApp(type: JavaExec) { args "server", "$rootDir/polaris-server.yml" } +task runBootstrap(type: JavaExec) { + if (System.getenv("AWS_REGION") == null) { + environment "AWS_REGION", "us-west-2" + } + classpath = sourceSets.main.runtimeClasspath + mainClass = "io.polaris.service.PolarisApplication" + args "bootstrap", "$rootDir/polaris-server.yml" +} + application { mainClass = "io.polaris.service.PolarisApplication" } diff --git a/polaris-service/src/main/java/io/polaris/service/PolarisApplication.java b/polaris-service/src/main/java/io/polaris/service/PolarisApplication.java index 74bff74ca2..c5f5cab73e 100644 --- a/polaris-service/src/main/java/io/polaris/service/PolarisApplication.java +++ b/polaris-service/src/main/java/io/polaris/service/PolarisApplication.java @@ -114,28 +114,27 @@ public static void main(final String[] args) throws Exception { } private static void printAsciiArt() { - String bannerArt = - String.join( - "\n", - " @@@@ @@@ @ @ @@@@ @ @@@@ @@@@ @ @@@@@ @ @ @@@ @@@@ ", - " @ @ @ @ @ @ @ @ @ @ @@ @ @ @ @ @ @ @ @ @ @ ", - " @@@@ @ @ @ @@@@@ @@@@ @ @@ @ @@@@@ @ @@@@@ @ @ @ @ @@@", - " @ @@@ @@@@ @ @ @ @@ @ @@@@ @@@@ @ @ @ @@ @@ @@@@ @@@ @@@@ ", - " ", - " ", - " ", - " ", - " /////| ", - " //||///T||| ", - " ///|||////|||||| ", - " //||||T////||||||||| ", - " /T| //|||||T///T||//T|||||| ", - " //|||/////T||////||/////||||||| //|| ", - " //||||||T///////////////////T|||||||T||||| ", - " //||||/////T|//////////|///////T|||||T|||||||| ", - " //|||||/////|||T////////////////||||||/||||||||| ", - ",,..,,,..,,,..,//||||////////||||||||||/////////|||||///||||||||||,,,..,,..,,,..,,,.", - ",,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,.,,,..,,,..,"); + String bannerArt = String.join( + "\n", + " @@@@ @@@ @ @ @@@@ @ @@@@ @@@@ @ @@@@@ @ @ @@@ @@@@ ", + " @ @ @ @ @ @ @ @ @ @ @@ @ @ @ @ @ @ @ @ @ @ ", + " @@@@ @ @ @ @@@@@ @@@@ @ @@ @ @@@@@ @ @@@@@ @ @ @ @ @@@", + " @ @@@ @@@@ @ @ @ @@ @ @@@@ @@@@ @ @ @ @@ @@ @@@@ @@@ @@@@ ", + " ", + " ", + " ", + " ", + " /////| ", + " //||///T||| ", + " ///|||////|||||| ", + " //||||T////||||||||| ", + " /T| //|||||T///T||//T|||||| ", + " //|||/////T||////||/////||||||| //|| ", + " //||||||T///////////////////T|||||||T||||| ", + " //||||/////T|//////////|///////T|||||T|||||||| ", + " //|||||/////|||T////////////////||||||/||||||||| ", + ",,..,,,..,,,..,//||||////////||||||||||/////////|||||///||||||||||,,,..,,..,,,..,,,.", + ",,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,..,,,.,,,..,,,..,"); System.out.println(bannerArt.replaceAll("\\|", "\\\\")); } @@ -143,8 +142,8 @@ private static void printAsciiArt() { public void initialize(Bootstrap bootstrap) { // Enable variable substitution with environment variables EnvironmentVariableSubstitutor substitutor = new EnvironmentVariableSubstitutor(false); - SubstitutingSourceProvider provider = - new SubstitutingSourceProvider(bootstrap.getConfigurationSourceProvider(), substitutor); + SubstitutingSourceProvider provider = new SubstitutingSourceProvider(bootstrap.getConfigurationSourceProvider(), + substitutor); bootstrap.setConfigurationSourceProvider(provider); bootstrap.addCommand(new BootstrapRealmsCommand()); @@ -152,7 +151,8 @@ public void initialize(Bootstrap bootstrap) { @Override public void run(PolarisApplicationConfig configuration, Environment environment) { - // PolarisEntityManager will be used for Management APIs and optionally the core Catalog APIs + // PolarisEntityManager will be used for Management APIs and optionally the core + // Catalog APIs // depending on the value of the baseCatalogType config. MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory(); StsClientBuilder stsClientBuilder = StsClient.builder(); @@ -163,8 +163,8 @@ public void run(PolarisApplicationConfig configuration, Environment environment) metaStoreManagerFactory.setStorageIntegrationProvider( new PolarisStorageIntegrationProviderImpl(stsClientBuilder::build)); - PolarisMetricRegistry polarisMetricRegistry = - new PolarisMetricRegistry(new PrometheusMeterRegistry(PrometheusConfig.DEFAULT)); + PolarisMetricRegistry polarisMetricRegistry = new PolarisMetricRegistry( + new PrometheusMeterRegistry(PrometheusConfig.DEFAULT)); metaStoreManagerFactory.setMetricRegistry(polarisMetricRegistry); OpenTelemetry openTelemetry = setupTracing(); @@ -175,8 +175,7 @@ public void run(PolarisApplicationConfig configuration, Environment environment) if (metaStoreManagerFactory instanceof ConfigurationStoreAware) { ((ConfigurationStoreAware) metaStoreManagerFactory).setConfigurationStore(configurationStore); } - RealmEntityManagerFactory entityManagerFactory = - new RealmEntityManagerFactory(metaStoreManagerFactory); + RealmEntityManagerFactory entityManagerFactory = new RealmEntityManagerFactory(metaStoreManagerFactory); CallContextResolver callContextResolver = configuration.getCallContextResolver(); callContextResolver.setEntityManagerFactory(entityManagerFactory); if (callContextResolver instanceof ConfigurationStoreAware csa) { @@ -192,8 +191,7 @@ public void run(PolarisApplicationConfig configuration, Environment environment) .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); TaskHandlerConfiguration taskConfig = configuration.getTaskHandler(); - TaskExecutorImpl taskExecutor = - new TaskExecutorImpl(taskConfig.executorService(), metaStoreManagerFactory); + TaskExecutorImpl taskExecutor = new TaskExecutorImpl(taskConfig.executorService(), metaStoreManagerFactory); TaskFileIOSupplier fileIOSupplier = new TaskFileIOSupplier(metaStoreManagerFactory); taskExecutor.addTaskHandler( new TableCleanupTaskHandler(taskExecutor, metaStoreManagerFactory, fileIOSupplier)); @@ -219,13 +217,11 @@ public void run(PolarisApplicationConfig configuration, Environment environment) } PolarisAuthorizer authorizer = new PolarisAuthorizer(configurationStore); - IcebergCatalogAdapter catalogAdapter = - new IcebergCatalogAdapter(catalogFactory, entityManagerFactory, authorizer); + IcebergCatalogAdapter catalogAdapter = new IcebergCatalogAdapter(catalogFactory, entityManagerFactory, authorizer); environment.jersey().register(new IcebergRestCatalogApi(catalogAdapter)); environment.jersey().register(new IcebergRestConfigurationApi(catalogAdapter)); - FilterRegistration.Dynamic corsRegistration = - environment.servlets().addFilter("CORS", CrossOriginFilter.class); + FilterRegistration.Dynamic corsRegistration = environment.servlets().addFilter("CORS", CrossOriginFilter.class); corsRegistration.setInitParameter( CrossOriginFilter.ALLOWED_ORIGINS_PARAM, String.join(",", configuration.getCorsConfiguration().getAllowedOrigins())); @@ -252,14 +248,13 @@ public void run(PolarisApplicationConfig configuration, Environment environment) .servlets() .addFilter("tracing", new TracingFilter(openTelemetry)) .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - DiscoverableAuthenticator authenticator = - configuration.getPolarisAuthenticator(); + DiscoverableAuthenticator authenticator = configuration + .getPolarisAuthenticator(); authenticator.setEntityManagerFactory(entityManagerFactory); - AuthFilter oauthCredentialAuthFilter = - new OAuthCredentialAuthFilter.Builder() - .setAuthenticator(authenticator) - .setPrefix("Bearer") - .buildAuthFilter(); + AuthFilter oauthCredentialAuthFilter = new OAuthCredentialAuthFilter.Builder() + .setAuthenticator(authenticator) + .setPrefix("Bearer") + .buildAuthFilter(); environment.jersey().register(new AuthDynamicFeature(oauthCredentialAuthFilter)); environment.healthChecks().register("polaris", new PolarisHealthCheck()); OAuth2ApiService oauth2Service = configuration.getOauth2Service(); @@ -299,27 +294,28 @@ public void run(PolarisApplicationConfig configuration, Environment environment) .getPrometheusRegistry())) .addMapping("/metrics"); - // For in-memory metastore we need to bootstrap Service and Service principal at startup (for + // For in-memory metastore we need to bootstrap Service and Service principal at + // startup (for // default realm) - // We can not utilize dropwizard Bootstrap command as command and server will be running two + // We can not utilize dropwizard Bootstrap command as command and server will be + // running two // different processes - // and in-memory state will be lost b/w invocation of bootstrap command and running a server + // and in-memory state will be lost b/w invocation of bootstrap command and + // running a server if (metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory) { metaStoreManagerFactory.getOrCreateMetaStoreManager(configuration::getDefaultRealm); } } private static OpenTelemetry setupTracing() { - Resource resource = - Resource.getDefault().toBuilder() - .put(ServiceAttributes.SERVICE_NAME, "polaris") - .put(ServiceAttributes.SERVICE_VERSION, "0.1.0") - .build(); - SdkTracerProvider sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create())) - .setResource(resource) - .build(); + Resource resource = Resource.getDefault().toBuilder() + .put(ServiceAttributes.SERVICE_NAME, "polaris") + .put(ServiceAttributes.SERVICE_VERSION, "0.1.0") + .build(); + SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create())) + .setResource(resource) + .build(); return OpenTelemetrySdk.builder() .setTracerProvider(sdkTracerProvider) .setPropagators( @@ -329,7 +325,10 @@ private static OpenTelemetry setupTracing() { .build(); } - /** Resolves and sets ThreadLocal CallContext/RealmContext based on the request contents. */ + /** + * Resolves and sets ThreadLocal CallContext/RealmContext based on the request + * contents. + */ private static class ContextResolverFilter implements Filter { private final RealmContextResolver realmContextResolver; private final CallContextResolver callContextResolver; @@ -345,37 +344,31 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha throws IOException, ServletException { HttpServletRequest httpRequest = (HttpServletRequest) request; Stream headerNames = Collections.list(httpRequest.getHeaderNames()).stream(); - Map headers = - headerNames.collect(Collectors.toMap(Function.identity(), httpRequest::getHeader)); - RealmContext currentRealmContext = - realmContextResolver.resolveRealmContext( - httpRequest.getRequestURL().toString(), - httpRequest.getMethod(), - httpRequest.getRequestURI().substring(1), - request.getParameterMap().entrySet().stream() - .collect( - Collectors.toMap(Map.Entry::getKey, (e) -> ((String[]) e.getValue())[0])), - headers); - CallContext currentCallContext = - callContextResolver.resolveCallContext( - currentRealmContext, - httpRequest.getMethod(), - httpRequest.getRequestURI().substring(1), - request.getParameterMap().entrySet().stream() - .collect( - Collectors.toMap(Map.Entry::getKey, (e) -> ((String[]) e.getValue())[0])), - headers); + Map headers = headerNames.collect(Collectors.toMap(Function.identity(), httpRequest::getHeader)); + RealmContext currentRealmContext = realmContextResolver.resolveRealmContext( + httpRequest.getRequestURL().toString(), + httpRequest.getMethod(), + httpRequest.getRequestURI().substring(1), + request.getParameterMap().entrySet().stream() + .collect( + Collectors.toMap(Map.Entry::getKey, (e) -> ((String[]) e.getValue())[0])), + headers); + CallContext currentCallContext = callContextResolver.resolveCallContext( + currentRealmContext, + httpRequest.getMethod(), + httpRequest.getRequestURI().substring(1), + request.getParameterMap().entrySet().stream() + .collect( + Collectors.toMap(Map.Entry::getKey, (e) -> ((String[]) e.getValue())[0])), + headers); CallContext.setCurrentContext(currentCallContext); - try (MDC.MDCCloseable context = - MDC.putCloseable("realm", currentRealmContext.getRealmIdentifier()); - MDC.MDCCloseable requestId = - MDC.putCloseable("request_id", httpRequest.getHeader("request_id"))) { + try (MDC.MDCCloseable context = MDC.putCloseable("realm", currentRealmContext.getRealmIdentifier()); + MDC.MDCCloseable requestId = MDC.putCloseable("request_id", httpRequest.getHeader("request_id"))) { chain.doFilter(request, response); } finally { - Object contextCatalog = - currentCallContext - .contextVariables() - .get(CallContext.REQUEST_PATH_CATALOG_INSTANCE_KEY); + Object contextCatalog = currentCallContext + .contextVariables() + .get(CallContext.REQUEST_PATH_CATALOG_INSTANCE_KEY); if (contextCatalog != null && contextCatalog instanceof Closeable) { ((Closeable) contextCatalog).close(); } diff --git a/polaris-service/src/main/resources/META-INF/persistence.xml b/polaris-service/src/main/resources/META-INF/persistence.xml index db8c1c4bd3..11828b2848 100644 --- a/polaris-service/src/main/resources/META-INF/persistence.xml +++ b/polaris-service/src/main/resources/META-INF/persistence.xml @@ -20,7 +20,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://java.sun.com/xml/ns/persistence http://java.sun.com/xml/ns/persistence/persistence_2_0.xsd"> - + org.eclipse.persistence.jpa.PersistenceProvider io.polaris.core.persistence.models.ModelEntity io.polaris.core.persistence.models.ModelEntityActive