diff --git a/docker-compose.base.yaml b/docker-compose.base.yaml new file mode 100644 index 0000000000..8e905b5ce2 --- /dev/null +++ b/docker-compose.base.yaml @@ -0,0 +1,200 @@ +# Base docker-compose configuration with shared services +# This file is extended by docker-compose.yaml and docker-compose.consumer.yaml +networks: + default: + name: opentdf_platform + +services: + opentdfdb: + image: postgres:15-alpine + restart: always + user: postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: changeme + POSTGRES_DB: opentdf + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 5s + timeout: 5s + retries: 10 + ports: + - "${POSTGRES_EXPOSE_PORT:-5432}:5432" + + jaeger: + image: jaegertracing/all-in-one:latest + environment: + COLLECTOR_OTLP_ENABLED: "true" + ports: + - "16686:16686" # Web UI + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + - "14250:14250" # Model/collector gRPC + profiles: + - tracing + restart: always + + # Entity Resolution Service Testing Infrastructure + # + # Provides PostgreSQL and OpenLDAP services for comprehensive ERS testing + # Enables full multi-strategy ERS testing with SQL and LDAP providers + # + # Usage: + # docker-compose --profile ers-test up -d # Start ERS test services + # docker-compose --profile ers-admin up -d # Include LDAP admin UI + # ERS_TEST_POSTGRES_URL="postgres://ers_test_user:ers_test_pass@localhost:5433/ers_test?sslmode=disable" \ + # ERS_TEST_LDAP_URL="ldap://localhost:1389" \ + # go test ./service/entityresolution/integration -run TestMultiStrategy -v + # + # Services: + # - ers-postgres: PostgreSQL 16 on port 5433 with test schema and data + # - ers-ldap: OpenLDAP on port 1389 with organizational test data + # - ers-ldap-admin: phpLDAPadmin on port 6443 (ers-admin profile only) + + # PostgreSQL for ERS SQL provider testing + ers-postgres: + image: postgres:16-alpine + container_name: ers_test_postgres + profiles: + - ers-test + - ers-admin + ports: + - "5433:5432" # Different port to avoid conflicts with main opentdfdb + environment: + POSTGRES_DB: ers_test + POSTGRES_USER: ers_test_user + POSTGRES_PASSWORD: ers_test_pass + POSTGRES_INITDB_ARGS: "--encoding=UTF-8" + volumes: + - ers_postgres_data:/var/lib/postgresql/data + - ./service/entityresolution/integration/sql_test_data:/docker-entrypoint-initdb.d + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ers_test_user -d ers_test"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + # OpenLDAP for ERS LDAP provider testing + ers-ldap: + image: osixia/openldap:1.5.0 + container_name: ers_test_openldap + profiles: + - ers-test + - ers-admin + ports: + - "1389:389" # LDAP port (different from standard 389) + - "1636:636" # LDAPS port (different from standard 636) + environment: + LDAP_ORGANISATION: "OpenTDF Test" + LDAP_DOMAIN: "opentdf.test" + LDAP_ADMIN_PASSWORD: "admin_password" + LDAP_CONFIG_PASSWORD: "config_password" + LDAP_READONLY_USER: "true" + LDAP_READONLY_USER_USERNAME: "readonly" + LDAP_READONLY_USER_PASSWORD: "readonly_password" + LDAP_RFC2307BIS_SCHEMA: "false" + LDAP_BACKEND: "mdb" + LDAP_TLS: "true" + LDAP_TLS_ENFORCE: "false" + LDAP_REPLICATION: "false" + KEEP_EXISTING_CONFIG: "false" + LDAP_REMOVE_CONFIG_AFTER_SETUP: "true" + volumes: + - ers_ldap_data:/var/lib/ldap + - ers_ldap_config:/etc/ldap/slapd.d + - ./service/entityresolution/integration/ldap_test_data:/container/service/slapd/assets/config/bootstrap/ldif/custom + healthcheck: + test: ["CMD-SHELL", "ldapsearch -x -H ldap://localhost:389 -b dc=opentdf,dc=test -D cn=admin,dc=opentdf,dc=test -w admin_password '(objectclass=*)' dn"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + # phpLDAPadmin for LDAP management (admin profile only) + ers-ldap-admin: + image: osixia/phpldapadmin:latest + container_name: ers_ldap_admin + profiles: + - ers-admin # Only available with --profile ers-admin + ports: + - "6443:443" + environment: + PHPLDAPADMIN_LDAP_HOSTS: ers-ldap + PHPLDAPADMIN_HTTPS: "false" + depends_on: + ers-ldap: + condition: service_healthy + + # Base Keycloak configuration (extended by specific compose files) + keycloak: + image: keycloak/keycloak:25.0 + restart: always + command: + - "start-dev" + - "--verbose" + - "-Djavax.net.ssl.trustStorePassword=password" + - "-Djavax.net.ssl.HostnameVerifier=AllowAll" + - "-Djavax.net.ssl.trustStore=/truststore/truststore.jks" + - "--spi-truststore-file-hostname-verification-policy=ANY" + environment: + KC_PROXY: edge + KC_HTTP_RELATIVE_PATH: /auth + KC_DB_VENDOR: postgres + KC_DB_URL_HOST: keycloakdb + KC_DB_URL_PORT: 5432 + KC_DB_URL_DATABASE: keycloak + KC_DB_USERNAME: keycloak + KC_DB_PASSWORD: changeme + KC_HOSTNAME_STRICT: "false" + KC_HOSTNAME_STRICT_BACKCHANNEL: "false" + KC_HOSTNAME_STRICT_HTTPS: "false" + KC_HTTP_ENABLED: "true" + KC_HTTP_PORT: "8888" + KC_HTTPS_PORT: "8443" + KC_HTTP_MANAGEMENT_PORT: "9001" + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: changeme + KC_FEATURES: "preview,token-exchange" + KC_HEALTH_ENABLED: "true" + KC_HTTPS_KEY_STORE_PASSWORD: "password" + KC_HTTPS_KEY_STORE_FILE: "/truststore/truststore.jks" + KC_HTTPS_CERTIFICATE_FILE: "/etc/x509/tls/localhost.crt" + KC_HTTPS_CERTIFICATE_KEY_FILE: "/etc/x509/tls/localhost.key" + KC_HTTPS_CLIENT_AUTH: "request" + JAVA_OPTS_APPEND: "${JAVA_OPTS_APPEND:-}" + ports: + - "${KC_EXPOSE_PORT:-8443}:8443" + - "${KC_EXPOSE_PORT_HTTP:-8888}:8888" + - "${KC_EXPOSE_PORT_MGMT:-9001}:9001" + healthcheck: + test: + - CMD-SHELL + - | + [ -f /tmp/HealthCheck.java ] || echo "public class HealthCheck { + public static void main(String[] args) throws java.lang.Throwable { + javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true); + javax.net.ssl.SSLContext sc = javax.net.ssl.SSLContext.getInstance(\"SSL\"); + sc.init(null, new javax.net.ssl.TrustManager[]{ + new javax.net.ssl.X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; } + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) {} + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) {} + } + }, new java.security.SecureRandom()); + javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + java.net.HttpURLConnection conn = (java.net.HttpURLConnection)new java.net.URL(args[0]).openConnection(); + System.exit(java.net.HttpURLConnection.HTTP_OK == conn.getResponseCode() ? 0 : 1); + } + }" > /tmp/HealthCheck.java && java ${JAVA_OPTS_APPEND} /tmp/HealthCheck.java https://localhost:9001/auth/health/live + timeout: 10s + retries: 3 + start_period: 2m + +volumes: + ers_postgres_data: + name: ers_test_postgres_data + ers_ldap_data: + name: ers_test_ldap_data + ers_ldap_config: + name: ers_test_ldap_config diff --git a/docker-compose.consumer.yaml b/docker-compose.consumer.yaml new file mode 100644 index 0000000000..ecfe7b818f --- /dev/null +++ b/docker-compose.consumer.yaml @@ -0,0 +1,631 @@ +networks: + default: + name: opentdf_platform + +configs: + caddy_config: + content: | + { + log { + level INFO + output stdout + } + } + https://keycloak.opentdf.local:9443 { + tls internal + reverse_proxy keycloak:8888 + } + https://platform.opentdf.local:8443 { + tls internal + reverse_proxy { + to h2c://platform:8080 + transport http { + versions h2c 2 1.1 # Enable gRPC proxying + } + } + } + +services: + caddy: + image: caddy:alpine + command: ['caddy','run', '--config', '/etc/caddy/Caddyfile'] + configs: + - source: caddy_config + target: /etc/caddy/Caddyfile + ports: + - "8443:8443" + - "9443:9443" + - "2019:2019" + volumes: + - caddy_data:/data + depends_on: + ensure-permissions: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:2019/metrics"] + interval: 5s + timeout: 5s + retries: 3 + restart: unless-stopped + + check-certs: + image: bash:latest + volumes: + - type: volume + source: caddy_data + target: /etc/ssl/certs + volume: + subpath: caddy/certificates/local/keycloak.opentdf.local/ + command: + - bash + - -c + - | + echo "Checking certificates" + ls -alh /etc/ssl/certs + cat /etc/ssl/certs/keycloak.opentdf.local.crt + depends_on: + caddy: + condition: service_healthy + ensure-permissions: + condition: service_completed_successfully + restart: "no" + + ensure-permissions: + image: alpine + command: + - 'sh' + - '-c' + - | + chmod -R 755 /data + volumes: + - caddy_data:/data + restart: "no" + + keycloak: + volumes: + - keys:/keys:ro + image: keycloak/keycloak:25.0 + restart: always + depends_on: + fix-keys-permissions: + condition: service_completed_successfully + command: + - "start-dev" + - "--verbose" + - "-Djavax.net.ssl.trustStorePassword=password" + - "-Djavax.net.ssl.HostnameVerifier=AllowAll" + - "-Djavax.net.ssl.trustStore=/keys/ca.jks" + - "--spi-truststore-file-hostname-verification-policy=ANY" + environment: + KC_PROXY: edge + KC_HTTP_RELATIVE_PATH: /auth + KC_HOSTNAME_STRICT: "false" + KC_HOSTNAME_STRICT_BACKCHANNEL: "false" + KC_HOSTNAME_STRICT_HTTPS: "false" + KC_HTTP_ENABLED: "true" + KC_HTTP_PORT: "8888" + KC_HTTPS_PORT: "8443" + KC_HTTP_MANAGEMENT_PORT: "9001" + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: changeme + #KC_HOSTNAME_URL: http://localhost:8888/auth + KC_FEATURES: "preview,token-exchange" + KC_HEALTH_ENABLED: "true" + KC_HTTPS_KEY_STORE_PASSWORD: "password" + KC_HTTPS_KEY_STORE_FILE: "/keys/ca.jks" + KC_HTTPS_CERTIFICATE_FILE: "/keys/localhost.crt" + KC_HTTPS_CERTIFICATE_KEY_FILE: "/keys/localhost.key" + KC_HTTPS_CLIENT_AUTH: "request" + ### + # The following environment variable resolves SIGILL with Code 134 when running Java processes on Apple M4 chips + # + # On Apple Silicon (M4 chip): + # export JAVA_OPTS_APPEND="-XX:UseSVE=0" + # docker-compose up + # + # On other architectures: + # export JAVA_OPTS_APPEND="" + # docker-compose up + # + # Or set directly: JAVA_OPTS_APPEND="-XX:UseSVE=0" docker-compose up + JAVA_OPTS_APPEND: "${JAVA_OPTS_APPEND:-}" + ### + # ports: + # - "${KC_EXPOSE_PORT:-8443}:8443" + # - "${KC_EXPOSE_PORT_HTTP:-8888}:8888" + # - "${KC_EXPOSE_PORT_MGMT:-9001}:9001" + healthcheck: + test: + - CMD-SHELL + - | + [ -f /tmp/HealthCheck.java ] || echo "public class HealthCheck { + public static void main(String[] args) throws java.lang.Throwable { + javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true); + javax.net.ssl.SSLContext sc = javax.net.ssl.SSLContext.getInstance(\"SSL\"); + sc.init(null, new javax.net.ssl.TrustManager[]{ + new javax.net.ssl.X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; } + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) {} + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) {} + } + }, new java.security.SecureRandom()); + javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + java.net.HttpURLConnection conn = (java.net.HttpURLConnection)new java.net.URL(args[0]).openConnection(); + System.exit(java.net.HttpURLConnection.HTTP_OK == conn.getResponseCode() ? 0 : 1); + } + }" > /tmp/HealthCheck.java && java ${JAVA_OPTS_APPEND} /tmp/HealthCheck.java http://localhost:8888/auth 2>/dev/null + interval: 10s + timeout: 10s + retries: 10 + start_period: 3m + opentdfdb: + image: postgres:15-alpine + restart: always + user: postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: changeme + POSTGRES_DB: opentdf + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 5s + timeout: 5s + retries: 10 + ports: + - "${POSTGRES_EXPOSE_PORT:-5432}:5432" + + jaeger: + image: jaegertracing/all-in-one:latest + environment: + COLLECTOR_OTLP_ENABLED: "true" + ports: + - "16686:16686" # Web UI + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + - "14250:14250" # Model/collector gRPC + profiles: + - tracing + restart: always + + # Entity Resolution Service Testing Infrastructure + # + # Provides PostgreSQL and OpenLDAP services for comprehensive ERS testing + # Enables full multi-strategy ERS testing with SQL and LDAP providers + # + # Usage: + # docker-compose --profile ers-test up -d # Start ERS test services + # docker-compose --profile ers-admin up -d # Include LDAP admin UI + # ERS_TEST_POSTGRES_URL="postgres://ers_test_user:ers_test_pass@localhost:5433/ers_test?sslmode=disable" \ + # ERS_TEST_LDAP_URL="ldap://localhost:1389" \ + # go test ./service/entityresolution/integration -run TestMultiStrategy -v + # + # Services: + # - ers-postgres: PostgreSQL 16 on port 5433 with test schema and data + # - ers-ldap: OpenLDAP on port 1389 with organizational test data + # - ers-ldap-admin: phpLDAPadmin on port 6443 (ers-admin profile only) + + # PostgreSQL for ERS SQL provider testing + ers-postgres: + image: postgres:16-alpine + container_name: ers_test_postgres + profiles: + - ers-test + - ers-admin + ports: + - "5433:5432" # Different port to avoid conflicts with main opentdfdb + environment: + POSTGRES_DB: ers_test + POSTGRES_USER: ers_test_user + POSTGRES_PASSWORD: ers_test_pass + POSTGRES_INITDB_ARGS: "--encoding=UTF-8" + volumes: + - ers_postgres_data:/var/lib/postgresql/data + - ./service/entityresolution/integration/sql_test_data:/docker-entrypoint-initdb.d + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ers_test_user -d ers_test"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + # OpenLDAP for ERS LDAP provider testing + ers-ldap: + image: osixia/openldap:1.5.0 + container_name: ers_test_openldap + profiles: + - ers-test + - ers-admin + ports: + - "1389:389" # LDAP port (different from standard 389) + - "1636:636" # LDAPS port (different from standard 636) + environment: + LDAP_ORGANISATION: "OpenTDF Test" + LDAP_DOMAIN: "opentdf.test" + LDAP_ADMIN_PASSWORD: "admin_password" + LDAP_CONFIG_PASSWORD: "config_password" + LDAP_READONLY_USER: "true" + LDAP_READONLY_USER_USERNAME: "readonly" + LDAP_READONLY_USER_PASSWORD: "readonly_password" + LDAP_RFC2307BIS_SCHEMA: "false" + LDAP_BACKEND: "mdb" + LDAP_TLS: "true" + LDAP_TLS_ENFORCE: "false" + LDAP_REPLICATION: "false" + KEEP_EXISTING_CONFIG: "false" + LDAP_REMOVE_CONFIG_AFTER_SETUP: "true" + volumes: + - ers_ldap_data:/var/lib/ldap + - ers_ldap_config:/etc/ldap/slapd.d + - ./service/entityresolution/integration/ldap_test_data:/container/service/slapd/assets/config/bootstrap/ldif/custom + healthcheck: + test: ["CMD-SHELL", "ldapsearch -x -H ldap://localhost:389 -b dc=opentdf,dc=test -D cn=admin,dc=opentdf,dc=test -w admin_password '(objectclass=*)' dn"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + # phpLDAPadmin for LDAP management (admin profile only) + ers-ldap-admin: + image: osixia/phpldapadmin:latest + container_name: ers_ldap_admin + profiles: + - ers-admin # Only available with --profile ers-admin + ports: + - "6443:443" + environment: + PHPLDAPADMIN_LDAP_HOSTS: ers-ldap + PHPLDAPADMIN_HTTPS: "false" + depends_on: + ers-ldap: + condition: service_healthy + + # Provision Keycloak with initial configuration + platform-provision-keycloak: + image: registry.opentdf.io/platform:nightly + command: ["provision", "keycloak", "-e", "https://keycloak.opentdf.local:9443/auth", "-f", "/configs/keycloak_data.yaml"] + depends_on: + keycloak: + condition: service_healthy + opentdfdb: + condition: service_healthy + patch-platform-config: + condition: service_completed_successfully + download-keycloak-data: + condition: service_completed_successfully + generate-keys: + condition: service_completed_successfully + volumes: + - configs:/configs:ro + - keys:/keys:ro + environment: + - OPENTDF_CONFIG_FILE=/configs/opentdf.yaml + restart: "no" + extra_hosts: + - "keycloak.opentdf.local:host-gateway" + # Prepare fixtures directory structure - create symlink to expected location + prepare-fixtures: + image: alpine:latest + volumes: + - configs:/configs + depends_on: + download-fixtures: + condition: service_completed_successfully + command: + - sh + - -c + - | + mkdir -p /configs/service/internal/fixtures + cd /configs + ln -sf /configs/service/internal/fixtures ./service + restart: "no" + + # Add sample attributes and metadata + platform-provision-fixtures: + image: registry.opentdf.io/platform:nightly + command: ["provision", "fixtures", "--config-file", "/configs/opentdf.yaml"] + working_dir: /configs + depends_on: + platform-provision-keycloak: + condition: service_completed_successfully + opentdfdb: + condition: service_healthy + prepare-fixtures: + condition: service_completed_successfully + generate-keys: + condition: service_completed_successfully + volumes: + - configs:/configs:ro + - keys:/keys:ro + restart: "no" + + # Prepare CA certificates bundle with Caddy cert + prepare-ca-certs: + image: alpine:latest + volumes: + - type: volume + source: caddy_data + target: /caddy-certs + read_only: true + volume: + subpath: caddy/certificates/local/keycloak.opentdf.local + - platform_certs:/etc/ssl/certs + depends_on: + caddy: + condition: service_healthy + command: + - sh + - -c + - | + # Install ca-certificates package + apk add --no-cache ca-certificates + # Copy Caddy certificate to CA bundle + cp /caddy-certs/keycloak.opentdf.local.crt /usr/local/share/ca-certificates/ + update-ca-certificates + # Copy the updated CA bundle to shared volume + cp -r /etc/ssl/certs/* /etc/ssl/certs/ + echo "CA certificates prepared successfully" + restart: "no" + + # Main OpenTDF Platform server + platform: + image: registry.opentdf.io/platform:nightly + command: ["start", "--config-file", "/configs/opentdf.yaml"] + depends_on: + platform-provision-fixtures: + condition: service_completed_successfully + keycloak: + condition: service_healthy + opentdfdb: + condition: service_healthy + generate-keys: + condition: service_completed_successfully + prepare-ca-certs: + condition: service_completed_successfully + ports: + - "8080:8080" + volumes: + - configs:/configs:ro + - keys:/keys:ro + - platform_certs:/etc/ssl/certs:ro + extra_hosts: + - "keycloak.opentdf.local:host-gateway" + restart: unless-stopped + + # Initialize volume permissions + init-volumes: + image: alpine:latest + volumes: + - configs:/configs + - keys:/keys + command: + - sh + - -c + - | + chmod 777 /configs /keys + mkdir -p /configs/service/internal/fixtures + chmod -R 777 /configs + restart: "no" + + # Fix keys permissions after generation + fix-keys-permissions: + image: alpine:latest + volumes: + - keys:/keys + depends_on: + generate-keys: + condition: service_completed_successfully + command: + - sh + - -c + - | + chmod -R 755 /keys + chmod 644 /keys/* + restart: "no" + + # Download platform configuration file + download-platform-config: + image: curlimages/curl:latest + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + command: ['-o', '/configs/opentdf.yaml', 'https://raw.githubusercontent.com/opentdf/platform/main/opentdf-example.yaml'] + restart: "no" + + # Patch platform configuration to use keycloak.opentdf.local:9443 + patch-platform-config: + image: alpine:latest + volumes: + - configs:/configs + depends_on: + download-platform-config: + condition: service_completed_successfully + command: + - sh + - -c + - | + apk add --no-cache sed + sed -i 's|http://keycloak:8888|https://keycloak.opentdf.local:9443|g' /configs/opentdf.yaml + echo "Patched opentdf.yaml to use keycloak.opentdf.local:9443" + restart: "no" + + # Download Keycloak provisioning data + download-keycloak-data: + image: curlimages/curl:latest + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + entrypoint: /bin/sh + command: + - -c + - | + URL='https://raw.githubusercontent.com/opentdf/platform/main/service/cmd/keycloak_data.yaml' + OUTPUT='/configs/keycloak_data.yaml' + MAX_ATTEMPTS=3 + + for i in $$(seq 1 $$MAX_ATTEMPTS); do + echo "Attempt $$i of $$MAX_ATTEMPTS: Downloading keycloak_data.yaml..." + + if curl -f -o "$$OUTPUT" "$$URL"; then + echo "Download successful" + + # Validate the downloaded file + if [ -f "$$OUTPUT" ] && [ -s "$$OUTPUT" ]; then + if head -1 "$$OUTPUT" | grep -q -E '^(---|\w+:)'; then + echo "Validation passed: File exists, non-empty, and appears to be valid YAML" + exit 0 + else + echo "Validation failed: File does not appear to be valid YAML" + rm -f "$$OUTPUT" + fi + else + echo "Validation failed: File is missing or empty" + fi + else + echo "Download failed (attempt $$i)" + fi + + if [ $$i -lt $$MAX_ATTEMPTS ]; then + echo "Retrying in 2 seconds..." + sleep 2 + fi + done + + echo "ERROR: Failed to download and validate keycloak_data.yaml after $$MAX_ATTEMPTS attempts" + exit 1 + restart: "no" + + # Download fixtures data + download-fixtures: + image: curlimages/curl:latest + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + command: ['-o', '/configs/service/internal/fixtures/policy_fixtures.yaml', 'https://raw.githubusercontent.com/opentdf/platform/main/service/internal/fixtures/policy_fixtures.yaml'] + restart: "no" + + # Download init-temp-keys script + download-init-script: + image: curlimages/curl:latest + volumes: + - configs:/configs + depends_on: + init-volumes: + condition: service_completed_successfully + entrypoint: /bin/sh + command: + - -c + - | + URL='https://raw.githubusercontent.com/opentdf/platform/main/.github/scripts/init-temp-keys.sh' + OUTPUT='/configs/init-temp-keys.sh' + MAX_ATTEMPTS=3 + + for i in $$(seq 1 $$MAX_ATTEMPTS); do + echo "Attempt $$i of $$MAX_ATTEMPTS: Downloading init-temp-keys.sh..." + + if curl -f -o "$$OUTPUT" "$$URL"; then + echo "Download successful" + + # Validate the downloaded file + if [ -f "$$OUTPUT" ] && [ -s "$$OUTPUT" ]; then + if head -1 "$$OUTPUT" | grep -q '^#!/'; then + echo "Validation passed: File exists, non-empty, and appears to be a shell script" + exit 0 + else + echo "Validation failed: File does not appear to be a valid shell script" + rm -f "$$OUTPUT" + fi + else + echo "Validation failed: File is missing or empty" + fi + else + echo "Download failed (attempt $$i)" + fi + + if [ $$i -lt $$MAX_ATTEMPTS ]; then + echo "Retrying in 2 seconds..." + sleep 2 + fi + done + + echo "ERROR: Failed to download and validate init-temp-keys.sh after $$MAX_ATTEMPTS attempts" + exit 1 + restart: "no" + + # Generate keys without Docker dependency + generate-keys: + image: alpine:latest + volumes: + - configs:/configs + - keys:/keys + depends_on: + download-init-script: + condition: service_completed_successfully + init-volumes: + condition: service_completed_successfully + entrypoint: /bin/sh + command: + - -c + - | + apk add --no-cache openssl openjdk11-jre bash + cd /keys + + # Generate KAS RSA private key + openssl genpkey -algorithm RSA -out /keys/kas-private.pem -pkeyopt rsa_keygen_bits:2048 + openssl rsa -in /keys/kas-private.pem -pubout -out /keys/kas-cert.pem + + # Generate ECC Key + openssl ecparam -name prime256v1 > /tmp/ecparams.tmp + openssl req -x509 -nodes -newkey ec:/tmp/ecparams.tmp -subj "/CN=kas" -keyout /keys/kas-ec-private.pem -out /keys/kas-ec-cert.pem -days 365 + + # Generate CA + openssl req -x509 -nodes -newkey RSA:2048 -subj "/CN=ca" -keyout /keys/keycloak-ca-private.pem -out /keys/keycloak-ca.pem -days 365 + + # Generate localhost certificate + printf "subjectAltName=DNS:localhost,IP:127.0.0.1" > /tmp/sanX509.conf + printf "[req]\ndistinguished_name=req_distinguished_name\n[req_distinguished_name]\n[alt_names]\nDNS.1=localhost\nIP.1=127.0.0.1" > /tmp/req.conf + openssl req -new -nodes -newkey rsa:2048 -keyout /keys/localhost.key -out /tmp/localhost.req -batch -subj "/CN=localhost" -config /tmp/req.conf + openssl x509 -req -in /tmp/localhost.req -CA /keys/keycloak-ca.pem -CAkey /keys/keycloak-ca-private.pem -CAcreateserial -out /keys/localhost.crt -days 3650 -sha256 -extfile /tmp/sanX509.conf + + # Generate sample user certificate + openssl req -new -nodes -newkey rsa:2048 -keyout /keys/sampleuser.key -out /tmp/sampleuser.req -batch -subj "/CN=sampleuser" + openssl x509 -req -in /tmp/sampleuser.req -CA /keys/keycloak-ca.pem -CAkey /keys/keycloak-ca-private.pem -CAcreateserial -out /keys/sampleuser.crt -days 3650 + + # Convert to PKCS12 + openssl pkcs12 -export -in /keys/keycloak-ca.pem -inkey /keys/keycloak-ca-private.pem -out /keys/ca.p12 -nodes -passout pass:password + + # Convert PKCS12 to JKS using keytool (no Docker needed) + keytool -importkeystore \ + -srckeystore /keys/ca.p12 \ + -srcstoretype PKCS12 \ + -destkeystore /keys/ca.jks \ + -deststoretype JKS \ + -srcstorepass "password" \ + -deststorepass "password" \ + -noprompt + + echo "Keys generated successfully" + environment: + JAVA_OPTS_APPEND: "${JAVA_OPTS_APPEND:-}" + restart: "no" + +volumes: + keys: + name: opentdf_keys + configs: + name: opentdf_configs + caddy_data: + platform_certs: + name: opentdf_platform_certs + ers_postgres_data: + name: ers_test_postgres_data + ers_ldap_data: + name: ers_test_ldap_data + ers_ldap_config: + name: ers_test_ldap_config + diff --git a/docker-compose.yaml b/docker-compose.yaml index 54ebd8f793..ee031ad6b3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,213 +1,34 @@ -networks: - default: - name: opentdf_platform services: keycloak: + extends: + file: docker-compose.base.yaml + service: keycloak volumes: - ${KEYS_DIR:-./keys}/localhost.crt:/etc/x509/tls/localhost.crt - ${KEYS_DIR:-./keys}/localhost.key:/etc/x509/tls/localhost.key - ${KEYS_DIR:-./keys}/ca.jks:/truststore/truststore.jks - image: keycloak/keycloak:25.0 - restart: always - command: - - "start-dev" - - "--verbose" - - "-Djavax.net.ssl.trustStorePassword=password" - - "-Djavax.net.ssl.HostnameVerifier=AllowAll" - - "-Djavax.net.ssl.trustStore=/truststore/truststore.jks" - - "--spi-truststore-file-hostname-verification-policy=ANY" - environment: - KC_PROXY: edge - KC_HTTP_RELATIVE_PATH: /auth - KC_DB_VENDOR: postgres - KC_DB_URL_HOST: keycloakdb - KC_DB_URL_PORT: 5432 - KC_DB_URL_DATABASE: keycloak - KC_DB_USERNAME: keycloak - KC_DB_PASSWORD: changeme - KC_HOSTNAME_STRICT: "false" - KC_HOSTNAME_STRICT_BACKCHANNEL: "false" - KC_HOSTNAME_STRICT_HTTPS: "false" - KC_HTTP_ENABLED: "true" - KC_HTTP_PORT: "8888" - KC_HTTPS_PORT: "8443" - KC_HTTP_MANAGEMENT_PORT: "9001" - KEYCLOAK_ADMIN: admin - KEYCLOAK_ADMIN_PASSWORD: changeme - #KC_HOSTNAME_URL: http://localhost:8888/auth - KC_FEATURES: "preview,token-exchange" - KC_HEALTH_ENABLED: "true" - KC_HTTPS_KEY_STORE_PASSWORD: "password" - KC_HTTPS_KEY_STORE_FILE: "/truststore/truststore.jks" - KC_HTTPS_CERTIFICATE_FILE: "/etc/x509/tls/localhost.crt" - KC_HTTPS_CERTIFICATE_KEY_FILE: "/etc/x509/tls/localhost.key" - KC_HTTPS_CLIENT_AUTH: "request" - ### - # The following environment variable resolves SIGILL with Code 134 when running Java processes on Apple M4 chips - # - # On Apple Silicon (M4 chip): - # export JAVA_OPTS_APPEND="-XX:UseSVE=0" - # docker-compose up - # - # On other architectures: - # export JAVA_OPTS_APPEND="" - # docker-compose up - # - # Or set directly: JAVA_OPTS_APPEND="-XX:UseSVE=0" docker-compose up - JAVA_OPTS_APPEND: "${JAVA_OPTS_APPEND:-}" - ### - ports: - - "${KC_EXPOSE_PORT:-8443}:8443" - - "${KC_EXPOSE_PORT_HTTP:-8888}:8888" - - "${KC_EXPOSE_PORT_MGMT:-9001}:9001" - healthcheck: - test: - - CMD-SHELL - - | - [ -f /tmp/HealthCheck.java ] || echo "public class HealthCheck { - public static void main(String[] args) throws java.lang.Throwable { - javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true); - javax.net.ssl.SSLContext sc = javax.net.ssl.SSLContext.getInstance(\"SSL\"); - sc.init(null, new javax.net.ssl.TrustManager[]{ - new javax.net.ssl.X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; } - public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) {} - public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) {} - } - }, new java.security.SecureRandom()); - javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); - java.net.HttpURLConnection conn = (java.net.HttpURLConnection)new java.net.URL(args[0]).openConnection(); - System.exit(java.net.HttpURLConnection.HTTP_OK == conn.getResponseCode() ? 0 : 1); - } - }" > /tmp/HealthCheck.java && java ${JAVA_OPTS_APPEND} /tmp/HealthCheck.java https://localhost:9001/auth/health/live - timeout: 10s - retries: 3 - start_period: 2m + opentdfdb: - image: postgres:15-alpine - restart: always - user: postgres - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: changeme - POSTGRES_DB: opentdf - healthcheck: - test: ["CMD-SHELL", "pg_isready"] - interval: 5s - timeout: 5s - retries: 10 - ports: - - "${POSTGRES_EXPOSE_PORT:-5432}:5432" + extends: + file: docker-compose.base.yaml + service: opentdfdb jaeger: - image: jaegertracing/all-in-one:latest - environment: - COLLECTOR_OTLP_ENABLED: "true" - ports: - - "16686:16686" # Web UI - - "4317:4317" # OTLP gRPC - - "4318:4318" # OTLP HTTP - - "14250:14250" # Model/collector gRPC - profiles: - - tracing - restart: always - - # Entity Resolution Service Testing Infrastructure - # - # Provides PostgreSQL and OpenLDAP services for comprehensive ERS testing - # Enables full multi-strategy ERS testing with SQL and LDAP providers - # - # Usage: - # docker-compose --profile ers-test up -d # Start ERS test services - # docker-compose --profile ers-admin up -d # Include LDAP admin UI - # ERS_TEST_POSTGRES_URL="postgres://ers_test_user:ers_test_pass@localhost:5433/ers_test?sslmode=disable" \ - # ERS_TEST_LDAP_URL="ldap://localhost:1389" \ - # go test ./service/entityresolution/integration -run TestMultiStrategy -v - # - # Services: - # - ers-postgres: PostgreSQL 16 on port 5433 with test schema and data - # - ers-ldap: OpenLDAP on port 1389 with organizational test data - # - ers-ldap-admin: phpLDAPadmin on port 6443 (ers-admin profile only) + extends: + file: docker-compose.base.yaml + service: jaeger - # PostgreSQL for ERS SQL provider testing ers-postgres: - image: postgres:16-alpine - container_name: ers_test_postgres - profiles: - - ers-test - - ers-admin - ports: - - "5433:5432" # Different port to avoid conflicts with main opentdfdb - environment: - POSTGRES_DB: ers_test - POSTGRES_USER: ers_test_user - POSTGRES_PASSWORD: ers_test_pass - POSTGRES_INITDB_ARGS: "--encoding=UTF-8" - volumes: - - ers_postgres_data:/var/lib/postgresql/data - - ./service/entityresolution/integration/sql_test_data:/docker-entrypoint-initdb.d - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ers_test_user -d ers_test"] - interval: 5s - timeout: 5s - retries: 5 - restart: unless-stopped + extends: + file: docker-compose.base.yaml + service: ers-postgres - # OpenLDAP for ERS LDAP provider testing ers-ldap: - image: osixia/openldap:1.5.0 - container_name: ers_test_openldap - profiles: - - ers-test - - ers-admin - ports: - - "1389:389" # LDAP port (different from standard 389) - - "1636:636" # LDAPS port (different from standard 636) - environment: - LDAP_ORGANISATION: "OpenTDF Test" - LDAP_DOMAIN: "opentdf.test" - LDAP_ADMIN_PASSWORD: "admin_password" - LDAP_CONFIG_PASSWORD: "config_password" - LDAP_READONLY_USER: "true" - LDAP_READONLY_USER_USERNAME: "readonly" - LDAP_READONLY_USER_PASSWORD: "readonly_password" - LDAP_RFC2307BIS_SCHEMA: "false" - LDAP_BACKEND: "mdb" - LDAP_TLS: "true" - LDAP_TLS_ENFORCE: "false" - LDAP_REPLICATION: "false" - KEEP_EXISTING_CONFIG: "false" - LDAP_REMOVE_CONFIG_AFTER_SETUP: "true" - volumes: - - ers_ldap_data:/var/lib/ldap - - ers_ldap_config:/etc/ldap/slapd.d - - ./service/entityresolution/integration/ldap_test_data:/container/service/slapd/assets/config/bootstrap/ldif/custom - healthcheck: - test: ["CMD-SHELL", "ldapsearch -x -H ldap://localhost:389 -b dc=opentdf,dc=test -D cn=admin,dc=opentdf,dc=test -w admin_password '(objectclass=*)' dn"] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped + extends: + file: docker-compose.base.yaml + service: ers-ldap - # phpLDAPadmin for LDAP management (admin profile only) ers-ldap-admin: - image: osixia/phpldapadmin:latest - container_name: ers_ldap_admin - profiles: - - ers-admin # Only available with --profile ers-admin - ports: - - "6443:443" - environment: - PHPLDAPADMIN_LDAP_HOSTS: ers-ldap - PHPLDAPADMIN_HTTPS: "false" - depends_on: - ers-ldap: - condition: service_healthy - -volumes: - ers_postgres_data: - name: ers_test_postgres_data - ers_ldap_data: - name: ers_test_ldap_data - ers_ldap_config: - name: ers_test_ldap_config \ No newline at end of file + extends: + file: docker-compose.base.yaml + service: ers-ldap-admin \ No newline at end of file diff --git a/docs/Consuming.md b/docs/Consuming.md index 77b554c297..8ae466c098 100644 --- a/docs/Consuming.md +++ b/docs/Consuming.md @@ -10,64 +10,65 @@ To contribute/develop, see [here](./Contributing.md). > This quickstart guide is intended for development and testing purposes only. The OpenTDF platform team does not > provide recommendations for production deployments. -To get started with the OpenTDF platform make sure you are running the same Go version found in the `go.mod` file. +**Update `/etc/hosts`** - -https://github.com/opentdf/platform/blob/main/service/go.mod#L3 -> **Note for Apple M4 chip users:** -> If you are running on an Apple M4 chip, set the Java environment variable before running any commands: -> ```sh -> export JAVA_OPTS_APPEND="-XX:UseSVE=0" -> ``` -> This resolves SIGILL with Code 134 errors when running Java processes. +In order for the services to communicate correctly you will need to update your `/etc/hosts` file. +```shell +echo -e "127.0.0.1 platform.opentdf.local\n127.0.0.1 keycloak.opentdf.local" | sudo tee -a /etc/hosts +``` -1. **Initialize Platform Configuration** - ```shell - cp opentdf-dev.yaml opentdf.yaml +**Start Platform Services** + +Start all services including automated provisioning with [compose-spec](https://compose-spec.io). - # Generate development keys/certs for the platform infrastructure. - ./.github/scripts/init-temp-keys.sh +```shell +# If running on Apple M4 chip +JAVA_OPTS_APPEND="-XX:UseSVE=0" docker compose --file docker-compose.consumer.yaml up -d - # The following command is for macOS to trust the local certificate. - # For Linux, you may need to use a different command, e.g.: - # sudo cp ./keys/localhost.crt /usr/local/share/ca-certificates/ && sudo update-ca-certificates - sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ./keys/localhost.crt - ``` - - Optional: Update the [configuration](./Configuring.md) as needed. - - Optional: To remove the certificate, run: - ```shell - sudo security delete-certificate -c "localhost" - ``` -2. **Start Background Services** - - Start the required infrastructure with [compose-spec](https://compose-spec.io). +# Or on other architectures +docker compose --file docker-compose.consumer.yaml up -d +``` - ```shell - docker compose up - ``` -3. **Provision Keycloak** - ```shell - go run ./service provision keycloak - ``` -4. **Add Sample Attributes and Metadata** - ```shell - go run ./service provision fixtures - ``` -5. **Start Server** - ```shell - go run ./service start - ``` +This will automatically: +- Download configuration files from GitHub +- Generate development keys and certificates +- Start the background services (Keycloak, PostgreSQL) +- Provision Keycloak with initial configuration +- Add sample attributes and metadata +- Start the OpenTDF Platform server ## 🎉 Your platform is ready to use! -You can now access platform services at http://localhost:8080/ , and Keycloak at http://localhost:8888/auth/ . +You can now access the platform services. + +* Find platform configuration at https://platform.opentdf.local:8443/.well-known/opentdf-configuration , and +* Find Keycloak at https://keycloak.opentdf.local:9443/auth/ . + +### Optional: Trust the Local Certificate + +If you want to trust the auto-generated certificate on your host machine: + +```shell +# For macOS +docker compose cp keycloak:/keys/localhost.crt ./ +sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ./localhost.crt + +# For Linux +docker compose cp keycloak:/keys/localhost.crt ./ +sudo cp ./localhost.crt /usr/local/share/ca-certificates/ && sudo update-ca-certificates +``` + +To remove the certificate later: +```shell +sudo security delete-certificate -c "localhost" # macOS +``` ## Next steps * Try out our CLI (`otdfctl`): https://github.com/opentdf/otdfctl ```sh - otdfctl auth client-credentials --host http://localhost:8080 --client-id opentdf --client-secret secret + otdfctl auth client-credentials --host https://platform.opentdf.local:8443 --tls-no-verify ``` * Join our slack channel ([click here](https://join.slack.com/t/opentdf/shared_invite/zt-1e3jhnedw-wjviK~qRH_T1zG4dfaa~3A)) * Connect with the team