diff --git a/.github/actions/start-opensearch-with-one-plugin/action.yml b/.github/actions/start-opensearch-with-one-plugin/action.yml new file mode 100644 index 0000000000..1df1abaf2c --- /dev/null +++ b/.github/actions/start-opensearch-with-one-plugin/action.yml @@ -0,0 +1,120 @@ +name: 'Launch OpenSearch with a single plugin installed' +description: 'Downloads latest build of OpenSearch, installs a plugin, executes a script and then starts OpenSearch on localhost:9200' + +inputs: + opensearch-version: + description: 'The version of OpenSearch that should be used, e.g "3.0.0"' + required: true + + plugin-name: + description: 'The name of the plugin to use, such as opensearch-security' + required: true + + setup-script-name: + description: 'The name of the setup script you want to run i.e. "setup" (do not include file extension). Leave empty to indicate one should not be run.' + required: false + +runs: + using: "composite" + steps: + + # Configure longpath names if on Windows + - name: Enable Longpaths if on Windows + if: ${{ runner.os == 'Windows' }} + run: git config --system core.longpaths true + shell: pwsh + + # Download OpenSearch + - name: Download OpenSearch for Windows + uses: peternied/download-file@v1 + if: ${{ runner.os == 'Windows' }} + with: + url: https://artifacts.opensearch.org/snapshots/core/opensearch/${{ inputs.opensearch-version }}-SNAPSHOT/opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-windows-x64-latest.zip + + + - name: Download OpenSearch for Linux + uses: peternied/download-file@v1 + if: ${{ runner.os == 'Linux' }} + with: + url: https://artifacts.opensearch.org/snapshots/core/opensearch/${{ inputs.opensearch-version }}-SNAPSHOT/opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-linux-x64-latest.tar.gz + + # Extract downloaded zip + - name: Extract downloaded tar + if: ${{ runner.os == 'Linux' }} + run: | + tar -xzf opensearch-*.tar.gz + rm -f opensearch-*.tar.gz + shell: bash + + - name: Extract downloaded zip + if: ${{ runner.os == 'Windows' }} + run: | + tar -xzf opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-windows-x64-latest.zip + del opensearch-min-${{ inputs.opensearch-version }}-SNAPSHOT-windows-x64-latest.zip + shell: pwsh + + # Move and rename the plugin for installation + - name: Move and rename the plugin for installation + run: mv ./build/distributions/${{ inputs.plugin-name }}-*-SNAPSHOT.zip ${{ inputs.plugin-name }}.zip + shell: bash + + # Install the plugin + - name: Install Plugin into OpenSearch for Linux + if: ${{ runner.os == 'Linux'}} + run: | + chmod +x ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT/bin/opensearch-plugin + /bin/bash -c "yes | ./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT/bin/opensearch-plugin install file:$(pwd)/opensearch-security.zip" + shell: bash + + - name: Install Plugin into OpenSearch for Windows + if: ${{ runner.os == 'Windows'}} + run: | + 'y' | .\opensearch-${{ inputs.opensearch-version }}-SNAPSHOT\bin\opensearch-plugin.bat install file:$(pwd)\${{ inputs.plugin-name }}.zip + shell: pwsh + + # Run any configuration scripts + - name: Run Setup Script for Linux + if: ${{ runner.os == 'Linux' && inputs.setup-script-name != '' }} + run: | + echo "running linux setup" + chmod +x ./${{ inputs.setup-script-name }}.sh + ./${{ inputs.setup-script-name }}.sh + shell: bash + + - name: Run Setup Script for Windows + if: ${{ runner.os == 'Windows' && inputs.setup-script-name != '' }} + run: .\${{ inputs.setup-script-name }}.bat + shell: pwsh + + # Run OpenSearch + - name: Run OpenSearch with plugin on Linux + if: ${{ runner.os == 'Linux'}} + run: /bin/bash -c "./opensearch-${{ inputs.opensearch-version }}-SNAPSHOT/bin/opensearch &" + shell: bash + + - name: Run OpenSearch with plugin on Windows + if: ${{ runner.os == 'Windows'}} + run: start .\opensearch-${{ inputs.opensearch-version }}-SNAPSHOT\bin\opensearch.bat + shell: pwsh + + # Give the OpenSearch process some time to boot up before sending any requires, might need to increase the default time! + - name: Sleep while OpenSearch starts + uses: peternied/action-sleep@v1 + with: + seconds: 30 + + # Verify that the server is operational + - name: Check OpenSearch Running on Linux + if: ${{ runner.os != 'Windows'}} + run: curl https://localhost:9200/_cat/plugins -u 'admin:admin' -k -v + shell: bash + + - name: Check OpenSearch Running on Windows + if: ${{ runner.os == 'Windows'}} + run: | + $credentialBytes = [Text.Encoding]::ASCII.GetBytes("admin:admin") + $encodedCredentials = [Convert]::ToBase64String($credentialBytes) + $baseCredentials = "Basic $encodedCredentials" + $Headers = @{ Authorization = $baseCredentials } + Invoke-WebRequest -SkipCertificateCheck -Uri 'https://localhost:9200/_cat/plugins' -Headers $Headers; + shell: pwsh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7245b0195..c33f97f5da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,98 +8,57 @@ env: jobs: build: name: build - runs-on: ubuntu-latest strategy: fail-fast: false matrix: - jdk: [8, 11, 14] + jdk: [8, 11] + platform: ["ubuntu-latest", "windows-latest"] + runs-on: ${{ matrix.platform }} steps: - - - name: Set up JDK - uses: actions/setup-java@v1 + - name: Set up JDK for build and test + uses: actions/setup-java@v2 with: + distribution: temurin # Temurin is a distribution of adoptium java-version: ${{ matrix.jdk }} - name: Checkout security uses: actions/checkout@v2 - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + - name: Build and Test + uses: gradle/gradle-build-action@v2 with: - languages: java - - - name: Cache Gradle packages - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - - name: Checkstyle - run: ./gradlew clean checkstyleMain checkstyleTest - - - name: Package - run: ./gradlew clean build -Dbuild.snapshot=false -x test - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 - - - name: Test - run: OPENDISTRO_SECURITY_TEST_OPENSSL_OPT=true ./gradlew test -i + arguments: | + build test -Dbuild.snapshot=false + -x checkstyleMain + -x checkstyleTest - name: Coverage uses: codecov/codecov-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./build/jacoco/test/jacocoTestReport.xml + files: ./build/reports/jacoco/test/jacocoTestReport.xml + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports + path: | + ./build/reports/ - build-artifact-names: + - name: check archive for debugging + if: always() + run: echo "Check the artifact ${{ matrix.platform }}-JDK${{ matrix.jdk }}-reports for detailed test results" + + code-ql: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-java@v1 with: java-version: 11 - - - run: | - security_plugin_version=$(./gradlew properties -q | grep -E '^version:' | awk '{print $2}') - security_plugin_version_no_snapshot=$(echo $security_plugin_version | sed 's/-SNAPSHOT//g') - security_plugin_version_only_number=$(echo $security_plugin_version_no_snapshot | cut -d- -f1) - test_qualifier=alpha2 - - echo "SECURITY_PLUGIN_VERSION=$security_plugin_version" >> $GITHUB_ENV - echo "SECURITY_PLUGIN_VERSION_NO_SNAPSHOT=$security_plugin_version_no_snapshot" >> $GITHUB_ENV - echo "SECURITY_PLUGIN_VERSION_ONLY_NUMBER=$security_plugin_version_only_number" >> $GITHUB_ENV - echo "TEST_QUALIFIER=$test_qualifier" >> $GITHUB_ENV - - - run: | - echo ${{ env.SECURITY_PLUGIN_VERSION }} - echo ${{ env.SECURITY_PLUGIN_VERSION_NO_SNAPSHOT }} - echo ${{ env.SECURITY_PLUGIN_VERSION_ONLY_NUMBER }} - echo ${{ env.TEST_QUALIFIER }} - - - run: ./gradlew clean assemble && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION }}.jar - - - run: ./gradlew clean assemble -Dbuild.snapshot=false && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION_NO_SNAPSHOT }}.jar - - - run: ./gradlew clean assemble -Dbuild.snapshot=false -Dbuild.version_qualifier=${{ env.TEST_QUALIFIER }} && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION_ONLY_NUMBER }}-${{ env.TEST_QUALIFIER }}.jar - - - run: ./gradlew clean assemble -Dbuild.version_qualifier=${{ env.TEST_QUALIFIER }} && test -s ./build/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION_ONLY_NUMBER }}-${{ env.TEST_QUALIFIER }}-SNAPSHOT.jar - - - run: | - ## EXISTING_OS_VERSION outputs the major version, example as 2 - EXISTING_OS_VERSION=$(./gradlew properties | grep opensearch.version | cut -d':' -f2- | awk '{$1=$1};1' | cut -d '-' -f1 | cut -d '.' -f1) - ## INCREMENT_OS_VERSION in an increment of 1, example if EXISTING_OS_VERSION is 2, INCREMENT_OS_VERSION is 3 - INCREMENT_OS_VERSION=$((++EXISTING_OS_VERSION)) - ./gradlew clean updateVersion -DnewVersion=$INCREMENT_OS_VERSION.0.0-SNAPSHOT - test `./gradlew properties | grep opensearch.version | cut -d':' -f2- | awk '{$1=$1};1'` = $INCREMENT_OS_VERSION.0.0-SNAPSHOT - - - name: List files in the build directory if there was an error - run: ls -al ./build/ - if: failure() + - uses: github/codeql-action/init@v1 + with: + languages: java + - run: ./gradlew clean build -Dbuild.snapshot=false -x test + - uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/plugin_install.yml b/.github/workflows/plugin_install.yml index 96b7b318e5..62be7f6b1f 100644 --- a/.github/workflows/plugin_install.yml +++ b/.github/workflows/plugin_install.yml @@ -2,118 +2,51 @@ name: Plugin Install on: [push, pull_request, workflow_dispatch] -jobs: - - linux-install: +env: + OPENSEARCH_VERSION: 1.3.7 + PLUGIN_NAME: opensearch-security - runs-on: ubuntu-latest +jobs: + plugin-install: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] #windows-latest does not currently have a min distribution + jdk: [11, 17] + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - name: Set up JDK uses: actions/setup-java@v1 with: - java-version: 11 - - - name: Build - run: ./gradlew clean assemble -Dbuild.snapshot=false -x bundleSecurityAdminStandalone - - - name: Download OpenSearch Core - run: | - opensearch_version=`./gradlew properties -q | grep "opensearch_version:" | awk '{print $2}' | sed 's/-SNAPSHOT//g'` - wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$opensearch_version/latest/linux/x64/tar/builds/opensearch/dist/opensearch-min-$opensearch_version-linux-x64.tar.gz - tar -xzf opensearch-*.tar.gz - rm -f opensearch-*.tar.gz - - - name: Move and rename security plugin for installation - run: mv build/distributions/opensearch-security-*.zip opensearch-security.zip - - - name: Run OpenSearch with plugin - run: | - cat > os-ep.sh < setup.sh <<'EOF' + chmod +x ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh + /bin/bash -c "yes | ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh" + EOF - - name: Run OpenSearch with plugin + - name: Create Setup Script + if: ${{ runner.os == 'Windows' }} run: | - cd .. - start .\Opensearch\bin\opensearch.bat + New-Item .\setup.bat -type file + Set-Content .\setup.bat -Value "powershell.exe -noexit -command `"echo 'y' | .\opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT\plugins\${{ env.PLUGIN_NAME }}\tools\install_demo_configuration.bat`"" + Get-Content .\setup.bat - - name: Sleep while OpenSearch finishes starting up - run: Start-Sleep -s 30 - - - name: Check OpenSearch Running - run: | - $credentialBytes = [Text.Encoding]::ASCII.GetBytes("admin:admin") - $encodedCredentials = [Convert]::ToBase64String($credentialBytes) - $baseCredentials = "Basic $encodedCredentials" - $Headers = @{ Authorization = $baseCredentials } - Invoke-WebRequest -SkipCertificateCheck -Uri 'https://localhost:9200' -Headers $Headers + - name: Run Opensearch with A Single Plugin + uses: ./.github/actions/start-opensearch-with-one-plugin + with: + opensearch-version: ${{ env.OPENSEARCH_VERSION }} + plugin-name: ${{ env.PLUGIN_NAME }} + setup-script-name: setup diff --git a/build.gradle b/build.gradle index 4162538228..f5f07d1c37 100644 --- a/build.gradle +++ b/build.gradle @@ -42,6 +42,7 @@ plugins { id "nebula.ospackage" version "9.0.0" id "com.google.osdetector" version "1.7.0" id "org.gradle.test-retry" version "1.3.1" + id "com.github.spotbugs" version "5.0.13" } import org.gradle.crypto.checksum.Checksum @@ -223,6 +224,13 @@ testsJar { libsDirName = '.' } +spotbugs { + includeFilter = file('spotbugs-include.xml') +} + +spotbugsTest { + enabled = false +} test { maxParallelForks = 3 diff --git a/spotbugs-include.xml b/spotbugs-include.xml new file mode 100644 index 0000000000..dd6062700d --- /dev/null +++ b/spotbugs-include.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java b/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java index 25c547e4e2..c2791a02b8 100644 --- a/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java +++ b/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.security.AccessController; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; @@ -155,7 +156,7 @@ private AuthTokenProcessorAction.Response handleImpl(RestRequest restRequest, Re SettingsException { if (token_log.isDebugEnabled()) { try { - token_log.debug("SAMLResponse for {}\n{}", samlRequestId, new String(Util.base64decoder(samlResponseBase64), "UTF-8")); + token_log.debug("SAMLResponse for {}\n{}", samlRequestId, new String(Util.base64decoder(samlResponseBase64), StandardCharsets.UTF_8)); } catch (Exception e) { token_log.warn( "SAMLResponse for {} cannot be decoded from base64\n{}", diff --git a/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java b/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java index 04dd285b5c..bd479c0db2 100644 --- a/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java +++ b/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java @@ -454,7 +454,7 @@ public void logDocumentWritten(ShardId shardId, GetResult originalResult, Index try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, originalResult.internalSourceRef(), XContentType.JSON)) { Object base64 = parser.map().values().iterator().next(); if(base64 instanceof String) { - originalSource = (new String(BaseEncoding.base64().decode((String) base64))); + originalSource = (new String(BaseEncoding.base64().decode((String) base64), StandardCharsets.UTF_8)); } else { originalSource = XContentHelper.convertToJson(originalResult.internalSourceRef(), false, XContentType.JSON); } @@ -465,7 +465,7 @@ public void logDocumentWritten(ShardId shardId, GetResult originalResult, Index try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, currentIndex.source(), XContentType.JSON)) { Object base64 = parser.map().values().iterator().next(); if(base64 instanceof String) { - currentSource = (new String(BaseEncoding.base64().decode((String) base64))); + currentSource = new String(BaseEncoding.base64().decode((String) base64), StandardCharsets.UTF_8); } else { currentSource = XContentHelper.convertToJson(currentIndex.source(), false, XContentType.JSON); } @@ -492,7 +492,7 @@ public void logDocumentWritten(ShardId shardId, GetResult originalResult, Index try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, currentIndex.source(), XContentType.JSON)) { Object base64 = parser.map().values().iterator().next(); if(base64 instanceof String) { - msg.addSecurityConfigContentToRequestBody(new String(BaseEncoding.base64().decode((String) base64)), id); + msg.addSecurityConfigContentToRequestBody(new String(BaseEncoding.base64().decode((String) base64), StandardCharsets.UTF_8), id); } else { msg.addSecurityConfigTupleToRequestBody(new Tuple(XContentType.JSON, currentIndex.source()), id); } diff --git a/src/main/java/org/opensearch/security/configuration/ConfigurationLoaderSecurity7.java b/src/main/java/org/opensearch/security/configuration/ConfigurationLoaderSecurity7.java index 0043373c47..8b98b1e039 100644 --- a/src/main/java/org/opensearch/security/configuration/ConfigurationLoaderSecurity7.java +++ b/src/main/java/org/opensearch/security/configuration/ConfigurationLoaderSecurity7.java @@ -31,6 +31,7 @@ package org.opensearch.security.configuration; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -275,7 +276,7 @@ private SecurityDynamicConfiguration toConfig(GetResponse singleGetResponse, parser.nextToken(); - final String jsonAsString = SecurityUtils.replaceEnvVars(new String(parser.binaryValue()), settings); + final String jsonAsString = SecurityUtils.replaceEnvVars(new String(parser.binaryValue(), StandardCharsets.UTF_8), settings); final JsonNode jsonNode = DefaultObjectMapper.readTree(jsonAsString); int configVersion = 1; diff --git a/src/main/java/org/opensearch/security/ssl/DefaultSecurityKeyStore.java b/src/main/java/org/opensearch/security/ssl/DefaultSecurityKeyStore.java index c84e8a05de..982364fd73 100644 --- a/src/main/java/org/opensearch/security/ssl/DefaultSecurityKeyStore.java +++ b/src/main/java/org/opensearch/security/ssl/DefaultSecurityKeyStore.java @@ -28,6 +28,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.File; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; diff --git a/src/main/java/org/opensearch/security/support/ConfigHelper.java b/src/main/java/org/opensearch/security/support/ConfigHelper.java index 17f6ea5135..de3e772d8b 100644 --- a/src/main/java/org/opensearch/security/support/ConfigHelper.java +++ b/src/main/java/org/opensearch/security/support/ConfigHelper.java @@ -31,10 +31,13 @@ package org.opensearch.security.support; import java.io.File; +import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; +import java.io.InputStreamReader; import java.io.Reader; import java.io.StringReader; +import java.nio.charset.StandardCharsets; import org.opensearch.security.securityconf.impl.Meta; import org.apache.logging.log4j.Logger; @@ -96,7 +99,7 @@ public static void uploadFile(Client tc, String filepath, String index, CType cT public static Reader createFileOrStringReader(CType cType, int configVersion, String filepath, boolean populateEmptyIfFileMissing) throws Exception { Reader reader; if (!populateEmptyIfFileMissing || new File(filepath).exists()) { - reader = new FileReader(filepath); + reader = new InputStreamReader(new FileInputStream(filepath), StandardCharsets.UTF_8); } else { reader = new StringReader(createEmptySdcYaml(cType, configVersion)); } @@ -148,7 +151,7 @@ public static SecurityDynamicConfiguration fromYamlReader(Reader yamlRead } public static SecurityDynamicConfiguration fromYamlFile(String filepath, CType ctype, int version, long seqNo, long primaryTerm) throws IOException { - return fromYamlReader(new FileReader(filepath), ctype, version, seqNo, primaryTerm); + return fromYamlReader(new InputStreamReader(new FileInputStream(filepath), StandardCharsets.UTF_8), ctype, version, seqNo, primaryTerm); } public static SecurityDynamicConfiguration fromYamlString(String yamlString, CType ctype, int version, long seqNo, long primaryTerm) throws IOException { diff --git a/src/main/java/org/opensearch/security/support/SecurityUtils.java b/src/main/java/org/opensearch/security/support/SecurityUtils.java index 2da1d0e252..58cbdb9d84 100644 --- a/src/main/java/org/opensearch/security/support/SecurityUtils.java +++ b/src/main/java/org/opensearch/security/support/SecurityUtils.java @@ -50,9 +50,10 @@ public final class SecurityUtils { protected final static Logger log = LogManager.getLogger(SecurityUtils.class); - private static final Pattern ENV_PATTERN = Pattern.compile("\\$\\{env\\.([\\w]+)((\\:\\-)?[\\w]*)\\}"); - private static final Pattern ENVBC_PATTERN = Pattern.compile("\\$\\{envbc\\.([\\w]+)((\\:\\-)?[\\w]*)\\}"); - private static final Pattern ENVBASE64_PATTERN = Pattern.compile("\\$\\{envbase64\\.([\\w]+)((\\:\\-)?[\\w]*)\\}"); + private static final String ENV_PATTERN_SUFFIX = "\\.([\\w=():\\-_]+?)(\\:\\-[\\w=():\\-_]*)?\\}"; + static final Pattern ENV_PATTERN = Pattern.compile("\\$\\{env" + ENV_PATTERN_SUFFIX); + static final Pattern ENVBC_PATTERN = Pattern.compile("\\$\\{envbc" + ENV_PATTERN_SUFFIX); + static final Pattern ENVBASE64_PATTERN = Pattern.compile("\\$\\{envbase64" + ENV_PATTERN_SUFFIX); public static Locale EN_Locale = forEN(); diff --git a/src/main/java/org/opensearch/security/tools/SecurityAdmin.java b/src/main/java/org/opensearch/security/tools/SecurityAdmin.java index a80f60c560..5681efbf6e 100644 --- a/src/main/java/org/opensearch/security/tools/SecurityAdmin.java +++ b/src/main/java/org/opensearch/security/tools/SecurityAdmin.java @@ -33,8 +33,10 @@ import java.io.ByteArrayInputStream; import java.io.Console; import java.io.File; +import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; +import java.io.OutputStreamWriter; import java.io.Reader; import java.io.Writer; import java.net.InetSocketAddress; @@ -911,8 +913,8 @@ private static boolean retrieveFile(final Client tc, final String filepath, fina } - System.out.println("Will retrieve '"+type+"/" +id+"' into "+filepath+" "+(legacy?"(legacy mode)":"")); - try (Writer writer = new FileWriter(filepath)) { + System.out.println("Will retrieve '"+"/" +id+"' into "+filepath+" "+(legacy?"(legacy mode)":"")); + try (Writer writer = new OutputStreamWriter(new FileOutputStream(filepath), StandardCharsets.UTF_8)) { final GetResponse response = tc.get(new GetRequest(index).type(type).id(id).refresh(true).realtime(false)).actionGet(); diff --git a/src/test/java/org/opensearch/security/auth/limiting/HeapBasedRateTrackerTest.java b/src/test/java/org/opensearch/security/auth/limiting/HeapBasedRateTrackerTest.java index 3637211eec..ae0fa500d2 100644 --- a/src/test/java/org/opensearch/security/auth/limiting/HeapBasedRateTrackerTest.java +++ b/src/test/java/org/opensearch/security/auth/limiting/HeapBasedRateTrackerTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import org.junit.Ignore; import org.junit.Test; import org.opensearch.security.util.ratetracking.HeapBasedRateTracker; @@ -39,6 +40,7 @@ public void simpleTest() throws Exception { } @Test + @Ignore // https://github.com/opensearch-project/security/issues/2193 public void expiryTest() throws Exception { HeapBasedRateTracker tracker = new HeapBasedRateTracker<>(100, 5, 100_000); @@ -78,6 +80,7 @@ public void expiryTest() throws Exception { } @Test + @Ignore // https://github.com/opensearch-project/security/issues/2193 public void maxTwoTriesTest() throws Exception { HeapBasedRateTracker tracker = new HeapBasedRateTracker<>(100, 2, 100_000); diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java index 3d4f290652..cf1cb18eb2 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/DlsTest.java @@ -375,7 +375,7 @@ public void testDlsWithMinDocCountZeroAggregations() throws Exception { // Significant Text Aggregation is not impacted. // Non-admin user with setting "min_doc_count=0". Expected to only have access to buckets for dept_manager". - String query3 = "{\"aggregations\":{\"significant_termX\":{\"significant_terms\":{\"field\":\"termX.keyword\",\"min_doc_count\":0}}}}"; + String query3 = "{\"size\":100,\"aggregations\":{\"significant_termX\":{\"significant_terms\":{\"field\":\"termX.keyword\",\"min_doc_count\":0}}}}"; HttpResponse response5 = rh.executePostRequest("logs*/_search", query3, encodeBasicHeader("dept_manager", "password")); Assert.assertEquals(HttpStatus.SC_OK, response5.getStatusCode()); @@ -386,7 +386,7 @@ public void testDlsWithMinDocCountZeroAggregations() throws Exception { Assert.assertFalse(response5.getBody(), response5.getBody().contains("\"termX\":\"E\"")); // Non-admin user without setting "min_doc_count". Expected to only have access to buckets for dept_manager". - String query4 = "{\"aggregations\":{\"significant_termX\":{\"significant_terms\":{\"field\":\"termX.keyword\"}}}}"; + String query4 = "{\"size\":100,\"aggregations\":{\"significant_termX\":{\"significant_terms\":{\"field\":\"termX.keyword\"}}}}"; HttpResponse response6 = rh.executePostRequest("logs*/_search", query4, encodeBasicHeader("dept_manager", "password")); @@ -419,7 +419,7 @@ public void testDlsWithMinDocCountZeroAggregations() throws Exception { // Histogram Aggregation is not impacted. // Non-admin user with setting "min_doc_count=0". Expected to only have access to buckets for dept_manager". - String query5 = "{\"aggs\":{\"amount\":{\"histogram\":{\"field\":\"amount\",\"interval\":1,\"min_doc_count\":0}}}}"; + String query5 = "{\"size\":100,\"aggs\":{\"amount\":{\"histogram\":{\"field\":\"amount\",\"interval\":1,\"min_doc_count\":0}}}}"; HttpResponse response9 = rh.executePostRequest("logs*/_search", query5, encodeBasicHeader("dept_manager", "password")); @@ -431,7 +431,7 @@ public void testDlsWithMinDocCountZeroAggregations() throws Exception { Assert.assertFalse(response9.getBody(), response9.getBody().contains("\"termX\":\"E\"")); // Non-admin user without setting "min_doc_count". Expected to only have access to buckets for dept_manager". - String query6 = "{\"aggs\":{\"amount\":{\"histogram\":{\"field\":\"amount\",\"interval\":1}}}}"; + String query6 = "{\"size\":100,\"aggs\":{\"amount\":{\"histogram\":{\"field\":\"amount\",\"interval\":1}}}}"; HttpResponse response10 = rh.executePostRequest("logs*/_search", query6, encodeBasicHeader("dept_manager", "password")); @@ -465,7 +465,7 @@ public void testDlsWithMinDocCountZeroAggregations() throws Exception { // Date Histogram Aggregation is not impacted. // Non-admin user with setting "min_doc_count=0". Expected to only have access to buckets for dept_manager". - String query7 = "{\"aggs\":{\"timestamp\":{\"date_histogram\":{\"field\":\"timestamp\",\"calendar_interval\":\"month\",\"min_doc_count\":0}}}}"; + String query7 = "{\"size\":100,\"aggs\":{\"timestamp\":{\"date_histogram\":{\"field\":\"timestamp\",\"calendar_interval\":\"month\",\"min_doc_count\":0}}}}"; HttpResponse response13 = rh.executePostRequest("logs*/_search", query7, encodeBasicHeader("dept_manager", "password")); @@ -477,7 +477,7 @@ public void testDlsWithMinDocCountZeroAggregations() throws Exception { Assert.assertFalse(response13.getBody(), response13.getBody().contains("\"termX\":\"E\"")); // Non-admin user without setting "min_doc_count". Expected to only have access to buckets for dept_manager". - String query8 = "{\"aggs\":{\"timestamp\":{\"date_histogram\":{\"field\":\"timestamp\",\"calendar_interval\":\"month\"}}}}"; + String query8 = "{\"size\":100,\"aggs\":{\"timestamp\":{\"date_histogram\":{\"field\":\"timestamp\",\"calendar_interval\":\"month\"}}}}"; HttpResponse response14 = rh.executePostRequest("logs*/_search", query8, encodeBasicHeader("dept_manager", "password")); diff --git a/src/test/java/org/opensearch/security/ssl/OpenSSLTest.java b/src/test/java/org/opensearch/security/ssl/OpenSSLTest.java index 3474901274..0b74dfb5d2 100644 --- a/src/test/java/org/opensearch/security/ssl/OpenSSLTest.java +++ b/src/test/java/org/opensearch/security/ssl/OpenSSLTest.java @@ -65,23 +65,10 @@ public static void restoreNettyDefaultAllocator() { @Before public void setup() { + Assume.assumeFalse(PlatformDependent.isWindows()); allowOpenSSL = true; } - @Test - public void testEnsureOpenSSLAvailability() { - //Assert.assertTrue("OpenSSL not available: "+String.valueOf(OpenSsl.unavailabilityCause()), OpenSsl.isAvailable()); - - final String openSSLOptional = System.getenv("OPENDISTRO_SECURITY_TEST_OPENSSL_OPT"); - System.out.println("OPENDISTRO_SECURITY_TEST_OPENSSL_OPT "+openSSLOptional); - if(!Boolean.parseBoolean(openSSLOptional)) { - System.out.println("OpenSSL must be available"); - Assert.assertTrue("OpenSSL not available: "+String.valueOf(OpenSsl.unavailabilityCause()), OpenSsl.isAvailable()); - } else { - System.out.println("OpenSSL can be available"); - } - } - @Override @Test public void testHttps() throws Exception { diff --git a/src/test/java/org/opensearch/security/support/SecurityUtilsTest.java b/src/test/java/org/opensearch/security/support/SecurityUtilsTest.java new file mode 100644 index 0000000000..b688b8c574 --- /dev/null +++ b/src/test/java/org/opensearch/security/support/SecurityUtilsTest.java @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.security.support; + +import java.util.Arrays; +import java.util.Collection; +import java.util.function.Predicate; +import java.util.regex.Pattern; + +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.security.support.SecurityUtils.ENVBASE64_PATTERN; +import static org.opensearch.security.support.SecurityUtils.ENVBC_PATTERN; +import static org.opensearch.security.support.SecurityUtils.ENV_PATTERN; + +public class SecurityUtilsTest { + + private final Collection interestingEnvKeyNames = Arrays.asList( + "=ExitCode", + "=C:", + "ProgramFiles(x86)", + "INPUT_GRADLE-HOME-CACHE-CLEANUP", + "MYENV", + "MYENV:", + "MYENV::" + ); + private final Collection namesFromThisRuntimeEnvironment = System.getenv().keySet(); + + @Test + public void checkInterestingNamesForEnvPattern() { + checkKeysWithPredicate(interestingEnvKeyNames, "env", asMatchPredicate(ENV_PATTERN)); + } + + @Test + public void checkRuntimeKeyNamesForEnvPattern() { + checkKeysWithPredicate(namesFromThisRuntimeEnvironment, "env", asMatchPredicate(ENV_PATTERN)); + } + + @Test + public void checkInterestingNamesForEnvbcPattern() { + checkKeysWithPredicate(interestingEnvKeyNames, "envbc", asMatchPredicate(ENVBC_PATTERN)); + } + + @Test + public void checkInterestingNamesForEnvBase64Pattern() { + checkKeysWithPredicate(interestingEnvKeyNames, "envbase64", asMatchPredicate(ENVBASE64_PATTERN)); + } + + private Predicate asMatchPredicate(final Pattern p) { + return (String s) -> p.matcher(s).matches(); + } + + private void checkKeysWithPredicate(Collection keys, String predicateName, Predicate predicate) { + keys.forEach(envKeyName -> { + final String prefixWithKeyName = "${" + predicateName + "." + envKeyName; + + final String baseKeyName = prefixWithKeyName + "}"; + assertThat("Testing " + envKeyName + ", " + baseKeyName, + predicate.test(baseKeyName), + equalTo(true)); + + final String baseKeyNameWithDefault = prefixWithKeyName + ":-tTt}"; + assertThat("Testing " + envKeyName + " with defaultValue, " + baseKeyNameWithDefault, + predicate.test(baseKeyNameWithDefault), + equalTo(true)); + }); + } +} diff --git a/tools/install_demo_configuration.bat b/tools/install_demo_configuration.bat index 32c1e76947..ef921b40ad 100755 --- a/tools/install_demo_configuration.bat +++ b/tools/install_demo_configuration.bat @@ -343,14 +343,14 @@ echo. > securityadmin_demo.bat echo %OPENSEARCH_PLUGINS_DIR%opensearch-security\tools\securityadmin.bat -cd %OPENSEARCH_CONF_DIR%opensearch-security -icl -key %OPENSEARCH_CONF_DIR%kirk-key.pem -cert %OPENSEARCH_CONF_DIR%kirk.pem -cacert %OPENSEARCH_CONF_DIR%root-ca.pem -nhnv >> securityadmin_demo.bat if %initsecurity% == 0 ( - echo ### After the whole cluster is up execute: + echo ### After the whole cluster is up execute: type securityadmin_demo.bat echo ### or run ./securityadmin_demo.bat echo ### After that you can also use the Security Plugin ConfigurationGUI ) else ( echo ### OpenSearch Security will be automatically initialized. - echo ### If you like to change the runtime configuration - echo ### change the files in ../../../config/opensearch-security and execute: + echo ### If you like to change the runtime configuration + echo ### change the files in ../../../config/opensearch-security and execute: type securityadmin_demo.bat echo ### or run ./securityadmin_demo.bat echo ### To use the Security Plugin ConfigurationGUI