diff --git a/opensearch-observability/.codecov.yml b/opensearch-observability/.codecov.yml new file mode 100644 index 000000000..7adfbc9d2 --- /dev/null +++ b/opensearch-observability/.codecov.yml @@ -0,0 +1,40 @@ +--- +## +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +## + +## +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# +## + +codecov: + require_ci_to_pass: yes + +coverage: + precision: 2 + round: down + range: "70...100" + status: + project: + default: + target: 70% # the required coverage value + threshold: 1% # the leniency in hitting the target diff --git a/opensearch-observability/.editorconfig b/opensearch-observability/.editorconfig new file mode 100644 index 000000000..de2de324e --- /dev/null +++ b/opensearch-observability/.editorconfig @@ -0,0 +1,38 @@ +## +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +## + +## +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# +## + +# config for Pinterest ktlint. https://github.com/pinterest/ktlint + +root = true + +[*.{kt,kts}] +# possible values: number (e.g. 2), "unset" (makes ktlint ignore indentation completely) +indent_size=4 + +# we have detekt also checking for max line length. Disable the linter and use only one tool to check for max line length. +# See https://github.com/arturbosch/detekt +max_line_length=off diff --git a/opensearch-observability/.gitignore b/opensearch-observability/.gitignore new file mode 100644 index 000000000..de6e7bf2c --- /dev/null +++ b/opensearch-observability/.gitignore @@ -0,0 +1,323 @@ +.DS_Store +.project +.classpath + +*.iml +.gradle +out +local-test +.local-* + +npm-debug.log* +node_modules +/build/ +/public/app.css +.idea/ +.vscode/ +yarn-error.log +/coverage/ +.history/ +.eslintcache + +# OpenSearch Dashboards +.empty + +### Java ### +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +### Kotlin ### +# Compiled class file + +# Log file + +# BlueJ files + +# Mobile Tools for Java (J2ME) + +# Package Files # + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml + +### Gradle ### +.gradle +build/ + +# Ignore Gradle GUI config +gradle-app.setting + +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar + +# Cache of project +.gradletasknamecache + +# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# gradle/wrapper/gradle-wrapper.properties + +### Gradle Patch ### +**/build/ + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +### Intellij ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +.idea/artifacts +.idea/compiler.xml +.idea/jarRepositories.xml +.idea/modules.xml +.idea/*.iml +.idea/modules +*.iml +*.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# AWS plugin +.idea/aws.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Intellij Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + + +### Maven ### +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +# https://github.com/takari/maven-wrapper#usage-without-binary-jar +.mvn/wrapper/maven-wrapper.jar + + +### Eclipse ### +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + +### Eclipse Patch ### +# Spring Boot Tooling +.sts4-cache/ + +local-test +.local-* diff --git a/opensearch-observability/build-tools/opensearchplugin-coverage.gradle b/opensearch-observability/build-tools/opensearchplugin-coverage.gradle new file mode 100644 index 000000000..d2d745a12 --- /dev/null +++ b/opensearch-observability/build-tools/opensearchplugin-coverage.gradle @@ -0,0 +1,94 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +/** + * OpenSearch Plugin build tools don't work with the Gradle Jacoco Plugin to report coverage out of the box. + * https://github.com/elastic/elasticsearch/issues/28867. + * + * This code sets up coverage reporting manually for OpenSearch plugin tests. This is complicated because: + * 1. The OpenSearch integTest Task doesn't implement Gradle's JavaForkOptions so we have to manually start the jacoco agent with the test JVM + * 2. The cluster nodes are stopped using 'kill -9' which means jacoco can't dump it's execution output to a file on VM shutdown + * 3. The Java Security Manager prevents JMX from writing execution output to the file. + * + * To workaround these we start the cluster with jmx enabled and then use Jacoco's JMX MBean to get the execution data before the + * cluster is stopped and dump it to a file. Luckily our current security policy seems to allow this. This will also probably + * break if there are multiple nodes in the integTestCluster. But for now... it sorta works. + */ + +// Get gradle to generate the required jvm agent arg for us using a dummy tasks of type Test. Unfortunately Elastic's +// testing tasks don't derive from Test so the jacoco plugin can't do this automatically. +def jacocoDir = "${buildDir}/jacoco" + +task dummyTest(type: Test) { + enabled = false + workingDir = file("/") // Force absolute path to jacoco agent jar + jacoco { + destinationFile = file("${jacocoDir}/test.exec") + destinationFile.parentFile.mkdirs() + jmx = true + } +} + +task dummyIntegTest(type: Test) { + enabled = false + workingDir = file("/") // Force absolute path to jacoco agent jar + jacoco { + destinationFile = file("${jacocoDir}/integTest.exec") + destinationFile.parentFile.mkdirs() + jmx = true + } +} + +integTest { + systemProperty 'jacoco.dir', "${jacocoDir}" +} + +jacocoTestReport { + dependsOn integTest, test + executionData dummyTest.jacoco.destinationFile, dummyIntegTest.jacoco.destinationFile + sourceDirectories.from = "src/main/kotlin" + classDirectories.from = sourceSets.main.output + reports { + html.enabled = true // human readable + xml.enabled = true // for coverlay + } +} + +allprojects{ + afterEvaluate { + jacocoTestReport.dependsOn integTest + + testClusters.integTest { + jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('javaagent:','javaagent:/') + systemProperty 'com.sun.management.jmxremote', "true" + systemProperty 'com.sun.management.jmxremote.authenticate', "false" + systemProperty 'com.sun.management.jmxremote.port', "7777" + systemProperty 'com.sun.management.jmxremote.ssl', "false" + systemProperty 'java.rmi.server.hostname', "127.0.0.1" + } + } +} diff --git a/opensearch-observability/build-tools/pkgbuild.gradle b/opensearch-observability/build-tools/pkgbuild.gradle new file mode 100644 index 000000000..50c762890 --- /dev/null +++ b/opensearch-observability/build-tools/pkgbuild.gradle @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +apply plugin: 'nebula.ospackage' + +// This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name +afterEvaluate { + ospackage { + packageName = "${name}" + release = isSnapshot ? "0.1" : '1' + version = "${project.version}" - "-SNAPSHOT" + + into '/usr/share/opensearch/plugins' + from(zipTree(bundlePlugin.archivePath)) { + into opensearchplugin.name + } + + user 'root' + permissionGroup 'root' + fileMode 0644 + dirMode 0755 + + requires('opensearch-oss', versions.opensearch, EQUAL) + packager = 'Amazon' + vendor = 'Amazon' + os = 'LINUX' + prefix '/usr' + + license 'ASL-2.0' + maintainer 'OpenSearch Team ' + url 'https://opensearch.org/' + summary ''' + OpenSearch Observability. + Reference documentation can be found at https://opensearch.org/docs/. + '''.stripIndent().replace('\n', ' ').trim() + } + + buildRpm { + arch = 'NOARCH' + archiveName "${packageName}-${version}.rpm" + dependsOn 'assemble' + } + + buildDeb { + arch = 'amd64' + archiveName "${packageName}-${version}.deb" + dependsOn 'assemble' + } +} diff --git a/opensearch-observability/build.gradle b/opensearch-observability/build.gradle new file mode 100644 index 000000000..5c974aaf5 --- /dev/null +++ b/opensearch-observability/build.gradle @@ -0,0 +1,306 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import org.opensearch.gradle.test.RestIntegTestTask +import java.util.concurrent.Callable + +buildscript { + ext { + opensearch_version = System.getProperty("opensearch.version", "1.1.0-SNAPSHOT") + // 1.0.0 -> 1.0.0.0, and 1.0.0-SNAPSHOT -> 1.0.0.0-SNAPSHOT + opensearch_build = opensearch_version.replaceAll(/(\.\d)([^\d]*)$/, '$1.0$2') + common_utils_version = System.getProperty("common_utils.version", opensearch_build) + kotlin_version = System.getProperty("kotlin.version", "1.4.0") + } + + repositories { + mavenLocal() + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } + jcenter() + } + + dependencies { + classpath "org.opensearch.gradle:build-tools:${opensearch_version}" + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}" + classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}" + classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.12.0" + classpath "org.jacoco:org.jacoco.agent:0.8.5" + } +} + +plugins { + id 'nebula.ospackage' version "8.3.0" + id "com.dorongold.task-tree" version "1.5" + id 'java-library' +} + +apply plugin: 'java' +apply plugin: 'jacoco' +apply plugin: 'idea' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.testclusters' +apply plugin: 'io.gitlab.arturbosch.detekt' +apply plugin: 'org.jetbrains.kotlin.jvm' +apply plugin: 'org.jetbrains.kotlin.plugin.allopen' + +def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster') +def usingMultiNode = project.properties.containsKey('numNodes') + +check.dependsOn jacocoTestReport + +opensearchplugin { + name 'opensearch-observability' + description 'OpenSearch Plugin for OpenSearch Dashboards Observability' + classname "org.opensearch.observability.ObservabilityPlugin" +} + +allOpen { + annotation("org.opensearch.observability.util.OpenForTesting") +} + +configurations { + ktlint +} + +detekt { + config = files("detekt.yml") + buildUponDefaultConfig = true +} + +configurations.testCompile { + exclude module: "securemock" +} + +configurations.all { + if (it.state != Configuration.State.UNRESOLVED) return + resolutionStrategy { + force "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" + force "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}" + force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.11.4" + } +} + +repositories { + mavenLocal() + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } + jcenter() +} + +ext { + projectSubstitutions = [:] + licenseFile = rootProject.file('LICENSE.txt') + noticeFile = rootProject.file('NOTICE.txt') + isSnapshot = "true" == System.getProperty("build.snapshot", "true") +} + +plugins.withId('java') { + sourceCompatibility = targetCompatibility = "1.8" +} + +plugins.withId('org.jetbrains.kotlin.jvm') { + compileKotlin.kotlinOptions.jvmTarget = compileTestKotlin.kotlinOptions.jvmTarget = "1.8" +} + +allprojects { + group = "org.opensearch" + + version = "${opensearch_version}" - "-SNAPSHOT" + ".0" + if (isSnapshot) { + version += "-SNAPSHOT" + } + + plugins.withId('java') { + sourceCompatibility = targetCompatibility = "1.8" + } +} + +dependencies { + compile "org.opensearch:opensearch:${opensearch_version}" + compile "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" + compile "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}" + compile "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.9" + compile "${group}:common-utils:${common_utils_version}" + compile group: 'com.google.guava', name: 'guava', version: '15.0' + testImplementation( + 'org.assertj:assertj-core:3.16.1', + 'org.junit.jupiter:junit-jupiter-api:5.6.2' + ) + testRuntime('org.junit.jupiter:junit-jupiter-engine:5.6.2') + testCompile "org.opensearch.test:framework:${opensearch_version}" + testCompile "org.jetbrains.kotlin:kotlin-test:${kotlin_version}" + testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0" + testCompile "org.mockito:mockito-core:2.23.0" + testCompile "com.google.code.gson:gson:2.8.6" + + ktlint "com.pinterest:ktlint:0.41.0" +} + +javadoc.enabled = false // turn off javadoc as it barfs on Kotlin code +licenseHeaders.enabled = true +// no need to validate pom, as we do not upload to maven/sonatype +validateNebulaPom.enabled = false +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +// Allow @Test to be used in test classes not inherited from LuceneTestCase. +// see https://github.com/elastic/elasticsearch/blob/master/buildSrc/src/main/resources/forbidden/es-test-signatures.txt +forbiddenApis.ignoreFailures = true +// Allow test cases to be named Tests without having to be inherited from LuceneTestCase. +// see https://github.com/elastic/elasticsearch/blob/323f312bbc829a63056a79ebe45adced5099f6e6/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java +testingConventions.enabled = false +loggerUsageCheck.enabled = false + +test { + systemProperty 'tests.security.manager', 'false' + useJUnitPlatform() +} + +File repo = file("$buildDir/testclusters/repo") +def _numNodes = findProperty('numNodes') as Integer ?: 1 + +def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile +es_tmp_dir.mkdirs() + +// As of ES 7.7 the sample-extension-plugin is being added to the list of plugins for the testCluster during build before +// the job-scheduler plugin (not a dependency for opensearch-observability) is causing build failures. +// The job-scheduler zip is added explicitly above but the sample-extension-plugin is added implicitly at some time during evaluation. +// Will need to do a deep dive to find out exactly what task adds the sample-extension-plugin and add job-scheduler there but a temporary hack is to +// reorder the plugins list after evaluation but prior to task execution when the plugins are installed. +afterEvaluate { + testClusters.integTest.nodes.each { node -> + def plugins = node.plugins + def firstPlugin = plugins.get(0) + plugins.remove(0) + plugins.add(firstPlugin) + } +} + +tasks.withType(licenseHeaders.class) { + additionalLicense 'AL ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")' +} + +task integTest(type: RestIntegTestTask) { + description = "Run tests against a cluster that has security enabled" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +tasks.named("check").configure { dependsOn(integTest) } + +integTest { + systemProperty 'tests.security.manager', 'false' + systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath + + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for + // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. + doFirst { + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can + // use longer timeouts for requests. + def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null + systemProperty 'cluster.debug', isDebuggingCluster + // Set number of nodes system property to be used in tests + systemProperty 'cluster.number_of_nodes', "${_numNodes}" + // There seems to be an issue when running multi node run or integ tasks with unicast_hosts + // not being written, the waitForAllConditions ensures it's written + getClusters().forEach { cluster -> + cluster.waitForAllConditions() + } + } + + // The -Dcluster.debug option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000' + } + + // https://github.com/opensearch-project/index-management/pull/93 + if (System.getProperty("tests.clustername") != null) { + exclude 'org/opensearch/observability/ObservabilityPluginIT.class' + } +} + + +Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin"); +integTest.dependsOn(bundle) +integTest.getClusters().forEach { c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile())) } + +testClusters.integTest { + testDistribution = "INTEG_TEST" + // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 + if (_numNodes > 1) numberOfNodes = _numNodes + // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore + // i.e. we have to use a custom property to flag when we want to debug opensearch JVM + // since we also support multi node integration tests we increase debugPort per node + if (System.getProperty("cluster.debug") != null) { + def debugPort = 5005 + nodes.forEach { node -> + node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}") + debugPort += 1 + } + } + setting 'path.repo', repo.absolutePath +} + +run { + doFirst { + // There seems to be an issue when running multi node run or integ tasks with unicast_hosts + // not being written, the waitForAllConditions ensures it's written + getClusters().forEach { cluster -> + cluster.waitForAllConditions() + } + } + useCluster testClusters.integTest +} + +task ktlint(type: JavaExec, group: "verification") { + description = "Check Kotlin code style." + main = "com.pinterest.ktlint.Main" + classpath = configurations.ktlint + args "src/**/*.kt" + // to generate report in checkstyle format prepend following args: + // "--reporter=plain", "--reporter=checkstyle,output=${buildDir}/ktlint.xml" + // see https://github.com/pinterest/ktlint#usage for more +} + +check.dependsOn ktlint + +task ktlintFormat(type: JavaExec, group: "formatting") { + description = "Fix Kotlin code style deviations." + main = "com.pinterest.ktlint.Main" + classpath = configurations.ktlint + args "-F", "src/**/*.kt" +} + +compileKotlin { kotlinOptions.freeCompilerArgs = ['-Xjsr305=strict'] } + +// Only apply jacoco test coverage if we are running a local single node cluster +if (!usingRemoteCluster && !usingMultiNode) { + apply from: 'build-tools/opensearchplugin-coverage.gradle' +} + +apply from: 'build-tools/pkgbuild.gradle' diff --git a/opensearch-observability/config/checkstyle/checkstyle.xml b/opensearch-observability/config/checkstyle/checkstyle.xml new file mode 100644 index 000000000..286aef81d --- /dev/null +++ b/opensearch-observability/config/checkstyle/checkstyle.xml @@ -0,0 +1,147 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/opensearch-observability/config/checkstyle/google_checks.xml b/opensearch-observability/config/checkstyle/google_checks.xml new file mode 100644 index 000000000..662740ff6 --- /dev/null +++ b/opensearch-observability/config/checkstyle/google_checks.xml @@ -0,0 +1,291 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/opensearch-observability/config/checkstyle/suppressions.xml b/opensearch-observability/config/checkstyle/suppressions.xml new file mode 100644 index 000000000..5926732a0 --- /dev/null +++ b/opensearch-observability/config/checkstyle/suppressions.xml @@ -0,0 +1,35 @@ + + + + + + + + + + diff --git a/opensearch-observability/detekt.yml b/opensearch-observability/detekt.yml new file mode 100644 index 000000000..dbcca8ab6 --- /dev/null +++ b/opensearch-observability/detekt.yml @@ -0,0 +1,39 @@ +--- +## +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +## + +## +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# +## + +style: + ForbiddenComment: + active: false + MaxLineLength: + maxLineLength: 150 + ThrowsCount: + active: true + max: 10 + ReturnCount: + active: true + max: 10 diff --git a/opensearch-observability/gradle/wrapper/gradle-wrapper.jar b/opensearch-observability/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..62d4c0535 Binary files /dev/null and b/opensearch-observability/gradle/wrapper/gradle-wrapper.jar differ diff --git a/opensearch-observability/gradle/wrapper/gradle-wrapper.properties b/opensearch-observability/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..db803810d --- /dev/null +++ b/opensearch-observability/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,33 @@ +## +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +## + +# +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# + +#Wed Jul 29 13:30:55 PDT 2020 +distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStorePath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME diff --git a/opensearch-observability/gradlew b/opensearch-observability/gradlew new file mode 100755 index 000000000..2fe81a7d9 --- /dev/null +++ b/opensearch-observability/gradlew @@ -0,0 +1,183 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/opensearch-observability/gradlew.bat b/opensearch-observability/gradlew.bat new file mode 100644 index 000000000..24467a141 --- /dev/null +++ b/opensearch-observability/gradlew.bat @@ -0,0 +1,100 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/opensearch-observability/src/main/config/observability.yml b/opensearch-observability/src/main/config/observability.yml new file mode 100644 index 000000000..18acb0654 --- /dev/null +++ b/opensearch-observability/src/main/config/observability.yml @@ -0,0 +1,50 @@ +--- +## +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +## + +## +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# +## + +# configuration file for the observability plugin +opensearch.notebooks: + general: + operationTimeoutMs: 60000 # 60 seconds, Minimum 100ms + defaultItemsQueryCount: 100 # default number of items to query + polling: + jobLockDurationSeconds: 300 # 5 Minutes, Minimum 10 seconds + minPollingDurationSeconds: 300 # 5 Minutes, Minimum 60 seconds + maxPollingDurationSeconds: 900 # 15 Minutes, Minimum 5 Minutes + maxLockRetries: 1 # Max number of retries to retry locking + access: + adminAccess: "AllNotebooks" + # adminAccess values: + ## Standard -> Admin user access follows standard user + ## AllNotebooks -> Admin user with "all_access" role can see all observability objects of all users. + filterBy: "NoFilter" # Applied when tenant != __user__ + # filterBy values: + ## NoFilter -> everyone see each other's observability objects + ## User -> observability objects are visible to only themselves + ## Roles -> observability objects are visible to users having any one of the role of creator + ## BackendRoles -> observability objects are visible to users having any one of the backend role of creator + ignoreRoles: ["own_index", "kibana_user", "notebooks_full_access", "notebooks_read_access"] diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/ObservabilityPlugin.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/ObservabilityPlugin.kt new file mode 100644 index 000000000..0ed142035 --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/ObservabilityPlugin.kt @@ -0,0 +1,141 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ +package org.opensearch.observability + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionResponse +import org.opensearch.client.Client +import org.opensearch.cluster.metadata.IndexNameExpressionResolver +import org.opensearch.cluster.node.DiscoveryNodes +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.io.stream.NamedWriteableRegistry +import org.opensearch.common.settings.ClusterSettings +import org.opensearch.common.settings.IndexScopedSettings +import org.opensearch.common.settings.Setting +import org.opensearch.common.settings.Settings +import org.opensearch.common.settings.SettingsFilter +import org.opensearch.common.xcontent.NamedXContentRegistry +import org.opensearch.env.Environment +import org.opensearch.env.NodeEnvironment +import org.opensearch.observability.action.CreateObservabilityObjectAction +import org.opensearch.observability.action.DeleteObservabilityObjectAction +import org.opensearch.observability.action.GetObservabilityObjectAction +import org.opensearch.observability.action.UpdateObservabilityObjectAction +import org.opensearch.observability.index.ObservabilityIndex +import org.opensearch.observability.resthandler.ObservabilityRestHandler +import org.opensearch.observability.settings.PluginSettings +import org.opensearch.plugins.ActionPlugin +import org.opensearch.plugins.Plugin +import org.opensearch.repositories.RepositoriesService +import org.opensearch.rest.RestController +import org.opensearch.rest.RestHandler +import org.opensearch.script.ScriptService +import org.opensearch.threadpool.ThreadPool +import org.opensearch.watcher.ResourceWatcherService +import java.util.function.Supplier + +/** + * Entry point of the OpenSearch Observability plugin. + * This class initializes the rest handlers. + */ +class ObservabilityPlugin : Plugin(), ActionPlugin { + + companion object { + const val PLUGIN_NAME = "opensearch-observability" + const val LOG_PREFIX = "observability" + const val BASE_OBSERVABILITY_URI = "/_plugins/_observability" + } + + /** + * {@inheritDoc} + */ + override fun getSettings(): List> { + return PluginSettings.getAllSettings() + } + + /** + * {@inheritDoc} + */ + override fun createComponents( + client: Client, + clusterService: ClusterService, + threadPool: ThreadPool, + resourceWatcherService: ResourceWatcherService, + scriptService: ScriptService, + xContentRegistry: NamedXContentRegistry, + environment: Environment, + nodeEnvironment: NodeEnvironment, + namedWriteableRegistry: NamedWriteableRegistry, + indexNameExpressionResolver: IndexNameExpressionResolver, + repositoriesServiceSupplier: Supplier + ): Collection { + PluginSettings.addSettingsUpdateConsumer(clusterService) + ObservabilityIndex.initialize(client, clusterService) + return emptyList() + } + + /** + * {@inheritDoc} + */ + override fun getRestHandlers( + settings: Settings, + restController: RestController, + clusterSettings: ClusterSettings, + indexScopedSettings: IndexScopedSettings, + settingsFilter: SettingsFilter, + indexNameExpressionResolver: IndexNameExpressionResolver, + nodesInCluster: Supplier + ): List { + return listOf( + ObservabilityRestHandler() + ) + } + + /** + * {@inheritDoc} + */ + override fun getActions(): List> { + return listOf( + ActionPlugin.ActionHandler( + CreateObservabilityObjectAction.ACTION_TYPE, + CreateObservabilityObjectAction::class.java + ), + ActionPlugin.ActionHandler( + DeleteObservabilityObjectAction.ACTION_TYPE, + DeleteObservabilityObjectAction::class.java + ), + ActionPlugin.ActionHandler( + GetObservabilityObjectAction.ACTION_TYPE, + GetObservabilityObjectAction::class.java + ), + ActionPlugin.ActionHandler( + UpdateObservabilityObjectAction.ACTION_TYPE, + UpdateObservabilityObjectAction::class.java + ) + ) + } +} diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/BaseModel.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/BaseModel.kt new file mode 100644 index 000000000..bc937685f --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/BaseModel.kt @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.observability.model + +import org.opensearch.common.io.stream.Writeable +import org.opensearch.common.xcontent.ToXContent + +/** + * interface for representing objects. + */ +interface BaseModel : Writeable, ToXContent diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/BaseObjectData.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/BaseObjectData.kt new file mode 100644 index 000000000..8cffefab2 --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/BaseObjectData.kt @@ -0,0 +1,17 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.observability.model + +/** + * Marker interface for Channel Data + */ +interface BaseObjectData : BaseModel diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/BaseResponse.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/BaseResponse.kt new file mode 100644 index 000000000..6bbe09a4d --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/BaseResponse.kt @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package org.opensearch.observability.model + +import org.opensearch.action.ActionResponse +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.ToXContentObject +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.rest.RestStatus +import java.io.IOException + +/** + * Base response which give REST status. + */ +internal abstract class BaseResponse : ActionResponse, ToXContentObject { + + /** + * constructor for creating the class + */ + constructor() + + /** + * {@inheritDoc} + */ + @Throws(IOException::class) + constructor(input: StreamInput) : super(input) + + /** + * {@inheritDoc} + */ + fun toXContent(params: ToXContent.Params = ToXContent.EMPTY_PARAMS): XContentBuilder { + return toXContent(XContentFactory.jsonBuilder(), params) + } + + /** + * get rest status for the response. Useful override for multi-status response. + * @return RestStatus for the response + */ + open fun getStatus(): RestStatus { + return RestStatus.OK + } +} diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/XParser.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/XParser.kt new file mode 100644 index 000000000..8aebce510 --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/model/XParser.kt @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.observability.model + +import org.opensearch.common.xcontent.XContentParser + +/** + * Functional interface to create object using XContentParser + */ +fun interface XParser { + /** + * Creator used in REST communication. + * @param parser XContentParser to deserialize data from. + */ + fun parse(parser: XContentParser): V +} diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/resthandler/RestResponseToXContentListener.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/resthandler/RestResponseToXContentListener.kt new file mode 100644 index 000000000..d87f4745c --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/resthandler/RestResponseToXContentListener.kt @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package org.opensearch.observability.resthandler + +import org.opensearch.observability.model.BaseResponse +import org.opensearch.rest.RestChannel +import org.opensearch.rest.RestStatus +import org.opensearch.rest.action.RestToXContentListener + +/** + * Overrides RestToXContentListener REST based action listener that assumes the response is of type + * {@link ToXContent} and automatically builds an XContent based response + * (wrapping the toXContent in startObject/endObject). + */ +internal class RestResponseToXContentListener(channel: RestChannel) : RestToXContentListener(channel) { + /** + * {@inheritDoc} + */ + override fun getStatus(response: Response): RestStatus { + return response.getStatus() + } +} diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/security/SecurityAccess.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/security/SecurityAccess.kt new file mode 100644 index 000000000..eb4abf381 --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/security/SecurityAccess.kt @@ -0,0 +1,53 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package org.opensearch.observability.security + +import org.opensearch.SpecialPermission +import java.security.AccessController +import java.security.PrivilegedActionException +import java.security.PrivilegedExceptionAction + +/** + * Class for providing the elevated permission for the function call. + * Ref: + * https://www.elastic.co/guide/en/elasticsearch/plugins/current/plugin-authors.html#_java_security_permissions + */ +internal object SecurityAccess { + /** + * Execute the operation in privileged mode. + */ + @Throws(Exception::class) + fun doPrivileged(operation: PrivilegedExceptionAction?): T { + SpecialPermission.check() + return try { + AccessController.doPrivileged(operation) + } catch (e: PrivilegedActionException) { + throw (e.cause as Exception?)!! + } + } +} diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/security/UserAccessManager.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/security/UserAccessManager.kt new file mode 100644 index 000000000..c2e80cdb3 --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/security/UserAccessManager.kt @@ -0,0 +1,207 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package org.opensearch.observability.security + +import org.opensearch.OpenSearchStatusException +import org.opensearch.commons.authuser.User +import org.opensearch.observability.settings.PluginSettings +import org.opensearch.observability.settings.PluginSettings.FilterBy +import org.opensearch.rest.RestStatus +import java.util.stream.Collectors + +/** + * Class for checking/filtering user access. + */ +internal object UserAccessManager { + private const val USER_TAG = "User:" + private const val ROLE_TAG = "Role:" + private const val BACKEND_ROLE_TAG = "BERole:" + private const val ALL_ACCESS_ROLE = "all_access" + private const val OPENSEARCH_DASHBOARDS_SERVER_USER = "opensearchdashboardsserver" // TODO: Change it to background user when created. + private const val PRIVATE_TENANT = "__user__" + const val DEFAULT_TENANT = "" + + /** + * Validate User if eligible to do operation + * If filterBy == NoFilter + * -> No validation + * If filterBy == User + * -> User name should be present + * If filterBy == Roles + * -> roles should be present + * If filterBy == BackendRoles + * -> backend roles should be present + */ + fun validateUser(user: User?) { + if (isUserPrivateTenant(user) && user?.name == null) { + throw OpenSearchStatusException( + "User name not provided for private tenant access", + RestStatus.FORBIDDEN + ) + } + when (PluginSettings.filterBy) { + FilterBy.NoFilter -> { // No validation + } + FilterBy.User -> { // User name must be present + user?.name + ?: throw OpenSearchStatusException( + "Filter-by enabled with security disabled", + RestStatus.FORBIDDEN + ) + } + FilterBy.Roles -> { // backend roles must be present + if (user == null || user.roles.isNullOrEmpty()) { + throw OpenSearchStatusException( + "User doesn't have roles configured. Contact administrator.", + RestStatus.FORBIDDEN + ) + } else if (user.roles.stream().filter { !PluginSettings.ignoredRoles.contains(it) }.count() == 0L) { + throw OpenSearchStatusException( + "No distinguishing roles configured. Contact administrator.", + RestStatus.FORBIDDEN + ) + } + } + FilterBy.BackendRoles -> { // backend roles must be present + if (user?.backendRoles.isNullOrEmpty()) { + throw OpenSearchStatusException( + "User doesn't have backend roles configured. Contact administrator.", + RestStatus.FORBIDDEN + ) + } + } + } + } + + /** + * validate if user has access to polling actions + */ + fun validatePollingUser(user: User?) { + if (user != null) { // Check only if security is enabled + if (user.name != OPENSEARCH_DASHBOARDS_SERVER_USER) { + throw OpenSearchStatusException("Permission denied", RestStatus.FORBIDDEN) + } + } + } + + /** + * Get tenant info from user object. + */ + fun getUserTenant(user: User?): String { + return when (val requestedTenant = user?.requestedTenant) { + null -> DEFAULT_TENANT + else -> requestedTenant + } + } + + /** + * Get all user access info from user object. + */ + fun getAllAccessInfo(user: User?): List { + if (user == null) { // Security is disabled + return listOf() + } + val retList: MutableList = mutableListOf() + if (user.name != null) { + retList.add("$USER_TAG${user.name}") + } + user.roles.forEach { retList.add("$ROLE_TAG$it") } + user.backendRoles.forEach { retList.add("$BACKEND_ROLE_TAG$it") } + return retList + } + + /** + * Get access info for search filtering + */ + fun getSearchAccessInfo(user: User?): List { + if (user == null) { // Security is disabled + return listOf() + } + if (isUserPrivateTenant(user)) { + return listOf("$USER_TAG${user.name}") // No sharing allowed in private tenant. + } + if (canAdminViewAllItems(user)) { + return listOf() + } + return when (PluginSettings.filterBy) { + FilterBy.NoFilter -> listOf() + FilterBy.User -> listOf("$USER_TAG${user.name}") + FilterBy.Roles -> user.roles.stream() + .filter { !PluginSettings.ignoredRoles.contains(it) } + .map { "$ROLE_TAG$it" } + .collect(Collectors.toList()) + FilterBy.BackendRoles -> user.backendRoles.map { "$BACKEND_ROLE_TAG$it" } + } + } + + /** + * validate if user has access based on given access list + */ + fun doesUserHasAccess(user: User?, tenant: String, access: List): Boolean { + if (user == null) { // Security is disabled + return true + } + if (getUserTenant(user) != tenant) { + return false + } + if (canAdminViewAllItems(user)) { + return true + } + return when (PluginSettings.filterBy) { + FilterBy.NoFilter -> true + FilterBy.User -> access.contains("$USER_TAG${user.name}") + FilterBy.Roles -> user.roles.stream() + .filter { !PluginSettings.ignoredRoles.contains(it) } + .map { "$ROLE_TAG$it" } + .anyMatch { it in access } + FilterBy.BackendRoles -> user.backendRoles.map { "$BACKEND_ROLE_TAG$it" }.any { it in access } + } + } + + /** + * Check if user has all info access. + */ + fun hasAllInfoAccess(user: User?): Boolean { + if (user == null) { // Security is disabled + return true + } + return isAdminUser(user) + } + + private fun canAdminViewAllItems(user: User): Boolean { + return PluginSettings.adminAccess == PluginSettings.AdminAccess.AllNotebooks && isAdminUser(user) + } + + private fun isAdminUser(user: User): Boolean { + return user.roles.contains(ALL_ACCESS_ROLE) + } + + private fun isUserPrivateTenant(user: User?): Boolean { + return getUserTenant(user) == PRIVATE_TENANT + } +} diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/settings/PluginSettings.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/settings/PluginSettings.kt new file mode 100644 index 000000000..49afb5f22 --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/settings/PluginSettings.kt @@ -0,0 +1,499 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package org.opensearch.observability.settings + +import org.apache.logging.log4j.LogManager +import org.opensearch.bootstrap.BootstrapInfo +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.settings.Setting +import org.opensearch.common.settings.Setting.Property.Dynamic +import org.opensearch.common.settings.Setting.Property.NodeScope +import org.opensearch.common.settings.Settings +import org.opensearch.observability.ObservabilityPlugin.Companion.LOG_PREFIX +import org.opensearch.observability.ObservabilityPlugin.Companion.PLUGIN_NAME +import java.io.IOException +import java.nio.file.Path + +/** + * settings specific to observability Plugin. + */ +internal object PluginSettings { + + /** + * Settings Key prefix for this plugin. + */ + private const val KEY_PREFIX = "opensearch.observability" + + /** + * General settings Key prefix. + */ + private const val GENERAL_KEY_PREFIX = "$KEY_PREFIX.general" + + /** + * Polling settings Key prefix. + */ + private const val POLLING_KEY_PREFIX = "$KEY_PREFIX.polling" + + /** + * Access settings Key prefix. + */ + private const val ACCESS_KEY_PREFIX = "$KEY_PREFIX.access" + + /** + * Operation timeout for network operations. + */ + private const val OPERATION_TIMEOUT_MS_KEY = "$GENERAL_KEY_PREFIX.operationTimeoutMs" + + /** + * Setting to choose Job lock duration. + */ + private const val JOB_LOCK_DURATION_S_KEY = "$POLLING_KEY_PREFIX.jobLockDurationSeconds" + + /** + * Setting to choose Minimum polling duration. + */ + private const val MIN_POLLING_DURATION_S_KEY = "$POLLING_KEY_PREFIX.minPollingDurationSeconds" + + /** + * Setting to choose Maximum polling duration. + */ + private const val MAX_POLLING_DURATION_S_KEY = "$POLLING_KEY_PREFIX.maxPollingDurationSeconds" + + /** + * Setting to choose Maximum number of retries to try locking. + */ + private const val MAX_LOCK_RETRIES_KEY = "$POLLING_KEY_PREFIX.maxLockRetries" + + /** + * Setting to choose default number of items to query. + */ + private const val DEFAULT_ITEMS_QUERY_COUNT_KEY = "$GENERAL_KEY_PREFIX.defaultItemsQueryCount" + + /** + * Setting to choose admin access restriction. + */ + private const val ADMIN_ACCESS_KEY = "$ACCESS_KEY_PREFIX.adminAccess" + + /** + * Setting to choose filter method. + */ + private const val FILTER_BY_KEY = "$ACCESS_KEY_PREFIX.filterBy" + + /** + * Setting to choose ignored roles for filtering. + */ + private const val IGNORE_ROLE_KEY = "$ACCESS_KEY_PREFIX.ignoreRoles" + + /** + * Default operation timeout for network operations. + */ + private const val DEFAULT_OPERATION_TIMEOUT_MS = 60000L + + /** + * Minimum operation timeout for network operations. + */ + private const val MINIMUM_OPERATION_TIMEOUT_MS = 100L + + /** + * Default Job lock duration. + */ + private const val DEFAULT_JOB_LOCK_DURATION_S = 300 + + /** + * Minimum Job lock duration. + */ + private const val MINIMUM_JOB_LOCK_DURATION_S = 10 + + /** + * Default Minimum polling duration. + */ + private const val DEFAULT_MIN_POLLING_DURATION_S = 300 + + /** + * Minimum Min polling duration. + */ + private const val MINIMUM_MIN_POLLING_DURATION_S = 60 + + /** + * Default Maximum polling duration. + */ + private const val DEFAULT_MAX_POLLING_DURATION_S = 900 + + /** + * Minimum Maximum polling duration. + */ + private const val MINIMUM_MAX_POLLING_DURATION_S = 300 + + /** + * Default number of retries to try locking. + */ + private const val DEFAULT_MAX_LOCK_RETRIES = 4 + + /** + * Minimum number of retries to try locking. + */ + private const val MINIMUM_LOCK_RETRIES = 1 + + /** + * Default number of items to query. + */ + private const val DEFAULT_ITEMS_QUERY_COUNT_VALUE = 1000 + + /** + * Minimum number of items to query. + */ + private const val MINIMUM_ITEMS_QUERY_COUNT = 10 + + /** + * Default admin access method. + */ + private const val DEFAULT_ADMIN_ACCESS_METHOD = "AllNotebooks" + + /** + * Default filter-by method. + */ + private const val DEFAULT_FILTER_BY_METHOD = "NoFilter" + + /** + * Default filter-by method. + */ + private val DEFAULT_IGNORED_ROLES = listOf( + "own_index", + "opensearch_dashboards_user", + "notebooks_full_access", + "notebooks_read_access" + ) + + /** + * Operation timeout setting in ms for I/O operations + */ + @Volatile + var operationTimeoutMs: Long + + /** + * Job lock duration + */ + @Volatile + var jobLockDurationSeconds: Int + + /** + * Minimum polling duration + */ + @Volatile + var minPollingDurationSeconds: Int + + /** + * Maximum polling duration. + */ + @Volatile + var maxPollingDurationSeconds: Int + + /** + * Max number of retries to try locking. + */ + @Volatile + var maxLockRetries: Int + + /** + * Default number of items to query. + */ + @Volatile + var defaultItemsQueryCount: Int + + /** + * admin access method. + */ + @Volatile + var adminAccess: AdminAccess + + /** + * Filter-by method. + */ + @Volatile + var filterBy: FilterBy + + /** + * list of ignored roles. + */ + @Volatile + var ignoredRoles: List + + /** + * Enum for types of admin access + * "Standard" -> Admin user access follows standard user + * "AllNotebooks" -> Admin user with "all_access" role can see all notebooks of all users. + */ + internal enum class AdminAccess { Standard, AllNotebooks } + + /** + * Enum for types of filterBy options + * NoFilter -> everyone see each other's notebooks + * User -> notebooks are visible to only themselves + * Roles -> notebooks are visible to users having any one of the role of creator + * BackendRoles -> notebooks are visible to users having any one of the backend role of creator + */ + internal enum class FilterBy { NoFilter, User, Roles, BackendRoles } + + private const val DECIMAL_RADIX: Int = 10 + + private val log = LogManager.getLogger(javaClass) + private val defaultSettings: Map + + init { + var settings: Settings? = null + val configDirName = BootstrapInfo.getSystemProperties()?.get("opensearch.path.conf")?.toString() + if (configDirName != null) { + val defaultSettingYmlFile = Path.of(configDirName, PLUGIN_NAME, "observability.yml") + try { + settings = Settings.builder().loadFromPath(defaultSettingYmlFile).build() + } catch (exception: IOException) { + log.warn("$LOG_PREFIX:Failed to load ${defaultSettingYmlFile.toAbsolutePath()}") + } + } + // Initialize the settings values to default values + operationTimeoutMs = (settings?.get(OPERATION_TIMEOUT_MS_KEY)?.toLong()) ?: DEFAULT_OPERATION_TIMEOUT_MS + jobLockDurationSeconds = (settings?.get(JOB_LOCK_DURATION_S_KEY)?.toInt()) ?: DEFAULT_JOB_LOCK_DURATION_S + minPollingDurationSeconds = (settings?.get(MIN_POLLING_DURATION_S_KEY)?.toInt()) + ?: DEFAULT_MIN_POLLING_DURATION_S + maxPollingDurationSeconds = (settings?.get(MAX_POLLING_DURATION_S_KEY)?.toInt()) + ?: DEFAULT_MAX_POLLING_DURATION_S + maxLockRetries = (settings?.get(MAX_LOCK_RETRIES_KEY)?.toInt()) ?: DEFAULT_MAX_LOCK_RETRIES + defaultItemsQueryCount = (settings?.get(DEFAULT_ITEMS_QUERY_COUNT_KEY)?.toInt()) + ?: DEFAULT_ITEMS_QUERY_COUNT_VALUE + adminAccess = AdminAccess.valueOf(settings?.get(ADMIN_ACCESS_KEY) ?: DEFAULT_ADMIN_ACCESS_METHOD) + filterBy = FilterBy.valueOf(settings?.get(FILTER_BY_KEY) ?: DEFAULT_FILTER_BY_METHOD) + ignoredRoles = settings?.getAsList(IGNORE_ROLE_KEY) ?: DEFAULT_IGNORED_ROLES + + defaultSettings = mapOf( + OPERATION_TIMEOUT_MS_KEY to operationTimeoutMs.toString(DECIMAL_RADIX), + JOB_LOCK_DURATION_S_KEY to jobLockDurationSeconds.toString(DECIMAL_RADIX), + MIN_POLLING_DURATION_S_KEY to minPollingDurationSeconds.toString(DECIMAL_RADIX), + MAX_POLLING_DURATION_S_KEY to maxPollingDurationSeconds.toString(DECIMAL_RADIX), + MAX_LOCK_RETRIES_KEY to maxLockRetries.toString(DECIMAL_RADIX), + DEFAULT_ITEMS_QUERY_COUNT_KEY to defaultItemsQueryCount.toString(DECIMAL_RADIX), + ADMIN_ACCESS_KEY to adminAccess.name, + FILTER_BY_KEY to filterBy.name + ) + } + + private val OPERATION_TIMEOUT_MS: Setting = Setting.longSetting( + OPERATION_TIMEOUT_MS_KEY, + defaultSettings[OPERATION_TIMEOUT_MS_KEY]!!.toLong(), + MINIMUM_OPERATION_TIMEOUT_MS, + NodeScope, Dynamic + ) + + private val JOB_LOCK_DURATION_S: Setting = Setting.intSetting( + JOB_LOCK_DURATION_S_KEY, + defaultSettings[JOB_LOCK_DURATION_S_KEY]!!.toInt(), + MINIMUM_JOB_LOCK_DURATION_S, + NodeScope, Dynamic + ) + + private val MIN_POLLING_DURATION_S: Setting = Setting.intSetting( + MIN_POLLING_DURATION_S_KEY, + defaultSettings[MIN_POLLING_DURATION_S_KEY]!!.toInt(), + MINIMUM_MIN_POLLING_DURATION_S, + NodeScope, Dynamic + ) + + private val MAX_POLLING_DURATION_S: Setting = Setting.intSetting( + MAX_POLLING_DURATION_S_KEY, + defaultSettings[MAX_POLLING_DURATION_S_KEY]!!.toInt(), + MINIMUM_MAX_POLLING_DURATION_S, + NodeScope, Dynamic + ) + + private val MAX_LOCK_RETRIES: Setting = Setting.intSetting( + MAX_LOCK_RETRIES_KEY, + defaultSettings[MAX_LOCK_RETRIES_KEY]!!.toInt(), + MINIMUM_LOCK_RETRIES, + NodeScope, Dynamic + ) + + private val DEFAULT_ITEMS_QUERY_COUNT: Setting = Setting.intSetting( + DEFAULT_ITEMS_QUERY_COUNT_KEY, + defaultSettings[DEFAULT_ITEMS_QUERY_COUNT_KEY]!!.toInt(), + MINIMUM_ITEMS_QUERY_COUNT, + NodeScope, Dynamic + ) + + private val ADMIN_ACCESS: Setting = Setting.simpleString( + ADMIN_ACCESS_KEY, + defaultSettings[ADMIN_ACCESS_KEY]!!, + NodeScope, Dynamic + ) + + private val FILTER_BY: Setting = Setting.simpleString( + FILTER_BY_KEY, + defaultSettings[FILTER_BY_KEY]!!, + NodeScope, Dynamic + ) + + private val IGNORED_ROLES: Setting> = Setting.listSetting( + IGNORE_ROLE_KEY, + DEFAULT_IGNORED_ROLES, + { it }, + NodeScope, Dynamic + ) + + /** + * Returns list of additional settings available specific to this plugin. + * + * @return list of settings defined in this plugin + */ + fun getAllSettings(): List> { + return listOf( + OPERATION_TIMEOUT_MS, + JOB_LOCK_DURATION_S, + MIN_POLLING_DURATION_S, + MAX_POLLING_DURATION_S, + MAX_LOCK_RETRIES, + DEFAULT_ITEMS_QUERY_COUNT, + ADMIN_ACCESS, + FILTER_BY, + IGNORED_ROLES + ) + } + + /** + * Update the setting variables to setting values from local settings + * @param clusterService cluster service instance + */ + private fun updateSettingValuesFromLocal(clusterService: ClusterService) { + operationTimeoutMs = OPERATION_TIMEOUT_MS.get(clusterService.settings) + jobLockDurationSeconds = JOB_LOCK_DURATION_S.get(clusterService.settings) + minPollingDurationSeconds = MIN_POLLING_DURATION_S.get(clusterService.settings) + maxPollingDurationSeconds = MAX_POLLING_DURATION_S.get(clusterService.settings) + maxLockRetries = MAX_LOCK_RETRIES.get(clusterService.settings) + defaultItemsQueryCount = DEFAULT_ITEMS_QUERY_COUNT.get(clusterService.settings) + adminAccess = AdminAccess.valueOf(ADMIN_ACCESS.get(clusterService.settings)) + filterBy = FilterBy.valueOf(FILTER_BY.get(clusterService.settings)) + ignoredRoles = IGNORED_ROLES.get(clusterService.settings) + } + + /** + * Update the setting variables to setting values from cluster settings + * @param clusterService cluster service instance + */ + private fun updateSettingValuesFromCluster(clusterService: ClusterService) { + val clusterOperationTimeoutMs = clusterService.clusterSettings.get(OPERATION_TIMEOUT_MS) + if (clusterOperationTimeoutMs != null) { + log.debug("$LOG_PREFIX:$OPERATION_TIMEOUT_MS_KEY -autoUpdatedTo-> $clusterOperationTimeoutMs") + operationTimeoutMs = clusterOperationTimeoutMs + } + val clusterJobLockDurationSeconds = clusterService.clusterSettings.get(JOB_LOCK_DURATION_S) + if (clusterJobLockDurationSeconds != null) { + log.debug("$LOG_PREFIX:$JOB_LOCK_DURATION_S_KEY -autoUpdatedTo-> $clusterJobLockDurationSeconds") + jobLockDurationSeconds = clusterJobLockDurationSeconds + } + val clusterMinPollingDurationSeconds = clusterService.clusterSettings.get(MIN_POLLING_DURATION_S) + if (clusterMinPollingDurationSeconds != null) { + log.debug("$LOG_PREFIX:$MIN_POLLING_DURATION_S_KEY -autoUpdatedTo-> $clusterMinPollingDurationSeconds") + minPollingDurationSeconds = clusterMinPollingDurationSeconds + } + val clusterMaxPollingDurationSeconds = clusterService.clusterSettings.get(MAX_POLLING_DURATION_S) + if (clusterMaxPollingDurationSeconds != null) { + log.debug("$LOG_PREFIX:$MAX_POLLING_DURATION_S_KEY -autoUpdatedTo-> $clusterMaxPollingDurationSeconds") + maxPollingDurationSeconds = clusterMaxPollingDurationSeconds + } + val clusterMaxLockRetries = clusterService.clusterSettings.get(MAX_LOCK_RETRIES) + if (clusterMaxLockRetries != null) { + log.debug("$LOG_PREFIX:$MAX_LOCK_RETRIES_KEY -autoUpdatedTo-> $clusterMaxLockRetries") + maxLockRetries = clusterMaxLockRetries + } + val clusterDefaultItemsQueryCount = clusterService.clusterSettings.get(DEFAULT_ITEMS_QUERY_COUNT) + if (clusterDefaultItemsQueryCount != null) { + log.debug("$LOG_PREFIX:$DEFAULT_ITEMS_QUERY_COUNT_KEY -autoUpdatedTo-> $clusterDefaultItemsQueryCount") + defaultItemsQueryCount = clusterDefaultItemsQueryCount + } + val clusterAdminAccess = clusterService.clusterSettings.get(ADMIN_ACCESS) + if (clusterAdminAccess != null) { + log.debug("$LOG_PREFIX:$ADMIN_ACCESS_KEY -autoUpdatedTo-> $clusterAdminAccess") + adminAccess = AdminAccess.valueOf(clusterAdminAccess) + } + val clusterFilterBy = clusterService.clusterSettings.get(FILTER_BY) + if (clusterFilterBy != null) { + log.debug("$LOG_PREFIX:$FILTER_BY_KEY -autoUpdatedTo-> $clusterFilterBy") + filterBy = FilterBy.valueOf(clusterFilterBy) + } + val clusterIgnoredRoles = clusterService.clusterSettings.get(IGNORED_ROLES) + if (clusterIgnoredRoles != null) { + log.debug("$LOG_PREFIX:$IGNORE_ROLE_KEY -autoUpdatedTo-> $clusterIgnoredRoles") + ignoredRoles = clusterIgnoredRoles + } + } + + /** + * adds Settings update listeners to all settings. + * @param clusterService cluster service instance + */ + fun addSettingsUpdateConsumer(clusterService: ClusterService) { + updateSettingValuesFromLocal(clusterService) + // Update the variables to cluster setting values + // If the cluster is not yet started then we get default values again + updateSettingValuesFromCluster(clusterService) + + clusterService.clusterSettings.addSettingsUpdateConsumer(OPERATION_TIMEOUT_MS) { + operationTimeoutMs = it + log.info("$LOG_PREFIX:$OPERATION_TIMEOUT_MS_KEY -updatedTo-> $it") + } + clusterService.clusterSettings.addSettingsUpdateConsumer(JOB_LOCK_DURATION_S) { + jobLockDurationSeconds = it + log.info("$LOG_PREFIX:$JOB_LOCK_DURATION_S_KEY -updatedTo-> $it") + } + clusterService.clusterSettings.addSettingsUpdateConsumer(MIN_POLLING_DURATION_S) { + minPollingDurationSeconds = it + log.info("$LOG_PREFIX:$MIN_POLLING_DURATION_S_KEY -updatedTo-> $it") + } + clusterService.clusterSettings.addSettingsUpdateConsumer(MAX_POLLING_DURATION_S) { + maxPollingDurationSeconds = it + log.info("$LOG_PREFIX:$MAX_POLLING_DURATION_S_KEY -updatedTo-> $it") + } + clusterService.clusterSettings.addSettingsUpdateConsumer(MAX_LOCK_RETRIES) { + maxLockRetries = it + log.info("$LOG_PREFIX:$MAX_LOCK_RETRIES_KEY -updatedTo-> $it") + } + clusterService.clusterSettings.addSettingsUpdateConsumer(DEFAULT_ITEMS_QUERY_COUNT) { + defaultItemsQueryCount = it + log.info("$LOG_PREFIX:$DEFAULT_ITEMS_QUERY_COUNT_KEY -updatedTo-> $it") + } + clusterService.clusterSettings.addSettingsUpdateConsumer(ADMIN_ACCESS) { + adminAccess = AdminAccess.valueOf(it) + log.info("$LOG_PREFIX:$ADMIN_ACCESS_KEY -updatedTo-> $it") + } + clusterService.clusterSettings.addSettingsUpdateConsumer(FILTER_BY) { + filterBy = FilterBy.valueOf(it) + log.info("$LOG_PREFIX:$FILTER_BY_KEY -updatedTo-> $it") + } + clusterService.clusterSettings.addSettingsUpdateConsumer(IGNORED_ROLES) { + ignoredRoles = it + log.info("$LOG_PREFIX:$IGNORE_ROLE_KEY -updatedTo-> $it") + } + } +} diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/util/Helpers.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/util/Helpers.kt new file mode 100644 index 000000000..6d0a5248d --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/util/Helpers.kt @@ -0,0 +1,80 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package org.opensearch.observability.util + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.opensearch.common.io.stream.StreamInput +import org.opensearch.common.xcontent.DeprecationHandler +import org.opensearch.common.xcontent.NamedXContentRegistry +import org.opensearch.common.xcontent.ToXContent +import org.opensearch.common.xcontent.ToXContentObject +import org.opensearch.common.xcontent.XContentBuilder +import org.opensearch.common.xcontent.XContentParser +import org.opensearch.common.xcontent.XContentParser.Token +import org.opensearch.common.xcontent.XContentParserUtils +import org.opensearch.common.xcontent.XContentType +import org.opensearch.rest.RestRequest + +internal fun StreamInput.createJsonParser(): XContentParser { + return XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.IGNORE_DEPRECATIONS, this) +} + +internal fun RestRequest.contentParserNextToken(): XContentParser { + val parser = this.contentParser() + parser.nextToken() + return parser +} + +internal fun XContentParser.stringList(): List { + val retList: MutableList = mutableListOf() + XContentParserUtils.ensureExpectedToken(Token.START_ARRAY, currentToken(), this) + while (nextToken() != Token.END_ARRAY) { + retList.add(text()) + } + return retList +} + +internal fun logger(forClass: Class): Lazy { + return lazy { LogManager.getLogger(forClass) } +} + +internal fun XContentBuilder.fieldIfNotNull(name: String, value: Any?): XContentBuilder { + if (value != null) { + this.field(name, value) + } + return this +} + +internal fun XContentBuilder.objectIfNotNull(name: String, xContentObject: ToXContentObject?): XContentBuilder { + if (xContentObject != null) { + this.field(name) + xContentObject.toXContent(this, ToXContent.EMPTY_PARAMS) + } + return this +} diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/util/OpenForTesting.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/util/OpenForTesting.kt new file mode 100644 index 000000000..05e6a2e38 --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/util/OpenForTesting.kt @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package org.opensearch.observability.util + +/** + * Annotation to elevate access for testing purpose. + */ +annotation class OpenForTesting diff --git a/opensearch-observability/src/main/kotlin/org/opensearch/observability/util/SecureIndexClient.kt b/opensearch-observability/src/main/kotlin/org/opensearch/observability/util/SecureIndexClient.kt new file mode 100644 index 000000000..76b217750 --- /dev/null +++ b/opensearch-observability/src/main/kotlin/org/opensearch/observability/util/SecureIndexClient.kt @@ -0,0 +1,325 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package org.opensearch.observability.util + +import org.opensearch.action.ActionFuture +import org.opensearch.action.ActionListener +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionResponse +import org.opensearch.action.ActionType +import org.opensearch.action.bulk.BulkRequest +import org.opensearch.action.bulk.BulkResponse +import org.opensearch.action.delete.DeleteRequest +import org.opensearch.action.delete.DeleteResponse +import org.opensearch.action.explain.ExplainRequest +import org.opensearch.action.explain.ExplainResponse +import org.opensearch.action.fieldcaps.FieldCapabilitiesRequest +import org.opensearch.action.fieldcaps.FieldCapabilitiesResponse +import org.opensearch.action.get.GetRequest +import org.opensearch.action.get.GetResponse +import org.opensearch.action.get.MultiGetRequest +import org.opensearch.action.get.MultiGetResponse +import org.opensearch.action.index.IndexRequest +import org.opensearch.action.index.IndexResponse +import org.opensearch.action.search.ClearScrollRequest +import org.opensearch.action.search.ClearScrollResponse +import org.opensearch.action.search.MultiSearchRequest +import org.opensearch.action.search.MultiSearchResponse +import org.opensearch.action.search.SearchRequest +import org.opensearch.action.search.SearchResponse +import org.opensearch.action.search.SearchScrollRequest +import org.opensearch.action.termvectors.MultiTermVectorsRequest +import org.opensearch.action.termvectors.MultiTermVectorsResponse +import org.opensearch.action.termvectors.TermVectorsRequest +import org.opensearch.action.termvectors.TermVectorsResponse +import org.opensearch.action.update.UpdateRequest +import org.opensearch.action.update.UpdateResponse +import org.opensearch.client.Client +import org.opensearch.common.util.concurrent.ThreadContext + +/** + * Wrapper class on [Client] with security context removed. + */ +@Suppress("TooManyFunctions") +internal class SecureIndexClient(private val client: Client) : Client by client { + /** + * {@inheritDoc} + */ + override fun execute( + action: ActionType, + request: Request + ): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.execute(action, request) } + } + + /** + * {@inheritDoc} + */ + override fun execute( + action: ActionType, + request: Request, + listener: ActionListener + ) { + client.threadPool().threadContext.stashContext().use { return client.execute(action, request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun index(request: IndexRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.index(request) } + } + + /** + * {@inheritDoc} + */ + override fun index(request: IndexRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.index(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun update(request: UpdateRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.update(request) } + } + + /** + * {@inheritDoc} + */ + override fun update(request: UpdateRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.update(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun delete(request: DeleteRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.delete(request) } + } + + /** + * {@inheritDoc} + */ + override fun delete(request: DeleteRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.delete(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun bulk(request: BulkRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.bulk(request) } + } + + /** + * {@inheritDoc} + */ + override fun bulk(request: BulkRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.bulk(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun get(request: GetRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.get(request) } + } + + /** + * {@inheritDoc} + */ + override fun get(request: GetRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.get(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun multiGet(request: MultiGetRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.multiGet(request) } + } + + /** + * {@inheritDoc} + */ + override fun multiGet(request: MultiGetRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.multiGet(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun search(request: SearchRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.search(request) } + } + + /** + * {@inheritDoc} + */ + override fun search(request: SearchRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.search(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun searchScroll(request: SearchScrollRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.searchScroll(request) } + } + + /** + * {@inheritDoc} + */ + override fun searchScroll(request: SearchScrollRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.searchScroll(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun multiSearch(request: MultiSearchRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.multiSearch(request) } + } + + /** + * {@inheritDoc} + */ + override fun multiSearch(request: MultiSearchRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.multiSearch(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun termVectors(request: TermVectorsRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.termVectors(request) } + } + + /** + * {@inheritDoc} + */ + override fun termVectors(request: TermVectorsRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.termVectors(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun multiTermVectors(request: MultiTermVectorsRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.multiTermVectors(request) } + } + + /** + * {@inheritDoc} + */ + override fun multiTermVectors(request: MultiTermVectorsRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.multiTermVectors(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun explain(request: ExplainRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.explain(request) } + } + + /** + * {@inheritDoc} + */ + override fun explain(request: ExplainRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.explain(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun clearScroll(request: ClearScrollRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.clearScroll(request) } + } + + /** + * {@inheritDoc} + */ + override fun clearScroll(request: ClearScrollRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.clearScroll(request, listener) } + } + + /** + * {@inheritDoc} + */ + override fun fieldCaps(request: FieldCapabilitiesRequest): ActionFuture { + client.threadPool().threadContext.stashContext().use { return client.fieldCaps(request) } + } + + /** + * {@inheritDoc} + */ + override fun fieldCaps(request: FieldCapabilitiesRequest, listener: ActionListener) { + client.threadPool().threadContext.stashContext().use { return client.fieldCaps(request, listener) } + } + + /** + * Executes the given [block] function on this resource and then closes it down correctly whether an exception + * is thrown or not. + * + * In case if the resource is being closed due to an exception occurred in [block], and the closing also fails with an exception, + * the latter is added to the [suppressed][java.lang.Throwable.addSuppressed] exceptions of the former. + * + * @param block a function to process this [AutoCloseable] resource. + * @return the result of [block] function invoked on this resource. + */ + @Suppress("TooGenericExceptionCaught") + private inline fun T.use(block: (T) -> R): R { + var exception: Throwable? = null + try { + return block(this) + } catch (e: Throwable) { + exception = e + throw e + } finally { + closeFinally(exception) + } + } + + /** + * Closes this [AutoCloseable], suppressing possible exception or error thrown by [AutoCloseable.close] function when + * it's being closed due to some other [cause] exception occurred. + * + * The suppressed exception is added to the list of suppressed exceptions of [cause] exception. + */ + @Suppress("TooGenericExceptionCaught") + private fun ThreadContext.StoredContext.closeFinally(cause: Throwable?) = when (cause) { + null -> close() + else -> try { + close() + } catch (closeException: Throwable) { + cause.addSuppressed(closeException) + } + } +} diff --git a/opensearch-observability/src/main/plugin-metadata/plugin-security.policy b/opensearch-observability/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 000000000..9e033ea01 --- /dev/null +++ b/opensearch-observability/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +grant { + // needed to find the classloader to load whitelisted classes. + permission java.lang.RuntimePermission "createClassLoader"; + permission java.lang.RuntimePermission "getClassLoader"; + + permission java.net.SocketPermission "*", "connect,resolve"; + permission java.net.NetPermission "getProxySelector"; +}; diff --git a/opensearch-observability/src/main/resources/META-INF/services/com.amazon.opendistroforelasticsearch.jobscheduler.spi.JobSchedulerExtension b/opensearch-observability/src/main/resources/META-INF/services/com.amazon.opendistroforelasticsearch.jobscheduler.spi.JobSchedulerExtension new file mode 100644 index 000000000..c4d972fb0 --- /dev/null +++ b/opensearch-observability/src/main/resources/META-INF/services/com.amazon.opendistroforelasticsearch.jobscheduler.spi.JobSchedulerExtension @@ -0,0 +1 @@ +org.opensearch.observability.ObservabilityPlugin diff --git a/opensearch-observability/src/main/resources/observability-mapping.yml b/opensearch-observability/src/main/resources/observability-mapping.yml new file mode 100644 index 000000000..a47081d81 --- /dev/null +++ b/opensearch-observability/src/main/resources/observability-mapping.yml @@ -0,0 +1,86 @@ +--- +## +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +## + +## +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# +## + +# Schema file for the observability index +# Since we only search based on "access", sort on lastUpdatedTimeMs & createdTimeMs, +# other fields are not used in mapping to avoid index on those fields. +# Also "dynamic" is set to "false" so that other fields can be added. +dynamic: false +properties: + lastUpdatedTimeMs: + type: date + format: epoch_millis + createdTimeMs: + type: date + format: epoch_millis + tenant: + type: keyword + access: # Array of access details like user,role,backend_role etc + type: keyword + type: + type: keyword + notebook: + type: object + properties: + name: + type: text + fields: + keyword: + type: keyword + savedQuery: + type: object + properties: + name: + type: text + fields: + keyword: + type: keyword + savedVisualization: + type: object + properties: + name: + type: text + fields: + keyword: + type: keyword + operationalPanel: + type: object + properties: + name: + type: text + fields: + keyword: + type: keyword + timestamp: + type: object + properties: + name: + type: text + fields: + keyword: + type: keyword \ No newline at end of file diff --git a/opensearch-observability/src/main/resources/observability-settings.yml b/opensearch-observability/src/main/resources/observability-settings.yml new file mode 100644 index 000000000..1a3ea0a38 --- /dev/null +++ b/opensearch-observability/src/main/resources/observability-settings.yml @@ -0,0 +1,33 @@ +--- +## +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +## + +## +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# +## + +# Settings file for the observability index +index: + number_of_shards: "1" + auto_expand_replicas: "0-1" + number_of_replicas: "0"