Add OpenSearch KNN vector field support and more

.github/workflows/master.yml

@@ -9,7 +9,7 @@ on:
       - '*.yml'
     branches:
       - master
-      - release/*
+      - opensearch
 
 jobs:
   scala-2_12:
@@ -35,7 +35,7 @@ jobs:
         id: import_gpg
         uses: crazy-max/ghaction-import-gpg@v3
         with:
-          gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
+          gpg-private-key: ${{ secrets.PGP_SECRET }}
           passphrase: ${{ secrets.PGP_PASSPHRASE }}
 
       - name: GPG user IDs
@@ -48,8 +48,8 @@ jobs:
       - name: publish snapshot
         run: sbt ++2.12.12 publish
         env:
-          OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
-          OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
+          OSSRH_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+          OSSRH_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
 
   scala-2_13:
     runs-on: ubuntu-latest
@@ -74,7 +74,7 @@ jobs:
         id: import_gpg
         uses: crazy-max/ghaction-import-gpg@v3
         with:
-          gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
+          gpg-private-key: ${{ secrets.PGP_SECRET }}
           passphrase: ${{ secrets.PGP_PASSPHRASE }}
 
       - name: GPG user IDs
@@ -87,8 +87,8 @@ jobs:
       - name: publish snapshot
         run: sbt ++2.13.4 publish
         env:
-          OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
-          OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
+          OSSRH_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+          OSSRH_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
 
 
   scala-3_0:
@@ -114,7 +114,7 @@ jobs:
         id: import_gpg
         uses: crazy-max/ghaction-import-gpg@v3
         with:
-          gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
+          gpg-private-key: ${{ secrets.PGP_SECRET }}
           passphrase: ${{ secrets.PGP_PASSPHRASE }}
 
       - name: GPG user IDs
@@ -127,5 +127,5 @@ jobs:
       - name: publish snapshot
         run: sbt ++3.3.0 elastic4s-scala3/publish
         env:
-          OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
-          OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
+          OSSRH_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+          OSSRH_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}

.github/workflows/release.yml

@@ -1,5 +1,7 @@
 name: release
 
+run-name: Release - ${{ inputs.version }} on ${{ inputs.branch }}
+
 on:
   workflow_dispatch:
     inputs:
@@ -9,7 +11,7 @@ on:
       branch:
         description: "The branch to release from"
         required: true
-        default: 'master'
+        default: 'opensearch'
 
 jobs:
   release:
@@ -32,7 +34,7 @@ jobs:
         id: import_gpg
         uses: crazy-max/ghaction-import-gpg@v3
         with:
-          gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
+          gpg-private-key: ${{ secrets.PGP_SECRET }}
           passphrase: ${{ secrets.PGP_PASSPHRASE }}
 
       - name: GPG user IDs
@@ -46,22 +48,22 @@ jobs:
         run: sbt ++2.12.12 publishSigned
         env:
           RELEASE_VERSION: ${{ github.event.inputs.version }}
-          OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
-          OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
+          OSSRH_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+          OSSRH_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
 
       - name: publish 2.13 release
         run: sbt ++2.13.4 publishSigned
         env:
           RELEASE_VERSION: ${{ github.event.inputs.version }}
-          OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
-          OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
+          OSSRH_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+          OSSRH_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
 
       - name: publish 3.0 release
         run: sbt ++3.3.0 elastic4s-scala3/publishSigned
         env:
           RELEASE_VERSION: ${{ github.event.inputs.version }}
-          OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
-          OSSRH_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
+          OSSRH_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+          OSSRH_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
 
       - name: tag release
         run: |

.scalafmt.conf

@@ -0,0 +1 @@
+version = 2.7.5

build.sbt

@@ -1,6 +1,6 @@
 import Dependencies._
 
-ThisBuild / organizationName := "com.sksamuel.elastic4s"
+ThisBuild / organizationName := "io.github.t83714"
 
 // Required due to dependency conflict in SBT
 // See https://github.com/sbt/sbt/issues/6997
@@ -15,7 +15,7 @@ def releaseVersion: String = sys.env.getOrElse("RELEASE_VERSION", "")
 def isRelease = releaseVersion != ""
 
 // the version to use to publish - either from release version or a snapshot run number
-def publishVersion = if (isRelease) releaseVersion else "8.6.0." + githubRunNumber + "-SNAPSHOT"
+def publishVersion = if (isRelease) releaseVersion else "8.11.6." + githubRunNumber + "-SNAPSHOT"
 
 // set by github actions and used as the snapshot build number
 def githubRunNumber = sys.env.getOrElse("GITHUB_RUN_NUMBER", "local")
@@ -40,7 +40,7 @@ lazy val warnUnusedImport = Seq(
 )
 
 lazy val commonSettings = Seq(
-  organization := "com.sksamuel.elastic4s",
+  organization := "io.github.t83714",
   version := publishVersion,
   resolvers ++= Seq(Resolver.mavenLocal),
   Test / parallelExecution := false,
@@ -76,15 +76,15 @@ lazy val commonJvmSettings = Seq(
 
 
 lazy val pomSettings = Seq(
-  homepage := Some(url("https://github.com/sksamuel/elastic4s")),
+  homepage := Some(url("https://github.com/t83714/elastic4s")),
   licenses := Seq("Apache 2" -> url("http://www.apache.org/licenses/LICENSE-2.0")),
-  scmInfo := Some(ScmInfo(url("https://github.com/sksamuel/elastic4s"), "scm:git:[email protected]:sksamuel/elastic4s.git")),
-  apiURL := Some(url("https://github.com/sksamuel/elastic4s/")),
+  scmInfo := Some(ScmInfo(url("https://github.com/t83714/elastic4s"), "scm:git:[email protected]:t83714/elastic4s.git")),
+  apiURL := Some(url("https://github.com/t83714/elastic4s/")),
   pomExtra := <developers>
     <developer>
-      <id>sksamuel</id>
-      <name>Sam Samuel</name>
-      <url>https://github.com/sksamuel</url>
+      <id>t83714</id>
+      <name>Jacky Jiang</name>
+      <url>https://github.com/t83714</url>
     </developer>
   </developers>
 )

elastic4s-core/src/main/scala/com/sksamuel/elastic4s/ElasticDsl.scala

@@ -26,6 +26,7 @@ import com.sksamuel.elastic4s.handlers.update.UpdateHandlers
 import com.sksamuel.elastic4s.handlers.validate.ValidateHandlers
 import com.sksamuel.elastic4s.json.XContentBuilder
 import com.sksamuel.elastic4s.requests.ingest.IngestHandlers
+import com.sksamuel.elastic4s.requests.searchPipeline.SearchPipelineHandlers
 import com.sksamuel.elastic4s.requests.searches.aggs.AbstractAggregation
 import com.sksamuel.elastic4s.requests.searches.template.SearchTemplateHandlers
 import com.sksamuel.elastic4s.requests.searches.{SearchHandlers, SearchScrollHandlers, defaultCustomAggregationHandler}
@@ -47,6 +48,7 @@ with IndexAliasHandlers
 with IndexStatsHandlers
 with IndexTemplateHandlers
 with IngestHandlers
+with SearchPipelineHandlers
 with LocksHandlers
 with MappingHandlers
 with NodesHandlers

elastic4s-core/src/main/scala/com/sksamuel/elastic4s/requests/searchPipeline/DeleteSearchPipelineRequest.scala

@@ -0,0 +1,3 @@
+package com.sksamuel.elastic4s.requests.searchPipeline
+
+case class DeleteSearchPipelineRequest(id: String)

elastic4s-core/src/main/scala/com/sksamuel/elastic4s/requests/searchPipeline/DeleteSearchPipelineResponse.scala

@@ -0,0 +1,3 @@
+package com.sksamuel.elastic4s.requests.searchPipeline
+
+case class DeleteSearchPipelineResponse(acknowledged: Boolean)

elastic4s-core/src/main/scala/com/sksamuel/elastic4s/requests/searchPipeline/GetSearchPipelineRequest.scala

@@ -0,0 +1,3 @@
+package com.sksamuel.elastic4s.requests.searchPipeline
+
+case class GetSearchPipelineRequest(id: String)

elastic4s-core/src/main/scala/com/sksamuel/elastic4s/requests/searchPipeline/GetSearchPipelineResponse.scala

@@ -0,0 +1,3 @@
+package com.sksamuel.elastic4s.requests.searchPipeline
+
+case class GetSearchPipelineResponse(data: SearchPipeline)

elastic4s-core/src/main/scala/com/sksamuel/elastic4s/requests/searchPipeline/NormalizationProcessor.scala

@@ -0,0 +1,140 @@
+package com.sksamuel.elastic4s.requests.searchPipeline
+
+import com.sksamuel.elastic4s.json.{XContentBuilder, XContentFactory}
+
+sealed trait NormalizationTechniqueType {
+  def name: String
+}
+
+object NormalizationTechniqueType {
+
+  val defaultValue: NormalizationTechniqueType = minMax
+
+  val values = Set(minMax, l2)
+
+  def withName(name: String): Option[NormalizationTechniqueType] =
+    values.map(v => v.name -> v).toMap.get(name)
+
+  case object minMax extends NormalizationTechniqueType {
+    val name = "min_max"
+  }
+
+  case object l2 extends NormalizationTechniqueType {
+    val name = "l2"
+  }
+}
+
+sealed trait CombinationTechniqueType {
+  def name: String
+}
+
+object CombinationTechniqueType {
+
+  val defaultValue: CombinationTechniqueType = arithmeticMean
+
+  val values = Set(arithmeticMean, geometricMean, harmonicNean)
+
+  def withName(name: String): Option[CombinationTechniqueType] =
+    values.map(v => v.name -> v).toMap.get(name)
+
+  case object arithmeticMean extends CombinationTechniqueType {
+    val name = "arithmetic_mean"
+  }
+
+  case object geometricMean extends CombinationTechniqueType {
+    val name = "geometric_mean"
+  }
+
+  case object harmonicNean extends CombinationTechniqueType {
+    val name = " harmonic_mean"
+  }
+}
+
+case class CombinationTechnique(
+    techniqueType: Option[CombinationTechniqueType] = None,
+    weights: Option[Seq[Double]] = None
+)
+
+case class NormalizationProcessor(
+    normalizationTechnique: Option[NormalizationTechniqueType] = None,
+    combinationTechnique: Option[CombinationTechnique] = None,
+    tag: Option[String] = None,
+    description: Option[String] = None
+) extends SearchPipelineProcessor {
+  val processorType = SearchPipelineProcessorType.SearchPhaseResultsProcessor
+  // for NormalizationProcessor, this value is ignored. If the processor fails, the pipeline always fails and returns an error.
+  val ignoreFailure = None
+
+  def builderFn(): XContentBuilder = {
+    val xcb = XContentFactory.jsonBuilder()
+    xcb.startObject("normalization-processor")
+    tag.foreach(v => xcb.field("tag", v))
+    description.foreach(v => xcb.field("description", v))
+    normalizationTechnique.foreach(v => {
+      xcb.startObject("normalization")
+      xcb.field("technique", v.name)
+      xcb.endObject()
+    })
+    combinationTechnique.foreach(ct => {
+      xcb.startObject("combination")
+      ct.techniqueType.foreach(t => xcb.field("technique", t.name))
+      if (ct.weights.nonEmpty && ct.weights.get.length > 0) {
+        xcb.startObject("parameters")
+        xcb.array("weights", ct.weights.get.toArray)
+        xcb.endObject()
+      }
+      xcb.endObject()
+    })
+    xcb.endObject()
+  }
+}
+
+object NormalizationProcessor {
+  def fromRawResponse(resp: Map[String, Any]): NormalizationProcessor = {
+    if (
+      resp.isEmpty || resp.get("normalization-processor").isEmpty || !resp
+        .get("normalization-processor")
+        .get
+        .isInstanceOf[Map[String, Any]]
+    ) {
+      NormalizationProcessor()
+    } else {
+      val pData =
+        resp.get("normalization-processor").get.asInstanceOf[Map[String, Any]]
+      NormalizationProcessor(
+        tag = pData.get("tag").map(_.asInstanceOf[String]),
+        description = pData.get("description").map(_.asInstanceOf[String]),
+        normalizationTechnique = pData
+          .get("normalization")
+          .flatMap {
+            case v: Map[String, String] =>
+              v.get("technique")
+            case _ => None.asInstanceOf[Option[String]]
+          }
+          .flatMap(NormalizationTechniqueType.withName(_)),
+        combinationTechnique = pData
+          .get("combination")
+          .flatMap {
+            case v: Map[String, Any] =>
+              Some(
+                CombinationTechnique(
+                  techniqueType = v.get("technique").flatMap {
+                    case v: String => CombinationTechniqueType.withName(v)
+                    case _         => None
+                  },
+                  weights = v.get("parameters").flatMap {
+                    case p: Map[String, Any] =>
+                      p.get("weights").map {
+                        case ws: Seq[Any] => ws.map(_.asInstanceOf[Double])
+                        case _            => Seq()
+                      }
+                    case _ => None
+                  }
+                )
+              )
+            case _ => None
+          }
+      )
+    }
+  }
+}

elastic4s-core/src/main/scala/com/sksamuel/elastic4s/requests/searchPipeline/PutSearchPipelineRequest.scala

@@ -0,0 +1,3 @@
+package com.sksamuel.elastic4s.requests.searchPipeline
+
+case class PutSearchPipelineRequest(data: SearchPipeline)

elastic4s-core/src/main/scala/com/sksamuel/elastic4s/requests/searchPipeline/PutSearchPipelineResponse.scala

@@ -0,0 +1,3 @@
+package com.sksamuel.elastic4s.requests.searchPipeline
+
+case class PutSearchPipelineResponse(acknowledged: Boolean)