Merge branch 'master' into interns/magnus/xmlintellij

.buildkite/factory-command.sh

@@ -8,47 +8,24 @@ if (( $# < 1 )); then
 fi
 
 COMMAND=$1
-FACTORY_API="https://factory.vespa.aws-us-east-1a.vespa.oath.cloud/api/factory/v1"
-COOKIEJAR=$(pwd)/jar.txt
-# shellcheck disable=2064
-trap "rm -f $COOKIEJAR" EXIT
+FACTORY_API="https://api.factory.vespa.ai/factory/v1"
 
-SESSION_TOKEN=null
-WAIT_UNTIL=$(( $(date +%s) + 120 ))
-set +e
-while [[ $SESSION_TOKEN == null ]]; do
-  SESSION_TOKEN=$(curl -s -H 'Content-Type: application/json' -H 'Accept: application/json' -d "{ \"username\": \"svc-okta-vespa-factory\", \"password\": \"$SVC_OKTA_VESPA_FACTORY_TOKEN\" }" https://ouryahoo.okta.com/api/v1/authn | jq -re '.sessionToken')
-
-  if [[ $SESSION_TOKEN == null ]]; then
-    if [[ $(date +%s) -ge $WAIT_UNTIL ]]; then
-      echo "Could not fetch session token from Okta: SESSION_TOKEN=$SESSION_TOKEN"
-      exit 1
-    else
-      echo "Invalid SESSION_TOKEN=$SESSION_TOKEN . Trying again ..." >&2
-      sleep 3
-    fi
-  fi
-done
-set -e
-
-LOCATION=$(curl -s -i -c "$COOKIEJAR" "https://factory.vespa.aws-us-east-1a.vespa.oath.cloud/login" | grep location | awk '{print $2}' | tr -d '\r')
-curl -sL -b "$COOKIEJAR" -c "$COOKIEJAR" "$LOCATION&sessionToken=$SESSION_TOKEN" &> /dev/null
-
-CURL="curl -sL -b $COOKIEJAR"
+CURL="curl -sL --key /workspace/identity/key --cert /workspace/identity/cert"
+TOKEN=$(curl -sL --key /workspace/identity/key --cert /workspace/identity/cert -X POST -H "Content-Type: application/x-www-form-urlencoded" -d"grant_type=client_credentials&scope=vespa.factory%3Adomain" "https://zts.athenz.vespa-cloud.com:4443/zts/v1/oauth2/token" | jq -re '.access_token')
 
 shift
 case $COMMAND in
   get-version)
     VERSION=$1
     if [[ -z $VERSION ]]; then echo "Usage: $0 $COMMAND <version>"; exit 1; fi
-    $CURL "$FACTORY_API/versions/$VERSION"
+    $CURL -H "Authorization: Bearer $TOKEN" "$FACTORY_API/versions/$VERSION"
     ;;
   create-build)
     FACTORY_PIPELINE_ID=$1
     FACTORY_PLATFORM=$2
     if [[ -z $FACTORY_PIPELINE_ID ]]; then echo "Usage: $0 $COMMAND <pipeline id> [factory platform]"; exit 1; fi
     if [[ -z $FACTORY_PLATFORM ]]; then FACTORY_PLATFORM="opensource_centos7"; fi
-    $CURL -d "{
+    $CURL -H "Authorization: Bearer $TOKEN" -d "{
         \"startSeconds\": $(date +%s),
         \"sdApiUrl\": \"https://api.buildkite.com/\",
         \"pipelineId\": $FACTORY_PIPELINE_ID,
@@ -59,7 +36,7 @@ case $COMMAND in
     "$FACTORY_API/builds"
     ;;
   create-release)
-    $CURL -d "{
+    $CURL -H "Authorization: Bearer $TOKEN" -d "{
         \"startSeconds\": $(date +%s),
         \"systemName\": \"opensource\"
     }" \
@@ -74,7 +51,7 @@ case $COMMAND in
       echo "Usage: $0 $COMMAND <pipeline id> <status> <description>"
       exit 1
     fi
-    $CURL -d "{
+    $CURL -H "Authorization: Bearer $TOKEN" -d "{
         \"updatedSeconds\": $(date +%s),
         \"sdApiUrl\": \"https://api.buildkite.com/\",
         \"pipelineId\": $FACTORY_PIPELINE_ID,
@@ -88,7 +65,7 @@ case $COMMAND in
   update-released-time)
     VERSION=$1
     if [[ -z $VERSION ]]; then echo "Usage: $0 $COMMAND <version>"; exit 1; fi
-    $CURL -d "{
+    $CURL -H "Authorization: Bearer $TOKEN" -d "{
         \"releasedSeconds\": $(date +%s),
         \"systemName\": \"opensource\"
     }" \

CONTRIBUTING.md

@@ -13,7 +13,7 @@ This document tells you what you need to know to contribute.
 All work on Vespa happens directly on GitHub,
 using the [GitHub flow model](https://docs.github.com/en/get-started/using-github/github-flow).
 We release the master branch four times a week, and you should expect it to always work.
-The continuous build of Vespa is at [https://factory.vespa.oath.cloud](https://factory.vespa.oath.cloud).
+The continuous build of Vespa is at [https://factory.vespa.ai](https://factory.vespa.ai).
 You can follow the fate of each commit there.
 
 All pull requests must be approved by a

README.md

@@ -23,7 +23,7 @@ A new release of Vespa is made from this repository's master branch every mornin
 
 - Home page: [https://vespa.ai](https://vespa.ai)
 - Documentation: [https://docs.vespa.ai](https://docs.vespa.ai)
-- Continuous build: [https://factory.vespa.oath.cloud](https://factory.vespa.oath.cloud)
+- Continuous build: [https://factory.vespa.ai](https://factory.vespa.ai)
 - Run applications in the cloud for free: [https://cloud.vespa.ai](https://cloud.vespa.ai)
 
 ## Table of contents

client/go/internal/vespa/deploy.go

@@ -203,7 +203,30 @@ func fetchFromConfigServer(deployment DeploymentOptions, path string) error {
 	if err := zipDir(dir, zipFile, &ignore.List{}); err != nil {
 		return err
 	}
-	return os.Rename(zipFile, path)
+	if err = renameOrCopyTmpFile(zipFile, path); err != nil {
+		return fmt.Errorf("Could neither rename nor copy %s to %s: %w", zipFile, path, err)
+	}
+	return err
+}
+
+func renameOrCopyTmpFile(srcPath, dstPath string) error {
+	if err := os.Rename(srcPath, dstPath); err == nil {
+		return err
+	}
+	src, err := os.Open(srcPath)
+	if err != nil {
+		return err
+	}
+	stat, err := os.Stat(srcPath)
+	if err != nil {
+		return err
+	}
+	dst, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY, stat.Mode())
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(dst, src)
+	return err
 }
 
 func fetchFilesFromConfigServer(deployment DeploymentOptions, contentURL *url.URL, path string) error {

config-model-api/abi-spec.json

@@ -1390,6 +1390,7 @@
       "public java.util.Optional endpointCertificateSecrets()",
       "public java.util.Optional athenzDomain()",
       "public com.yahoo.config.model.api.Quota quota()",
+      "public java.util.List tenantVaults()",
       "public java.util.List tenantSecretStores()",
       "public java.lang.String jvmGCOptions()",
       "public abstract java.lang.String jvmGCOptions(java.util.Optional)",
@@ -1798,6 +1799,44 @@
     ],
     "fields" : [ ]
   },
+  "com.yahoo.config.model.api.TenantVault$Secret" : {
+    "superClass" : "java.lang.Record",
+    "interfaces" : [ ],
+    "attributes" : [
+      "public",
+      "final",
+      "record"
+    ],
+    "methods" : [
+      "public void <init>(java.lang.String, java.lang.String)",
+      "public final java.lang.String toString()",
+      "public final int hashCode()",
+      "public final boolean equals(java.lang.Object)",
+      "public java.lang.String name()",
+      "public java.lang.String id()"
+    ],
+    "fields" : [ ]
+  },
+  "com.yahoo.config.model.api.TenantVault" : {
+    "superClass" : "java.lang.Record",
+    "interfaces" : [ ],
+    "attributes" : [
+      "public",
+      "final",
+      "record"
+    ],
+    "methods" : [
+      "public void <init>(java.lang.String, java.lang.String, java.lang.String, java.util.List)",
+      "public final java.lang.String toString()",
+      "public final int hashCode()",
+      "public final boolean equals(java.lang.Object)",
+      "public java.lang.String id()",
+      "public java.lang.String name()",
+      "public java.lang.String externalId()",
+      "public java.util.List secrets()"
+    ],
+    "fields" : [ ]
+  },
   "com.yahoo.config.model.api.ValidationParameters$CheckRouting" : {
     "superClass" : "java.lang.Enum",
     "interfaces" : [ ],

config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java

@@ -91,7 +91,6 @@ public class DeploymentSpecXmlReader {
     private static final String idAttribute = "id";
     private static final String athenzServiceAttribute = "athenz-service";
     private static final String athenzDomainAttribute = "athenz-domain";
-    private static final String testerFlavorAttribute = "tester-flavor";
     private static final String testerTag = "tester";
     private static final String nodesTag = "nodes";
     private static final String majorVersionAttribute = "major-version";

config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java

@@ -150,6 +150,8 @@ interface Properties {
 
         default Quota quota() { return Quota.unlimited(); }
 
+        default List<TenantVault> tenantVaults() { return List.of(); }
+
         default List<TenantSecretStore> tenantSecretStores() { return List.of(); }
 
         // Default setting for the gc-options attribute if not specified explicit by application

config-model-api/src/main/java/com/yahoo/config/model/api/TenantVault.java

@@ -0,0 +1,13 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.api;
+
+import java.util.List;
+
+/**
+ * @author gjoranv
+ */
+public record TenantVault(String id, String name, String externalId, List<Secret> secrets) {
+
+    public record Secret(String id, String name) { }
+
+}

config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java

@@ -1607,7 +1607,7 @@ public void testDeployableHash() {
                                             <deployment>
                                               <instance id='default' tags='  '>
                                                 <test />
-                                                <staging tester-flavor='2-8-50' />
+                                                <staging />
                                                 <block-change days='mon' />
                                                 <upgrade policy='canary' revision-target='next' revision-change='when-clear' rollout='simultaneous' />
                                                 <prod />

config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java

@@ -78,7 +78,9 @@ public BuilderV2() {
 
         @Override
         public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) {
-            if (modelContext.getDeployState().isHosted()) { // admin v4 is used on hosted: Build a default V4 instead
+            // admin v4 is used on hosted: Build a default V4 instead. We want to allow version 2.0 so
+            // that self-hosted apps deploy without changes. TODO: Warn if tags from version 2.0 are used (and ignored)
+            if (modelContext.getDeployState().isHosted()) {
                 new BuilderV4().doBuild(model, adminElement, modelContext);
                 return;
             }

config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java

@@ -7,6 +7,7 @@
 import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.Quota;
 import com.yahoo.config.model.api.TenantSecretStore;
+import com.yahoo.config.model.api.TenantVault;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.CloudAccount;
@@ -54,6 +55,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private double feedConcurrency = 0.5;
     private double feedNiceness = 0.0;
     private int maxActivationInhibitedOutOfSyncGroups = 0;
+    private List<TenantVault> tenantVaults = List.of();
     private List<TenantSecretStore> tenantSecretStores = List.of();
     private boolean allowDisableMtls = true;
     private List<X509Certificate> operatorCertificates = List.of();
@@ -114,6 +116,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public double feedConcurrency() { return feedConcurrency; }
     @Override public double feedNiceness() { return feedNiceness; }
     @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; }
+    @Override public List<TenantVault> tenantVaults() { return tenantVaults; }
     @Override public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; }
     @Override public boolean allowDisableMtls() { return allowDisableMtls; }
     @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; }
@@ -278,6 +281,11 @@ public TestProperties maxActivationInhibitedOutOfSyncGroups(int nGroups) {
         return this;
     }
 
+    public TestProperties setTenantVaults(List<TenantVault> tenantVaults) {
+        this.tenantVaults = List.copyOf(tenantVaults);
+        return this;
+    }
+
     public TestProperties setTenantSecretStores(List<TenantSecretStore> secretStores) {
         this.tenantSecretStores = List.copyOf(secretStores);
         return this;

config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java

@@ -151,7 +151,6 @@ public static boolean isModelIntegrationClass(String className) {
             com.yahoo.search.searchchain.ForkingSearcher.class.getName(),
             com.yahoo.search.searchers.CacheControlSearcher.class.getName(),
             com.yahoo.search.searchers.RateLimitingSearcher.class.getName(),
-            com.yahoo.vespa.streamingvisitors.MetricsSearcher.class.getName(),
             com.yahoo.vespa.streamingvisitors.StreamingBackend.class.getName(),
             ai.vespa.search.llm.LLMSearcher.class.getName(),
             ai.vespa.search.llm.RAGSearcher.class.getName()