Extension Signing request endpoint

.circleci/config.yml

@@ -264,7 +264,8 @@ jobs:
           name: build and test Docker image
           command: |
             ./gradlew --no-daemon --parallel "-Pbranch=${CIRCLE_BRANCH}" testDocker
-
+      - store_test_results:
+            path: docker/reports
   publishDockerAmd64:
     executor: machine_executor_amd64
     steps:

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Next Version
 
+### Features Added
+- Added endpoint `/api/v1/eth2/ext/sign/:identifier` which is enabled using cli option `--Xsigning-ext-enabled=true`. This endpoint allows signing of additional data not covered by the remoting API specs. [#982](https://github.com/Consensys/web3signer/pull/982)
+
 ### Bugs fixed
 - Update transitive dependency threetenbp and google cloud secretmanager library to fix CVE-2024-23082, CVE-2024-23081
 - Update bouncycastle libraries to fix CVE-2024-29857, CVE-2024-30171, CVE-2024-30172
@@ -11,6 +14,7 @@
 - Update Postgresql JDBC driver to fix CVE-2024-1597
 - Fix cached gvr to be thread-safe during first boot. [#978](https://github.com/Consensys/web3signer/issues/978)
 
+---
 ## 24.2.0
 
 This is a required update for Mainnet users containing the configuration for the Deneb upgrade on March 13th. This update is required for Gnosis Deneb network upgrade on March 11th. For all other networks, this update is optional.

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/Signer.java

@@ -66,6 +66,8 @@ public class Signer extends FilecoinJsonRpcEndpoint {
   public static final String ETH2_PUBLIC_KEYS = "/api/v1/eth2/publicKeys"; // bls keys
   public static final String RELOAD_ENDPOINT = "/reload";
 
+  public static final String SIGN_EXT_ENDPOINT = "/api/v1/eth2/ext/sign/{identifier}";
+
   public static final ObjectMapper ETH_2_INTERFACE_OBJECT_MAPPER =
       SigningObjectMapperFactory.createObjectMapper().setSerializationInclusion(Include.NON_NULL);
   private static final String METRICS_ENDPOINT = "/metrics";
@@ -175,6 +177,17 @@ public Response eth2Sign(
         .post(signPath(BLS));
   }
 
+  public Response signExtensionPayload(
+      final String publicKey, final String payload, final ContentType acceptMediaType) {
+    return given()
+        .baseUri(getUrl())
+        .contentType(ContentType.JSON)
+        .accept(acceptMediaType)
+        .pathParam("identifier", publicKey)
+        .body(payload)
+        .post(SIGN_EXT_ENDPOINT);
+  }
+
   public Response callApiPublicKeys(final KeyType keyType) {
     return given().baseUri(getUrl()).get(publicKeysPath(keyType));
   }

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfiguration.java

@@ -79,6 +79,8 @@ public class SignerConfiguration {
   private final ChainIdProvider chainIdProvider;
   private final Optional<KeystoresParameters> v3KeystoresBulkloadParameters;
 
+  private final boolean signingExtEnabled;
+
   public SignerConfiguration(
       final String hostname,
       final int httpRpcPort,
@@ -123,7 +125,8 @@ public SignerConfiguration(
       final int downstreamHttpPort,
       final Optional<ClientTlsOptions> downstreamTlsOptions,
       final ChainIdProvider chainIdProvider,
-      final Optional<KeystoresParameters> v3KeystoresBulkloadParameters) {
+      final Optional<KeystoresParameters> v3KeystoresBulkloadParameters,
+      final boolean signingExtEnabled) {
     this.hostname = hostname;
     this.logLevel = logLevel;
     this.httpRpcPort = httpRpcPort;
@@ -168,6 +171,7 @@ public SignerConfiguration(
     this.downstreamTlsOptions = downstreamTlsOptions;
     this.chainIdProvider = chainIdProvider;
     this.v3KeystoresBulkloadParameters = v3KeystoresBulkloadParameters;
+    this.signingExtEnabled = signingExtEnabled;
   }
 
   public String hostname() {
@@ -353,4 +357,8 @@ public ChainIdProvider getChainIdProvider() {
   public Optional<KeystoresParameters> getV3KeystoresBulkloadParameters() {
     return v3KeystoresBulkloadParameters;
   }
+
+  public boolean isSigningExtEnabled() {
+    return signingExtEnabled;
+  }
 }

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/SignerConfigurationBuilder.java

@@ -83,6 +83,8 @@ public class SignerConfigurationBuilder {
 
   private KeystoresParameters v3KeystoresBulkloadParameters;
 
+  private boolean signingExtEnabled;
+
   public SignerConfigurationBuilder withLogLevel(final Level logLevel) {
     this.logLevel = logLevel;
     return this;
@@ -318,6 +320,11 @@ public SignerConfigurationBuilder withV3KeystoresBulkloadParameters(
     return this;
   }
 
+  public SignerConfigurationBuilder withSigningExtEnabled(final boolean signingExtEnabled) {
+    this.signingExtEnabled = signingExtEnabled;
+    return this;
+  }
+
   public SignerConfiguration build() {
     if (mode == null) {
       throw new IllegalArgumentException("Mode cannot be null");
@@ -366,6 +373,7 @@ public SignerConfiguration build() {
         downstreamHttpPort,
         Optional.ofNullable(downstreamTlsOptions),
         chainIdProvider,
-        Optional.ofNullable(v3KeystoresBulkloadParameters));
+        Optional.ofNullable(v3KeystoresBulkloadParameters),
+        signingExtEnabled);
   }
 }

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java

@@ -158,6 +158,11 @@ public List<String> createCmdLineParams() {
           .getGcpParameters()
           .ifPresent(gcpParameters -> yamlConfig.append(gcpBulkLoadingOptions(gcpParameters)));
 
+      if (signerConfig.isSigningExtEnabled()) {
+        yamlConfig.append(
+            String.format(YAML_BOOLEAN_FMT, "eth2.Xsigning-ext-enabled", Boolean.TRUE));
+      }
+
       final CommandArgs subCommandArgs = createSubCommandArgs();
       params.addAll(subCommandArgs.params);
       yamlConfig.append(subCommandArgs.yamlConfig);

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java

@@ -134,6 +134,10 @@ public List<String> createCmdLineParams() {
       signerConfig
           .getGcpParameters()
           .ifPresent(gcpParams -> params.addAll(gcpSecretManagerBulkLoadingOptions(gcpParams)));
+
+      if (signerConfig.isSigningExtEnabled()) {
+        params.add("--Xsigning-ext-enabled=true");
+      }
     } else if (signerConfig.getMode().equals("eth1")) {
       params.add("--downstream-http-port");
       params.add(Integer.toString(signerConfig.getDownstreamHttpPort()));

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/signing/ProofOfValidationSigningExtAcceptanceTest.java

@@ -0,0 +1,187 @@
+/*
+ * Copyright 2024 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.web3signer.tests.signing;
+
+import static io.restassured.http.ContentType.JSON;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.assertj.core.api.Assertions.assertThat;
+import static tech.pegasys.teku.spec.SpecMilestone.DENEB;
+
+import tech.pegasys.teku.bls.BLS;
+import tech.pegasys.teku.bls.BLSKeyPair;
+import tech.pegasys.teku.bls.BLSPublicKey;
+import tech.pegasys.teku.bls.BLSSecretKey;
+import tech.pegasys.teku.bls.BLSSignature;
+import tech.pegasys.teku.spec.networks.Eth2Network;
+import tech.pegasys.web3signer.core.service.http.SigningObjectMapperFactory;
+import tech.pegasys.web3signer.core.service.http.handlers.signing.ProofOfValidationBody;
+import tech.pegasys.web3signer.core.service.http.handlers.signing.SigningExtensionType;
+import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder;
+import tech.pegasys.web3signer.dsl.utils.MetadataFileHelpers;
+import tech.pegasys.web3signer.signing.KeyType;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.restassured.http.ContentType;
+import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.bytes.Bytes32;
+import org.apache.tuweni.units.bigints.UInt64;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class ProofOfValidationSigningExtAcceptanceTest extends SigningAcceptanceTestBase {
+
+  private static final String PRIVATE_KEY =
+      "3ee2224386c82ffea477e2adf28a2929f5c349165a4196158c7f3a2ecca40f35";
+  private static final MetadataFileHelpers METADATA_FILE_HELPERS = new MetadataFileHelpers();
+  private static final BLSSecretKey KEY =
+      BLSSecretKey.fromBytes(Bytes32.fromHexString(PRIVATE_KEY));
+  private static final BLSKeyPair KEY_PAIR = new BLSKeyPair(KEY);
+  private static final BLSPublicKey PUBLIC_KEY = KEY_PAIR.getPublicKey();
+  private static final ObjectMapper JSON_MAPPER = SigningObjectMapperFactory.createObjectMapper();
+
+  @BeforeEach
+  void setup() {
+    final String configFilename = PUBLIC_KEY.toString().substring(2);
+    final Path keyConfigFile = testDirectory.resolve(configFilename + ".yaml");
+    METADATA_FILE_HELPERS.createUnencryptedYamlFileAt(keyConfigFile, PRIVATE_KEY, KeyType.BLS);
+
+    setForkEpochsAndStartSigner(
+        new SignerConfigurationBuilder()
+            .withKeyStoreDirectory(testDirectory)
+            .withMode("eth2")
+            .withNetwork(Eth2Network.MINIMAL.configName())
+            .withSigningExtEnabled(true),
+        DENEB);
+  }
+
+  @ParameterizedTest(name = "{index} - Testing Accept Media Type: {0}")
+  @EnumSource(
+      value = ContentType.class,
+      names = {"ANY", "JSON", "TEXT"})
+  void extensionSigningData(final ContentType acceptMediaType) throws Exception {
+    final var signingExtensionBody =
+        new ProofOfValidationBody(
+            SigningExtensionType.PROOF_OF_VALIDATION,
+            "AT",
+            UInt64.valueOf(System.currentTimeMillis()));
+    final var payload = JSON_MAPPER.writeValueAsString(signingExtensionBody);
+
+    final var response =
+        signer.signExtensionPayload(PUBLIC_KEY.toString(), payload, acceptMediaType);
+
+    response.then().statusCode(200).contentType(JSON);
+
+    final var signatureResponse =
+        JSON_MAPPER.readValue(response.asByteArray(), ProofOfValidationResponse.class);
+
+    // assert that the signature is valid
+    final var blsSignature =
+        BLSSignature.fromBytesCompressed(Bytes.fromHexString(signatureResponse.signature));
+
+    final var isValidBLSSig =
+        BLS.verify(PUBLIC_KEY, Bytes.wrap(payload.getBytes(UTF_8)), blsSignature);
+    assertThat(isValidBLSSig).isTrue();
+
+    // assert that Base64 encoded payload is correct
+    assertThat(signatureResponse.payload)
+        .isEqualTo(Bytes.wrap(payload.getBytes(UTF_8)).toBase64String());
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"1634025600000", "\"1634025600000\""})
+  void timestampAsStringAndNumberResultsInValidSignature(final String timestampValue)
+      throws IOException {
+    final var payloadFormat =
+        """
+        {
+          "type": "PROOF_OF_VALIDATION",
+          "platform": "AT",
+          "timestamp": %s
+        }
+        """;
+    final var payload = String.format(payloadFormat, timestampValue);
+
+    final var response = signer.signExtensionPayload(PUBLIC_KEY.toString(), payload, JSON);
+    response.then().statusCode(200).contentType(JSON);
+
+    final var signatureResponse =
+        JSON_MAPPER.readValue(response.asByteArray(), ProofOfValidationResponse.class);
+
+    // assert that the signature is valid
+    final var blsSignature =
+        BLSSignature.fromBytesCompressed(Bytes.fromHexString(signatureResponse.signature));
+
+    final var isValidBLSSig =
+        BLS.verify(PUBLIC_KEY, Bytes.wrap(payload.getBytes(UTF_8)), blsSignature);
+    assertThat(isValidBLSSig).isTrue();
+
+    // assert that Base64 encoded payload is correct
+    assertThat(signatureResponse.payload)
+        .isEqualTo(Bytes.wrap(payload.getBytes(UTF_8)).toBase64String());
+  }
+
+  @Test
+  void invalidIdentifierCausesNotFound() throws Exception {
+    final ProofOfValidationBody proofOfValidationBody =
+        new ProofOfValidationBody(
+            SigningExtensionType.PROOF_OF_VALIDATION,
+            "AT",
+            UInt64.valueOf(System.currentTimeMillis()));
+    final String data = JSON_MAPPER.writeValueAsString(proofOfValidationBody);
+
+    signer.signExtensionPayload("0x1234", data, JSON).then().statusCode(404);
+  }
+
+  @ParameterizedTest(name = "{index} - Testing Invalid Body: {0}")
+  @ValueSource(strings = {"", "invalid", "{}", "{\"data\": \"invalid\"}"})
+  void invalidBodyCausesBadRequestStatusCode(final String data) {
+    signer.signExtensionPayload(PUBLIC_KEY.toString(), data, JSON).then().statusCode(400);
+  }
+
+  @Test
+  void invalidSignExtensionTypeCausesBadRequestStatusCode() throws Exception {
+    final ProofOfValidationBody proofOfValidationBody =
+        new ProofOfValidationBody(
+            SigningExtensionType.PROOF_OF_VALIDATION,
+            "AT",
+            UInt64.valueOf(System.currentTimeMillis()));
+    var payload = JSON_MAPPER.writeValueAsString(proofOfValidationBody);
+    payload = payload.replace("PROOF_OF_VALIDATION", "INVALID_TYPE");
+
+    signer.signExtensionPayload(PUBLIC_KEY.toString(), payload, JSON).then().statusCode(400);
+  }
+
+  @Test
+  void extraJsonFieldsCausesBadRequestStatusCode() throws Exception {
+    final ProofOfValidationBody proofOfValidationBody =
+        new ProofOfValidationBody(
+            SigningExtensionType.PROOF_OF_VALIDATION,
+            "AT",
+            UInt64.valueOf(System.currentTimeMillis()));
+    var payload = JSON_MAPPER.writeValueAsString(proofOfValidationBody);
+    payload = payload.replace("}", ",\"extraField\": \"extraValue\"}");
+
+    signer.signExtensionPayload(PUBLIC_KEY.toString(), payload, JSON).then().statusCode(400);
+  }
+
+  record ProofOfValidationResponse(
+      @JsonProperty(value = "payload", required = true) String payload,
+      @JsonProperty(value = "signature", required = true) String signature) {}
+}

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/signing/SigningAcceptanceTestBase.java

@@ -49,19 +49,13 @@ protected void setupEth2Signer(final Eth2Network eth2Network, final SpecMileston
         .withKeyStoreDirectory(testDirectory)
         .withMode("eth2")
         .withNetwork(eth2Network.configName());
-
-    setForkEpochs(specMilestone, builder);
-
-    startSigner(builder.build());
+    setForkEpochsAndStartSigner(builder, specMilestone);
   }
 
   protected void setupEth2Signer(final Path networkConfigFile, final SpecMilestone specMilestone) {
     final SignerConfigurationBuilder builder = new SignerConfigurationBuilder();
     builder.withKeyStoreDirectory(testDirectory).withMode("eth2").withNetwork(networkConfigFile);
-
-    setForkEpochs(specMilestone, builder);
-
-    startSigner(builder.build());
+    setForkEpochsAndStartSigner(builder, specMilestone);
   }
 
   protected void setupEth2SignerWithCustomNetworkConfig(final Path networkConfigFile) {
@@ -73,6 +67,12 @@ protected void setupEth2SignerWithCustomNetworkConfig(final Path networkConfigFi
     startSigner(builder.build());
   }
 
+  protected void setForkEpochsAndStartSigner(
+      final SignerConfigurationBuilder builder, final SpecMilestone specMilestone) {
+    setForkEpochs(specMilestone, builder);
+    startSigner(builder.build());
+  }
+
   private void setForkEpochs(
       final SpecMilestone specMilestone, final SignerConfigurationBuilder builder) {
     switch (specMilestone) {

commandline/src/main/java/tech/pegasys/web3signer/commandline/Web3SignerBaseCommand.java

@@ -213,7 +213,7 @@ public class Web3SignerBaseCommand implements BaseConfig, Runnable {
       paramLabel = INTEGER_FORMAT_HELP)
   private Integer vertxWorkerPoolSize = null;
 
-  @Deprecated
+  @Deprecated(forRemoval = true)
   @Option(names = "--Xworker-pool-size", hidden = true)
   private Integer deprecatedWorkerPoolSize = null;