Watcher: Never return credentials after watch creation... (elastic/x-pack-elasticsearch#3581)

... yet support updates. This commit introduces a few changes of how
watches are put.

The GET Watch API will never return credentials like basic auth
passwords, but a placeholder instead now. If the watcher is enabled to
encrypt sensitive settings, then the original encrypted value is
returned otherwise a "::es_redacted::" place holder.

There have been several Put Watch API changes.

The API now internally uses the Update API and versioning. This has
several implications. First if no version is supplied, we assume an
initial creation. This will work as before, however if a credential is
marked as redacted we will reject storing the watch, so users do not
accidentally store the wrong watch.

The watch xcontent parser now has an additional methods to tell the
caller if redacted passwords have been found. Based on this information
an error can be thrown.

If the user now wants to store a watch that contains a password marked
as redacted, this password will not be part of the toXContent
representation of the watch and in combinatination with update request
the existing password will be merged in. If the encrypted password is
supplied this one will be stored.

The serialization for GetWatchResponse/PutWatchRequest has changed.
The version checks for this will be put into the 6.x branch.

The Watcher UI now needs specify the version, when it wants to store a
watch. This also prevents last-write-wins scenarios and is the reason
why the put/get watch response now contains the internal version.

relates elastic/x-pack-elasticsearch#3089

Original commit: elastic/x-pack-elasticsearch@bb63be9

Author: spinscale

docs/en/rest-api/watcher/ack-watch.asciidoc

@@ -86,6 +86,7 @@ The action state of a newly-created watch is `awaits_successful_execution`:
 --------------------------------------------------
 {
   "found": true,
+  "_version": 1,
   "_id": "my_watch",
   "status": {
     "version": 1,
@@ -129,6 +130,7 @@ and the action is now in `ackable` state:
 {
   "found": true,
   "_id": "my_watch",
+  "_version": 2,
   "status": {
     "version": 2,
     "actions": {
@@ -177,6 +179,7 @@ GET _xpack/watcher/watch/my_watch
 {
   "found": true,
   "_id": "my_watch",
+  "_version": 3,
   "status": {
     "version": 3,
     "actions": {

docs/en/rest-api/watcher/activate-watch.asciidoc

@@ -41,6 +41,7 @@ GET _xpack/watcher/watch/my_watch
 {
   "found": true,
   "_id": "my_watch",
+  "_version": 1,
   "status": {
     "state" : {
       "active" : false,

docs/en/rest-api/watcher/deactivate-watch.asciidoc

@@ -40,6 +40,7 @@ GET _xpack/watcher/watch/my_watch
 {
   "found": true,
   "_id": "my_watch",
+  "_version": 1,
   "status": {
     "state" : {
       "active" : true,

docs/en/rest-api/watcher/get-watch.asciidoc

@@ -41,6 +41,7 @@ Response:
 {
   "found": true,
   "_id": "my_watch",
+  "_version": 1,
   "status": { <1>
     "version": 1,
     "state": {

plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatchSourceBuilder.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.core.watcher.input.Input;
 import org.elasticsearch.xpack.core.watcher.input.none.NoneInput;
 import org.elasticsearch.xpack.core.watcher.support.Exceptions;
+import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transform.Transform;
 import org.elasticsearch.xpack.core.watcher.trigger.Trigger;
@@ -175,7 +176,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
      */
     public final BytesReference buildAsBytes(XContentType contentType) {
         try {
-            return XContentHelper.toXContent(this, contentType, false);
+            WatcherParams params = WatcherParams.builder().hideSecrets(false).build();
+            return XContentHelper.toXContent(this, contentType,  params,false);
         } catch (Exception e) {
             throw new ElasticsearchException("Failed to build ToXContent", e);
         }

plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java

@@ -39,7 +39,7 @@ public class CryptoService extends AbstractComponent {
     public static final String KEY_ALGO = "HmacSHA512";
     public static final int KEY_SIZE = 1024;
 
-    static final String ENCRYPTED_TEXT_PREFIX = "::es_encrypted::";
+    public static final String ENCRYPTED_TEXT_PREFIX = "::es_encrypted::";
 
     // the encryption used in this class was picked when Java 7 was still the min. supported
     // version. The use of counter mode was chosen to simplify the need to deal with padding

plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherParams.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.core.watcher.support.xcontent;
 
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.xpack.core.watcher.watch.Watch;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -17,9 +18,9 @@ public class WatcherParams extends ToXContent.DelegatingMapParams {
 
     public static final WatcherParams HIDE_SECRETS = WatcherParams.builder().hideSecrets(true).build();
 
-    static final String HIDE_SECRETS_KEY = "hide_secrets";
-    public static final String HIDE_HEADERS = "hide_headers";
-    static final String DEBUG_KEY = "debug";
+    private static final String HIDE_SECRETS_KEY = "hide_secrets";
+    private static final String HIDE_HEADERS = "hide_headers";
+    private static final String DEBUG_KEY = "debug";
 
     public static boolean hideSecrets(ToXContent.Params params) {
         return wrap(params).hideSecrets();
@@ -29,19 +30,23 @@ public static boolean debug(ToXContent.Params params) {
         return wrap(params).debug();
     }
 
+    public static boolean hideHeaders(ToXContent.Params params) {
+        return wrap(params).hideHeaders();
+    }
+
     private WatcherParams(Map<String, String> params, ToXContent.Params delegate) {
         super(params, delegate);
     }
 
-    public boolean hideSecrets() {
-        return paramAsBoolean(HIDE_SECRETS_KEY, false);
+    private boolean hideSecrets() {
+        return paramAsBoolean(HIDE_SECRETS_KEY, true);
     }
 
-    public boolean debug() {
+    private boolean debug() {
         return paramAsBoolean(DEBUG_KEY, false);
     }
 
-    public boolean hideHeaders() {
+    private boolean hideHeaders() {
         return paramAsBoolean(HIDE_HEADERS, true);
     }
 
@@ -83,6 +88,11 @@ public Builder debug(boolean debug) {
             return this;
         }
 
+        public Builder includeStatus(boolean includeStatus) {
+            params.put(Watch.INCLUDE_STATUS_KEY, String.valueOf(includeStatus));
+            return this;
+        }
+
         public Builder put(String key, Object value) {
             params.put(key, String.valueOf(value));
             return this;

plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java

@@ -7,6 +7,7 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -33,41 +34,47 @@
  */
 public class WatcherXContentParser implements XContentParser {
 
-    public static Secret secret(XContentParser parser) throws IOException {
-        char[] chars = parser.text().toCharArray();
-        if (parser instanceof WatcherXContentParser) {
-            WatcherXContentParser watcherParser = (WatcherXContentParser) parser;
-            if (watcherParser.cryptoService != null) {
-                chars = watcherParser.cryptoService.encrypt(chars);
-            }
-        }
-        return new Secret(chars);
-    }
+    public static final String REDACTED_PASSWORD = "::es_redacted::";
 
     public static Secret secretOrNull(XContentParser parser) throws IOException {
         String text = parser.textOrNull();
         if (text == null) {
             return null;
         }
-        char[] chars = parser.text().toCharArray();
+
+        char[] chars = text.toCharArray();
+        boolean isEncryptedAlready = text.startsWith(CryptoService.ENCRYPTED_TEXT_PREFIX);
+        if (isEncryptedAlready) {
+            return new Secret(chars);
+        }
+
         if (parser instanceof WatcherXContentParser) {
             WatcherXContentParser watcherParser = (WatcherXContentParser) parser;
-            if (watcherParser.cryptoService != null) {
-                chars = watcherParser.cryptoService.encrypt(text.toCharArray());
+            if (REDACTED_PASSWORD.equals(text)) {
+                if (watcherParser.allowRedactedPasswords) {
+                    return null;
+                } else {
+                    throw new ElasticsearchParseException("found redacted password in field [{}]", parser.currentName());
+                }
+            } else if (watcherParser.cryptoService != null) {
+                return new Secret(watcherParser.cryptoService.encrypt(chars));
             }
-            return new Secret(chars);
         }
+
         return new Secret(chars);
     }
 
     private final DateTime parseTime;
     private final XContentParser parser;
     @Nullable private final CryptoService cryptoService;
+    private final boolean allowRedactedPasswords;
 
-    public WatcherXContentParser(XContentParser parser, DateTime parseTime, @Nullable CryptoService cryptoService) {
+    public WatcherXContentParser(XContentParser parser, DateTime parseTime, @Nullable CryptoService cryptoService,
+                                 boolean allowRedactedPasswords) {
         this.parseTime = parseTime;
         this.parser = parser;
         this.cryptoService = cryptoService;
+        this.allowRedactedPasswords = allowRedactedPasswords;
     }
 
     public DateTime getParseDateTime() { return parseTime; }

plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchResponse.java

@@ -5,10 +5,12 @@
  */
 package org.elasticsearch.xpack.core.watcher.transport.actions.get;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.watch.WatchStatus;
@@ -21,6 +23,7 @@ public class GetWatchResponse extends ActionResponse {
     private WatchStatus status;
     private boolean found = false;
     private XContentSource source;
+    private long version;
 
     public GetWatchResponse() {
     }
@@ -32,16 +35,18 @@ public GetWatchResponse(String id) {
         this.id = id;
         this.found = false;
         this.source = null;
+        version = Versions.NOT_FOUND;
     }
 
     /**
      * ctor for found watch
      */
-    public GetWatchResponse(String id, WatchStatus status, BytesReference source, XContentType contentType) {
+    public GetWatchResponse(String id, long version, WatchStatus status, BytesReference source, XContentType contentType) {
         this.id = id;
         this.status = status;
         this.found = true;
         this.source = new XContentSource(source, contentType);
+        this.version = version;
     }
 
     public String getId() {
@@ -60,6 +65,10 @@ public XContentSource getSource() {
         return source;
     }
 
+    public long getVersion() {
+        return version;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -68,6 +77,7 @@ public void readFrom(StreamInput in) throws IOException {
         if (found) {
             status = WatchStatus.read(in);
             source = XContentSource.readFrom(in);
+            version = in.readZLong();
         }
     }
 
@@ -79,6 +89,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (found) {
             status.writeTo(out);
             XContentSource.writeTo(source, out);
+            out.writeZLong(version);
         }
     }
 }

plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchRequest.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.support.WatcherUtils;
@@ -28,14 +29,11 @@ public class PutWatchRequest extends ActionRequest {
     private BytesReference source;
     private boolean active = true;
     private XContentType xContentType = XContentType.JSON;
+    private long version = Versions.MATCH_ANY;
 
     public PutWatchRequest() {
     }
 
-    public PutWatchRequest(String id, WatchSourceBuilder source) {
-        this(id, source.buildAsBytes(XContentType.JSON), XContentType.JSON);
-    }
-
     public PutWatchRequest(String id, BytesReference source, XContentType xContentType) {
         this.id = id;
         this.source = source;
@@ -48,6 +46,7 @@ public PutWatchRequest(StreamInput in) throws IOException {
         source = in.readBytesReference();
         active = in.readBoolean();
         xContentType = XContentType.readFrom(in);
+        version = in.readZLong();
     }
 
     @Override
@@ -57,6 +56,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBytesReference(source);
         out.writeBoolean(active);
         xContentType.writeTo(out);
+        out.writeZLong(version);
     }
 
     /**
@@ -116,6 +116,14 @@ public XContentType xContentType() {
         return xContentType;
     }
 
+    public long getVersion() {
+        return version;
+    }
+
+    public void setVersion(long version) {
+        this.version = version;
+    }
+
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;