Split etcdctl into etcdctl (public API access) & etcdutl (direct surgery on files)

CHANGELOG-3.5.md

@@ -225,6 +225,8 @@ Note that any `etcd_debugging_*` metrics are experimental and subject to change.
 - Add [`etcdctl get --count-only`](https://github.com/etcd-io/etcd/pull/11743) flag for output type `fields`.
 - Add [`etcdctl member list -w=json --hex`](https://github.com/etcd-io/etcd/pull/11812) flag to print memberListResponse in hex format json.
 - Changed [`etcdctl lock <lockname> exec-command`](https://github.com/etcd-io/etcd/pull/12829) to return exit code of exec-command.
+- [New tool: `etcdutl`](https://github.com/etcd-io/etcd/pull/12971) incorporated functionality of: `etcdctl snapshot status|restore`, `etcdctl backup`, `etcdctl defrag --data-dir ...`.
+- [ETCDCTL_API=2 `etcdctl migrate`](https://github.com/etcd-io/etcd/pull/12971) has been decomissioned. Use etcd <=v3.4 to restore v2 storage.
 
 ### gRPC gateway
 

Dockerfile-release.amd64

@@ -2,6 +2,7 @@ FROM k8s.gcr.io/build-image/debian-base:buster-v1.4.0
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/
+ADD etcdutl /usr/local/bin/
 RUN mkdir -p /var/etcd/
 RUN mkdir -p /var/lib/etcd/
 

Dockerfile-release.arm64

@@ -2,6 +2,7 @@ FROM k8s.gcr.io/build-image/debian-base-arm64:buster-v1.4.0
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/
+ADD etcdutl /usr/local/bin/
 ADD var/etcd /var/etcd
 ADD var/lib/etcd /var/lib/etcd
 ENV ETCD_UNSUPPORTED_ARCH=arm64

Dockerfile-release.ppc64le

@@ -2,6 +2,7 @@ FROM k8s.gcr.io/build-image/debian-base-ppc64le:buster-v1.4.0
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/
+ADD etcdutl /usr/local/bin/
 ADD var/etcd /var/etcd
 ADD var/lib/etcd /var/lib/etcd
 

Dockerfile-release.s390x

@@ -2,6 +2,7 @@ FROM k8s.gcr.io/build-image/debian-base-s390x:buster-v1.4.0
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/
+ADD etcdutl /usr/local/bin/
 ADD var/etcd /var/etcd
 ADD var/lib/etcd /var/lib/etcd
 

Makefile

@@ -25,6 +25,7 @@ build:
 	GO_BUILD_FLAGS="-v" ./build.sh
 	./bin/etcd --version
 	./bin/etcdctl version
+	./bin/etcdutl version
 
 clean:
 	rm -f ./codecov
@@ -210,7 +211,7 @@ build-docker-release-main:
 	docker run \
 	  --rm \
 	  gcr.io/etcd-development/etcd:$(ETCD_VERSION) \
-	  /bin/sh -c "/usr/local/bin/etcd --version && /usr/local/bin/etcdctl version"
+	  /bin/sh -c "/usr/local/bin/etcd --version && /usr/local/bin/etcdctl version && /usr/local/bin/etcdutl version"
 
 push-docker-release-main:
 	$(info ETCD_VERSION: $(ETCD_VERSION))
@@ -529,6 +530,7 @@ build-docker-functional:
 	  /bin/bash -c "./bin/etcd --version && \
 	   ./bin/etcd-failpoints --version && \
 	   ./bin/etcdctl version && \
+	   ./bin/etcdutl version && \
 	   ./bin/etcd-agent -help || true && \
 	   ./bin/etcd-proxy -help || true && \
 	   ./bin/etcd-runner --help || true && \

bill-of-materials.json

@@ -485,6 +485,15 @@
 			}
 		]
 	},
+	{
+		"project": "go.etcd.io/etcd/etcdutl/v3",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
 	{
 		"project": "go.etcd.io/etcd/pkg/v3",
 		"licenses": [

build.sh

@@ -47,6 +47,16 @@ etcd_build() {
       -o="../${out}/etcd" . || return 2
   ) || return 2
 
+  run rm -f "${out}/etcdutl"
+  # shellcheck disable=SC2086
+  (
+    cd ./etcdutl
+    run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \
+      -installsuffix=cgo \
+      "-ldflags=${GO_LDFLAGS[*]}" \
+      -o="../${out}/etcdutl" . || return 2
+  ) || return 2
+
   run rm -f "${out}/etcdctl"
   # shellcheck disable=SC2086
   (

dummy.go

@@ -18,6 +18,8 @@ package main_test
 // module (e.g. for sake of 'bom' generation).
 // Thanks to this 'go mod tidy' is not removing that dependencies from go.mod.
 import (
-	_ "go.etcd.io/etcd/client/v2"            // keep
-	_ "go.etcd.io/etcd/tests/v3/integration" // keep
+	_ "go.etcd.io/etcd/client/v2"                // keep
+	_ "go.etcd.io/etcd/etcdctl/v3/ctlv3/command" // keep
+	_ "go.etcd.io/etcd/etcdutl/v3/etcdutl"       // keep
+	_ "go.etcd.io/etcd/tests/v3/integration"     // keep
 )

etcdctl/README.md

@@ -913,15 +913,17 @@ If NOSPACE alarm is present:
 
 ### DEFRAG [options]
 
-DEFRAG defragments the backend database file for a set of given endpoints while etcd is running, or directly defragments an etcd data directory while etcd is not running. When an etcd member reclaims storage space from deleted and compacted keys, the space is kept in a free list and the database file remains the same size. By defragmenting the database, the etcd member releases this free space back to the file system.
+DEFRAG defragments the backend database file for a set of given endpoints while etcd is running, ~~or directly defragments an etcd data directory while etcd is not running~~. When an etcd member reclaims storage space from deleted and compacted keys, the space is kept in a free list and the database file remains the same size. By defragmenting the database, the etcd member releases this free space back to the file system.
+
+**Note: to defragment offline (`--data-dir` flag), use: `etcutl defrag` instead**
 
 **Note that defragmentation to a live member blocks the system from reading and writing data while rebuilding its states.**
 
 **Note that defragmentation request does not get replicated over cluster. That is, the request is only applied to the local node. Specify all members in `--endpoints` flag or `--cluster` flag to automatically find all cluster members.**
 
 #### Options
 
-- data-dir -- Optional. If present, defragments a data directory not in use by etcd.
+- data-dir -- Optional. **Deprecated**. If present, defragments a data directory not in use by etcd. To be removed in v3.6.
 
 #### Output
 
@@ -944,11 +946,12 @@ Finished defragmenting etcd member[http://127.0.0.1:22379]
 Finished defragmenting etcd member[http://127.0.0.1:32379]
 ```
 
-To defragment a data directory directly, use the `--data-dir` flag:
+To defragment a data directory directly, use the `etcdutl` with `--data-dir` flag 
+(`etcdctl` will remove this flag in v3.6):
 
 ``` bash
 # Defragment while etcd is not running
-./etcdctl defrag --data-dir default.etcd
+./etcdutl defrag --data-dir default.etcd
 # success (exit status 0)
 # Error: cannot open database at default.etcd/member/snap/db
 ```
@@ -978,6 +981,8 @@ Save a snapshot to "snapshot.db":
 
 ### SNAPSHOT RESTORE [options] \<filename\>
 
+Note: Deprecated. Use `etcdutl snapshot restore` instead. To be removed in v3.6.
+
 SNAPSHOT RESTORE creates an etcd data directory for an etcd cluster member from a backend database snapshot and a new cluster configuration. Restoring the snapshot into each member for a new cluster configuration will initialize a new etcd cluster preloaded by the snapshot data.
 
 #### Options
@@ -1021,6 +1026,8 @@ bin/etcd --name sshot3 --listen-client-urls http://127.0.0.1:32379 --advertise-c
 
 ### SNAPSHOT STATUS \<filename\>
 
+Note: Deprecated. Use `etcdutl snapshot restore` instead. To be removed in v3.6.
+
 SNAPSHOT STATUS lists information about a given backend database snapshot file.
 
 #### Output
@@ -1495,54 +1502,6 @@ The approximate total number of keys transferred to the destination cluster, upd
 
 [mirror]: ./doc/mirror_maker.md
 
-### MIGRATE [options]
-
-Migrates keys in a v2 store to a v3 mvcc store. Users should run migration command for all members in the cluster.
-
-#### Options
-
-- data-dir -- Path to the data directory
-
-- wal-dir -- Path to the WAL directory
-
-- transformer -- Path to the user-provided transformer program (default if not provided)
-
-#### Output
-
-No output on success.
-
-#### Default transformer
-
-If user does not provide a transformer program, migrate command will use the default transformer. The default transformer transforms `storev2` formatted keys into `mvcc` formatted keys according to the following Go program:
-
-```go
-func transform(n *storev2.Node) *mvccpb.KeyValue {
-	if n.Dir {
-		return nil
-	}
-	kv := &mvccpb.KeyValue{
-		Key:            []byte(n.Key),
-		Value:          []byte(n.Value),
-		CreateRevision: int64(n.CreatedIndex),
-		ModRevision:    int64(n.ModifiedIndex),
-		Version:        1,
-	}
-	return kv
-}
-```
-
-#### User-provided transformer
-
-Users can provide a customized 1:n transformer function that transforms a key from the v2 store to any number of keys in the mvcc store. The migration program writes JSON formatted [v2 store keys][v2key] to the transformer program's stdin, reads protobuf formatted [mvcc keys][v3key] back from the transformer program's stdout, and finishes migration by saving the transformed keys into the mvcc store.
-
-The provided transformer should read until EOF and flush the stdout before exiting to ensure data integrity.
-
-#### Example
-
-```
-./etcdctl migrate --data-dir=/var/etcd --transformer=k8s-transformer
-# finished transforming keys
-```
 
 ### VERSION