diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b4b82473d97..314ce185322 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,3 +4,17 @@ /helm/ @derekperkins @dkhenry /config/mycnf/ @morgo /go/vt/mysqlctl/mysqld.go @morgo +/go/test/endtoend/vtgate @harshit-gangal @systay +/go/vt/vtgate @harshit-gangal @systay +/go/vt/vttablet/tabletserver @harshit-gangal @systay +/go/vt/vttablet/tabletmanager/vstreamer @rohit-nayak-ps +/go/vt/sqlparser @harshit-gangal @systay +/go/vt/discovery @deepthi +/go/vt/vttablet/tabletmanager @deepthi +/go/vt/vttablet/tabletmanager/vreplication @rohit-nayak-ps +/go/vt/orchestrator @deepthi +/go/vt/vtctl @deepthi +/go/vt/wrangler @deepthi @rohit-nayak-ps +/go/vt/mysqlctl @deepthi +/web/vtctld2 @rohit-nayak-ps +/examples/local @rohit-nayak-ps diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index 401a147edb1..c188cf15f4f 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - name: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + name: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 26, 27] steps: - name: Set up Go diff --git a/.github/workflows/docker_test_1.yml b/.github/workflows/docker_test_1.yml index c963d1788b8..f15e3a3f9bd 100644 --- a/.github/workflows/docker_test_1.yml +++ b/.github/workflows/docker_test_1.yml @@ -16,6 +16,6 @@ jobs: - name: Check out code uses: actions/checkout@v2 - - name: Run tests which requires docker 1 + - name: Run tests which require docker - 1 run: | go run test.go -docker=true --follow -shard 10 \ No newline at end of file diff --git a/.github/workflows/docker_test_2.yml b/.github/workflows/docker_test_2.yml index 81433ea58a7..84ea569d5de 100644 --- a/.github/workflows/docker_test_2.yml +++ b/.github/workflows/docker_test_2.yml @@ -1,4 +1,4 @@ -name: docker test 2 +name: docker_test_2 on: [push, pull_request] jobs: @@ -16,6 +16,6 @@ jobs: - name: Check out code uses: actions/checkout@v2 - - name: Run tests which requires docker - 2 + - name: Run tests which require docker - 2 run: | go run test.go -docker=true --follow -shard 25 \ No newline at end of file diff --git a/.github/workflows/docker_test_3.yml b/.github/workflows/docker_test_3.yml new file mode 100644 index 00000000000..92c9e0cbe5f --- /dev/null +++ b/.github/workflows/docker_test_3.yml @@ -0,0 +1,21 @@ +name: docker_test_3 +on: [push, pull_request] +jobs: + + build: + name: Docker Test 3 + runs-on: ubuntu-latest + + steps: + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.13 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Run tests which require docker - 3 + run: | + go run test.go -docker=true --follow -shard 26 diff --git a/.github/workflows/region_example.yml b/.github/workflows/region_example.yml index 0da77d8c89e..bc74a0b13fb 100644 --- a/.github/workflows/region_example.yml +++ b/.github/workflows/region_example.yml @@ -51,5 +51,5 @@ jobs: fi # Make sure that testing is entirely non-reliant on config mv config config-moved - sed -i 's/user\/my-vitess/runner\/work\/vitess\/vitess/g' examples/region_sharding/main_vschema.json #set correct path to countries.json + sed -i 's/user\/my-vitess/runner\/work\/vitess\/vitess/g' examples/region_sharding/main_vschema_sharded.json #set correct path to countries.json eatmydata -- test/region_example.sh diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 4ac0a3104f5..f40497cf096 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -2,6 +2,7 @@ This page lists all active maintainers and their areas of expertise. This can be The following is the full list, alphabetically ordered. +* Alkin Tezuysal ([askdba](https://github.com/askdba)) alkin@planetscale.com * Andres Taylor ([systay](https://github.com/systay)) andres@planetscale.com * Anthony Yeh ([enisoc](https://github.com/enisoc)) enisoc@planetscale.com * Dan Kozlowski ([dkhenry](https://github.com/dkhenry)) koz@planetscale.com @@ -15,6 +16,7 @@ The following is the full list, alphabetically ordered. * Morgan Tocker ([morgo](https://github.com/morgo)) morgan@planetscale.com * Paul Hemberger ([pH14](https://github.com/pH14)) phemberger@hubspot.com * Rafael Chacon ([rafael](https://github.com/rafael)) rchacon@slack-corp.com +* Shlomi Noach ([shlomi-noach](https://github.com/shlomi-noach)) shlomi@planetscale.com * Sugu Sougoumarane ([sougou](https://github.com/sougou)) sougou@planetscale.com ## Areas of expertise @@ -29,10 +31,10 @@ dkhenry, morgo sougou, rafael, tirsen, dweitzman, systay ### Parser -sougou, dweitzman, deepthi, systay +sougou, dweitzman, deepthi, systay, harshit-gangal ### Cluster Management -deepthi, rafael, enisoc +deepthi, rafael, enisoc, shlomi-noach ### Java mpawliszyn, pH14, harshit-gangal diff --git a/Makefile b/Makefile index 4a04dbe2a12..d3473f19784 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,13 @@ ifdef VT_EXTRA_BUILD_FLAGS export EXTRA_BUILD_FLAGS := $(VT_EXTRA_BUILD_FLAGS) endif +# We now have CGO code in the build which throws warnings with newer gcc builds. +# See: https://github.com/mattn/go-sqlite3/issues/803 +# Work around by dropping optimization level from default -O2. +# Safe, since this code isn't performance critical. +export CGO_CFLAGS := -O1 + +# regenerate rice-box.go when any of the .cnf files change embed_config: cd go/vt/mysqlctl go run github.com/GeertJohan/go.rice/rice embed-go @@ -47,7 +54,8 @@ ifndef NOBANNER echo $$(date): Building source tree endif bash ./build.env - go install $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/... + go install $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/... && \ + (cd go/cmd/vttablet && go run github.com/GeertJohan/go.rice/rice append --exec=../../../bin/vttablet) debug: ifndef NOBANNER @@ -61,7 +69,7 @@ endif install: build # binaries mkdir -p "$${PREFIX}/bin" - cp "$${VTROOT}/bin/"{mysqlctld,vtctld,vtctlclient,vtgate,vttablet,vtworker,vtbackup} "$${PREFIX}/bin/" + cp "$${VTROOT}/bin/"{mysqlctld,vtorc,vtctld,vtctlclient,vtgate,vttablet,vtworker,vtbackup} "$${PREFIX}/bin/" # install copies the files needed to run test Vitess using vtcombo into the given directory tree. # Usage: make install PREFIX=/path/to/install/root @@ -155,14 +163,13 @@ ifndef NOBANNER echo $$(date): Compiling proto definitions endif -# TODO(sougou): find a better way around this temp hack. -VTTOP=$(VTROOT)/../../.. $(PROTO_GO_OUTS): install_protoc-gen-go proto/*.proto for name in $(PROTO_SRC_NAMES); do \ - cd $(VTTOP)/src && \ - $(VTROOT)/bin/protoc --go_out=plugins=grpc:. -Ivitess.io/vitess/proto vitess.io/vitess/proto/$${name}.proto && \ - goimports -w $(VTROOT)/go/vt/proto/$${name}/$${name}.pb.go; \ + $(VTROOT)/bin/protoc --go_out=plugins=grpc:. -Iproto proto/$${name}.proto && \ + goimports -w vitess.io/vitess/go/vt/proto/$${name}/$${name}.pb.go; \ done + cp -Rf vitess.io/vitess/go/vt/proto/* go/vt/proto + rm -rf vitess.io/vitess/go/vt/proto/ # Helper targets for building Docker images. # Please read docker/README.md to understand the different available images. diff --git a/bootstrap.sh b/bootstrap.sh index a2f7891dd0b..c539809bc8e 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -119,7 +119,7 @@ function install_zookeeper() { local dist="$2" zk="zookeeper-$version" - wget "https://apache.org/dist/zookeeper/$zk/$zk.tar.gz" + wget "https://archive.apache.org/dist/zookeeper/$zk/$zk.tar.gz" tar -xzf "$zk.tar.gz" ant -f "$zk/build.xml" package ant -f "$zk/zookeeper-contrib/zookeeper-contrib-fatjar/build.xml" jar diff --git a/config/mycnf/default-fast.cnf b/config/mycnf/default-fast.cnf index 1880704e05b..e57368a41db 100644 --- a/config/mycnf/default-fast.cnf +++ b/config/mycnf/default-fast.cnf @@ -20,3 +20,6 @@ innodb_doublewrite=0 # https://github.com/vitessio/vitess/issues/5396 sql_mode = STRICT_TRANS_TABLES + +# set a short heartbeat interval in order to detect failures quickly +slave_net_timeout = 4 diff --git a/config/orchestrator/default.json b/config/orchestrator/default.json new file mode 100644 index 00000000000..8fc7ab3747d --- /dev/null +++ b/config/orchestrator/default.json @@ -0,0 +1,8 @@ +{ + "Debug": true, + "MySQLTopologyUser": "orc_client_user", + "MySQLTopologyPassword": "orc_client_user_password", + "MySQLReplicaUser": "vt_repl", + "MySQLReplicaPassword": "", + "RecoveryPeriodBlockSeconds": 5 +} diff --git a/doc/TwoPhaseCommitDesign.md b/doc/TwoPhaseCommitDesign.md index 38b93b3f891..81eef23370a 100644 --- a/doc/TwoPhaseCommitDesign.md +++ b/doc/TwoPhaseCommitDesign.md @@ -142,7 +142,7 @@ The detailed design explains all the functionalities and interactions. Currently, transaction ids are issued by VTTablets (VTID), and those ids are considered local. In order to coordinate distributed transactions, a new system is needed to identify and track them. This is needed mainly so that the watchdog process can pick up an orphaned transaction and resolve it to completion. -The DTID will be generated by taking the VTID of the MM and prefixing it with the keyspace, shard info and a sequence to prevent to prevent collisions. If the MM’s VTID was ‘1234’ for keyspace ‘order’ and shard ‘40-80’, then the DTID would be ‘order:40-80:1234’. A collision could still happen if there is a failover and the new vttablet’s starting VTID had overlaps with the previous instance. To prevent this, the starting VTID of the vttablet will be adjusted to a value higher than any used by the prepared GTIDs. +The DTID will be generated by taking the VTID of the MM and prefixing it with the keyspace, shard info and a sequence to prevent collisions. If the MM’s VTID was ‘1234’ for keyspace ‘order’ and shard ‘40-80’, then the DTID would be ‘order:40-80:1234’. A collision could still happen if there is a failover and the new vttablet’s starting VTID had overlaps with the previous instance. To prevent this, the starting VTID of the vttablet will be adjusted to a value higher than any used by the prepared GTIDs. ## Prepare API diff --git a/doc/releasenotes/7_0_0_release_notes.md b/doc/releasenotes/7_0_0_release_notes.md new file mode 100644 index 00000000000..22fbaaebffc --- /dev/null +++ b/doc/releasenotes/7_0_0_release_notes.md @@ -0,0 +1,410 @@ +## Incompatible Changes + +*This release includes the following changes which may result in incompatibilities when upgrading from a previous release*. *It is important that Vitess components are* _[upgraded in the recommended order](https://vitess.io/docs/user-guides/upgrading/#upgrade-order)_. *This will change in the next release as documented in* *[VEP-3](https://github.com/vitessio/enhancements/blob/master/veps/vep-3.md).* + +* VTGate: The default gateway (-gateway_implementation flag value) is now "tabletgateway". The old default was “discoverygateway”. To use the old gateway, specify the flag in the command line. Anyone already using the flag will see no change; to use the new gateway, remove the -gateway_implementation flag. The new gateway is simpler and uses a new healthcheck module with no loss of functionality. #6155 + +* VTGate: VTGate incorrectly returned Warning Code field name as Type. This has now been fixed. Code or scripts that depend on the incorrect behavior might break. #6437 + +* VReplication: The "-cell" parameter to MoveTables has been renamed to “-cells”, since it now accepts multiple comma-separated cells or cellsAliases. The parameter specifies which cells can provide tablets as the source for vreplication streams. #6456 + +* VTGate: The possibility of sending OpenTracing trace spans through query comments has been changed. The old way of doing it was severely broken, so our assumption is that no-one has successfully been using this feature. Still, this is an API change, and we are reporting it as a breaking change. #6463 + +* VTTablet has been refactored to be generally more responsive to state changes and its environment: #6131 #6139 #6241 #6263 #6311 #6345 #6348 #6384 #6396 #6432 #6461 + + * Some initialization parameters are now mandatory: init_keyspace, init_shard and init_tablet_type. The flow where you could issue a vtctlclient InitTablet and then invoke vttablet without those init parameters is no longer supported. + + * Consequently vtctl's InitTablet and UpdateTabletAddrs are deprecated. + + * demote_master_type is deprecated: Now that init_tablet_type is mandatory, that value can act as the demote_master_type. There is no need for a separate flag. + + * VTTablet will publish the discovered MySQL port on startup, and will not update it during the life of the process. There is no known use case where a MySQL port is expected to change. So, there is no need to keep polling MySQL for its port. + + * Decouple enable_replication_watcher from disable_active_reparents. Currently, vttablet attempts to fix replication only if enable_replication_watcher is true and disable_active_reparents is false. This behavior has changed to depend only on the disable_active_reparents flag. + + * Transitioning from a serving type to SPARE used to go through a brief lameduck period specified by the serving_state_grace_period (default 0). This has been removed. The transition will now happen immediately even if a grace period is specified. + + * It was possible to enable health reporting and heartbeat at the same time. Performing the two at the same time is not useful. As of this release, enabling heartbeat automatically disables the polling mode of checking for replica health. + + * The ACL rules used to be reported on the vttablet status page. The design was using an incorrect framework. This has been removed. + +* Build: make targets "build_web" and “embed_static” have been removed. Instead there are new targets “web_bootstrap”, “web_build” and “web_start” #6473 + +## Deprecations + +* The following vtctl commands have been deprecated. Deprecated commands will continue to work until the next release (8.0). #6428 #6345 #6461 + + * StartSlave -> Replaced with StartReplication + + * StopSlave -> Replaced with StopReplication + + * ChangeSlaveType -> Replaced with ChangeTabletType + + * InitTablet + + * UpdateTabletAddrs + +* Various vtctl commands had a flag called "-wait_slave_timeout". It has been replaced with “-wait_replicas_timeout”. Scripts using the old flag will continue to work until the next major release (8.0). #6428 + +* Helm charts: The current helm charts have not been keeping up with the forward progress in vitess, and there has been no maintainer for it. Additionally, the operator has pulled ahead of what the helm charts could do. The operator can also be functionally more versatile than the helm charts because custom code can be written to manage complex workflows. To maintain focus and velocity in the project, we are deprecating the helm charts. #6439 + +* Consul: The usage of Consul has been problematic for vitess users. The main difficulty comes from the fact that it's non-trivial for a multi-cell deployment. Due to this, we want to discourage the use of consul as a topo server, and are deprecating support for it. #6440 + +## Bugs Fixed + +### VTGate / MySQL compatibility + +* Handle non-keyword tokens FORMAT, TREE and VITESS. (This was broken by #6110) #6445 + +* When using user-defined variables that have not been initialized first, a non-fatal panic occurs and the connection is dropped. #6294 #6308 + +* Setting system variable panics vtgate. #6162 #6163 + +* regression: USE does not work from MySQL client. #6153 #6157 + +* USE does not return an error if a keyspace doesn't exist. #3060 #6157 + +* Dotted table names are not escaped properly in lookup vindex flow. #6142 #6145 + +* Produce correct error code (1105 instead of 1062) when inserting duplicates into consistent_lookup vindex. #6144 + +* Rewriting expressions fail when used as function parameters. #6281 + +* Rollback after error clobbers updates. #6285 + +* Queries should not be wrapped into transactions if **autocommit=1** is set. #6264 + +* Prepared statements with bind variables in column specification produce ‘missing bind var’ error. #6287 #6298 + +* Prepared statement queries returning NULL results are not parsed. #6240 #6249 + +* DEFAULT function not working with status column. #6221 #6224 + +* VTGate panic kills connection when selecting database() when using OLAP workload. #6200 #6205 + +* Wrong column value can be inserted due to parameterization variable confusion. #6266 #6270 + +* Fix owned table name formatting and duplicate error code in consistent lookup. #6145 + +* Support for case-insensitive user defined variables. #6225 + +* Fix performance regression on VTGate. #6203 + +* Return NULL for non-existent user-defined variables. #6308 + +* MySQL connection closing should not inflate query counts. #6226 + +* show vitess_shards intermittently returns empty list. #5038 #5189 + +* Protect insert lookup queries from bind variable name clashes #6328 + +* CAST/CONVERT should work correctly #6304 + +* Allow charset introducers for vindex columns #6301 + +* Fix panic from EXPLAIN #6204 + +* Handle special comments that start with ! #6186 + +* Fix panic with uppercase types #6140 + +* Allow ‘$’ in identifiers #6136 + +### Other + +* VTTablet: Etcd query timeout because auto-committable transactions are not properly rolled back. #6258 #6299 + +* VTTablet: Fixed race in schema engine during notification of schema changes. #6268 + +* VTTablet: Reloading ACL config was throwing misleading error messages.his has now been fixed and the error messages improved. #6309 + +* VTTablet: Panic during backup caused an error in the pgzip library has been fixed by contributing a fix to pgzip and upgrading to a new release that includes the fix. #6189 + +* VTTablet: Improvement of error handling by retrying while uploading to S3. #6182 #6395 #6405 + +* VTTablet: builtinbackupengine was uploading a manifest despite errors from S3. Error collection during backup has been fixed to address this. #6349 #6350 + +* VTTablet: Fix dba connection leak in LockTables. #6424 + +* VTTablet: Fix connection pool leak in schema engine. #6426 + +* VTTablet: Fix regression where a heartbeat interval flag setting of 1s would have resulted in an internal value of 1000s #6356 #6357 + +* VTTablet: Fix panic from binlog_streamer when ‘-watch_replication_stream’ is set., #6253 + +* VTTablet: Add context to MysqlDaemon schema functions. This lets us handle disconnected RPCs instead of the client hanging forever. #6243 + +* VTTablet: fix panic from txlogz #6238 + +* VTTablet: Add back support for millisecond values for flags that was dropped when yaml config was introduced #6250 + +* Vtctld UI: Change all external links in vitess to use https #6170 #6169 + +* Ensure SrvVSchema gets rebuilt for all cells #6276 + +* Fix MariadbGTIDSet multi-domain support #6184 + +* mysqlctl: Fix connection leak during killConnection() #6245 + +* Fix release script to install examples in the correct directory #6159 + +* examples/operator: fix port-forward command #6418 + +* VStream client: should send current position events immediately #6391 #6397 + +## Functionality Added or Changed + +### VTGate/ MySQL compatibility + +* Transactions on read-only replicas are now supported. Start a transaction by first choosing the desired tablet_type ("use keyspace@replica"), then issuing a “BEGIN” statement. Internally, this will issue a “START TRANSACTION READ ONLY” on the underlying database. This is only available if using the new gateway_implementation of tabletgateway (which is now the default). #6166 #6244 + +* Create better plans for NULL comparisons. Instead of using a SelectScatter, now vtgate will produce empty results when the predicate is a comparison against literal NULLs. #6133 #6425 + +* Support IN, NOT IN and equality (=) with null values #6152 #6146 + +* Add EXPLAIN support for vtgate plans. View the produced vtgate query plan by using explain format=vitess #6110 + +* Allow users to perform savepoint related constructs. #6355 #6404 #6412 #6414 + +* Added support for user-level advisory lock functions like get_lock and release_lock. This is considered experimental and will be improved in future releases based on user feedback. #6367 #6370 #6470 + +* Allow users to change system variable settings at session level using set statements. This is a major change that was merged later in the development cycle. It is considered experimental and there may be edge cases. #6107 #6459 #6472 #6488 + +* "ignore_nulls" option to lookup vindexes #6147 #6222: There are situations where the "from" columns of a lookup vindex can be null. Such columns cannot be inserted in the lookup due to the uniqueness constraints of a lookup. The new “ignore_nulls” option of the lookup vindex allows you to accommodate null values in rows. Such rows can still be found through other means, like the primary vindex. + +* Add support for row_count() function. #6174 #6179 + +* XOR Operator is now supported. #6369 #6371 + +* Allow 'binary' for character_set_results. It is equivalent to null. #6237 + +* Improve logging by adding keyspace, table, and tablet_type. #6282 + +* VTGate now takes in optional flags for ‘max_payload_size’ and ‘warn_payload_size’. A payload size above the warning threshold will only produce a warning. A payload size that exceeds ‘max_payload_size’ will result in an error. The default behavior is not to check payload size, which is backwards-compatible. #6143 #6375 + +* A new comment directive (‘/*vt+ IGNORE_MAX_PAYLOAD_SIZE=1 */) has been added to provide the ability to override the payload size checking per query. #6143 + +* A new comment directive (‘/*vt+ IGNORE_MAX_MEMORY_ROWS=1 */) has been added to provide the ability to ignore the "-max_memory_rows" setting per query. #6430 + +* Support ‘select * from information_schema.processlist’. #6310 + +* Support batch lookup queries for integer and binary types. Other types will continue to require one query per row. #6420 + +* Support system settings at session level. #6149 + +* Prepare for future features by adding reserved connections. This is the ability for a connected client to have a dedicated connection instead of using the connection pool. #6303 #6313 #6402 #6441 #6451 + +* Allow charset introducers with vindex column values. #6236 #6314 + +* Support Union All between different shards. #6351 + +* Exclude KeyspaceId(s) from plan cache key.This improves the cache hit rate. #6233 + +* Reload TLS config on SIGHUP. #6215 + +* Drop vindex DDL. #6151 + +* show vitess_tablets now reports master_term_start_time. We ensure that this is reported correctly after reparents. #6135 #6292 #6293 + +* Allow parenthesized selects #6342 + +* Support DO statements #6239 + +* Support calling database() inside other functions #6291 + +* Support IN, NOT IN and equality (=) with null values #6152 #6146 + +### VReplication + +* VStream Copy extends the VStream API to allow streaming of the contents of databases or tables. Earlier you had to specify a position to start streaming events from. #6277 + +* VReplication workflows now accept "-cells" as an option. This can be a comma-separated list of cells or cellsAliases. The source tablet for vreplication will be chosen randomly among the tablets that belong to the specified cells/cellsAliases. If the option is not provided, we default to the cell that is local to the vreplication target. #6442, #6456 #6465 + +* Schema Versioning. Tablets now have a schema tracker that can be enabled to store schema versions as DDLs are applied. The _vt.schema_version table contains the schema with the corresponding ddl and gtid position when the ddl was applied. The tracked schemas are used by the new Historian service to provide vstreams with the correct schema while streaming old events. #6300 #6164 #6278 #6399 #6411 #6435 #6448 + +* Support for specifying multiple external sources to facilitate migration into vitess. + +* Foreign Keys are now supported on the target database during vreplication. In the copy phase we disable FK checks and re-enable them once the replication phase starts. #6284 + +* Materialize: added a drop_constraint option during table copy to strip constraints while creating target tables #6422 + +* Materialize: optimizations while creating target tables resulting in a significant improvement in performance when moving a large number of tables from an external MySQL instance #6207, #6251 + +* DropSources: Added ‘-rename_tables’ option to rename the source tables rather than dropping them entirely. #6383 + +* Savepoints: Vreplication streams are not able to handle savepoint events in the binlog. #6438 + +* Add keyspace, shard and workflow labels to VReplicationSecondsBehindMaster metrics #6279 + +* Correct command line help for MigrateServedTypes, MigrateServedFrom and SwitchWrites #6187 + +* VExec and Workflow commands have been added to vtctld as a layer above VReplicationExec. They allow listing/stopping/starting/deleting workflows and ability to run custom queries on the _vt.vreplication table for all shards in a workflow. #6410 + +### Point-in-time Recovery + +In the 4.0 release we introduced functionality to recover a keyspace into a snapshot based on a specified time. That feature only allowed you to restore a backup, but did not have the ability to automatically replay binlogs up to the specified time. The feature has now been completed so that true point-in-time recovery is possible. This is currently only supported for MySQL compatible installations, not MariaDB. #4857 #6267 #6408 + +### Healthcheck + +* VTGate can now use a new healthcheck that is logically simpler while providing the same functionality. This is the default. The older healthcheck has been deprecated but is still available to use if desired by setting "-gateway_implementation=discoverygateway" #5750 #6155 + +* VTCombo has been migrated to use the new healthcheck #6302 + +* VTExplain has been migrated to use new healthcheck #6460 + +### MiniVitess + +A docker container that can be used to bring up a self-contained vitess cluster, which can automatically resolve and self-configure to meet an external MySQL cluster. While this might sound very similar to VTCombo and vttestserver, it is not quite the same. + +MiniVitess spins up in docker. Given a MySQL server hostname, schema name & credentials, it runs an automated topology analysis via orchestrator, identifying the cluster members. It proceeds to bring up distinct tablet for each server and opens up vtgate connections. #6377 + +### Examples / Tutorials + +* Vitess operator example #6154 + +* Region sharding example #6209 #6275 + +* Change local example to use new command DropSources #6178 + +* Are-you-alive example now makes it easier to test multiple endpoints #6202 + +* Local docker tutorial added to website + +* Simplified helm chart for examples (however, helm is now deprecated) #6168 + +### Other + +* vtctlclient ListAllTablets now shows the MasterTermStartTime. This is shown as "" for read-only tablet types. In case of an old master that has not yet updated the topo record, TabletType will be reported as “unknown” #6135 + +* New debug handlers have been added to enable/disable golang block and mutex profiling. #6137 + +* Vitessdriver: Allow overriding driver name. Default stays as "vitess" #6138 + +* VTCombo: add options to allow starting MySQL within VTCombo. Note that vttestserver provides similar functionality. #6265 + +* Disable foreign key checks during preflight schema change tests #4696 + +* Fix labels for opentsdb #6289 + +* Allow empty shards when running ValidateSchemaKeyspace behind flag ‘-skip-no-master’ #6216 + +* Death by SIGPIPE on stdout/stderr can be avoided by setting the new flag ‘catch-sigpipe’ #6297 + +* Prepared Statement support added when using special vitess construct of use `:` by adding GetField support in engine plan execute #6132 + +* GTIDSet Union #6180 + +* Add relay log position and file based positions to ReplicationStatus #6217 + +* Function that can find errant GTIDs #6296 + +* VTTablet: Add StopReplicationMode to StopReplicationAndGetStatus RPC #6335 + +* VTTablet: DemoteMaster RPC returns full status instead of string representation of current position #6365 + +* VTTablet: WaitForPosition RPC can now accept either file-based or GTID-based positions #6374 + +## Documentation + +* Fix links to local example and sample client in vitessdriver godoc #6254 + +* Flag documentation has been improved #6261 + +* All flags are now documented in the [program reference](https://vitess.io/docs/reference/programs/) on the website. + +## Build Environment Changes + +* golangci-lint now runs as a pre-commit hook and CI check #6385 #6406 + +* CI test for region_sharding example #6275 + +* Removed staticcheck as a pre-commit hook because it is included in golangci-lint #6416 + +* Faster unit tests #6415 + +* Consul-api version upgraded to 1.8.0 #6358 + +* Upgrade pgzip to v1.2.4 #6189 + +* Upgrade log4j2 version to 2.13.3 #6403 + +* Upgrade netty version to 4.1.48.Final #6317 + +* Upgrade tcnative version to 2.0.30.Final #6317 + +* Add docker_local target to Makefile and a docker/local/run.sh script #6393 + +* Only check for k3s on Linux #6353 + +* UI build tooling was broken, it has now been fixed. #6473 + +## Functionality Neutral Changes + +* Refactor table filtering logic #6242 #6259 + +* VTGate: Replace Sprintf with string concatenation in normalizer, this reduces CPU usage on benchmarks #6127 + +* VTTablet: Reduce the number of packets sent from vttablet to MySQL #6130 + +* Fixed flakiness in External Connector Test: fixed race #6201 + +* Fixed flakiness in Message Test: test had not been updated after jitter functionality was added #6283 + +* Fix flakiness in TestConnectTimeout #6188 + +* Fix TLS test for Go 1.13+ #6185 + +* Fixed linter errors #6416 #6364 #6307 + +* More dollar-sign tests #6363 + +* VTExplain: use DiscoveryGateway (and old healthcheck) until migration can be completed #6248 + +* Deprecated RPCs from 6.0 have been deleted #6380 + +* Update references to orchestrator repo to point to openark instead of github #6360 + +* Terminology: deprecate or replace references to "slave" with “replica” except where referring to actual sql commands or fields returned from MySQL. #6428 #6392 #6379 + +* Terminology: replace user-facing references to "slave" with “replica” or “replication” in UI #6481 + +* Fix log format - use Infof instead of Info #6413 + +* Cleanup temp files used in tests #6400 + +* Add unit test for func stringMapToString #6280 + +* Check errors in etcd2topo unit test #6128 + +* Unit test for atomic.go #6120 + +* Unit test for ResolveIPv4Addrs #6230 + +* Enabled set statement tests #6167 + +* Replaced VTTablet ExecuteBatch api call with Execute for autocommit query. #6407 #6431 + +* Fixed flaky TestQueryPlanCache #6454 + +* Fix test compile errors #6434 + +* Scatter_conn unit test using new healthcheck #6458 + +* Test that vtexplain works with healthcheck #6252 + +* Tests: Removed error check that caused spurious test failures #6462 + +* More test cases for IGNORE_MAX_PAYLOAD_SIZE #6467 + +* VSCopy Test: Mods/bug fixes/test logging to make test repeatable/debuggable, fix flakiness #6341 #6235 + +* Cleanup scatter conn #6257 + +* Simplify vtgate executor #6256 + +* Add tests for SHOW COLUMNS #6192 + +* Added fuzzer #6175 + +* Simplified TxPool and ConnPool code #6150 + diff --git a/doc/releasenotes/7_0_1_release_notes.md b/doc/releasenotes/7_0_1_release_notes.md new file mode 100644 index 00000000000..5b0d3f30a62 --- /dev/null +++ b/doc/releasenotes/7_0_1_release_notes.md @@ -0,0 +1,19 @@ +This update fixes several regressions that were deemed significant enough to be backported to the release branch. +In addition, the experimental feature to set system variables is now behind a flag that disables it by default. + +## Configuration Changes +* Vtgate: added flag enable_system_settings to enable passthrough system settings, defaults to false #6562 + +## Bugs Fixed +* Vreplication: Materialize should only get schema from source tablets if target is missing tables #6623 +* Vtctld UI: mysql port is not displayed in tablet list #6597 #6600 +* Vtgate: transaction killer was closing reserved connection after the specified time. Only idle connections should be closed. Idle transactions should be rolled back. #6583 +* Vtcompose: docker-compose example was broken by tabletmanager changes #6587 +* Vreplication: tablet_picker should respect canceled context. Also, keep trying to find a tablet until context expires. #6567 +* Vtgate: vindex update/delete by destination #6541 #6572 +* Vtgate: some usages of EXPLAIN were broken #6574 #6581 +* Vtgate: handle multiple settings in single SET statement #6504 +* Vtgate: cleanup shard session for reserved connection on failure #6563 +* Vtgate: if executing DDL on reserved connection, start transaction if needed #6555 +* Vtgate: make sure lookup vindexes use same transaction if one is open #6505 +* Healthcheck: add diagnostic logging, do not add tablet if it is shutting down #6512 diff --git a/doc/releasenotes/7_0_2_release_notes.md b/doc/releasenotes/7_0_2_release_notes.md new file mode 100644 index 00000000000..33ceed95c30 --- /dev/null +++ b/doc/releasenotes/7_0_2_release_notes.md @@ -0,0 +1,12 @@ +This update fixes several regressions that were deemed significant enough to be backported to the release branch. + +## Bugs Fixed + +* vtgate : Operator precedence must take associativity into consideration #6764 +* vtgate: Fix reserved connection in autocommit mode on DML #6748 +* vDiff: fix panic for tables with a unicode_loose_md5 vindex #6745 +* vttablet: Turn off schema tracker by default #6746 +* restore: Do not change tablet type to RESTORE if not actually performing a restore #6687 +* vttablet : Tablet manager init should publish displayState #6686 +* vttablet: Tablet manager add more logging to checkMastership #6685 +* vttablet : Fix replmanager deadlock #6634 diff --git a/doc/releasenotes/8_0_0_release_notes.md b/doc/releasenotes/8_0_0_release_notes.md new file mode 100644 index 00000000000..26d50aab3dd --- /dev/null +++ b/doc/releasenotes/8_0_0_release_notes.md @@ -0,0 +1,274 @@ +This release complies with VEP-3 which removes the upgrade order requirement. Components can be upgraded in any order. It is recommended that the upgrade order should still be followed if possible, except to canary test the new version of VTGate before upgrading the rest of the components. + +## Incompatible Changes + +The following PRs made changes to behavior that clients might be relying on. They should be reviewed carefully so that client code can be changed in concert with a Vitess release deployment. +* Change error code from 1203 to 1153 for three specific error conditions. #6630 + The three errors being changed: +When a GRPC message coming back to the vtgate exceeds the configured size limit (configured via grpc_max_message_size) +If a write is attempting to make a change with a payload that is larger than the configured allowable size +If a scatter query is attempting to collect and process too many rows in memory within the vtgate to construct a response +* Zero auto-increment mode: This enables inserting a 0 value (and not just NULL) into an auto-increment column to generate sequence values. This matches the default behavior of MySQL. #6749 +* Turn off schema tracker by default #6712 + +## Bugs Fixed + +### VTGate / MySQL compatibility +* Fix where clause in information schema with correct database name #6599 +* Fix DDL execution on reserved connection without in active transaction #6514 +* Fix allow passed table_schema in information_schema queries to vttablet if the keyspace is not found #6653 +* Fix information schema for non-existent schema #6667 +* Fix reserved connection in autocommit mode on DML #6748 +* Fix dbname that needs escaping that we broke in fce4cfd894 #6794 +* MySQL CLI, mysql --ssl does not work in MySQL 8.0 #5556 +* OperationalError: (_mysql_exceptions.OperationalError) (2013, 'Lost connection to MySQL server during query') #6208 +* Fix support for `show tables where tables_in_db` to use keyspace name, not underlying db name #6446 +* Common empty result set query on MySQL returns a row in Vitess #6663 +* Operator precedence issue #6734 +* Table names out-of-order between replica and primary #6738 +* Close Idle reserved connections and Rollback Idle transactions #6552 +* Minor Vindex fixes: #6526 +* VTgate does not report autocommit system variable and SERVER_STATUS_IN_TRANS flag correctly #5825 +* MariaDB JDBC Connector fails to connect to vtgate with default options #5851 +* VTgate panics if there are no vttablets and vtctlds #6117 +* Remove shard session held in vtgate for reserved connection on connection failure #6522 +* Session variable does not work as expected in 7.0 #6559 +* Retry should not happen when in dedicated connection #6523 +* Correctly report AUTOCOMMIT status in network packets #6551 +* Close Idle reserved connections and Rollback Idle transactions #6552 +* Reset Session for Reserved Connection Query Failure #6673 +* Errors in multi-statement queries not fatal to statement sequence #6747 #6808 +* Lock wait timeout on DELETE from table using lookup index #6698 +* Application couldn't connect to external db giving error tabletserver.go:1522] Code: INVALID_ARGUMENT syntax error at position 33 near ':vtg2' (CallerID: gnocchi) : Sql: "select convert(:vtg1, CHAR(:vtg2)) as anon_1 from dual", #6466 +* The client got the wrong database name #6753 +* DBAPIError exception wrapped from (pymysql.err.InternalError) (1105, u'vtgate: http://gnocchi-gnocchizone-vtgate:15000/: reserved connections are not supported on old gen gateway') #6577 +* Provide support for “SELECT INTO OUTFILE S3” #6811 +* Fix error around breaking of multi statements #6824 +* Support SHOW TABLE STATUS FROM db type queries in vtgate #6354 +* SHOW FULL FIELDS doesn't follow routing rules #6803 +* VDiff/Tablet Picker: fix picker retry logic to handle canceled context #6954 +* Healthcheck: Use isIncluded correctly to fix replica/rdonly routing bug #6922 + +### Other +* VReplication tablet_picker should keep trying to find a tablet until context expires #6546 +* VReplication improve reverse workflow: update cells/tablet_types #6556 +* VDiff: fix panic for tables with a unicode_loose_md5 vindex #6640 +* Vtctld UI: Add mysqld port display back to vtctld web UI #6598 #6597 +* Vtctld: Fix workflow CopyState fetching (and thus printing). #6657 +* Fix unsaved event regression #6710 +* Change tracker to use allprivs instead of dba #6720 +* Fixes long wait filter keyspace #6721 +* s3 ListBackups doesn’t work from root directory #6732 +* Vtctld: ExecuteFetchAsDba and ExecuteFetchAsAllPrivs do not escape db names. #6545 +* Can't switch on restore_from_backup on existing shard with no tables #4896 +* Refactor conn.go #6818 + + +## Functionality Added or Changed + +### VTGate / MySQL compatibility + +* Add support for DELETE IGNORE #6722 +* Only take on simple dual queries in the vtgate #6666 +* Support mycli access to vtgate #4365 +* Add support for unicode_loose_xxhash #6457 +* Provide virtual information_schema #5394 +* information_schema.tables_schema comparison with dynamic value no longer works #6827 +* Mysqldump: Allow setting multiple variables in one statement #5401 +* Block lock function usage other than dual table #6470 +* Lock Session Support #6517 +* Make sure lookup vindexes are queryable inside transaction #6499 +* Add unicode_loose_xxhash Vindex type #6549 +* Support vindex update or delete when destination is provided by client in dml query #6554 +* Make sure to handle EXPLAIN on tablets #6579 +* Rewrite SHOW TABLES #6615 +* Make offset work in OLAP mode #6655 +* Support sql_calc_found_rows with limit in sharded keyspace #6680 +* Lock shard session heartbeat #6683 +* Mysql -ssl tag is deprecated in the later versions of mysql #6682 +* Use statement support in olap mode #6692 +* Delete row using consistent lookup vindex in where clause #6700 +* Add vtexplain support for non-evenly sharded keyspaces #6705 +* Make sure dual queries set found_rows correctly #6718 +* Add if not exists support to database ddl in parsing #6724 +* Allow Update of same vindex using same vindex lookup on consistent lookup vindex #6736 +* Update Vindex only on changes #6737 +* Add support for `SHOW (DATABASES|VITESS_SHARDS|VITESS_TABLETS) LIKE` #6750 +* 'CASCADE' support #6743 +* Mask database name #6791 +* Make sure to backtick schema names properly #6550 +* Transaction Limiter: fix quota leak if Begin() errors or if the tx killer runs #6731 +* Allow Load Data From S3 #6823 + +### OLAP Functioanlity +* Allow switching between OLAP and OLTP #6691 +* Use statement support in olap mode #6692 +* Fix error handling in olap mode #6940 + + +### Set Statement Support +Set statement support is added in Vitess. There are [some system variables](https://github.com/vitessio/vitess/blob/master/go/vt/sysvars/sysvars.go#L147,L190) which are disabled by default and can be enabled using flag `-enable_system_settings` on VTGate.These system variables are set on the backing MySQL instance, and will force the connection to be dedicated instead of part of the connection pool. + +* Disabled passthrough system variables by default. #6859 +* Allow switching workload between OLAP and OLTP #4086 #6691 +* Support for multiple session set statement syntax #6508 #6495 #6494 +* Allow enumeration in set system variables #6493 +* Handle boolean settings better #6501 +* Set statements refactored to use plan building #6487 +* Add more system settings #6486 +* Evaluate system variables #6708 + + + +### VReplication + +* VReplication: _vt.vreplication source column VARBINARY->BLOB, allowing more tables to participate in a workflow #6421 +* VReplication and vstreamer performance and error metrics #6519, #6132 +* VReplication: Find collations for char primary keys and cast comparison during catchup #6568 +* Varchar primary keys with collations can cause vreplication to miss rows #6622 +* VReplication: Timestamp w/ Heartbeat #6635 +* VReplication: Improve few error/log messages #6669 +* VReplication: handle SET statements in the binlog stream #6772 +* VReplication: e2e Tests added to CI #6730 +* VDiff logs: additional logging around vdiff/vstreamer for observability #6684 +* VStream: experimental POC of mysql protocol #6670 +* Materialize: Support rollups using a literal value as a column #6733 +* Materialization: Ignore non-participating tables that are renamed/deleted #6778 +* Materialize: add ability for maintaining accurate table-level aggregates #6767 +* Schema Tracker: Turn off schema tracker by default #6712 (breaking change) +* Workflow: Add List-All Command #6533 +* Workflow: List -> Show, and Expand Metadata #6544 +* Refactor: Cleanup Noisy Workflow Logging #6744 +* Schema tracker: Change tracker to use the allprivs user instead of dba #6720 +* vtctl: Adding VExec and Workflow commands #6410 +* vtctl Workflow cmd: don't expect all shards to have workflow streams #6576 +* Reshard: Sort table definitions before comparing them #6765 #6738 +* Expose a shard range calculator as a vtctl command GenerateShardRanges #6751 #6752 +* VStream: Field event now has all column attributes #6525 +* VStream events do not populate field "Flags" on FIELD events #5914 +* RFC: VReplication based SplitClone #4604 +* Experimental: Mysql Protocol support for VStream #6675 +* VReplication: Add metrics around vstreamer and vreplication metrics #6787 +* Fix issues with missing tables in vstreams #6789 +* Add validations and logs to fix common issues faced during MoveTables/Reshard #6814 +* Allow multiple blacklists for master #6816 + +### VTtablet +* VTtablet throttling #6668 +https://vitess.io/docs/reference/features/tablet-throttler/ +* TabletManager: publish displayState #6648 +* TabletManager: call setReadOnly inside ChangeTabletType #6762 +* TabletManager: change how SetReadOnly is called to avoid errors from externally managed tablets #6786 +* TabletManager: call SetReadOnly inside ChangeTabletType #6804 +* vttablet: Open and Close healthStreamer #6515 +* vttablet: create database if not present #6490 +* Restore: do not change tablet type to RESTORE if not actually performing a restore #6679 +* Restore: checkNoDB should not require tables to be present #6695 +* Heartbeat: Additional metrics/stats (lag histogram) #6528 #6634 +* VTtablet throttler feature flag: -enable-lag-throttler #6815 +* VTtablet two-phase commit design doc #6498 + +### VTorc - Orchestrator +Following PRs are experimental version of Vitess-native Orchestrator 'vtorc' is ready for users to try out. + +* Orchestrator: initial import of Orchestrator #6582 +* Orchestrator: vitess mode #6613 +* Orchestrator: more changes #6714 +* Orchestrator: use contexts with timeout for remote operations #6780 +* Orchestrator: fixed orchestrator govet errors #6781 +* Orchestrator: Add clusters_to_watch flag to. Defaults to all clusters. #6821 + +### Point in Time Recovery + +* PITR: Fix deadlock in ff code #6774 +* Add flags to allow PITR binlog client TLS support #6775 + +### Other +* Add realtime health stats to vtctld /api/keyspace/ks/tablets API #6569 +* vtctld UI: Fix logic for displaying vindexes #6603 +* Online schema changes [#6547](https://vitess.io/docs/user-guides/schema-changes/) +* Table lifecycle management #6719, [docs](https://vitess.io/docs/reference/features/table-lifecycle/) +* Online-DDL: migration uses low priority throttling #6837 +* Emergency Reparent Refactor #6449 + * Make emergency reparents more robust. #6206 +* Reparent test failures #6706 +* Update user-facing terminology in the vtctld2 UI #6481 +* GetSchema: Batch/parallel access to underlying mysqld for lower latency #6491 +* Prometheus interface: vtgate_buffer_* metrics cardinality #6326 +* Show more vindex details in vtctld /app/schema UI #6450 +* GRPC: update enforcement policy on server to match the one from the client #6629 + +## Examples / Tutorials + +* region_sharding: working resharding example #6565 +* vtcompose/docker-compose: fix InitShardMaster #6584 +* [docker] fix tablet hostname #6509 + +## Documentation + +* Replace more uses of slave with replica/replication #6520 +* Update some option docstrings for accuracy. #6590 +* Minor addition to Materialize help text. #6627 +* Remove auto-generation comment #6728 +* add skip flag that can skip comparing source & destination schema when run splitdiff #6477 +* Fix and clarify some help options #6817 + +## Build Environment Changes + +* Address #6631 by reducing optimization level #6632 +* Add front-end build tooling + fix build error #6473 +* Replace gogo proto with golang #6571 +* Improve make proto #6580 +* Remove chromium hard-coding #6636 +* Docker - allow BUILD_NUMBER variable override #6628 +* Docker/lite: Fix package URLs for ubi images. #6704 +* Helm & docker/k8s update #6723 +* Fix build errors in ./go/vt/vtgate for go1.15 #6800 +* Fix build error on vcursor_impl tests on newer versions of go #6799 +* [java] bump java version to 8.0.0-SNAPSHOT for next release #6478 +* Helm/docker: fix Docker builds + tag 7.0.2 #6773 +* Docker: Upgrade to Debian Buster (release-8.0) #6888 +* Docker: Pin mysql 5.7 and 8.0 versions for xtrabackup compatibility +* Docker: Revert "Docker - upgrade to Debian Buster (release-8.0)" #6929 +* Zookeper: Download zookeeper 3.4.14 from archive site #6867 + +## Functionality Neutral Changes + +* Routing cleanup: remove routeOptions #6531 +* PITR: testcase #6594 +* Replaced Error with a warning in case parsing of VT_SPAN_CONTEXT fails #6766 +* Reparent tests refactoring: setup/teardown for each test, reduce cut/paste, improve readability #6726 +* Decider: endtoend test infrastructure + tests #6770 +* Orchestrator: add more test cases #6801 +* Orchestrator: don't set DbName for tests that don't need it. #6812 +* Add unit test case to improve test coverage for go/sqltypes/named_result.go #6672 +* End to end: deflake sharding tests #6715 +* Workflow test: fix flaky test. remove obsolete code #6694 +* Add unit test for func Proto3ValuesEqual. #6649 +* Ensure tests for discoverygateway #6536 +* Convert syslog events to normal log events #6511 +* Add diagnostic logging to healthcheck. #6512 +* Healthcheck should receive healthcheck updates from all tablets in cells_to_watch #6852 +* Named columns and convenience SQL query interface #6543 +* Check http response code in vtctld API tests #6570 +* Reverting package-lock.json to the one pre-PR #6603 #6611 +* Removed sqlparser.preview to set logstats stmtType and use plan.type #6637 +* AST struct name rewording #6642 +* Fix: Remove SetMaster Query Expectation from ERS Test #6617 +* Fix path for vttablet query log in end to end test #6656 +* Fixes error log #6496 +* Add diagnostic logging to healthcheck. #6535 +* Try to install apt keys from list of different keyservers #6674 +* Trivial copy-pasta comment fixup #6592 +* Remove long-unused memcache and cacheservice #6596 +* Fix MoveTables docstring; was not valid JSON. #6606 +* Tablet gateway: unit tests #6608 +* Tablet gateway doesn't see all tablets in cells from cells_to_watch #6846 +* Add Sleep to ERS Unit Test #6610 +* Strings to enum #6729 +* Remove syslog dependency #3563 +* Operator precedence must take associativity into consideration #6758 +* Add optional ignore to delete statement #6802 +* Invalid trace payload causes vitess to fail the request and return an error #6759 +* Vttablet: minor logging fix #6564 +* Vttablet: add more logging to checkMastership #6618 diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index ecab9ab6c42..bc81925d79b 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -25,6 +25,9 @@ FROM vitess/bootstrap:mysql57 # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER + # Re-copy sources from working tree USER root COPY . /vt/src/vitess.io/vitess diff --git a/docker/base/Dockerfile.mariadb b/docker/base/Dockerfile.mariadb index a3f9771334f..42e09d1a9d2 100644 --- a/docker/base/Dockerfile.mariadb +++ b/docker/base/Dockerfile.mariadb @@ -1,5 +1,11 @@ FROM vitess/bootstrap:mariadb +# Allows some docker builds to disable CGO +ARG CGO_ENABLED=0 + +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER + # Re-copy sources from working tree USER root COPY . /vt/src/vitess.io/vitess diff --git a/docker/base/Dockerfile.mariadb103 b/docker/base/Dockerfile.mariadb103 index 75811dc68ba..cc86d5aede5 100644 --- a/docker/base/Dockerfile.mariadb103 +++ b/docker/base/Dockerfile.mariadb103 @@ -1,5 +1,11 @@ FROM vitess/bootstrap:mariadb103 +# Allows some docker builds to disable CGO +ARG CGO_ENABLED=0 + +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER + # Re-copy sources from working tree USER root COPY . /vt/src/vitess.io/vitess diff --git a/docker/base/Dockerfile.mysql56 b/docker/base/Dockerfile.mysql56 index 5d1f55079fa..efdbade8ced 100644 --- a/docker/base/Dockerfile.mysql56 +++ b/docker/base/Dockerfile.mysql56 @@ -1,5 +1,11 @@ FROM vitess/bootstrap:mysql56 +# Allows some docker builds to disable CGO +ARG CGO_ENABLED=0 + +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER + # Re-copy sources from working tree USER root COPY . /vt/src/vitess.io/vitess diff --git a/docker/base/Dockerfile.mysql80 b/docker/base/Dockerfile.mysql80 index 5b2fc5c91ae..4a798700060 100644 --- a/docker/base/Dockerfile.mysql80 +++ b/docker/base/Dockerfile.mysql80 @@ -1,5 +1,11 @@ FROM vitess/bootstrap:mysql80 +# Allows some docker builds to disable CGO +ARG CGO_ENABLED=0 + +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER + # Re-copy sources from working tree USER root COPY . /vt/src/vitess.io/vitess diff --git a/docker/base/Dockerfile.percona b/docker/base/Dockerfile.percona index 93a218ff059..5b488680126 100644 --- a/docker/base/Dockerfile.percona +++ b/docker/base/Dockerfile.percona @@ -1,5 +1,11 @@ FROM vitess/bootstrap:percona +# Allows some docker builds to disable CGO +ARG CGO_ENABLED=0 + +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER + # Re-copy sources from working tree USER root COPY . /vt/src/vitess.io/vitess diff --git a/docker/base/Dockerfile.percona57 b/docker/base/Dockerfile.percona57 index 4f396c9c86b..4a3aa6b0c7c 100644 --- a/docker/base/Dockerfile.percona57 +++ b/docker/base/Dockerfile.percona57 @@ -1,5 +1,11 @@ FROM vitess/bootstrap:percona57 +# Allows some docker builds to disable CGO +ARG CGO_ENABLED=0 + +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER + # Re-copy sources from working tree USER root COPY . /vt/src/vitess.io/vitess diff --git a/docker/base/Dockerfile.percona80 b/docker/base/Dockerfile.percona80 index 83672111166..f5997a16e1a 100644 --- a/docker/base/Dockerfile.percona80 +++ b/docker/base/Dockerfile.percona80 @@ -1,5 +1,11 @@ FROM vitess/bootstrap:percona80 +# Allows some docker builds to disable CGO +ARG CGO_ENABLED=0 + +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER + # Re-copy sources from working tree USER root COPY . /vt/src/vitess.io/vitess diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index 4ee1e190b96..92ca899f712 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -6,7 +6,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins automake \ bison \ bzip2 \ - chromium=70.0.3538.110-1~deb9u1 \ + chromium \ curl \ g++ \ git \ diff --git a/docker/k8s/Dockerfile b/docker/k8s/Dockerfile index 391c352e4f8..c6d28a79b6c 100644 --- a/docker/k8s/Dockerfile +++ b/docker/k8s/Dockerfile @@ -1,25 +1,27 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM vitess/base AS base +ARG VT_BASE_VER=latest -FROM debian:stretch-slim +FROM vitess/base:${VT_BASE_VER} AS base + +FROM debian:buster-slim # TODO: remove when https://github.com/vitessio/vitess/issues/3553 is fixed RUN apt-get update && \ apt-get upgrade -qq && \ - apt-get install mysql-client -qq --no-install-recommends && \ + apt-get install default-mysql-client -qq --no-install-recommends && \ apt-get autoremove && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/k8s/logrotate/Dockerfile b/docker/k8s/logrotate/Dockerfile index ba1e53d961c..0d63e97fce4 100644 --- a/docker/k8s/logrotate/Dockerfile +++ b/docker/k8s/logrotate/Dockerfile @@ -1,18 +1,18 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:stretch-slim +FROM debian:buster-slim ADD logrotate.conf /vt/logrotate.conf @@ -30,4 +30,4 @@ RUN mkdir -p /vt && \ chown -R vitess:vitess /vt && \ chmod +x /vt/rotate.sh -ENTRYPOINT [ "/vt/rotate.sh" ] \ No newline at end of file +ENTRYPOINT [ "/vt/rotate.sh" ] diff --git a/docker/k8s/logtail/Dockerfile b/docker/k8s/logtail/Dockerfile index a5428e2bd52..eca4775a135 100644 --- a/docker/k8s/logtail/Dockerfile +++ b/docker/k8s/logtail/Dockerfile @@ -1,18 +1,18 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:stretch-slim +FROM debian:buster-slim ENV TAIL_FILEPATH /dev/null @@ -21,7 +21,7 @@ ADD tail.sh /vt/tail.sh RUN mkdir -p /vt && \ apt-get update && \ apt-get upgrade -qq && \ - apt-get install mysql-client -qq --no-install-recommends && \ + apt-get install default-mysql-client -qq --no-install-recommends && \ apt-get autoremove -qq && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ @@ -30,4 +30,4 @@ RUN mkdir -p /vt && \ chown -R vitess:vitess /vt && \ chmod +x /vt/tail.sh -ENTRYPOINT [ "/vt/tail.sh" ] \ No newline at end of file +ENTRYPOINT [ "/vt/tail.sh" ] diff --git a/docker/k8s/mysqlctld/Dockerfile b/docker/k8s/mysqlctld/Dockerfile index 4dafb77bbdb..ff1ddc1215a 100644 --- a/docker/k8s/mysqlctld/Dockerfile +++ b/docker/k8s/mysqlctld/Dockerfile @@ -1,20 +1,22 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM vitess/k8s AS k8s +ARG VT_BASE_VER=latest -FROM debian:stretch-slim +FROM vitess/k8s:${VT_BASE_VER} AS k8s + +FROM debian:buster-slim RUN apt-get update && \ apt-get upgrade -qq && \ @@ -44,4 +46,4 @@ COPY --from=k8s /vt/config /vt/config RUN groupadd -r --gid 2000 vitess && \ useradd -r -g vitess --uid 1000 vitess && \ chown -R vitess:vitess /vt && \ - chown -R vitess:vitess /vtdataroot \ No newline at end of file + chown -R vitess:vitess /vtdataroot diff --git a/docker/k8s/orchestrator/Dockerfile b/docker/k8s/orchestrator/Dockerfile index 68f4889123c..a56a2295cd8 100644 --- a/docker/k8s/orchestrator/Dockerfile +++ b/docker/k8s/orchestrator/Dockerfile @@ -1,27 +1,29 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM vitess/k8s AS k8s +ARG VT_BASE_VER=latest +FROM vitess/k8s:${VT_BASE_VER} AS k8s -FROM debian:stretch-slim +FROM debian:buster-slim +ARG ORC_VER='3.2.3' RUN apt-get update && \ apt-get upgrade -qq && \ apt-get install wget ca-certificates jq -qq --no-install-recommends && \ - wget https://github.com/openark/orchestrator/releases/download/v3.1.1/orchestrator_3.1.0_amd64.deb && \ - dpkg -i orchestrator_3.1.0_amd64.deb && \ - rm orchestrator_3.1.0_amd64.deb && \ + wget https://github.com/openark/orchestrator/releases/download/v${ORC_VER}/orchestrator_${ORC_VER}_amd64.deb && \ + dpkg -i orchestrator_${ORC_VER}_amd64.deb && \ + rm orchestrator_${ORC_VER}_amd64.deb && \ apt-get purge wget -qq && \ apt-get autoremove -qq && \ apt-get clean && \ diff --git a/docker/k8s/pmm-client/Dockerfile b/docker/k8s/pmm-client/Dockerfile index db5b1a92119..4fb986a8b7c 100644 --- a/docker/k8s/pmm-client/Dockerfile +++ b/docker/k8s/pmm-client/Dockerfile @@ -1,27 +1,29 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM vitess/k8s AS k8s +ARG VT_BASE_VER=latest +FROM vitess/k8s:${VT_BASE_VER} AS k8s -FROM debian:stretch-slim +FROM debian:buster-slim +ARG PMM_CLIENT_VER='1.17.4' RUN apt-get update && \ apt-get upgrade -qq && \ apt-get install procps wget ca-certificates -qq --no-install-recommends && \ - wget https://www.percona.com/redir/downloads/pmm-client/1.17.0/binary/debian/stretch/x86_64/pmm-client_1.17.0-1.stretch_amd64.deb && \ - dpkg -i pmm-client_1.17.0-1.stretch_amd64.deb && \ - rm pmm-client_1.17.0-1.stretch_amd64.deb && \ + wget https://www.percona.com/redir/downloads/pmm-client/${PMM_CLIENT_VER}/binary/debian/buster/x86_64/pmm-client_${PMM_CLIENT_VER}-1.buster_amd64.deb && \ + dpkg -i pmm-client_${PMM_CLIENT_VER}-1.buster_amd64.deb && \ + rm pmm-client_${PMM_CLIENT_VER}-1.buster_amd64.deb && \ apt-get purge wget ca-certificates -qq && \ apt-get autoremove -qq && \ apt-get clean && \ diff --git a/docker/k8s/vtbackup/Dockerfile b/docker/k8s/vtbackup/Dockerfile index 306911457a2..2768017e247 100644 --- a/docker/k8s/vtbackup/Dockerfile +++ b/docker/k8s/vtbackup/Dockerfile @@ -1,20 +1,22 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM vitess/k8s AS k8s +ARG VT_BASE_VER=latest -FROM debian:stretch-slim +FROM vitess/k8s:${VT_BASE_VER} AS k8s + +FROM debian:buster-slim # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTROOT /vt diff --git a/docker/k8s/vtctl/Dockerfile b/docker/k8s/vtctl/Dockerfile index b260f167dc4..2d9c717a9c5 100644 --- a/docker/k8s/vtctl/Dockerfile +++ b/docker/k8s/vtctl/Dockerfile @@ -1,20 +1,22 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM vitess/k8s AS k8s +ARG VT_BASE_VER=latest -FROM debian:stretch-slim +FROM vitess/k8s:${VT_BASE_VER} AS k8s + +FROM debian:buster-slim # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTROOT /vt @@ -32,4 +34,4 @@ COPY --from=k8s /vt/bin/vtctl /vt/bin/ RUN groupadd -r --gid 2000 vitess && \ useradd -r -g vitess --uid 1000 vitess && \ chown -R vitess:vitess /vt && \ - chown -R vitess:vitess /vtdataroot \ No newline at end of file + chown -R vitess:vitess /vtdataroot diff --git a/docker/k8s/vtctlclient/Dockerfile b/docker/k8s/vtctlclient/Dockerfile index 0cf4956691d..4f50a333ea9 100644 --- a/docker/k8s/vtctlclient/Dockerfile +++ b/docker/k8s/vtctlclient/Dockerfile @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM vitess/k8s AS k8s +ARG VT_BASE_VER=latest -FROM debian:stretch-slim +FROM vitess/k8s:${VT_BASE_VER} AS k8s + +FROM debian:buster-slim RUN apt-get update && \ apt-get upgrade -qq && \ diff --git a/docker/k8s/vtctld/Dockerfile b/docker/k8s/vtctld/Dockerfile index 8122e3f4e77..cc64990c05f 100644 --- a/docker/k8s/vtctld/Dockerfile +++ b/docker/k8s/vtctld/Dockerfile @@ -1,20 +1,22 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM vitess/k8s AS k8s +ARG VT_BASE_VER=latest -FROM debian:stretch-slim +FROM vitess/k8s:${VT_BASE_VER} AS k8s + +FROM debian:buster-slim # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTROOT /vt @@ -36,4 +38,4 @@ COPY --from=k8s /vt/web /vt/web RUN groupadd -r --gid 2000 vitess && \ useradd -r -g vitess --uid 1000 vitess && \ chown -R vitess:vitess /vt && \ - chown -R vitess:vitess /vtdataroot \ No newline at end of file + chown -R vitess:vitess /vtdataroot diff --git a/docker/k8s/vtexplain/Dockerfile b/docker/k8s/vtexplain/Dockerfile index 1dda593c113..0fcdf4599a2 100644 --- a/docker/k8s/vtexplain/Dockerfile +++ b/docker/k8s/vtexplain/Dockerfile @@ -1,11 +1,11 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,7 +14,7 @@ FROM vitess/base AS base -FROM debian:stretch-slim +FROM debian:buster-slim # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTROOT /vt diff --git a/docker/k8s/vtgate/Dockerfile b/docker/k8s/vtgate/Dockerfile index 2734e08f464..5f5bee0fade 100644 --- a/docker/k8s/vtgate/Dockerfile +++ b/docker/k8s/vtgate/Dockerfile @@ -1,20 +1,22 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM vitess/k8s AS k8s +ARG VT_BASE_VER=latest -FROM debian:stretch-slim +FROM vitess/k8s:${VT_BASE_VER} AS k8s + +FROM debian:buster-slim # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTROOT /vt @@ -32,4 +34,4 @@ COPY --from=k8s /vt/bin/vtgate /vt/bin/ RUN groupadd -r --gid 2000 vitess && \ useradd -r -g vitess --uid 1000 vitess && \ chown -R vitess:vitess /vt && \ - chown -R vitess:vitess /vtdataroot \ No newline at end of file + chown -R vitess:vitess /vtdataroot diff --git a/docker/k8s/vttablet/Dockerfile b/docker/k8s/vttablet/Dockerfile index bbc56cebe99..97d40a0b6e2 100644 --- a/docker/k8s/vttablet/Dockerfile +++ b/docker/k8s/vttablet/Dockerfile @@ -1,25 +1,27 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM vitess/k8s AS k8s +ARG VT_BASE_VER=latest -FROM debian:stretch-slim +FROM vitess/k8s:${VT_BASE_VER} AS k8s + +FROM debian:buster-slim # TODO: remove when https://github.com/vitessio/vitess/issues/3553 is fixed RUN apt-get update && \ apt-get upgrade -qq && \ - apt-get install wget mysql-client jq -qq --no-install-recommends && \ + apt-get install wget default-mysql-client jq -qq --no-install-recommends && \ apt-get autoremove && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/k8s/vtworker/Dockerfile b/docker/k8s/vtworker/Dockerfile index d43d00819bb..b5628b0d69a 100644 --- a/docker/k8s/vtworker/Dockerfile +++ b/docker/k8s/vtworker/Dockerfile @@ -1,20 +1,22 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM vitess/k8s AS k8s +ARG VT_BASE_VER=latest -FROM debian:stretch-slim +FROM vitess/k8s:${VT_BASE_VER} AS k8s + +FROM debian:buster-slim # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTROOT /vt @@ -32,4 +34,4 @@ COPY --from=k8s /vt/bin/vtworker /vt/bin/ RUN groupadd -r --gid 2000 vitess && \ useradd -r -g vitess --uid 1000 vitess && \ chown -R vitess:vitess /vt && \ - chown -R vitess:vitess /vtdataroot \ No newline at end of file + chown -R vitess:vitess /vtdataroot diff --git a/docker/lite/Dockerfile.alpine b/docker/lite/Dockerfile.alpine index 4082bb04b67..aac7caba428 100644 --- a/docker/lite/Dockerfile.alpine +++ b/docker/lite/Dockerfile.alpine @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:mariadb103 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -48,6 +48,7 @@ ENV MYSQL_FLAVOR MariaDB103 # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot diff --git a/docker/lite/Dockerfile.mariadb b/docker/lite/Dockerfile.mariadb index ce9c8cf4b4f..2299fa0866f 100644 --- a/docker/lite/Dockerfile.mariadb +++ b/docker/lite/Dockerfile.mariadb @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:mariadb AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -48,6 +48,7 @@ ENV MYSQL_FLAVOR MariaDB # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot diff --git a/docker/lite/Dockerfile.mariadb103 b/docker/lite/Dockerfile.mariadb103 index f0833f81f51..825035a66a6 100644 --- a/docker/lite/Dockerfile.mariadb103 +++ b/docker/lite/Dockerfile.mariadb103 @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:mariadb103 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -48,6 +48,7 @@ ENV MYSQL_FLAVOR MariaDB103 # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot diff --git a/docker/lite/Dockerfile.mysql56 b/docker/lite/Dockerfile.mysql56 index 7b78af0aa89..a8d5a76de68 100644 --- a/docker/lite/Dockerfile.mysql56 +++ b/docker/lite/Dockerfile.mysql56 @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:mysql56 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -47,6 +47,7 @@ ENV PATH $VTROOT/bin:$PATH # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot diff --git a/docker/lite/Dockerfile.mysql57 b/docker/lite/Dockerfile.mysql57 index 25535464af8..11013292b49 100644 --- a/docker/lite/Dockerfile.mysql57 +++ b/docker/lite/Dockerfile.mysql57 @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:mysql57 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -47,6 +47,7 @@ ENV PATH $VTROOT/bin:$PATH # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot diff --git a/docker/lite/Dockerfile.mysql80 b/docker/lite/Dockerfile.mysql80 index 4d0ad6d6ec8..99752bf11b7 100644 --- a/docker/lite/Dockerfile.mysql80 +++ b/docker/lite/Dockerfile.mysql80 @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:mysql80 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -48,6 +48,7 @@ ENV MYSQL_FLAVOR MySQL80 # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot diff --git a/docker/lite/Dockerfile.percona b/docker/lite/Dockerfile.percona index a3e77350672..b93f0dffe4a 100644 --- a/docker/lite/Dockerfile.percona +++ b/docker/lite/Dockerfile.percona @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:percona AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -47,6 +47,7 @@ ENV PATH $VTROOT/bin:$PATH # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot diff --git a/docker/lite/Dockerfile.percona57 b/docker/lite/Dockerfile.percona57 index 3ac6d264f8b..a1091434a88 100644 --- a/docker/lite/Dockerfile.percona57 +++ b/docker/lite/Dockerfile.percona57 @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:percona57 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -47,6 +47,7 @@ ENV PATH $VTROOT/bin:$PATH # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80 index 71f258d7e03..dfa875a51db 100644 --- a/docker/lite/Dockerfile.percona80 +++ b/docker/lite/Dockerfile.percona80 @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:percona80 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -48,6 +48,7 @@ ENV MYSQL_FLAVOR MySQL80 # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot diff --git a/docker/lite/Dockerfile.testing b/docker/lite/Dockerfile.testing index 5bcad5bcfbf..19214800013 100644 --- a/docker/lite/Dockerfile.testing +++ b/docker/lite/Dockerfile.testing @@ -1,11 +1,11 @@ # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:mysql57 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -47,6 +47,7 @@ ENV PATH $VTROOT/bin:$PATH # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot diff --git a/docker/lite/Dockerfile.ubi7.mysql57 b/docker/lite/Dockerfile.ubi7.mysql57 index d4dbe1df4f1..c8675f48382 100644 --- a/docker/lite/Dockerfile.ubi7.mysql57 +++ b/docker/lite/Dockerfile.ubi7.mysql57 @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:mysql57 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -49,9 +49,9 @@ RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && rm -f /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm RUN curl -L --retry-delay 10 --retry 3 -o /tmp/libev.rpm http://mirror.centos.org/centos/7/extras/x86_64/Packages/libev-4.15-7.el7.x86_64.rpm \ && curl -L --retry-delay 10 --retry 3 -o /tmp/gperf.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/gperftools-libs-2.6.1-1.el7.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/updates/x86_64/Packages/numactl-libs-2.0.12-3.el7_7.1.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/updates/x86_64/Packages/sysstat-10.1.5-18.el7_7.1.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.12-9.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/numactl-libs-2.0.12-5.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/sysstat-10.1.5-19.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-4.el7.x86_64.rpm \ && rpm -i --nodeps /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm \ && rm -f /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm RUN yum update \ @@ -81,6 +81,7 @@ ENV PATH $VTROOT/bin:$PATH # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator RUN mkdir -p /licenses COPY LICENSE /licenses diff --git a/docker/lite/Dockerfile.ubi7.mysql80 b/docker/lite/Dockerfile.ubi7.mysql80 index 9da03ca65a6..9c9be8c68f5 100644 --- a/docker/lite/Dockerfile.ubi7.mysql80 +++ b/docker/lite/Dockerfile.ubi7.mysql80 @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:mysql80 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -49,9 +49,9 @@ RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && rm -f /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm RUN curl -L --retry-delay 10 --retry 3 -o /tmp/libev.rpm http://mirror.centos.org/centos/7/extras/x86_64/Packages/libev-4.15-7.el7.x86_64.rpm \ && curl -L --retry-delay 10 --retry 3 -o /tmp/gperf.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/gperftools-libs-2.6.1-1.el7.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/updates/x86_64/Packages/numactl-libs-2.0.12-3.el7_7.1.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/updates/x86_64/Packages/sysstat-10.1.5-18.el7_7.1.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.12-9.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/numactl-libs-2.0.12-5.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/sysstat-10.1.5-19.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-4.el7.x86_64.rpm \ && rpm -i --nodeps /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm \ && rm -f /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm RUN yum update \ @@ -82,6 +82,7 @@ ENV MYSQL_FLAVOR MySQL80 # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator RUN mkdir -p /licenses COPY LICENSE /licenses diff --git a/docker/lite/Dockerfile.ubi7.percona57 b/docker/lite/Dockerfile.ubi7.percona57 index c05e8150cfc..a630262f006 100644 --- a/docker/lite/Dockerfile.ubi7.percona57 +++ b/docker/lite/Dockerfile.ubi7.percona57 @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:percona57 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -47,9 +47,9 @@ RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && rm -f /tmp/perconarepo.rpm RUN curl -L --retry-delay 10 --retry 3 -o /tmp/libev.rpm http://mirror.centos.org/centos/7/extras/x86_64/Packages/libev-4.15-7.el7.x86_64.rpm \ && curl -L --retry-delay 10 --retry 3 -o /tmp/gperf.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/gperftools-libs-2.6.1-1.el7.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/updates/x86_64/Packages/numactl-libs-2.0.12-3.el7_7.1.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/updates/x86_64/Packages/sysstat-10.1.5-18.el7_7.1.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.12-9.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/numactl-libs-2.0.12-5.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/sysstat-10.1.5-19.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-4.el7.x86_64.rpm \ && rpm -i --nodeps /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm \ && rm -f /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm RUN yum update \ @@ -72,6 +72,7 @@ ENV PATH $VTROOT/bin:$PATH # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator RUN mkdir -p /licenses COPY LICENSE /licenses diff --git a/docker/lite/Dockerfile.ubi7.percona80 b/docker/lite/Dockerfile.ubi7.percona80 index 828b0a79db6..6e993a85ebd 100644 --- a/docker/lite/Dockerfile.ubi7.percona80 +++ b/docker/lite/Dockerfile.ubi7.percona80 @@ -19,8 +19,8 @@ # Use a temporary layer for the build stage. FROM vitess/bootstrap:percona80 AS builder -# Allows some docker builds to disable CGO -ARG CGO_ENABLED=0 +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER # Re-copy sources from working tree. COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess @@ -47,9 +47,9 @@ RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ && rm -f /tmp/perconarepo.rpm RUN curl -L --retry-delay 10 --retry 3 -o /tmp/libev.rpm http://mirror.centos.org/centos/7/extras/x86_64/Packages/libev-4.15-7.el7.x86_64.rpm \ && curl -L --retry-delay 10 --retry 3 -o /tmp/gperf.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/gperftools-libs-2.6.1-1.el7.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/updates/x86_64/Packages/numactl-libs-2.0.12-3.el7_7.1.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/updates/x86_64/Packages/sysstat-10.1.5-18.el7_7.1.x86_64.rpm \ - && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.12-9.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/numactl-libs-2.0.12-5.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/sysstat-10.1.5-19.el7.x86_64.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-4.el7.x86_64.rpm \ && rpm -i --nodeps /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm \ && rm -f /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm RUN yum update \ @@ -77,6 +77,7 @@ ENV MYSQL_FLAVOR MySQL80 # Copy artifacts from builder layer. COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator RUN mkdir -p /licenses COPY LICENSE /licenses diff --git a/docker/lite/install_dependencies.sh b/docker/lite/install_dependencies.sh index 986d80df05d..dcc89a483cf 100755 --- a/docker/lite/install_dependencies.sh +++ b/docker/lite/install_dependencies.sh @@ -10,12 +10,22 @@ set -euo pipefail FLAVOR="$1" export DEBIAN_FRONTEND=noninteractive -retry() { - for i in $(seq 1 10); do - if "$@"; then return; fi +KEYSERVERS=( + ha.pool.sks-keyservers.net + keyserver.ubuntu.com + hkp://p80.pool.sks-keyservers.net:80 + hkp://keyserver.ubuntu.com:80 +) + +add_apt_key() { + for i in {1..3}; do + for keyserver in "${KEYSERVERS[@]}"; do + if apt-key adv --no-tty --keyserver "${keyserver}" --recv-keys "$1"; then return; fi + done done } + # Install base packages that are common to all flavors. BASE_PACKAGES=( bzip2 @@ -50,18 +60,38 @@ mysql56) ) ;; mysql57) + mysql57_version=5.7.31 + wget https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/libmysqlclient20_${mysql57_version}-1debian9_amd64.deb -O /tmp/libmysqlclient20_${mysql57_version}-1debian9_amd64.deb + wget https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-community-client_${mysql57_version}-1debian9_amd64.deb -O /tmp/mysql-community-client_${mysql57_version}-1debian9_amd64.deb + wget https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-client_${mysql57_version}-1debian9_amd64.deb -O /tmp/mysql-client_${mysql57_version}-1debian9_amd64.deb + wget https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-community-server_${mysql57_version}-1debian9_amd64.deb -O /tmp/mysql-community-server_${mysql57_version}-1debian9_amd64.deb + wget https://repo.mysql.com/apt/debian/pool/mysql-5.7/m/mysql-community/mysql-server_${mysql57_version}-1debian9_amd64.deb -O /tmp/mysql-server_${mysql57_version}-1debian9_amd64.deb PACKAGES=( - libmysqlclient20 - mysql-client - mysql-server + /tmp/libmysqlclient20_${mysql57_version}-1debian9_amd64.deb + /tmp/mysql-community-client_${mysql57_version}-1debian9_amd64.deb + /tmp/mysql-client_${mysql57_version}-1debian9_amd64.deb + /tmp/mysql-community-server_${mysql57_version}-1debian9_amd64.deb + /tmp/mysql-server_${mysql57_version}-1debian9_amd64.deb percona-xtrabackup-24 ) ;; mysql80) + mysql8_version=8.0.21 + wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/libmysqlclient21_${mysql8_version}-1debian9_amd64.deb -O /tmp/libmysqlclient21_${mysql8_version}-1debian9_amd64.deb + wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-client-core_${mysql8_version}-1debian9_amd64.deb -O /tmp/mysql-community-client-core_${mysql8_version}-1debian9_amd64.deb + wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-client_${mysql8_version}-1debian9_amd64.deb -O /tmp/mysql-community-client_${mysql8_version}-1debian9_amd64.deb + wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-client_${mysql8_version}-1debian9_amd64.deb -O /tmp/mysql-client_${mysql8_version}-1debian9_amd64.deb + wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-server-core_${mysql8_version}-1debian9_amd64.deb -O /tmp/mysql-community-server-core_${mysql8_version}-1debian9_amd64.deb + wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-community-server_${mysql8_version}-1debian9_amd64.deb -O /tmp/mysql-community-server_${mysql8_version}-1debian9_amd64.deb + wget https://repo.mysql.com/apt/debian/pool/mysql-8.0/m/mysql-community/mysql-server_${mysql8_version}-1debian9_amd64.deb -O /tmp/mysql-server_${mysql8_version}-1debian9_amd64.deb PACKAGES=( - libmysqlclient21 - mysql-client - mysql-server + /tmp/libmysqlclient21_${mysql8_version}-1debian9_amd64.deb + /tmp/mysql-community-client-core_${mysql8_version}-1debian9_amd64.deb + /tmp/mysql-community-client_${mysql8_version}-1debian9_amd64.deb + /tmp/mysql-client_${mysql8_version}-1debian9_amd64.deb + /tmp/mysql-community-server-core_${mysql8_version}-1debian9_amd64.deb + /tmp/mysql-community-server_${mysql8_version}-1debian9_amd64.deb + /tmp/mysql-server_${mysql8_version}-1debian9_amd64.deb percona-xtrabackup-80 ) ;; @@ -81,7 +111,7 @@ percona57) percona80) PACKAGES=( libperconaserverclient21 - percona-server-rocksdb + percona-server-rocksdb percona-server-server percona-server-tokudb percona-xtrabackup-80 @@ -102,16 +132,16 @@ esac case "${FLAVOR}" in mysql56|mysql57|mysql80) # repo.mysql.com - retry apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 8C718D3B5072E1F5 + add_apt_key 8C718D3B5072E1F5 ;; mariadb|mariadb103) # digitalocean.com - retry apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys F1656F24C74CD1D8 + add_apt_key F1656F24C74CD1D8 ;; esac # All flavors (except mariadb*) include Percona XtraBackup (from repo.percona.com). -retry apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 +add_apt_key 9334A25F8507EFA5 # Add extra apt repositories for MySQL. case "${FLAVOR}" in @@ -171,7 +201,8 @@ esac # Install flavor-specific packages apt-get update apt-get install -y --no-install-recommends "${PACKAGES[@]}" - - # Clean up files we won't need in the final image. - rm -rf /var/lib/apt/lists/* - rm -rf /var/lib/mysql/ + +# Clean up files we won't need in the final image. +rm -rf /var/lib/apt/lists/* +rm -rf /var/lib/mysql/ +rm -rf /tmp/*.deb diff --git a/docker/mini/vttablet-mini-up.sh b/docker/mini/vttablet-mini-up.sh index 6d31ccc9b9c..065f428ddb5 100755 --- a/docker/mini/vttablet-mini-up.sh +++ b/docker/mini/vttablet-mini-up.sh @@ -44,14 +44,12 @@ vttablet \ -log_dir $VTDATAROOT/tmp \ -log_queries_to_file $VTDATAROOT/tmp/$tablet_logfile \ -tablet-path $alias \ - -tablet_hostname "$tablet_hostname" \ + -tablet_hostname "$hostname" \ -init_db_name_override "$keyspace" \ -init_keyspace $keyspace \ -init_shard $shard \ -init_tablet_type $tablet_type \ -health_check_interval 5s \ - -heartbeat_enable \ - -heartbeat_interval 500ms \ -enable_semi_sync \ -enable_replication_reporter \ -backup_storage_implementation file \ diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index 412542d8cef..5337d87b2f9 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -434,6 +434,7 @@ func applyKeyspaceDependentPatches( if keyspaceData.shards < 2 { tabAlias = tabAlias + 100 dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap, opts) + dockerComposeFile = applyShardPatches(dockerComposeFile, tabAlias, shard, keyspaceData, opts) } else { // Determine shard range for i := 0; i < keyspaceData.shards; i++ { @@ -446,6 +447,7 @@ func applyKeyspaceDependentPatches( } tabAlias = tabAlias + 100 dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap, opts) + dockerComposeFile = applyShardPatches(dockerComposeFile, tabAlias, shard, keyspaceData, opts) } } @@ -475,6 +477,39 @@ func applyDockerComposePatches( return dockerComposeFile } +func applyShardPatches( + dockerComposeFile []byte, + tabAlias int, + shard string, + keyspaceData keyspaceInfo, + opts vtOptions, +) []byte { + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultShard(tabAlias, shard, keyspaceData, opts)) + return dockerComposeFile +} + +func generateDefaultShard(tabAlias int, shard string, keyspaceData keyspaceInfo, opts vtOptions) string { + aliases := []int{tabAlias + 1} // master alias, e.g. 201 + for i := 0; i < keyspaceData.replicaTablets; i++ { + aliases = append(aliases, tabAlias+2+i) // replica aliases, e.g. 202, 203, ... + } + tabletDepends := make([]string, len(aliases)) + for i, tabletId := range aliases { + tabletDepends[i] = fmt.Sprintf("vttablet%d: {condition : service_healthy}", tabletId) + } + // Wait on all shard tablets to be healthy + dependsOn := "depends_on: {" + strings.Join(tabletDepends, ", ") + "}" + + return fmt.Sprintf(` +- op: add + path: /services/init_shard_master%[2]d + value: + image: vitess/base + command: ["sh", "-c", "$$VTROOT/bin/vtctl %[5]s InitShardMaster -force %[4]s/%[3]s %[6]s-%[2]d "] + %[1]s +`, dependsOn, aliases[0], shard, keyspaceData.keyspace, opts.topologyFlags, opts.cell) +} + func applyTabletPatches( dockerComposeFile []byte, tabAlias int, @@ -674,7 +709,7 @@ func generatePrimaryVIndex(tableName, column, name string) string { return fmt.Sprintf(` [{"op": "add", "path": "/tables/%[1]s", -"value": +"value": {"column_vindexes": [ { "column": "%[2]s", diff --git a/examples/compose/vttablet-up.sh b/examples/compose/vttablet-up.sh index 7284fc37d7e..cfc9b89b44d 100755 --- a/examples/compose/vttablet-up.sh +++ b/examples/compose/vttablet-up.sh @@ -1,13 +1,13 @@ #!/bin/bash # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -120,8 +120,6 @@ $VTROOT/bin/vtctl $TOPOLOGY_FLAGS AddCellInfo -root vitess/$CELL -server_address $VTROOT/bin/vtctl $TOPOLOGY_FLAGS CreateKeyspace $keyspace || true $VTROOT/bin/vtctl $TOPOLOGY_FLAGS CreateShard $keyspace/$shard || true -$VTROOT/bin/vtctl $TOPOLOGY_FLAGS InitTablet -shard $shard -keyspace $keyspace -grpc_port $grpc_port -port $web_port -allow_master_override $alias $tablet_role - #Populate external db conditional args if [ "$external" = "1" ]; then if [ $role = "master" ]; then @@ -166,6 +164,8 @@ exec $VTROOT/bin/vttablet \ -health_check_interval 5s \ -enable_semi_sync \ -enable_replication_reporter \ + -heartbeat_enable \ + -heartbeat_interval 250ms \ -port $web_port \ -grpc_port $grpc_port \ -binlog_use_v3_resharding_mode=true \ diff --git a/examples/local/orc_test.sh b/examples/local/orc_test.sh new file mode 100755 index 00000000000..0817a470770 --- /dev/null +++ b/examples/local/orc_test.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script brings up zookeeper and all the vitess components +# required for a single shard deployment. + +source ./env.sh + +# start topo server +if [ "${TOPO}" = "zk2" ]; then + CELL=zone1 ./scripts/zk-up.sh +elif [ "${TOPO}" = "k8s" ]; then + CELL=zone1 ./scripts/k3s-up.sh +else + CELL=zone1 ./scripts/etcd-up.sh +fi + +# start vtctld +CELL=zone1 ./scripts/vtctld-up.sh + +vtctlclient AddCellInfo -root /vitess/zone2 -server_address "${ETCD_SERVER}" zone2 + +# start vttablets for keyspace commerce +for i in 100 101 102; do + CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh + CELL=zone1 KEYSPACE=commerce TABLET_UID=$i ./scripts/ovttablet-up.sh +done + +for i in 110 111; do + CELL=zone2 TABLET_UID=$i ./scripts/mysqlctl-up.sh + CELL=zone2 KEYSPACE=commerce TABLET_UID=$i ./scripts/ovttablet-up.sh +done + +# set one of the replicas to master +#vtctlclient InitShardMaster -force commerce/0 zone1-100 + +# create the schema +#vtctlclient ApplySchema -sql-file create_commerce_schema.sql commerce + +# create the vschema +#vtctlclient ApplyVSchema -vschema_file vschema_commerce_initial.json commerce + +# start vtgate +CELL=zone1 ./scripts/vtgate-up.sh diff --git a/examples/local/scripts/ovttablet-up.sh b/examples/local/scripts/ovttablet-up.sh new file mode 100755 index 00000000000..ad53b840034 --- /dev/null +++ b/examples/local/scripts/ovttablet-up.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ./env.sh + +cell=${CELL:-'test'} +keyspace=${KEYSPACE:-'test_keyspace'} +shard=${SHARD:-'0'} +uid=$TABLET_UID +mysql_port=$[17000 + $uid] +port=$[15000 + $uid] +grpc_port=$[16000 + $uid] +printf -v alias '%s-%010d' $cell $uid +printf -v tablet_dir 'vt_%010d' $uid +tablet_hostname='' +printf -v tablet_logfile 'vttablet_%010d_querylog.txt' $uid + +tablet_type=replica +if [[ "${uid: -1}" -gt 1 ]]; then + tablet_type=rdonly +fi + +echo "Starting vttablet for $alias..." +# shellcheck disable=SC2086 +vttablet \ + $TOPOLOGY_FLAGS \ + -log_dir $VTDATAROOT/tmp \ + -log_queries_to_file $VTDATAROOT/tmp/$tablet_logfile \ + -tablet-path $alias \ + -tablet_hostname "$tablet_hostname" \ + -init_keyspace $keyspace \ + -init_shard $shard \ + -init_tablet_type $tablet_type \ + -health_check_interval 5s \ + -enable_replication_reporter \ + -backup_storage_implementation file \ + -file_backup_storage_root $VTDATAROOT/backups \ + -restore_from_backup \ + -port $port \ + -grpc_port $grpc_port \ + -service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \ + -pid_file $VTDATAROOT/$tablet_dir/vttablet.pid \ + -vtctld_addr http://$hostname:$vtctld_web_port/ \ + -disable_active_reparents \ + > $VTDATAROOT/$tablet_dir/vttablet.out 2>&1 & + +# Block waiting for the tablet to be listening +# Not the same as healthy + +for i in $(seq 0 300); do + curl -I "http://$hostname:$port/debug/status" >/dev/null 2>&1 && break + sleep 0.1 +done + +# check one last time +curl -I "http://$hostname:$port/debug/status" || fail "tablet could not be started!" diff --git a/examples/operator/orc_cluster.yaml b/examples/operator/orc_cluster.yaml new file mode 100644 index 00000000000..1cff45c99d2 --- /dev/null +++ b/examples/operator/orc_cluster.yaml @@ -0,0 +1,212 @@ +# The following example is minimalist. The security policies +# and resource specifications are not meant to be used in production. +# Please refer to the operator documentation for recommendations on +# production settings. +apiVersion: planetscale.com/v2 +kind: VitessCluster +metadata: + name: example +spec: + images: + vtctld: vitess/orctest:v1 + orchestrator: vitess/orctest:v1 + vtgate: vitess/orctest:v1 + vttablet: vitess/orctest:v1 + vtbackup: vitess/orctest:v1 + mysqld: + mysql56Compatible: vitess/lite:v6.0.20-20200429 + mysqldExporter: prom/mysqld-exporter:v0.11.0 + cells: + - name: zone1 + gateway: + authentication: + static: + secret: + name: example-cluster-config + key: users.json + replicas: 1 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 256Mi + vitessDashboard: + cells: + - zone1 + extraFlags: + security_policy: read-only + replicas: 1 + resources: + limits: + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + vitessOrchestrator: + cells: + - zone1 + configSecret: + name: example-cluster-config + key: orc_config.json + replicas: 1 + resources: + limits: + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + + keyspaces: + - name: commerce + turndownPolicy: Immediate + partitionings: + - equal: + parts: 1 + shardTemplate: + databaseInitScriptSecret: + name: example-cluster-config + key: init_db.sql + replication: + enforceSemiSync: false + tabletPools: + - cell: zone1 + type: replica + replicas: 2 + vttablet: + extraFlags: + db_charset: utf8mb4 + disable_active_reparents: "true" + resources: + requests: + cpu: 100m + memory: 256Mi + mysqld: + resources: + requests: + cpu: 100m + memory: 256Mi + dataVolumeClaimTemplate: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi + updateStrategy: + type: Immediate +--- +apiVersion: v1 +kind: Secret +metadata: + name: example-cluster-config +type: Opaque +stringData: + users.json: | + { + "user": [{ + "UserData": "user", + "Password": "" + }] + } + init_db.sql: | + # This file is executed immediately after mysql_install_db, + # to initialize a fresh data directory. + + ############################################################################### + # Equivalent of mysql_secure_installation + ############################################################################### + + # Changes during the init db should not make it to the binlog. + # They could potentially create errant transactions on replicas. + SET sql_log_bin = 0; + # Remove anonymous users. + DELETE FROM mysql.user WHERE User = ''; + + # Disable remote root access (only allow UNIX socket). + DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost'; + + # Remove test database. + DROP DATABASE IF EXISTS test; + + ############################################################################### + # Vitess defaults + ############################################################################### + + # Vitess-internal database. + CREATE DATABASE IF NOT EXISTS _vt; + # Note that definitions of local_metadata and shard_metadata should be the same + # as in production which is defined in go/vt/mysqlctl/metadata_tables.go. + CREATE TABLE IF NOT EXISTS _vt.local_metadata ( + name VARCHAR(255) NOT NULL, + value VARCHAR(255) NOT NULL, + db_name VARBINARY(255) NOT NULL, + PRIMARY KEY (db_name, name) + ) ENGINE=InnoDB; + CREATE TABLE IF NOT EXISTS _vt.shard_metadata ( + name VARCHAR(255) NOT NULL, + value MEDIUMBLOB NOT NULL, + db_name VARBINARY(255) NOT NULL, + PRIMARY KEY (db_name, name) + ) ENGINE=InnoDB; + + # Admin user with all privileges. + CREATE USER 'vt_dba'@'localhost'; + GRANT ALL ON *.* TO 'vt_dba'@'localhost'; + GRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost'; + + # User for app traffic, with global read-write access. + CREATE USER 'vt_app'@'localhost'; + GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, + REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES, + LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW, + SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER + ON *.* TO 'vt_app'@'localhost'; + + # User for app debug traffic, with global read access. + CREATE USER 'vt_appdebug'@'localhost'; + GRANT SELECT, SHOW DATABASES, PROCESS ON *.* TO 'vt_appdebug'@'localhost'; + + # User for administrative operations that need to be executed as non-SUPER. + # Same permissions as vt_app here. + CREATE USER 'vt_allprivs'@'localhost'; + GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, + REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES, + LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW, + SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER + ON *.* TO 'vt_allprivs'@'localhost'; + + # User for slave replication connections. + # TODO: Should we set a password on this since it allows remote connections? + CREATE USER 'vt_repl'@'%'; + GRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%'; + + # User for Vitess filtered replication (binlog player). + # Same permissions as vt_app. + CREATE USER 'vt_filtered'@'localhost'; + GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, + REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES, + LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW, + SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER + ON *.* TO 'vt_filtered'@'localhost'; + + # User for Orchestrator (https://github.com/openark/orchestrator). + # TODO: Reenable when the password is randomly generated. + CREATE USER 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_user_password'; + GRANT SUPER, PROCESS, REPLICATION SLAVE, RELOAD + ON *.* TO 'orc_client_user'@'%'; + GRANT SELECT + ON _vt.* TO 'orc_client_user'@'%'; + + FLUSH PRIVILEGES; + + RESET SLAVE ALL; + RESET MASTER; + orc_config.json: | + { + "Debug": true, + "Durability": "none", + "MySQLTopologyUser": "orc_client_user", + "MySQLTopologyPassword": "orc_client_user_password", + "MySQLReplicaUser": "vt_repl", + "MySQLReplicaPassword": "", + "RecoveryPeriodBlockSeconds": 5 + } diff --git a/examples/operator/orcpf.sh b/examples/operator/orcpf.sh new file mode 100755 index 00000000000..692cab3e74e --- /dev/null +++ b/examples/operator/orcpf.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +kubectl port-forward --address localhost "$(kubectl get service --selector="planetscale.com/component=vtctld" -o name | head -n1)" 15000 15999 & +process_id1=$! +kubectl port-forward --address localhost "$(kubectl get service --selector="planetscale.com/component=vtgate,!planetscale.com/cell" -o name | head -n1)" 15306:3306 & +process_id2=$! +kubectl port-forward --address localhost "$(kubectl get service --selector="planetscale.com/component=orchestrator" -o name | head -n1)" 3000 & +process_id3=$! +sleep 2 +echo "You may point your browser to http://localhost:15000 for vtctld, http://localhost:3000 for orchestrator, and use the following aliases as shortcuts:" +echo 'alias vtctlclient="vtctlclient -server=localhost:15999 -logtostderr"' +echo 'alias mysql="mysql -h 127.0.0.1 -P 15306 -u user"' +echo "Hit Ctrl-C to stop the port forwards" +wait $process_id1 +wait $process_id2 +wait $process_id3 diff --git a/examples/region_sharding/101_initial_cluster.sh b/examples/region_sharding/101_initial_cluster.sh index 46ed082886c..cf06cff0867 100755 --- a/examples/region_sharding/101_initial_cluster.sh +++ b/examples/region_sharding/101_initial_cluster.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# this script brings up zookeeper and all the vitess components +# this script brings up topo server and all the vitess components # required for a single shard deployment. source ./env.sh @@ -31,27 +31,17 @@ fi # start vtctld CELL=zone1 ./scripts/vtctld-up.sh -# start vttablets for main keyspace. we start only one tablet each (master) +# start unsharded keyspace and tablet CELL=zone1 TABLET_UID=100 ./scripts/mysqlctl-up.sh -SHARD=-40 CELL=zone1 KEYSPACE=main TABLET_UID=100 ./scripts/vttablet-up.sh -CELL=zone1 TABLET_UID=200 ./scripts/mysqlctl-up.sh -SHARD=40-80 CELL=zone1 KEYSPACE=main TABLET_UID=200 ./scripts/vttablet-up.sh -CELL=zone1 TABLET_UID=300 ./scripts/mysqlctl-up.sh -SHARD=80-c0 CELL=zone1 KEYSPACE=main TABLET_UID=300 ./scripts/vttablet-up.sh -CELL=zone1 TABLET_UID=400 ./scripts/mysqlctl-up.sh -SHARD=c0- CELL=zone1 KEYSPACE=main TABLET_UID=400 ./scripts/vttablet-up.sh - -# set master -vtctlclient InitShardMaster -force main/-40 zone1-100 -vtctlclient InitShardMaster -force main/40-80 zone1-200 -vtctlclient InitShardMaster -force main/80-c0 zone1-300 -vtctlclient InitShardMaster -force main/c0- zone1-400 +SHARD=0 CELL=zone1 KEYSPACE=main TABLET_UID=100 ./scripts/vttablet-up.sh + +vtctlclient InitShardMaster -force main/0 zone1-100 # create the schema vtctlclient ApplySchema -sql-file create_main_schema.sql main # create the vschema -vtctlclient ApplyVSchema -vschema_file main_vschema.json main +vtctlclient ApplyVSchema -vschema_file main_vschema_initial.json main # start vtgate CELL=zone1 ./scripts/vtgate-up.sh diff --git a/examples/region_sharding/201_main_sharded.sh b/examples/region_sharding/201_main_sharded.sh new file mode 100755 index 00000000000..2b59971d9d9 --- /dev/null +++ b/examples/region_sharding/201_main_sharded.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ./env.sh + +# apply sharding vschema +vtctlclient ApplyVSchema -vschema_file main_vschema_sharded.json main + +# optional: create the schema needed for lookup vindex +#vtctlclient ApplySchema -sql-file create_lookup_schema.sql main + +# create the lookup vindex +vtctlclient CreateLookupVindex -tablet_types=MASTER main "$(cat lookup_vindex.json)" + +# we have to wait for replication to catch up +# Can see on vttablet status page Vreplication that copy is complete +sleep 5 + +#externalize vindex +vtctlclient ExternalizeVindex main.customer_region_lookup diff --git a/examples/region_sharding/202_new_tablets.sh b/examples/region_sharding/202_new_tablets.sh new file mode 100755 index 00000000000..5e385ab622e --- /dev/null +++ b/examples/region_sharding/202_new_tablets.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ./env.sh + +# start vttablets for new shards. we start only one tablet each (master) +CELL=zone1 TABLET_UID=200 ./scripts/mysqlctl-up.sh +SHARD=-40 CELL=zone1 KEYSPACE=main TABLET_UID=200 ./scripts/vttablet-up.sh +CELL=zone1 TABLET_UID=300 ./scripts/mysqlctl-up.sh +SHARD=40-80 CELL=zone1 KEYSPACE=main TABLET_UID=300 ./scripts/vttablet-up.sh +CELL=zone1 TABLET_UID=400 ./scripts/mysqlctl-up.sh +SHARD=80-c0 CELL=zone1 KEYSPACE=main TABLET_UID=400 ./scripts/vttablet-up.sh +CELL=zone1 TABLET_UID=500 ./scripts/mysqlctl-up.sh +SHARD=c0- CELL=zone1 KEYSPACE=main TABLET_UID=500 ./scripts/vttablet-up.sh + +# set master +vtctlclient InitShardMaster -force main/-40 zone1-200 +vtctlclient InitShardMaster -force main/40-80 zone1-300 +vtctlclient InitShardMaster -force main/80-c0 zone1-400 +vtctlclient InitShardMaster -force main/c0- zone1-500 diff --git a/examples/region_sharding/203_reshard.sh b/examples/region_sharding/203_reshard.sh new file mode 100755 index 00000000000..b868885aa39 --- /dev/null +++ b/examples/region_sharding/203_reshard.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ./env.sh + +vtctlclient Reshard -tablet_types=MASTER main.main2regions '0' '-40,40-80,80-c0,c0-' diff --git a/examples/region_sharding/204_switch_reads.sh b/examples/region_sharding/204_switch_reads.sh new file mode 100755 index 00000000000..8ff123cf8e0 --- /dev/null +++ b/examples/region_sharding/204_switch_reads.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script migrates traffic for the rdonly and replica tablets + +source ./env.sh + +vtctlclient SwitchReads -tablet_type=rdonly main.main2regions +vtctlclient SwitchReads -tablet_type=replica main.main2regions diff --git a/examples/region_sharding/205_switch_writes.sh b/examples/region_sharding/205_switch_writes.sh new file mode 100755 index 00000000000..a7f22f430c4 --- /dev/null +++ b/examples/region_sharding/205_switch_writes.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2020 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script migrates traffic for the master tablet + +source ./env.sh + +vtctlclient SwitchWrites main.main2regions + +# to go back to unsharded +# call SwitchReads and SwitchWrites with workflow main.main2regions_reverse +# delete vreplication rows from sharded tablets +# drop all the tables +# change vschema back to unsharded +# drop lookup table + diff --git a/examples/region_sharding/206_down_shard_0.sh b/examples/region_sharding/206_down_shard_0.sh new file mode 100755 index 00000000000..d302a071a46 --- /dev/null +++ b/examples/region_sharding/206_down_shard_0.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# this script brings down the tablets for main/0 keyspace/shard + +source ./env.sh + +CELL=zone1 TABLET_UID=100 ./scripts/vttablet-down.sh +CELL=zone1 TABLET_UID=100 ./scripts/mysqlctl-down.sh diff --git a/examples/region_sharding/207_delete_shard_0.sh b/examples/region_sharding/207_delete_shard_0.sh new file mode 100755 index 00000000000..b626dcbf1ac --- /dev/null +++ b/examples/region_sharding/207_delete_shard_0.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2019 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script deletes the old shard 0 which has been replaced by 4 shards + +source ./env.sh + +vtctlclient DeleteShard -recursive main/0 diff --git a/examples/region_sharding/201_teardown.sh b/examples/region_sharding/301_teardown.sh similarity index 96% rename from examples/region_sharding/201_teardown.sh rename to examples/region_sharding/301_teardown.sh index 4f8e869ffeb..1c7d7f304d2 100755 --- a/examples/region_sharding/201_teardown.sh +++ b/examples/region_sharding/301_teardown.sh @@ -21,7 +21,7 @@ source ./env.sh ./scripts/vtgate-down.sh -for tablet in 100 200 300 400; do +for tablet in 200 300 400 500; do if vtctlclient -server localhost:15999 GetTablet zone1-$tablet >/dev/null 2>&1; then echo "Shutting down tablet zone1-$tablet" CELL=zone1 TABLET_UID=$tablet ./scripts/vttablet-down.sh @@ -45,7 +45,7 @@ fi if [ ! -z "$VTDATAROOT" ]; then if pgrep -f -l "$VTDATAROOT" >/dev/null; then - echo "ERROR: Stale processes detected! It is recommended to manuallly kill them:" + echo "ERROR: Stale processes detected! It is recommended to manually kill them:" pgrep -f -l "$VTDATAROOT" else echo "All good! It looks like every process has shut down" diff --git a/examples/region_sharding/README.md b/examples/region_sharding/README.md index 6392463553e..21c53f1b497 100644 --- a/examples/region_sharding/README.md +++ b/examples/region_sharding/README.md @@ -10,13 +10,45 @@ This document contains the summary of the commands to be run. "region_map": "/home/user/vitess/examples/region_sharding/countries.json", -# Bring up initial cluster and main keyspace +# Bring up initial cluster and main keyspace (unsharded) ./101_initial_cluster.sh +# setup aliases +alias mysql="command mysql -h 127.0.0.1 -P 15306" +alias vtctlclient="command vtctlclient -server localhost:15999 -log_dir ${VTDATAROOT}/tmp -alsologtostderr" + # Insert and verify data mysql < insert_customers.sql +mysql --table < show_initial_data.sql + +# create schema and vschema for sharding (+lookup vindex) +./201_main_sharded.sh + +# bring up shards and tablets +./202_new_tablets.sh + +# reshard +./203_reshard.sh + +# SwitchReads +./204_switch_reads.sh + +# run script to create traffic before switching writes +#./client.sh + +# SwitchWrites +./205_switch_writes.sh +# show no / minimal write errors during switch + +# verify sharded data mysql --table < show_data.sql +# down shard +./206_down_shard_0.sh + +# delete shard 0 +./207_delete_shard_0.sh + # Down cluster -./201_teardown.sh +./301_teardown.sh ``` diff --git a/examples/region_sharding/create_lookup_schema.sql b/examples/region_sharding/create_lookup_schema.sql new file mode 100644 index 00000000000..550a1e399e6 --- /dev/null +++ b/examples/region_sharding/create_lookup_schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE customer_lookup ( + id int NOT NULL, + keyspace_id varbinary(128), + primary key(id) + ); diff --git a/examples/region_sharding/create_main_schema.sql b/examples/region_sharding/create_main_schema.sql index 2e41cb35e54..6a26ce660e8 100644 --- a/examples/region_sharding/create_main_schema.sql +++ b/examples/region_sharding/create_main_schema.sql @@ -5,8 +5,3 @@ CREATE TABLE customer ( country varbinary(256), primary key(id) ); -CREATE TABLE customer_lookup ( - id int NOT NULL, - keyspace_id varbinary(256), - primary key(id) - ); diff --git a/examples/region_sharding/lookup_vindex.json b/examples/region_sharding/lookup_vindex.json new file mode 100644 index 00000000000..17f3b4471c9 --- /dev/null +++ b/examples/region_sharding/lookup_vindex.json @@ -0,0 +1,24 @@ +{ + "sharded": true, + "vindexes": { + "customer_region_lookup": { + "type": "consistent_lookup_unique", + "params": { + "table": "main.customer_lookup", + "from": "id", + "to": "keyspace_id" + }, + "owner": "customer" + } + }, + "tables": { + "customer": { + "column_vindexes": [ + { + "column": "id", + "name": "customer_region_lookup" + } + ] + } + } +} diff --git a/examples/region_sharding/main_vschema.json b/examples/region_sharding/main_vschema.json deleted file mode 100644 index 48a8f93dce4..00000000000 --- a/examples/region_sharding/main_vschema.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "sharded": true, - "vindexes": { - "region_vdx": { - "type": "region_json", - "params": { - "region_map": "/home/user/my-vitess/examples/region_sharding/countries.json", - "region_bytes": "1" - } - }, - "customer_region_lookup": { - "type": "consistent_lookup_unique", - "params": { - "table": "customer_lookup", - "from": "id", - "to": "keyspace_id" - }, - "owner": "customer" - }, - "identity": { - "type": "binary" - } - }, - "tables": { - "customer_lookup": { - "column_vindexes": [ - { - "column": "keyspace_id", - "name": "identity" - } - ] - }, - "customer": { - "column_vindexes": [ - { - "columns": ["id", "country"], - "name": "region_vdx" - }, - { - "column": "id", - "name": "customer_region_lookup" - } - ] - } - } -} diff --git a/examples/region_sharding/main_vschema_initial.json b/examples/region_sharding/main_vschema_initial.json new file mode 100644 index 00000000000..b8c11210f8d --- /dev/null +++ b/examples/region_sharding/main_vschema_initial.json @@ -0,0 +1,5 @@ +{ + "tables": { + "customer": {} + } +} \ No newline at end of file diff --git a/examples/region_sharding/main_vschema_sharded.json b/examples/region_sharding/main_vschema_sharded.json new file mode 100644 index 00000000000..94770e57ea7 --- /dev/null +++ b/examples/region_sharding/main_vschema_sharded.json @@ -0,0 +1,22 @@ +{ + "sharded": true, + "vindexes": { + "region_vdx": { + "type": "region_json", + "params": { + "region_map": "/home/user/my-vitess/examples/region_sharding/countries.json", + "region_bytes": "1" + } + } + }, + "tables": { + "customer": { + "column_vindexes": [ + { + "columns": ["id", "country"], + "name": "region_vdx" + } + ] + } + } +} diff --git a/examples/region_sharding/show_initial_data.sql b/examples/region_sharding/show_initial_data.sql new file mode 100644 index 00000000000..c85ce491404 --- /dev/null +++ b/examples/region_sharding/show_initial_data.sql @@ -0,0 +1,2 @@ +use main; +select * from customer; diff --git a/go.mod b/go.mod index 1ff58dc17cc..f1b2d8186d2 100644 --- a/go.mod +++ b/go.mod @@ -8,37 +8,43 @@ require ( github.com/Azure/go-autorest/autorest v0.10.0 // indirect github.com/GeertJohan/go.rice v1.0.0 github.com/PuerkitoBio/goquery v1.5.1 - github.com/TylerBrock/colorjson v0.0.0-20180527164720-95ec53f28296 - github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect + github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 + github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 github.com/aws/aws-sdk-go v1.28.8 github.com/buger/jsonparser v0.0.0-20200322175846-f7e751efca13 github.com/cespare/xxhash/v2 v2.1.1 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect + github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect github.com/coreos/bbolt v1.3.2 // indirect github.com/coreos/etcd v3.3.10+incompatible github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/corpix/uarand v0.1.1 // indirect + github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 github.com/evanphx/json-patch v4.5.0+incompatible + github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab github.com/go-sql-driver/mysql v1.5.0 - github.com/gogo/protobuf v1.3.1 + github.com/gogo/protobuf v1.3.1 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/mock v1.3.1 github.com/golang/protobuf v1.3.2 github.com/golang/snappy v0.0.1 github.com/google/go-cmp v0.4.0 + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 + github.com/google/uuid v1.1.1 github.com/googleapis/gnostic v0.2.0 // indirect github.com/gorilla/websocket v1.4.0 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/hashicorp/consul/api v1.5.0 github.com/hashicorp/go-immutable-radix v1.1.0 // indirect - github.com/hashicorp/go-msgpack v0.5.5 // indirect + github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/go-sockaddr v1.0.2 // indirect github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/hashicorp/serf v0.9.2 // indirect + github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428 github.com/imdario/mergo v0.3.6 // indirect github.com/klauspost/compress v1.4.1 // indirect @@ -47,25 +53,33 @@ require ( github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/krishicks/yaml-patch v0.0.10 github.com/magiconair/properties v1.8.1 - github.com/manifoldco/promptui v0.7.0 + github.com/martini-contrib/auth v0.0.0-20150219114609-fa62c19b7ae8 + github.com/martini-contrib/gzip v0.0.0-20151124214156-6c035326b43f + github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11 + github.com/mattn/go-sqlite3 v1.14.0 github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 github.com/mitchellh/go-ps v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.14.0 // indirect github.com/mitchellh/mapstructure v1.2.3 // indirect + github.com/montanaflynn/stats v0.6.3 github.com/olekukonko/tablewriter v0.0.5-0.20200416053754-163badb3bac6 github.com/onsi/ginkgo v1.10.3 // indirect github.com/onsi/gomega v1.7.1 // indirect github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 github.com/opentracing/opentracing-go v1.1.0 + github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.0 github.com/philhofer/fwd v1.0.0 // indirect github.com/pires/go-proxyproto v0.0.0-20191211124218-517ecdf5bb2b github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.4.1 github.com/prometheus/common v0.9.1 + github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 + github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e github.com/satori/go.uuid v1.2.0 // indirect + github.com/sjmudd/stopwatch v0.0.0-20170613150411-f380bf8a9be1 github.com/smartystreets/goconvey v1.6.4 // indirect - github.com/spf13/cobra v0.0.5 github.com/stretchr/testify v1.4.0 github.com/tchap/go-patricia v0.0.0-20160729071656-dd168db6051b github.com/tebeka/selenium v0.9.9 @@ -78,7 +92,7 @@ require ( github.com/z-division/go-zookeeper v0.0.0-20190128072838-6d7457066b9b golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 golang.org/x/lint v0.0.0-20190409202823-959b441ac422 - golang.org/x/net v0.0.0-20200202094626-16171245cfb2 + golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/text v0.3.2 @@ -89,12 +103,15 @@ require ( google.golang.org/grpc v1.24.0 gopkg.in/DataDog/dd-trace-go.v1 v1.17.0 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect + gopkg.in/gcfg.v1 v1.2.3 gopkg.in/ini.v1 v1.51.0 // indirect gopkg.in/ldap.v2 v2.5.0 + gopkg.in/warnings.v0 v0.1.2 // indirect gotest.tools v2.2.0+incompatible honnef.co/go/tools v0.0.1-2019.2.3 k8s.io/apiextensions-apiserver v0.17.3 k8s.io/apimachinery v0.17.3 k8s.io/client-go v0.17.3 + k8s.io/utils v0.0.0-20191114184206-e782cd3c129f sigs.k8s.io/yaml v1.1.0 ) diff --git a/go.sum b/go.sum index d017a2f9d3b..c700b4a7578 100644 --- a/go.sum +++ b/go.sum @@ -54,8 +54,6 @@ github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/TylerBrock/colorjson v0.0.0-20180527164720-95ec53f28296 h1:JYWTroLXcNzSCgu66NMgdjwoMHQRbv2SoOVNFb4kRkE= -github.com/TylerBrock/colorjson v0.0.0-20180527164720-95ec53f28296/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= @@ -70,6 +68,7 @@ github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5z github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= @@ -97,10 +96,6 @@ github.com/buger/jsonparser v0.0.0-20200322175846-f7e751efca13 h1:+qUNY4VRkEH46b github.com/buger/jsonparser v0.0.0-20200322175846-f7e751efca13/go.mod h1:tgcrVJ81GPSF0mz+0nu1Xaz0fazGPrmmJfJtxjbHhUQ= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible h1:C29Ae4G5GtYyYMm1aztcyj/J5ckgJm2zwdDajFbx1NY= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj7ug5D7I/orNUA= @@ -110,6 +105,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= @@ -132,6 +129,8 @@ github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 h1:M5QgkYacWj0Xs8MhpIK/5uwU02icXpEoSo9sM2aRCps= +github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432/go.mod h1:xwIwAxMvYnVrGJPe2FKx5prTrnAjGOD8zvDOnxnrrkM= github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY= github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -168,6 +167,8 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab h1:xveKWz2iaueeTaUgdetzel+U7exyigDYBryyVfV/rZk= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -256,6 +257,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= @@ -334,6 +337,8 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/serf v0.9.0/go.mod h1:YL0HO+FifKOW2u1ke99DGVu1zhcpZzNwrLIqBC7vbYU= github.com/hashicorp/serf v0.9.2 h1:yJoyfZXo4Pk2p/M/viW+YLibBFiIbKoP79gu7kDAFP0= github.com/hashicorp/serf v0.9.2/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c h1:aY2hhxLhjEAbfXOx2nRJxCXezC6CO2V/yN+OCr1srtk= +github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428 h1:Mo9W14pwbO9VfRe+ygqZ8dFbPpoIK1HFrG/zjTuQ+nc= @@ -360,8 +365,6 @@ github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpR github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU= -github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -388,8 +391,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/krishicks/yaml-patch v0.0.10 h1:H4FcHpnNwVmw8u0MjPRjWyIXtco6zM2F78t+57oNM3E= github.com/krishicks/yaml-patch v0.0.10/go.mod h1:Sm5TchwZS6sm7RJoyg87tzxm2ZcKzdRE4Q7TjNhPrME= -github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw= -github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= @@ -400,8 +401,12 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/manifoldco/promptui v0.7.0 h1:3l11YT8tm9MnwGFQ4kETwkzpAwY2Jt9lCrumCUW4+z4= -github.com/manifoldco/promptui v0.7.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ= +github.com/martini-contrib/auth v0.0.0-20150219114609-fa62c19b7ae8 h1:1ded5x5QpCLsyTH5ct62Rh1RXPFnn0/dubCqAeh+stU= +github.com/martini-contrib/auth v0.0.0-20150219114609-fa62c19b7ae8/go.mod h1:ahTFgV/NtzY/CALneRrC67m1dis5arHTQDfyIhKk69E= +github.com/martini-contrib/gzip v0.0.0-20151124214156-6c035326b43f h1:wVDxEVZP1eiPIlHVaafUAEUDtyl6ytjHv3egJVbyfOk= +github.com/martini-contrib/gzip v0.0.0-20151124214156-6c035326b43f/go.mod h1:jhUB0rZB2TPWqy0yGugKRRictO591eSO7If7O4MfCaA= +github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11 h1:YFh+sjyJTMQSYjKwM4dFKhJPJC/wfo98tPUc17HdoYw= +github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11/go.mod h1:Ah2dBMoxZEqk118as2T4u4fjfXarE0pPnMJaArZQZsI= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -419,9 +424,10 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= @@ -453,6 +459,8 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.6.3 h1:F8446DrvIF5V5smZfZ8K9nrmmix0AFgevPdLruGOmzk= +github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -462,8 +470,6 @@ github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 h1:58+kh9C6jJVXYjt8IE48G2eWl6BjwU5Gj0gqY84fy78= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/olekukonko/tablewriter v0.0.5-0.20200416053754-163badb3bac6 h1:F721VBMijn0OBFZ5wUSuMVVLQj2IJiiupn6UNd7UbBE= github.com/olekukonko/tablewriter v0.0.5-0.20200416053754-163badb3bac6/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -479,9 +485,13 @@ github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 h1:0R5 github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= +github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= @@ -523,6 +533,8 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -530,6 +542,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e h1:CGjiMQ0wMH4wtNWrlj6kiTbkPt2F3rbYnhGX6TWLfco= +github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= @@ -538,6 +552,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sjmudd/stopwatch v0.0.0-20170613150411-f380bf8a9be1 h1:acClJNSOjUrAUKW+ZneCZymCFDWtSaJG5YQl8FoOlyI= +github.com/sjmudd/stopwatch v0.0.0-20170613150411-f380bf8a9be1/go.mod h1:Pgf1sZ2KrHK8vdRTV5UHGp80LT7HMUKuNAiKC402abY= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -685,6 +701,9 @@ golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ym golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -693,7 +712,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -705,7 +723,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -726,11 +743,11 @@ golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -766,7 +783,6 @@ golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191219041853-979b82bfef62 h1:vDaiisQl0rGVXqk3wT2yc43gSnwlj4haEG5J78IGZP4= golang.org/x/tools v0.0.0-20191219041853-979b82bfef62/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -774,7 +790,6 @@ gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0 h1:9sdfJOzWlkqPltHAuzT2Cp+yrBeY1KRVYgms8soxMwM= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0 h1:jbyannxz0XFD3zdjgrSUsaJbgpH4eTrkdhRChkHPfO8= @@ -790,15 +805,12 @@ google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190626174449-989357319d63/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64 h1:iKtrH9Y8mcbADOP0YFaEMth7OfuHY9xHOwNj4znpM1A= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190926190326-7ee9db18f195 h1:dWzgMaXfaHsnkRKZ1l3iJLDmTEB40JMl/dqRbJX4D/o= google.golang.org/genproto v0.0.0-20190926190326-7ee9db18f195/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= @@ -817,25 +829,25 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE= gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ldap.v2 v2.5.0 h1:1rO3ojzsHUk+gq4ZYhC4Pg+EzWaaKIV8+DJwExS5/QQ= gopkg.in/ldap.v2 v2.5.0/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= @@ -844,9 +856,7 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a h1:LJwr7TCTghdatWv40WobzlKXc9c4s8oGa7QKJUtHhWA= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/go/cacheservice/cacheservice.go b/go/cacheservice/cacheservice.go deleted file mode 100644 index 9a7dbcabaec..00000000000 --- a/go/cacheservice/cacheservice.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cacheservice - -import ( - "fmt" - "sync" - "time" -) - -// NewConnFunc is a factory method that creates a CacheService instance -// using given CacheServiceConfig. -type NewConnFunc func(config Config) (CacheService, error) - -var ( - // services stores all supported cache service. - services = make(map[string]NewConnFunc) - - mu sync.Mutex - - // DefaultCacheService decides the default cache service connection. - DefaultCacheService string -) - -// Config carries config data for CacheService. -type Config struct { - Address string - Timeout time.Duration -} - -// Result gives the cached data. -type Result struct { - Key string - Value []byte - Flags uint16 - Cas uint64 -} - -// CacheService defines functions to use a cache service. -type CacheService interface { - // Get returns cached data for given keys. - Get(keys ...string) (results []Result, err error) - // Gets returns cached data for given keys, it is an alternative Get api - // for using with CAS. Gets returns a CAS identifier with the item. If - // the item's CAS value has changed since you Gets'ed it, it will not be stored. - Gets(keys ...string) (results []Result, err error) - // Set sets the value with specified cache key. - Set(key string, flags uint16, timeout uint64, value []byte) (stored bool, err error) - // Add stores the value only if it does not already exist. - Add(key string, flags uint16, timeout uint64, value []byte) (stored bool, err error) - // Replace replaces the value, only if the value already exists, - // for the specified cache key. - Replace(key string, flags uint16, timeout uint64, value []byte) (stored bool, err error) - // Append appends the value after the last bytes in an existing item. - Append(key string, flags uint16, timeout uint64, value []byte) (stored bool, err error) - // Prepend prepends the value before existing value. - Prepend(key string, flags uint16, timeout uint64, value []byte) (stored bool, err error) - // Cas stores the value only if no one else has updated the data since you read it last. - Cas(key string, flags uint16, timeout uint64, value []byte, cas uint64) (stored bool, err error) - // Delete deletes the value for the specified cache key. - Delete(key string) (deleted bool, err error) - // FlushAll purges the entire cache. - FlushAll() (err error) - // Stats returns a list of basic stats. - Stats(argument string) (result []byte, err error) - // Close closes the CacheService - Close() -} - -// Register a db connection. -func Register(name string, fn NewConnFunc) { - mu.Lock() - defer mu.Unlock() - if _, ok := services[name]; ok { - panic(fmt.Sprintf("register a registered key: %s", name)) - } - services[name] = fn -} - -// Connect returns a CacheService using the given config. -func Connect(config Config) (CacheService, error) { - mu.Lock() - defer mu.Unlock() - if DefaultCacheService == "" { - if len(services) == 1 { - for _, fn := range services { - return fn(config) - } - } - panic("there are more than one service connect func " + - "registered but no default cache service has been specified.") - } - fn, ok := services[DefaultCacheService] - if !ok { - panic(fmt.Sprintf("service connect function for given default cache service: %s is not found.", DefaultCacheService)) - } - return fn(config) -} diff --git a/go/cmd/query_analyzer/query_analyzer.go b/go/cmd/query_analyzer/query_analyzer.go index dde5be1cca4..caeb1928270 100644 --- a/go/cmd/query_analyzer/query_analyzer.go +++ b/go/cmd/query_analyzer/query_analyzer.go @@ -113,7 +113,7 @@ func analyze(line []byte) { } func formatWithBind(buf *sqlparser.TrackedBuffer, node sqlparser.SQLNode) { - v, ok := node.(*sqlparser.SQLVal) + v, ok := node.(*sqlparser.Literal) if !ok { node.Format(buf) return diff --git a/go/cmd/vtbackup/vtbackup.go b/go/cmd/vtbackup/vtbackup.go index 5cb4d5be52c..f5117376a23 100644 --- a/go/cmd/vtbackup/vtbackup.go +++ b/go/cmd/vtbackup/vtbackup.go @@ -306,8 +306,6 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back return fmt.Errorf("no backup found; not starting up empty since -initial_backup flag was not enabled") } restorePos = mysql.Position{} - case mysqlctl.ErrExistingDB: - return fmt.Errorf("can't run vtbackup because data directory is not empty") default: return fmt.Errorf("can't restore from backup: %v", err) } diff --git a/go/cmd/vtexplain/vtexplain.go b/go/cmd/vtexplain/vtexplain.go index 3d1edb16519..88fd6bf15df 100644 --- a/go/cmd/vtexplain/vtexplain.go +++ b/go/cmd/vtexplain/vtexplain.go @@ -29,18 +29,20 @@ import ( ) var ( - sqlFlag = flag.String("sql", "", "A list of semicolon-delimited SQL commands to analyze") - sqlFileFlag = flag.String("sql-file", "", "Identifies the file that contains the SQL commands to analyze") - schemaFlag = flag.String("schema", "", "The SQL table schema") - schemaFileFlag = flag.String("schema-file", "", "Identifies the file that contains the SQL table schema") - vschemaFlag = flag.String("vschema", "", "Identifies the VTGate routing schema") - vschemaFileFlag = flag.String("vschema-file", "", "Identifies the VTGate routing schema file") - numShards = flag.Int("shards", 2, "Number of shards per keyspace") - executionMode = flag.String("execution-mode", "multi", "The execution mode to simulate -- must be set to multi, legacy-autocommit, or twopc") - replicationMode = flag.String("replication-mode", "ROW", "The replication mode to simulate -- must be set to either ROW or STATEMENT") - normalize = flag.Bool("normalize", false, "Whether to enable vtgate normalization") - outputMode = flag.String("output-mode", "text", "Output in human-friendly text or json") - dbName = flag.String("dbname", "", "Optional database target to override normal routing") + sqlFlag = flag.String("sql", "", "A list of semicolon-delimited SQL commands to analyze") + sqlFileFlag = flag.String("sql-file", "", "Identifies the file that contains the SQL commands to analyze") + schemaFlag = flag.String("schema", "", "The SQL table schema") + schemaFileFlag = flag.String("schema-file", "", "Identifies the file that contains the SQL table schema") + vschemaFlag = flag.String("vschema", "", "Identifies the VTGate routing schema") + vschemaFileFlag = flag.String("vschema-file", "", "Identifies the VTGate routing schema file") + ksShardMapFlag = flag.String("ks-shard-map", "", "JSON map of keyspace name -> shard name -> ShardReference object. The inner map is the same as the output of FindAllShardsInKeyspace") + ksShardMapFileFlag = flag.String("ks-shard-map-file", "", "File containing json blob of keyspace name -> shard name -> ShardReference object") + numShards = flag.Int("shards", 2, "Number of shards per keyspace. Passing -ks-shard-map/-ks-shard-map-file causes this flag to be ignored.") + executionMode = flag.String("execution-mode", "multi", "The execution mode to simulate -- must be set to multi, legacy-autocommit, or twopc") + replicationMode = flag.String("replication-mode", "ROW", "The replication mode to simulate -- must be set to either ROW or STATEMENT") + normalize = flag.Bool("normalize", false, "Whether to enable vtgate normalization") + outputMode = flag.String("output-mode", "text", "Output in human-friendly text or json") + dbName = flag.String("dbname", "", "Optional database target to override normal routing") // vtexplainFlags lists all the flags that should show in usage vtexplainFlags = []string{ @@ -54,6 +56,8 @@ var ( "sql-file", "vschema", "vschema-file", + "ks-shard-map", + "ks-shard-map-file", "dbname", "queryserver-config-passthrough-dmls", } @@ -104,7 +108,7 @@ func init() { // getFileParam returns a string containing either flag is not "", // or the content of the file named flagFile -func getFileParam(flag, flagFile, name string) (string, error) { +func getFileParam(flag, flagFile, name string, required bool) (string, error) { if flag != "" { if flagFile != "" { return "", fmt.Errorf("action requires only one of %v or %v-file", name, name) @@ -113,7 +117,11 @@ func getFileParam(flag, flagFile, name string) (string, error) { } if flagFile == "" { - return "", fmt.Errorf("action requires one of %v or %v-file", name, name) + if required { + return "", fmt.Errorf("action requires one of %v or %v-file", name, name) + } + + return "", nil } data, err := ioutil.ReadFile(flagFile) if err != nil { @@ -137,17 +145,22 @@ func main() { } func parseAndRun() error { - sql, err := getFileParam(*sqlFlag, *sqlFileFlag, "sql") + sql, err := getFileParam(*sqlFlag, *sqlFileFlag, "sql", true) + if err != nil { + return err + } + + schema, err := getFileParam(*schemaFlag, *schemaFileFlag, "schema", true) if err != nil { return err } - schema, err := getFileParam(*schemaFlag, *schemaFileFlag, "schema") + vschema, err := getFileParam(*vschemaFlag, *vschemaFileFlag, "vschema", true) if err != nil { return err } - vschema, err := getFileParam(*vschemaFlag, *vschemaFileFlag, "vschema") + ksShardMap, err := getFileParam(*ksShardMapFlag, *ksShardMapFileFlag, "ks-shard-map", false) if err != nil { return err } @@ -164,7 +177,7 @@ func parseAndRun() error { log.V(100).Infof("schema %s\n", schema) log.V(100).Infof("vschema %s\n", vschema) - err = vtexplain.Init(vschema, schema, opts) + err = vtexplain.Init(vschema, schema, ksShardMap, opts) if err != nil { return err } diff --git a/go/cmd/vtorc/main.go b/go/cmd/vtorc/main.go new file mode 100644 index 00000000000..19c7063f233 --- /dev/null +++ b/go/cmd/vtorc/main.go @@ -0,0 +1,131 @@ +/* + Copyright 2014 Outbrain Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "flag" + + "vitess.io/vitess/go/vt/orchestrator/app" + "vitess.io/vitess/go/vt/orchestrator/config" + "vitess.io/vitess/go/vt/orchestrator/external/golib/log" + "vitess.io/vitess/go/vt/orchestrator/inst" +) + +var AppVersion, GitCommit string + +// main is the application's entry point. It will either spawn a CLI or HTTP itnerfaces. +func main() { + // TODO(sougou): change this to use vitess servenv framework + // TODO(sougou): remove cli code + configFile := flag.String("config", "", "config file name") + sibling := flag.String("s", "", "sibling instance, host_fqdn[:port]") + destination := flag.String("d", "", "destination instance, host_fqdn[:port] (synonym to -s)") + discovery := flag.Bool("discovery", true, "auto discovery mode") + quiet := flag.Bool("quiet", false, "quiet") + verbose := flag.Bool("verbose", false, "verbose") + debug := flag.Bool("debug", false, "debug mode (very verbose)") + stack := flag.Bool("stack", false, "add stack trace upon error") + config.RuntimeCLIFlags.SkipBinlogSearch = flag.Bool("skip-binlog-search", false, "when matching via Pseudo-GTID, only use relay logs. This can save the hassle of searching for a non-existend pseudo-GTID entry, for example in servers with replication filters.") + config.RuntimeCLIFlags.SkipUnresolve = flag.Bool("skip-unresolve", false, "Do not unresolve a host name") + config.RuntimeCLIFlags.SkipUnresolveCheck = flag.Bool("skip-unresolve-check", false, "Skip/ignore checking an unresolve mapping (via hostname_unresolve table) resolves back to same hostname") + config.RuntimeCLIFlags.Noop = flag.Bool("noop", false, "Dry run; do not perform destructing operations") + config.RuntimeCLIFlags.BinlogFile = flag.String("binlog", "", "Binary log file name") + config.RuntimeCLIFlags.Statement = flag.String("statement", "", "Statement/hint") + config.RuntimeCLIFlags.GrabElection = flag.Bool("grab-election", false, "Grab leadership (only applies to continuous mode)") + config.RuntimeCLIFlags.PromotionRule = flag.String("promotion-rule", "prefer", "Promotion rule for register-andidate (prefer|neutral|prefer_not|must_not)") + config.RuntimeCLIFlags.SkipContinuousRegistration = flag.Bool("skip-continuous-registration", false, "Skip cli commands performaing continuous registration (to reduce orchestratrator backend db load") + config.RuntimeCLIFlags.EnableDatabaseUpdate = flag.Bool("enable-database-update", false, "Enable database update, overrides SkipOrchestratorDatabaseUpdate") + config.RuntimeCLIFlags.IgnoreRaftSetup = flag.Bool("ignore-raft-setup", false, "Override RaftEnabled for CLI invocation (CLI by default not allowed for raft setups). NOTE: operations by CLI invocation may not reflect in all raft nodes.") + config.RuntimeCLIFlags.Tag = flag.String("tag", "", "tag to add ('tagname' or 'tagname=tagvalue') or to search ('tagname' or 'tagname=tagvalue' or comma separated 'tag0,tag1=val1,tag2' for intersection of all)") + flag.Parse() + + if *destination != "" && *sibling != "" { + log.Fatalf("-s and -d are synonyms, yet both were specified. You're probably doing the wrong thing.") + } + switch *config.RuntimeCLIFlags.PromotionRule { + case "prefer", "neutral", "prefer_not", "must_not": + { + // OK + } + default: + { + log.Fatalf("-promotion-rule only supports prefer|neutral|prefer_not|must_not") + } + } + if *destination == "" { + *destination = *sibling + } + + log.SetLevel(log.ERROR) + if *verbose { + log.SetLevel(log.INFO) + } + if *debug { + log.SetLevel(log.DEBUG) + } + if *stack { + log.SetPrintStackTrace(*stack) + } + + startText := "starting orchestrator" + if AppVersion != "" { + startText += ", version: " + AppVersion + } + if GitCommit != "" { + startText += ", git commit: " + GitCommit + } + log.Info(startText) + + if len(*configFile) > 0 { + config.ForceRead(*configFile) + } else { + config.Read("/etc/orchestrator.conf.json", "conf/orchestrator.conf.json", "orchestrator.conf.json") + } + if *config.RuntimeCLIFlags.EnableDatabaseUpdate { + config.Config.SkipOrchestratorDatabaseUpdate = false + } + if config.Config.Debug { + log.SetLevel(log.DEBUG) + } + if *quiet { + // Override!! + log.SetLevel(log.ERROR) + } + if config.Config.EnableSyslog { + log.EnableSyslogWriter("orchestrator") + log.SetSyslogLevel(log.INFO) + } + if config.Config.AuditToSyslog { + inst.EnableAuditSyslog() + } + config.RuntimeCLIFlags.ConfiguredVersion = AppVersion + config.MarkConfigurationLoaded() + + helpTopic := "" + if flag.Arg(0) == "help" { + if flag.Arg(1) != "" { + helpTopic = flag.Arg(1) + } + } + + switch { + case helpTopic != "": + app.HelpCommand(helpTopic) + default: + app.Http(*discovery) + } +} diff --git a/go/cmd/vtorc/plugin_consultopo.go b/go/cmd/vtorc/plugin_consultopo.go new file mode 100644 index 00000000000..59d6774fdbc --- /dev/null +++ b/go/cmd/vtorc/plugin_consultopo.go @@ -0,0 +1,23 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// This plugin imports consultopo to register the consul implementation of TopoServer. + +import ( + _ "vitess.io/vitess/go/vt/topo/consultopo" +) diff --git a/go/cmd/vtorc/plugin_etcd2topo.go b/go/cmd/vtorc/plugin_etcd2topo.go new file mode 100644 index 00000000000..d99ef51d4af --- /dev/null +++ b/go/cmd/vtorc/plugin_etcd2topo.go @@ -0,0 +1,23 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// This plugin imports etcd2topo to register the etcd2 implementation of TopoServer. + +import ( + _ "vitess.io/vitess/go/vt/topo/etcd2topo" +) diff --git a/go/cmd/vtorc/plugin_grpctmclient.go b/go/cmd/vtorc/plugin_grpctmclient.go new file mode 100644 index 00000000000..ce554da96df --- /dev/null +++ b/go/cmd/vtorc/plugin_grpctmclient.go @@ -0,0 +1,23 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// Imports and register the gRPC tabletmanager client + +import ( + _ "vitess.io/vitess/go/vt/vttablet/grpctmclient" +) diff --git a/go/cmd/vtorc/plugin_kubernetestopo.go b/go/cmd/vtorc/plugin_kubernetestopo.go new file mode 100644 index 00000000000..671d0c8321f --- /dev/null +++ b/go/cmd/vtorc/plugin_kubernetestopo.go @@ -0,0 +1,23 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// This plugin imports k8stopo to register the kubernetes implementation of TopoServer. + +import ( + _ "vitess.io/vitess/go/vt/topo/k8stopo" +) diff --git a/go/cmd/vtorc/plugin_zk2topo.go b/go/cmd/vtorc/plugin_zk2topo.go new file mode 100644 index 00000000000..ebf385ec1af --- /dev/null +++ b/go/cmd/vtorc/plugin_zk2topo.go @@ -0,0 +1,23 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// Imports and register the zk2 TopologyServer + +import ( + _ "vitess.io/vitess/go/vt/topo/zk2topo" +) diff --git a/go/cmd/vttablet/vttablet.go b/go/cmd/vttablet/vttablet.go index 8fea9def342..50d857070ce 100644 --- a/go/cmd/vttablet/vttablet.go +++ b/go/cmd/vttablet/vttablet.go @@ -32,11 +32,14 @@ import ( "vitess.io/vitess/go/vt/tableacl/simpleacl" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vttablet/onlineddl" "vitess.io/vitess/go/vt/vttablet/tabletmanager" "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" "vitess.io/vitess/go/vt/vttablet/tabletserver" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" "vitess.io/vitess/go/yaml2" + + rice "github.com/GeertJohan/go.rice" ) var ( @@ -77,6 +80,10 @@ func main() { mysqld := mysqlctl.NewMysqld(config.DB) servenv.OnClose(mysqld.Close) + if err := extractOnlineDDL(); err != nil { + log.Exitf("failed to extract online DDL binaries: %v", err) + } + // Initialize and start tm. gRPCPort := int32(0) if servenv.GRPCPort != nil { @@ -157,6 +164,27 @@ func initConfig(tabletAlias *topodatapb.TabletAlias) (*tabletenv.TabletConfig, * return config, mycnf } +// extractOnlineDDL extracts the gh-ost binary from this executable. gh-ost is appended +// to vttablet executable by `make build` and via ricebox +func extractOnlineDDL() error { + riceBox, err := rice.FindBox("../../../resources/bin") + if err != nil { + return err + } + + if binaryFileName, isOverride := onlineddl.GhostBinaryFileName(); !isOverride { + ghostBinary, err := riceBox.Bytes("gh-ost") + if err != nil { + return err + } + if err := ioutil.WriteFile(binaryFileName, ghostBinary, 0755); err != nil { + return err + } + } + + return nil +} + func createTabletServer(config *tabletenv.TabletConfig, ts *topo.Server, tabletAlias *topodatapb.TabletAlias) *tabletserver.TabletServer { if *tableACLConfig != "" { // To override default simpleacl, other ACL plugins must set themselves to be default ACL factory diff --git a/go/event/syslogger/syslogger.go b/go/event/syslogger/syslogger.go index 2c26fee0839..1c8ff22136b 100644 --- a/go/event/syslogger/syslogger.go +++ b/go/event/syslogger/syslogger.go @@ -80,11 +80,6 @@ type syslogWriter interface { var writer syslogWriter func listener(ev Syslogger) { - if writer == nil { - log.Errorf("no connection, dropping syslog event: %#v", ev) - return - } - // Ask the event to convert itself to a syslog message. sev, msg := ev.Syslog() @@ -92,21 +87,53 @@ func listener(ev Syslogger) { var err error switch sev { case syslog.LOG_EMERG: - err = writer.Emerg(msg) + if writer != nil { + err = writer.Emerg(msg) + } else { + log.Errorf(msg) + } case syslog.LOG_ALERT: - err = writer.Alert(msg) + if writer != nil { + err = writer.Alert(msg) + } else { + log.Errorf(msg) + } case syslog.LOG_CRIT: - err = writer.Crit(msg) + if writer != nil { + err = writer.Crit(msg) + } else { + log.Errorf(msg) + } case syslog.LOG_ERR: - err = writer.Err(msg) + if writer != nil { + err = writer.Err(msg) + } else { + log.Errorf(msg) + } case syslog.LOG_WARNING: - err = writer.Warning(msg) + if writer != nil { + err = writer.Warning(msg) + } else { + log.Warningf(msg) + } case syslog.LOG_NOTICE: - err = writer.Notice(msg) + if writer != nil { + err = writer.Notice(msg) + } else { + log.Infof(msg) + } case syslog.LOG_INFO: - err = writer.Info(msg) + if writer != nil { + err = writer.Info(msg) + } else { + log.Infof(msg) + } case syslog.LOG_DEBUG: - err = writer.Debug(msg) + if writer != nil { + err = writer.Debug(msg) + } else { + log.Infof(msg) + } default: err = fmt.Errorf("invalid syslog severity: %v", sev) } diff --git a/go/event/syslogger/syslogger_test.go b/go/event/syslogger/syslogger_test.go index 111335d41f5..44176decbb3 100644 --- a/go/event/syslogger/syslogger_test.go +++ b/go/event/syslogger/syslogger_test.go @@ -19,9 +19,11 @@ package syslogger import ( "fmt" "log/syslog" + "strings" "testing" "vitess.io/vitess/go/event" + "vitess.io/vitess/go/vt/log" ) type TestEvent struct { @@ -61,6 +63,60 @@ func (fw *fakeWriter) Info(msg string) error { return fw.write(syslog.LOG_INF func (fw *fakeWriter) Notice(msg string) error { return fw.write(syslog.LOG_NOTICE, msg) } func (fw *fakeWriter) Warning(msg string) error { return fw.write(syslog.LOG_WARNING, msg) } +type loggerMsg struct { + msg string + level string +} +type testLogger struct { + logs []loggerMsg + savedInfof func(format string, args ...interface{}) + savedWarningf func(format string, args ...interface{}) + savedErrorf func(format string, args ...interface{}) +} + +func newTestLogger() *testLogger { + tl := &testLogger{ + savedInfof: log.Infof, + savedWarningf: log.Warningf, + savedErrorf: log.Errorf, + } + log.Infof = tl.recordInfof + log.Warningf = tl.recordWarningf + log.Errorf = tl.recordErrorf + return tl +} + +func (tl *testLogger) Close() { + log.Infof = tl.savedInfof + log.Warningf = tl.savedWarningf + log.Errorf = tl.savedErrorf +} + +func (tl *testLogger) recordInfof(format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + tl.logs = append(tl.logs, loggerMsg{msg, "INFO"}) + tl.savedInfof(msg) +} + +func (tl *testLogger) recordWarningf(format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + tl.logs = append(tl.logs, loggerMsg{msg, "WARNING"}) + tl.savedWarningf(msg) +} + +func (tl *testLogger) recordErrorf(format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + tl.logs = append(tl.logs, loggerMsg{msg, "ERROR"}) + tl.savedErrorf(msg) +} + +func (tl *testLogger) getLog() loggerMsg { + if len(tl.logs) > 0 { + return tl.logs[len(tl.logs)-1] + } + return loggerMsg{"no logs!", "ERROR"} +} + // TestSyslog checks that our callback works. func TestSyslog(t *testing.T) { writer = &fakeWriter{} @@ -73,14 +129,59 @@ func TestSyslog(t *testing.T) { } } -// TestBadWriter checks that we don't panic when the connection fails. +// TestBadWriter verifies we are still triggering (to normal logs) if +// the syslog connection failed func TestBadWriter(t *testing.T) { + tl := newTestLogger() + defer tl.Close() + writer = nil + wantMsg := "testing message" + wantLevel := "ERROR" + ev := &TestEvent{priority: syslog.LOG_ALERT, message: wantMsg} + event.Dispatch(ev) + if !strings.Contains(tl.getLog().msg, wantMsg) { + t.Errorf("error log msg [%s], want msg [%s]", tl.getLog().msg, wantMsg) + } + if !strings.Contains(tl.getLog().level, wantLevel) { + t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) + } + ev = &TestEvent{priority: syslog.LOG_CRIT, message: wantMsg} + event.Dispatch(ev) + if !strings.Contains(tl.getLog().level, wantLevel) { + t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) + } + ev = &TestEvent{priority: syslog.LOG_ERR, message: wantMsg} + event.Dispatch(ev) + if !strings.Contains(tl.getLog().level, wantLevel) { + t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) + } - ev := new(TestEvent) + wantLevel = "WARNING" + ev = &TestEvent{priority: syslog.LOG_WARNING, message: wantMsg} event.Dispatch(ev) + if !strings.Contains(tl.getLog().level, wantLevel) { + t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) + } - if ev.triggered { + wantLevel = "INFO" + ev = &TestEvent{priority: syslog.LOG_INFO, message: wantMsg} + event.Dispatch(ev) + if !strings.Contains(tl.getLog().level, wantLevel) { + t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) + } + ev = &TestEvent{priority: syslog.LOG_NOTICE, message: wantMsg} + event.Dispatch(ev) + if !strings.Contains(tl.getLog().level, wantLevel) { + t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) + } + ev = &TestEvent{priority: syslog.LOG_DEBUG, message: wantMsg} + event.Dispatch(ev) + if !strings.Contains(tl.getLog().level, wantLevel) { + t.Errorf("error log level [%s], want level [%s]", tl.getLog().level, wantLevel) + } + + if !ev.triggered { t.Errorf("passed nil writer to client") } } diff --git a/go/memcache/memcache.go b/go/memcache/memcache.go deleted file mode 100644 index f645ac80d5f..00000000000 --- a/go/memcache/memcache.go +++ /dev/null @@ -1,320 +0,0 @@ -/* -Copyright 2019 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package memcache is a client for memcached. -package memcache - -import ( - "bufio" - "fmt" - "io" - "net" - "strconv" - "strings" - "time" - - "vitess.io/vitess/go/cacheservice" -) - -// Connection is the connection to a memcache. -type Connection struct { - conn net.Conn - buffered bufio.ReadWriter - timeout time.Duration -} - -// Connect connects a memcache process. -func Connect(address string, timeout time.Duration) (conn *Connection, err error) { - var network string - if strings.Contains(address, "/") { - network = "unix" - } else { - network = "tcp" - } - var nc net.Conn - nc, err = net.DialTimeout(network, address, timeout) - if err != nil { - return nil, err - } - return &Connection{ - conn: nc, - buffered: bufio.ReadWriter{ - Reader: bufio.NewReader(nc), - Writer: bufio.NewWriter(nc), - }, - timeout: timeout, - }, nil -} - -// Close closes the memcache connection. -func (mc *Connection) Close() { - mc.conn.Close() -} - -// Get returns cached data for given keys. -func (mc *Connection) Get(keys ...string) (results []cacheservice.Result, err error) { - defer handleError(&err) - results = mc.get("get", keys) - return -} - -// Gets returns cached data for given keys, it is an alternative Get api -// for using with CAS. Gets returns a CAS identifier with the item. If -// the item's CAS value has changed since you Gets'ed it, it will not be stored. -func (mc *Connection) Gets(keys ...string) (results []cacheservice.Result, err error) { - defer handleError(&err) - results = mc.get("gets", keys) - return -} - -// Set sets the value with specified cache key. -func (mc *Connection) Set(key string, flags uint16, timeout uint64, value []byte) (stored bool, err error) { - defer handleError(&err) - return mc.store("set", key, flags, timeout, value, 0), nil -} - -// Add stores the value only if it does not already exist. -func (mc *Connection) Add(key string, flags uint16, timeout uint64, value []byte) (stored bool, err error) { - defer handleError(&err) - return mc.store("add", key, flags, timeout, value, 0), nil -} - -// Replace replaces the value, only if the value already exists, -// for the specified cache key. -func (mc *Connection) Replace(key string, flags uint16, timeout uint64, value []byte) (stored bool, err error) { - defer handleError(&err) - return mc.store("replace", key, flags, timeout, value, 0), nil -} - -// Append appends the value after the last bytes in an existing item. -func (mc *Connection) Append(key string, flags uint16, timeout uint64, value []byte) (stored bool, err error) { - defer handleError(&err) - return mc.store("append", key, flags, timeout, value, 0), nil -} - -// Prepend prepends the value before existing value. -func (mc *Connection) Prepend(key string, flags uint16, timeout uint64, value []byte) (stored bool, err error) { - defer handleError(&err) - return mc.store("prepend", key, flags, timeout, value, 0), nil -} - -// Cas stores the value only if no one else has updated the data since you read it last. -func (mc *Connection) Cas(key string, flags uint16, timeout uint64, value []byte, cas uint64) (stored bool, err error) { - defer handleError(&err) - return mc.store("cas", key, flags, timeout, value, cas), nil -} - -// Delete deletes the value for the specified cache key. -func (mc *Connection) Delete(key string) (deleted bool, err error) { - defer handleError(&err) - mc.setDeadline() - // delete [