From 993a20fa02ca55c59a51486b6ba6d01e6ce1d9c9 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Wed, 24 Jan 2018 13:19:57 -0800 Subject: [PATCH] WIP: Vitess Operator example --- examples/vitess/README.md | 126 +++++++++++ examples/vitess/hooks/etcd.libsonnet | 26 +++ examples/vitess/hooks/k8s.libsonnet | 68 ++++++ .../vitess/hooks/metacontroller.libsonnet | 98 +++++++++ examples/vitess/hooks/sync-cell.jsonnet | 132 ++++++++++++ examples/vitess/hooks/sync-cluster.jsonnet | 30 +++ examples/vitess/hooks/sync-keyspace.jsonnet | 34 +++ examples/vitess/hooks/sync-shard.jsonnet | 66 ++++++ examples/vitess/hooks/vitess.libsonnet | 189 +++++++++++++++++ examples/vitess/hooks/vtctlclient.libsonnet | 61 ++++++ examples/vitess/hooks/vtctld.libsonnet | 165 +++++++++++++++ examples/vitess/hooks/vtgate.libsonnet | 93 ++++++++ examples/vitess/hooks/vttablet.libsonnet | 198 ++++++++++++++++++ examples/vitess/my-vitess.yaml | 68 ++++++ examples/vitess/test/k8s_test.jsonnet | 108 ++++++++++ examples/vitess/test/vitess_test.jsonnet | 51 +++++ examples/vitess/update-hooks.sh | 14 ++ examples/vitess/vitess-operator.yaml | 183 ++++++++++++++++ 18 files changed, 1710 insertions(+) create mode 100644 examples/vitess/README.md create mode 100644 examples/vitess/hooks/etcd.libsonnet create mode 100644 examples/vitess/hooks/k8s.libsonnet create mode 100644 examples/vitess/hooks/metacontroller.libsonnet create mode 100644 examples/vitess/hooks/sync-cell.jsonnet create mode 100644 examples/vitess/hooks/sync-cluster.jsonnet create mode 100644 examples/vitess/hooks/sync-keyspace.jsonnet create mode 100644 examples/vitess/hooks/sync-shard.jsonnet create mode 100644 examples/vitess/hooks/vitess.libsonnet create mode 100644 examples/vitess/hooks/vtctlclient.libsonnet create mode 100644 examples/vitess/hooks/vtctld.libsonnet create mode 100644 examples/vitess/hooks/vtgate.libsonnet create mode 100644 examples/vitess/hooks/vttablet.libsonnet create mode 100644 examples/vitess/my-vitess.yaml create mode 100644 examples/vitess/test/k8s_test.jsonnet create mode 100644 examples/vitess/test/vitess_test.jsonnet create mode 100755 examples/vitess/update-hooks.sh create mode 100644 examples/vitess/vitess-operator.yaml diff --git a/examples/vitess/README.md b/examples/vitess/README.md new file mode 100644 index 0000000..a262ab9 --- /dev/null +++ b/examples/vitess/README.md @@ -0,0 +1,126 @@ +## Vitess Operator + +This is an example of an app-specific [Operator](https://coreos.com/operators/), +in this case for [Vitess](http://vitess.io), built with Metacontroller. + +It's meant to demonstrate the following patterns: + +* Building an Operator for a complex, stateful application out of a set of small + Lambda Controllers that each do one thing well. + * In addition to presenting a k8s-style API to users, this Operator uses + custom k8s API objects to coordinate within itself. + * Each controller manages one layer of the hierarchical Vitess cluster topology. + The user only needs to create and manage a single, top-level VitessCluster + object. +* Replacing static, client-side template rendering with Lambda Controllers, + which can adjust based on dynamic cluster state. + * Each controller aggregates status and orchestrates app-specific rolling + updates for its immediate children. + * The top-level object contains a continuously-updated, aggregate "Ready" + condition for the whole app, and can be directly edited to trigger rolling + updates throughout the app. +* Using a functional-style language ([Jsonnet](http://jsonnet.org)) to + define Lambda Controllers in terms of template-like transformations on JSON + objects. + * You can use any language to write a Lambda Controller webhook, but the + functional style is a good fit for a process that conceptually consists of + declarative input, declarative output, and no side effects. + * As a JSON templating language, Jsonnet is a particularly good fit for + generating k8s manifests, providing functionality missing from pure + JavaScript, such as first-class *merge* and *deep equal* operations. +* Using the "Apply" update strategy feature of CompositeController, which + emulates the behavior of `kubectl apply`, except that it attempts to do + pseudo-strategic merges for CRDs. + +### Vitess Components + +A typical VitessCluster might expand to the following tree once it's fully +deployed. +Objects in **bold** are custom resource kinds defined by this Operator. + +* **VitessCluster**: The top-level specification for a Vitess cluster. + This is the only one the user creates. + * **VitessCell**: Each Vitess [cell](http://vitess.io/overview/concepts/#cell-data-center) + represents an independent failure domain (e.g. a Zone or Availability Zone). + * EtcdCluster ([etcd-operator](https://github.com/coreos/etcd-operator)): + Vitess needs its own etcd cluster to coordinate its built-in load-balancing + and automatic shard routing. + * Deployment ([orchestrator](https://github.com/github/orchestrator)): + An optional automated failover tool that works with Vitess. + * Deployment ([vtctld](http://vitess.io/overview/#vtctld)): + A pool of stateless Vitess admin servers, which serve a dashboard UI as well + as being an endpoint for the Vitess CLI tool (vtctlclient). + * Deployment ([vtgate](http://vitess.io/overview/#vtgate)): + A pool of stateless Vitess query routers. + The client application can use any one of these vtgate Pods as the entry + point into Vitess, through a MySQL-compatible interface. + * **VitessKeyspace** (db1): Each Vitess [keyspace](http://vitess.io/overview/concepts/#keyspace) + is a logical database that may be composed of many MySQL databases (shards). + * **VitessShard** (db1/0): Each Vitess [shard](http://vitess.io/overview/concepts/#shard) + is a single-master tree of replicating MySQL instances. + * Pod(s) ([vttablet](http://vitess.io/overview/#vttablet)): Within a shard, there may be many Vitess [tablets](http://vitess.io/overview/concepts/#tablet) + (individual MySQL instances). + VitessShard acts like an app-specific [replacement for StatefulSet](https://github.com/GoogleCloudPlatform/kube-metacontroller/tree/master/examples/catset), + creating both Pods and PersistentVolumeClaims. + * PersistentVolumeClaim(s) + * **VitessShard** (db1/1) + * Pod(s) (vttablet) + * PersistentVolumeClaim(s) + * **VitessKeyspace** (db2) + * **VitessShard** (db2/0) + * Pod(s) (vttablet) + * PersistentVolumeClaim(s) + +### Prerequisites + +* Kubernetes 1.8+ is required for its improved CRD support, especially garbage + collection. + * This config currently requires a dynamic PersistentVolume provisioner and a + default StorageClass. + * The example `my-vitess.yaml` config results in a lot of Pods. + If the Pods don't schedule due to resource limits, you can try lowering the + limits, lowering `replicas` values, or removing the `batch` config under + `tablets`. +* Install [kube-metacontroller](https://github.com/GoogleCloudPlatform/kube-metacontroller). +* Install [etcd-operator](https://github.com/coreos/etcd-operator) in the + namespace where you plan to create a VitessCluster. + +### Deploy the Operator + +```sh +kubectl create configmap vitess-operator-hooks -n metacontroller --from-file=hooks +kubectl apply -f vitess-operator.yaml +``` + +### Create a VitessCluster + +```sh +kubectl apply -f my-vitess.yaml +``` + +### View the Vitess Dashboard + +Wait until the cluster is ready: + +```sh +kubectl get vitessclusters -o 'custom-columns=NAME:.metadata.name,READY:.status.conditions[?(@.type=="Ready")].status' +``` + +You should see: + +```console +NAME READY +vitess True +``` + +Start a kubectl proxy: + +```sh +kubectl proxy --port=8001 +``` + +Then visit: + +``` +http://localhost:8001/api/v1/namespaces/default/services/vitess-global-vtctld:web/proxy/app/ +``` diff --git a/examples/vitess/hooks/etcd.libsonnet b/examples/vitess/hooks/etcd.libsonnet new file mode 100644 index 0000000..12ec7d3 --- /dev/null +++ b/examples/vitess/hooks/etcd.libsonnet @@ -0,0 +1,26 @@ +local k8s = import "k8s.libsonnet"; +local metacontroller = import "metacontroller.libsonnet"; + +{ + local etcd = self, + + apiVersion: "etcd.database.coreos.com/v1beta2", + + // EtcdClusters + clusters(observed, specs):: + metacontroller.collection(observed, specs, etcd.apiVersion, "EtcdCluster", etcd.cluster), + + // Create/update an EtcdCluster child for a VitessCell parent. + cluster(observed, spec):: { + apiVersion: etcd.apiVersion, + kind: "EtcdCluster", + metadata: { + name: observed.parent.metadata.name + "-etcd", + labels: observed.parent.spec.template.metadata.labels, + }, + spec: { + version: spec.version, + size: spec.size, + } + }, +} diff --git a/examples/vitess/hooks/k8s.libsonnet b/examples/vitess/hooks/k8s.libsonnet new file mode 100644 index 0000000..980eafc --- /dev/null +++ b/examples/vitess/hooks/k8s.libsonnet @@ -0,0 +1,68 @@ +// Library for working with Kubernetes objects. +{ + local k8s = self, + + // Fill in a conventional status condition object. + condition(type, status):: { + type: type, + status: + if std.type(status) == "string" then ( + status + ) else if std.type(status) == "boolean" then ( + if status then "True" else "False" + ) else ( + "Unknown" + ), + }, + + // Extract the status of a given condition type. + // Returns null if the condition doesn't exist. + conditionStatus(obj, type):: + if obj != null && "status" in obj && "conditions" in obj.status then + // Filter conditions with matching "type" field. + local matches = [ + cond.status for cond in obj.status.conditions if cond.type == type + ]; + // Take the first one, if any. + if std.length(matches) > 0 then matches[0] else "" + else + null, + + // Returns only the objects from a given list that have the + // "Ready" condition set to "True". + filterReady(list):: + std.filter(function(x) self.conditionStatus(x, "Ready") == "True", list), + + // Returns only the objects from a given list that have the + // "Available" condition set to "True". + filterAvailable(list):: + std.filter(function(x) self.conditionStatus(x, "Available") == "True", list), + + // Returns whether the object matches the given label values. + matchLabels(obj, labels):: + local keys = std.objectFields(labels); + + "metadata" in obj && "labels" in obj.metadata && + [ + obj.metadata.labels[k] + for k in keys if k in obj.metadata.labels + ] + == + [labels[k] for k in keys], + + // Get the value of a label from object metadata. + // Returns null if the label doesn't exist. + getLabel(obj, key):: + if "metadata" in obj && "labels" in obj.metadata && key in obj.metadata.labels then + obj.metadata.labels[key] + else + null, + + // Get the value of an annotation from object metadata. + // Returns null if the annotation doesn't exist. + getAnnotation(obj, key):: + if "metadata" in obj && "annotations" in obj.metadata && key in obj.metadata.annotations then + obj.metadata.annotations[key] + else + null, +} diff --git a/examples/vitess/hooks/metacontroller.libsonnet b/examples/vitess/hooks/metacontroller.libsonnet new file mode 100644 index 0000000..079642b --- /dev/null +++ b/examples/vitess/hooks/metacontroller.libsonnet @@ -0,0 +1,98 @@ +local k8s = import "k8s.libsonnet"; + +// Library for working with kube-metacontroller. +{ + local metacontroller = self, + + // Extend a metacontroller request object with extra fields and functions. + observed(request):: request + { + children+: { + // Get a map of children of a given kind, by child name. + getMap(apiVersion, kind):: + self[kind + "." + apiVersion], + + // Get a list of children of a given kind. + getList(apiVersion, kind):: + local map = self.getMap(apiVersion, kind); + [map[key] for key in std.objectFields(map)], + + // Get a child object of a given kind and name. + get(apiVersion, kind, name):: + local map = self.getMap(apiVersion, kind); + if name in map then map[name] else null, + }, + }, + + // Helpers for managing spec, observed, and desired states + // for a collection of objects of a given Kind. + collection(observed, specs, apiVersion, kind, desired):: { + specs: if specs != null then specs else [], + + observed: observed.children.getList(apiVersion, kind), + desired: [ + {apiVersion: apiVersion, kind: kind} + desired(observed, spec) + for spec in self.specs + ], + + getObserved(name): observed.children.get(apiVersion, kind, name), + updateStrategy: + if "updateStrategy" in observed.controller.spec then + observed.controller.spec.updateStrategy + else + null, + }, + + // Mix-in for collection that filters observed objects. + // This may be needed if a given parent has multiple collections of children + // of the same Kind. + collectionFilter(filter):: { + observed: std.filter(filter, super.observed), + }, + + // Mix-in for collection that causes the child objects to be treated as + // "immutable". That is, they will be created if they don't exist, but left + // untouched if they do. + // + // Note that in the case of the "Apply" strategy, this means we return the + // last applied config. Metacontroller might still attempt an update in + // response, if a third party has mutated fields to which we've previously + // applied values. In other words, it continues to actively maintain the last + // applied config; it doesn't become totally passive. + collectionImmutable:: { + desired: [ + local observed = super.getObserved(desired.metadata.name); + if observed == null then + desired + else ( + if super.updateStrategy == "Apply" then + metacontroller.getLastApplied(observed) + else + observed + ) + for desired in super.desired + ], + }, + + // Unmarshal the Metacontroller last applied config annotation from an object. + // If the annotation doesn't exist, it returns an "empty" last applied config. + // Returning an empty config when using the "Apply" strategy will tell + // Metacontroller, "I want this to exist, but I don't care what's in it." + getLastApplied(obj):: + local lastApplied = k8s.getAnnotation(obj, "metacontroller.k8s.io/last-applied-configuration"); + if lastApplied != null then + metacontroller.jsonUnmarshal(lastApplied) + else { + apiVersion: obj.apiVersion, + kind: obj.kind, + metadata: {name: obj.metadata.name}, + }, + + // Unmarshal JSON into a Jsonnet value. + // This function is defined as a native extension in jsonnetd. + jsonUnmarshal(jsonString):: std.native("jsonUnmarshal")(jsonString), + + // Convert an integer string in the given base to "int" (actually double). + // Should be precise up to 2^53. + // This function is defined as a native extension in jsonnetd. + parseInt(intStr, base):: std.native("parseInt")(intStr, base), +} diff --git a/examples/vitess/hooks/sync-cell.jsonnet b/examples/vitess/hooks/sync-cell.jsonnet new file mode 100644 index 0000000..570d530 --- /dev/null +++ b/examples/vitess/hooks/sync-cell.jsonnet @@ -0,0 +1,132 @@ +local k8s = import "k8s.libsonnet"; +local etcd = import "etcd.libsonnet"; +local vitess = import "vitess.libsonnet"; +local vtctld = import "vtctld.libsonnet"; +local vtgate = import "vtgate.libsonnet"; +local vttablet = import "vttablet.libsonnet"; +local vtctlclient = import "vtctlclient.libsonnet"; +local metacontroller = import "metacontroller.libsonnet"; + +// Sync hook for VitessCell. +function(request) { + // Wrap the raw request object to add functions. + local observed = metacontroller.observed(request), + + // Whether a component is enabled in this cell. + local enabled = function(name) name in observed.parent.spec, + + // VitessKeyspaces live within VitessCells because a given keyspace + // doesn't have to be deployed across all cells. + // VitessCluster passes the relevant VitessKeyspaces to each VitessCell. + local keyspaces = vitess.keyspaces(observed, observed.parent.spec.keyspaces), + + // Vitess needs its own etcd cluster for internal coordination. + // This requires etcd-operator to be installed and running in the + // same namespace as your VitessCluster. + local etcdClusters = etcd.clusters(observed, + if enabled("etcd") then [observed.parent.spec.etcd]), + + // Vitess administrative server (vtctld). + local vtctldSpecs = + if enabled("vtctld") then [observed.parent.spec.vtctld], + local vtctldServices = vtctld.services(observed, vtctldSpecs), + local vtctldConfigMaps = vtctld.configMaps(observed, vtctldSpecs), + local vtctldDeployments = vtctld.deployments(observed, vtctldSpecs), + + // Vitess query routers (vtgate). + local vtgateSpecs = + if enabled("vtgate") then [observed.parent.spec.vtgate], + local vtgateServices = vtgate.services(observed, vtgateSpecs), + local vtgateDeployments = vtgate.deployments(observed, vtgateSpecs), + + // Vitess database instances (vttablet). + local vttabletServiceSpecs = + if observed.parent.spec.name == "global" then [] else [observed.parent.spec], + local vttabletServices = vttablet.services(observed, vttabletServiceSpecs), + + // vtctlclient Jobs + local topoAddr = vitess.topoFlags(observed.parent.spec.cluster).topo_global_server_address, + local cellName = observed.parent.spec.name, + local vtctlclientSpecs = [] + ( + // For now, just share the global etcd since we're in a single k8s cluster. + if cellName != "global" then [{ + name: "cell-info", + command: ["UpdateCellInfo", "-server_address", topoAddr, + "-root", "/vitess/" + cellName, cellName], + }] else [] + ), + local vtctlclientJobs = vtctlclient.jobs(observed, vtctlclientSpecs), + + // Aggregate status for a cell. + status: { + local status = self, + local specKeyspaceNames = [spec.name for spec in keyspaces.specs], + + keyspaces: + std.sort([ks.spec.name for ks in keyspaces.observed]), + readyKeyspaces: + std.sort([ks.spec.name for ks in k8s.filterReady(keyspaces.observed)]), + + etcd: { + clusters: std.length(etcdClusters.observed), + availableClusters: std.length(k8s.filterAvailable(etcdClusters.observed)), + }, + vtctld: { + services: std.length(vtctldServices.observed), + configMaps: std.length(vtctldConfigMaps.observed), + deployments: std.length(vtctldDeployments.observed), + availableDeployments: + std.length(k8s.filterAvailable(vtctldDeployments.observed)), + }, + vtgate: { + services: std.length(vtgateServices.observed), + deployments: std.length(vtgateDeployments.observed), + availableDeployments: + std.length(k8s.filterAvailable(vtgateDeployments.observed)), + }, + vttablet: { + services: std.length(vttabletServices.observed), + }, + cellInfoRegistered: + if cellName == "global" then + // Global is implicitly registered. + true + else + vtctlclientJobs.isComplete("cell-info"), + conditions: [ + k8s.condition("Ready", + local expected = { + [comp]: if enabled(comp) then 1 else 0 + for comp in ["etcd", "vtctld", "vtgate"] + }; + + status.readyKeyspaces == std.sort(specKeyspaceNames) && + status.etcd.availableClusters >= expected.etcd && + status.vtctld.services >= expected.vtctld && + status.vtctld.configMaps >= expected.vtctld && + status.vtctld.availableDeployments >= expected.vtctld && + status.vtgate.services >= expected.vtgate && + status.vtgate.availableDeployments >= expected.vtgate && + status.vttablet.services == std.length(vttabletServiceSpecs) && + status.cellInfoRegistered + ), + ], + }, + + // Child objects for this cell. + children: + keyspaces.desired + + + etcdClusters.desired + + + vtctldServices.desired + + vtctldConfigMaps.desired + + vtctldDeployments.desired + + + vtgateServices.desired + + vtgateDeployments.desired + + + vttabletServices.desired + + + vtctlclientJobs.desired, +} diff --git a/examples/vitess/hooks/sync-cluster.jsonnet b/examples/vitess/hooks/sync-cluster.jsonnet new file mode 100644 index 0000000..2d11248 --- /dev/null +++ b/examples/vitess/hooks/sync-cluster.jsonnet @@ -0,0 +1,30 @@ +local k8s = import "k8s.libsonnet"; +local vitess = import "vitess.libsonnet"; +local metacontroller = import "metacontroller.libsonnet"; + +// Sync hook for VitessCluster. +function(request) { + // Wrap the raw request object to add functions. + local observed = metacontroller.observed(request), + + // Everything lives in one of the VitessCells. + local cells = vitess.cells(observed, observed.parent.spec.cells), + + // Aggregate status of all VitessCells. + status: { + local status = self, + local specCellNames = [spec.name for spec in cells.specs], + + cells: + std.sort([cell.spec.name for cell in cells.observed]), + readyCells: + std.sort([cell.spec.name for cell in k8s.filterReady(cells.observed)]), + conditions: [ + k8s.condition("Ready", status.readyCells == std.sort(specCellNames)), + ], + }, + + // Children of this VitessCluster. + children: + cells.desired, +} diff --git a/examples/vitess/hooks/sync-keyspace.jsonnet b/examples/vitess/hooks/sync-keyspace.jsonnet new file mode 100644 index 0000000..e053463 --- /dev/null +++ b/examples/vitess/hooks/sync-keyspace.jsonnet @@ -0,0 +1,34 @@ +local k8s = import "k8s.libsonnet"; +local vitess = import "vitess.libsonnet"; +local metacontroller = import "metacontroller.libsonnet"; + +// Sync hook for VitessKeyspace. +function(request) { + // Wrap the raw request object to add functions. + local observed = metacontroller.observed(request), + + local shards = vitess.shards(observed, + if "shards" in observed.parent.spec then + observed.parent.spec.shards + else + vitess.unsharded), + + // Aggregate status of shards in the keyspace. + status: { + local status = self, + local specShardNames = [spec.name for spec in shards.specs], + + shards: + std.sort([s.spec.name for s in shards.observed]), + readyShards: + std.sort([s.spec.name for s in k8s.filterReady(shards.observed)]), + + conditions: [ + k8s.condition("Ready", + status.readyShards == std.sort(specShardNames) + ), + ], + }, + + children: shards.desired, +} diff --git a/examples/vitess/hooks/sync-shard.jsonnet b/examples/vitess/hooks/sync-shard.jsonnet new file mode 100644 index 0000000..c3d0e79 --- /dev/null +++ b/examples/vitess/hooks/sync-shard.jsonnet @@ -0,0 +1,66 @@ +local k8s = import "k8s.libsonnet"; +local vitess = import "vitess.libsonnet"; +local vttablet = import "vttablet.libsonnet"; +local metacontroller = import "metacontroller.libsonnet"; + +// Sync hook for VitessShard. +function(request) { + // Wrap the raw request object to add functions. + local observed = metacontroller.observed(request), + + // Defaults that apply to all tablet types. + local tabletDefaults = + if "defaults" in observed.parent.spec.tablets then + observed.parent.spec.tablets.defaults + else {}, + + // Generate individual tablet specs (not written by users explicitly). + local tabletSpec = function(spec) spec + { + cluster: observed.parent.spec.cluster, + cell: observed.parent.spec.cell, + keyspace: observed.parent.spec.keyspace, + shard: observed.parent.spec.name, + uid: vitess.tabletUid(self), + uidString: "%010d" % self.uid, + alias: self.cell + "-" + self.uidString, + subdomain: "%s-vttablet-%s" % [self.cluster, self.cell], + }, + local tabletSpecs = ( + // "replica" type tablets + if "masterEligible" in observed.parent.spec.tablets then + local spec = tabletDefaults + observed.parent.spec.tablets.masterEligible; + [tabletSpec(spec + {type: "replica", index: i}) for i in std.range(0, spec.replicas - 1)] + else [] + ) + ( + // "rdonly" type tablets + if "batch" in observed.parent.spec.tablets then + local spec = tabletDefaults + observed.parent.spec.tablets.batch; + [tabletSpec(spec + {type: "rdonly", index: i}) for i in std.range(0, spec.replicas - 1)] + else [] + ), + + // Collect observed and compute desired objects for tablets. + local volumes = vttablet.volumes(observed, tabletSpecs), + local pods = vttablet.pods(observed, tabletSpecs), + + // Aggregate status of tablets in the shard. + status: { + local status = self, + local specTabletUids = [spec.uidString for spec in tabletSpecs], + + tablets: + std.sort([vttablet.getUid(t) for t in pods.observed]), + readyTablets: + std.sort([vttablet.getUid(t) for t in k8s.filterReady(pods.observed)]), + + conditions: [ + k8s.condition("Ready", + status.readyTablets == std.sort(specTabletUids) + ), + ], + }, + + children: + volumes.desired + + pods.desired, +} diff --git a/examples/vitess/hooks/vitess.libsonnet b/examples/vitess/hooks/vitess.libsonnet new file mode 100644 index 0000000..f2bf9e4 --- /dev/null +++ b/examples/vitess/hooks/vitess.libsonnet @@ -0,0 +1,189 @@ +local k8s = import "k8s.libsonnet"; +local metacontroller = import "metacontroller.libsonnet"; + +// Library for working with Vitess objects. +{ + local vitess = self, + + apiVersion: "vitess.io/v1alpha1", + + // Default flags shared by multiple Vitess components. + baseFlags: { + logtostderr: true, + }, + serverFlags: vitess.baseFlags + { + port: 15000, + grpc_port: 15999, + }, + topoFlags(cluster):: { + topo_implementation: "etcd2", + topo_global_root: "/vitess/global", + topo_global_server_address: + "http://%s-global-etcd-client:2379" % cluster, + }, + + // Collections of Vitess objects. + cells(observed, specs):: + metacontroller.collection(observed, specs, vitess.apiVersion, "VitessCell", vitess.cell), + keyspaces(observed, specs):: + metacontroller.collection(observed, specs, vitess.apiVersion, "VitessKeyspace", vitess.keyspace), + shards(observed, specs):: + local shardSpecs = [vitess.shardSpec(spec) for spec in specs]; + metacontroller.collection(observed, shardSpecs, vitess.apiVersion, "VitessShard", vitess.shard), + + // Create/update a VitessCell child for a VitessCluster parent. + cell(observed, spec):: { + apiVersion: vitess.apiVersion, + kind: "VitessCell", + metadata: { + name: observed.parent.metadata.name + "-" + spec.name, + labels: observed.parent.spec.template.metadata.labels, + }, + // Each VitessCell spec starts from a VitessCluster.spec.cells item. + spec: spec + { + // Pass down info from the parent that children will need. + cluster: observed.parent.metadata.name, + backupFlags: + if "backupFlags" in observed.parent.spec && observed.parent.spec.backupFlags != null then + observed.parent.spec.backupFlags + else {}, + + // Propagate the selector and child template from the parent. + // Add a "cell" label that will be applied to all child objects + // of this VitessCell. + selector: observed.parent.spec.selector + { + matchLabels+: { + "vitess.io/cell": spec.name, + }, + }, + template: observed.parent.spec.template + { + metadata+: { + labels+: { + "vitess.io/cell": spec.name, + }, + }, + }, + + // Add keyspaces that are enabled in this cell. + // This way you don't have to duplicate the spec in every cell. + keyspaces: + std.filter( + function(ks) std.count(ks.cells, spec.name) > 0, observed.parent.spec.keyspaces), + }, + }, + + // Create/update a VitessKeyspace child for a VitessCell parent. + keyspace(observed, spec):: { + apiVersion: vitess.apiVersion, + kind: "VitessKeyspace", + metadata: { + name: observed.parent.metadata.name + "-" + spec.name, + labels: observed.parent.spec.template.metadata.labels, + }, + // Each VitessKeyspace spec starts from a VitessCell.spec.keyspaces item. + spec: spec + { + // Pass down info from the parent that children will need. + cluster: observed.parent.spec.cluster, + cell: observed.parent.spec.name, + backupFlags: observed.parent.spec.backupFlags, + + // Propagate the selector and child template from the parent. + // Add a "keyspace" label that will be applied to all child objects + // of this VitessKeyspace. + selector: observed.parent.spec.selector + { + matchLabels+: { + "vitess.io/keyspace": spec.name, + }, + }, + template: observed.parent.spec.template + { + metadata+: { + labels+: { + "vitess.io/keyspace": spec.name, + }, + }, + }, + }, + }, + + // Create/update a VitessShard child for a VitessKeyspace parent. + shard(observed, spec):: { + apiVersion: vitess.apiVersion, + kind: "VitessShard", + metadata: { + name: observed.parent.metadata.name + "-" + spec.kname, + labels: observed.parent.spec.template.metadata.labels, + }, + // Each VitessShard spec starts from a VitessKeyspace.spec.shards item. + spec: spec + { + // Pass down info from the parent that children will need. + cluster: observed.parent.spec.cluster, + cell: observed.parent.spec.cell, + keyspace: observed.parent.spec.name, + backupFlags: observed.parent.spec.backupFlags, + + // Propagate the selector and child template from the parent. + // Add a "shard" label that will be applied to all child objects + // of this VitessShard. + selector: observed.parent.spec.selector + { + matchLabels+: { + "vitess.io/shard": spec.kname, + }, + }, + template: observed.parent.spec.template + { + metadata+: { + labels+: { + "vitess.io/shard": spec.kname, + }, + }, + }, + + // Propagate tablet specs from the keyspace. + // This way you don't have to duplicate the spec in every shard. + tablets: observed.parent.spec.tablets, + }, + }, + + // Extend a shard spec. + shardSpec(spec):: + if "keyRange" in spec then + // Range-based shard. + spec + { + // The Vitess shard name. + name: self.keyRange.start + "-" + self.keyRange.end, + + // The Kubernetes-safe name (can't start or end with "-"). + kname:: + (if self.keyRange.start != "" then self.keyRange.start else "x") + + "-" + + (if self.keyRange.end != "" then self.keyRange.end else "x"), + } + else + // Custom shard or unsharded. + spec + { + // Custom shard names should just be integers. + kname:: self.name, + }, + + // Shard spec list for an "unsharded" keyspace. + unsharded: [{ + name: "0", + }], + + // Format key-value pairs (object fields) into + // a flags string for a Vitess binary. + formatFlags(flags):: + std.join(" ", [ + "-%s=\"%s\"" % [key,flags[key]] for key in std.objectFields(flags) + ]), + + // Compute a deterministic tablet UID (32-bit unsigned int) for a tablet spec. + // Note that this is an arbitrary algorithm, not necessarily shared by any + // other methods of deploying Vitess. + tabletUid(spec):: + // Make a string that's unique within a given cluster (even across cells). + local str = std.join("/", [spec.cell, spec.keyspace, spec.shard, spec.type]); + // Checksum the string, take the first 24 bits, convert to integer. + local hash = metacontroller.parseInt(std.md5(str)[:6], 16); + // Shift left 2 decimal digits, add index. + hash * 100 + spec.index, +} diff --git a/examples/vitess/hooks/vtctlclient.libsonnet b/examples/vitess/hooks/vtctlclient.libsonnet new file mode 100644 index 0000000..b085672 --- /dev/null +++ b/examples/vitess/hooks/vtctlclient.libsonnet @@ -0,0 +1,61 @@ +local k8s = import "k8s.libsonnet"; +local vitess = import "vitess.libsonnet"; +local metacontroller = import "metacontroller.libsonnet"; + +// Library for running vtctlclient commands as Jobs. +{ + local vtctlclient = self, + + image: "vitess/vtctlclient", + + // Filter for vtctlclient Jobs among other child Jobs. + matchJob(obj):: + k8s.matchLabels(obj, {"vitess.io/component": "vtctlclient"}), + + jobs(observed, specs):: + metacontroller.collection(observed, specs, "batch/v1", "Job", vtctlclient.job) + + metacontroller.collectionFilter(vtctlclient.matchJob) + + metacontroller.collectionImmutable + + { + isComplete(specName):: + local name = observed.parent.metadata.name + "-" + specName; + k8s.conditionStatus(super.getObserved(name), "Complete") == "True", + }, + + // Create/update a Job. + job(observed, spec):: { + // The metadata.name of the Job. + local labels = observed.parent.spec.template.metadata.labels + { + "vitess.io/component": "vtctlclient", + }, + local vtctldAddr = "%s-global-vtctld:%d" % + [observed.parent.spec.cluster, vitess.serverFlags.grpc_port], + + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: observed.parent.metadata.name + "-" + spec.name, + labels: labels, + }, + spec: { + activeDeadlineSeconds: 60, + backoffLimit: 10, + template: { + metadata: { + labels: labels, + }, + spec: { + restartPolicy: "OnFailure", + containers: [ + { + name: "vtctlclient", + image: vtctlclient.image, + command: ["vtctlclient", "-server", vtctldAddr] + spec.command, + }, + ], + + }, + }, + }, + }, +} diff --git a/examples/vitess/hooks/vtctld.libsonnet b/examples/vitess/hooks/vtctld.libsonnet new file mode 100644 index 0000000..0c3e5f8 --- /dev/null +++ b/examples/vitess/hooks/vtctld.libsonnet @@ -0,0 +1,165 @@ +local k8s = import "k8s.libsonnet"; +local vitess = import "vitess.libsonnet"; +local metacontroller = import "metacontroller.libsonnet"; + +{ + local vtctld = self, + + // Filter for vtctld objects among other children of VitessCell. + matchName(obj):: + std.endsWith(obj.metadata.name, "-vtctld"), + + // Collections of vtctld objects. + services(observed, specs):: + metacontroller.collection(observed, specs, "v1", "Service", vtctld.service) + + metacontroller.collectionFilter(vtctld.matchName), + configMaps(observed, specs):: + metacontroller.collection(observed, specs, "v1", "ConfigMap", vtctld.configMap) + + metacontroller.collectionFilter(vtctld.matchName), + deployments(observed, specs):: + metacontroller.collection(observed, specs, "apps/v1beta2", "Deployment", vtctld.deployment) + + metacontroller.collectionFilter(vtctld.matchName), + + // Create/update a Service for vtctld. + service(observed, spec):: { + apiVersion: "v1", + kind: "Service", + metadata: { + name: observed.parent.metadata.name + "-vtctld", + labels: observed.parent.spec.template.metadata.labels, + }, + spec: { + selector: observed.parent.spec.selector.matchLabels + { + "vitess.io/component": "vtctld", + }, + ports: [ + {name: "web", port: 15000}, + {name: "grpc", port: 15999}, + ], + }, + }, + + // Create/update a Deployment for vtctld. + deployment(observed, spec):: { + local vtctldFlags = { + cell: observed.parent.spec.name, + web_dir: "/vt/web/vtctld", + web_dir2: "/vt/web/vtctld2/app", + workflow_manager_init: true, + workflow_manager_use_election: true, + service_map: "grpc-vtctl", + }, + local flags = vitess.serverFlags + + vitess.topoFlags(observed.parent.spec.cluster) + + vtctldFlags + + observed.parent.spec.backupFlags + + (if "flags" in spec then spec.flags else {}), + + apiVersion: "apps/v1beta2", + kind: "Deployment", + metadata: { + name: observed.parent.metadata.name + "-vtctld", + labels: observed.parent.spec.template.metadata.labels, + }, + spec: { + replicas: spec.replicas, + selector: observed.parent.spec.selector + { + matchLabels+: { + "vitess.io/component": "vtctld", + }, + }, + template: { + metadata: { + labels: observed.parent.spec.template.metadata.labels + { + "vitess.io/component": "vtctld", + }, + }, + spec: { + securityContext: {runAsUser: 999, fsGroup: 999}, + initContainers: [ + { + name: "init-vtctld", + image: spec.image, + command: ["bash", "-c", ||| + set -ex + rm -rf /vt/web/* + cp -R $VTTOP/web/* vt/web/ + cp /mnt/config/config.js /vt/web/vtctld/ + |||], + volumeMounts: [ + {name: "config", mountPath: "/mnt/config"}, + {name: "web", mountPath: "/vt/web"}, + ], + }, + ], + containers: [ + { + name: "vtctld", + image: spec.image, + livenessProbe: { + httpGet: {path: "/debug/vars", port: 15000}, + initialDelaySeconds: 30, + timeoutSeconds: 5, + }, + volumeMounts: [ + {name: "vtdataroot", mountPath: "/vt/vtdataroot"}, + {name: "web", mountPath: "/vt/web"}, + ], + resources: spec.resources, + command: ["bash", "-c", + "set -ex; exec /vt/bin/vtctld " + + vitess.formatFlags(flags)], + }, + ], + volumes: [ + {name: "vtdataroot", emptyDir: {}}, + {name: "web", emptyDir: {}}, + { + name: "config", + configMap: { + name: observed.parent.metadata.name + "-vtctld", + }, + }, + ], + }, + }, + }, + }, + + configMap(observed, spec):: { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: observed.parent.metadata.name + "-vtctld", + labels: observed.parent.spec.template.metadata.labels, + }, + data: { + // Customize the vtctld web UI for Kubernetes. + "config.js": ||| + vtconfig = { + k8s_proxy_re: /(\/api\/v1\/namespaces\/.*)\/services\//, + tabletLinks: function(tablet) { + status_href = 'http://'+tablet.hostname+':'+tablet.port_map.vt+'/debug/status' + + // If we're going through the Kubernetes API server proxy, + // route tablet links through the proxy as well. + var match = window.location.pathname.match(vtconfig.k8s_proxy_re); + if (match) { + var hostname = tablet.hostname.split('.'); + var alias = hostname[0]; + var subdomain = hostname[1]; + status_href = match[1] + '/pods/' + subdomain + '-' + alias.split('-')[1] + ':' + tablet.port_map.vt + '/proxy/debug/status'; + } + + return [ + { + title: 'Status', + href: status_href + } + ]; + } + }; + |||, + }, + }, +} diff --git a/examples/vitess/hooks/vtgate.libsonnet b/examples/vitess/hooks/vtgate.libsonnet new file mode 100644 index 0000000..da0f864 --- /dev/null +++ b/examples/vitess/hooks/vtgate.libsonnet @@ -0,0 +1,93 @@ +local k8s = import "k8s.libsonnet"; +local vitess = import "vitess.libsonnet"; +local metacontroller = import "metacontroller.libsonnet"; + +{ + local vtgate = self, + + // Filter for vtgate objects among other children of VitessCell. + matchName(obj):: + std.endsWith(obj.metadata.name, "-vtgate"), + + // Collections of vtgate objects. + services(observed, specs):: + metacontroller.collection(observed, specs, "v1", "Service", vtgate.service) + + metacontroller.collectionFilter(vtgate.matchName), + deployments(observed, specs):: + metacontroller.collection(observed, specs, "apps/v1beta2", "Deployment", vtgate.deployment) + + metacontroller.collectionFilter(vtgate.matchName), + + // Create/update a Service for vtgate. + service(observed, spec):: { + apiVersion: "v1", + kind: "Service", + metadata: { + name: observed.parent.metadata.name + "-vtgate", + labels: observed.parent.spec.template.metadata.labels, + }, + spec: { + selector: observed.parent.spec.selector.matchLabels + { + "vitess.io/component": "vtgate", + }, + ports: [ + {name: "web", port: 15000}, + {name: "grpc", port: 15999}, + ], + }, + }, + + // Create/update a Deployment for vtgate. + deployment(observed, spec):: { + local vtgateFlags = { + cell: observed.parent.spec.name, + service_map: "grpc-vtgateservice", + cells_to_watch: self.cell, + tablet_types_to_wait: "MASTER,REPLICA", + gateway_implementation: "discoverygateway" + }, + local flags = vitess.serverFlags + + vitess.topoFlags(observed.parent.spec.cluster) + + vtgateFlags + + (if "flags" in spec then spec.flags else {}), + + apiVersion: "apps/v1beta2", + kind: "Deployment", + metadata: { + name: observed.parent.metadata.name + "-vtgate", + labels: observed.parent.spec.template.metadata.labels, + }, + spec: { + replicas: spec.replicas, + selector: observed.parent.spec.selector + { + matchLabels+: { + "vitess.io/component": "vtgate", + }, + }, + template: { + metadata: { + labels: observed.parent.spec.template.metadata.labels + { + "vitess.io/component": "vtgate", + }, + }, + spec: { + securityContext: {runAsUser: 999, fsGroup: 999}, + containers: [ + { + name: "vtgate", + image: spec.image, + livenessProbe: { + httpGet: {path: "/debug/vars", port: 15000}, + initialDelaySeconds: 30, + timeoutSeconds: 5, + }, + resources: spec.resources, + command: ["bash", "-c", + "set -ex; exec /vt/bin/vtgate " + + vitess.formatFlags(flags)], + }, + ], + }, + }, + }, + }, +} diff --git a/examples/vitess/hooks/vttablet.libsonnet b/examples/vitess/hooks/vttablet.libsonnet new file mode 100644 index 0000000..0dc82b1 --- /dev/null +++ b/examples/vitess/hooks/vttablet.libsonnet @@ -0,0 +1,198 @@ +local k8s = import "k8s.libsonnet"; +local vitess = import "vitess.libsonnet"; +local metacontroller = import "metacontroller.libsonnet"; + +{ + local vttablet = self, + + // Filter for vttablet-related objects among other children of VitessShard. + matchLabels(obj):: + k8s.matchLabels(obj, {"vitess.io/component": "vttablet"}), + + // Collections of vttablet objects. + pods(observed, specs):: + metacontroller.collection(observed, specs, "v1", "Pod", vttablet.pod) + + metacontroller.collectionFilter(vttablet.matchLabels) + + metacontroller.collectionImmutable, + volumes(observed, specs):: + metacontroller.collection(observed, specs, "v1", "PersistentVolumeClaim", vttablet.volume) + + metacontroller.collectionFilter(vttablet.matchLabels), + services(observed, specs):: + metacontroller.collection(observed, specs, "v1", "Service", vttablet.service) + + metacontroller.collectionFilter(vttablet.matchLabels), + + // Create/update a Pod for a tablet spec within a VitessShard parent. + pod(observed, spec):: { + local podName = observed.parent.spec.cluster + "-vttablet-" + spec.alias, + + // A shell expression for a running tablet to find its own hostname. + local hostnameExpr = "$(hostname -s)." + spec.subdomain, + + local defaultVttabletFlags = { + service_map: "grpc-queryservice,grpc-tabletmanager,grpc-updatestream", + "tablet-path": spec.alias, + tablet_hostname: hostnameExpr, + init_keyspace: spec.keyspace, + init_shard: spec.shard, + init_tablet_type: spec.type, + health_check_interval: "5s", + mysqlctl_socket: "$VTDATAROOT/mysqlctl.sock", + "db-config-app-uname": "vt_app", + "db-config-app-dbname": "vt_" + spec.keyspace, + "db-config-app-charset": "utf8", + "db-config-dba-uname": "vt_dba", + "db-config-dba-dbname": "vt_" + spec.keyspace, + "db-config-dba-charset": "utf8", + "db-config-repl-uname": "vt_repl", + "db-config-repl-dbname": "vt_" + spec.keyspace, + "db-config-repl-charset": "utf8", + "db-config-filtered-uname": "vt_filtered", + "db-config-filtered-dbname": "vt_" + spec.keyspace, + "db-config-filtered-charset": "utf8", + enable_semi_sync: true, + enable_replication_reporter: true, + orc_api_url: "http://%s-global-orchestrator/api" % spec.cluster, + orc_discover_interval: "5m", + restore_from_backup: "backup_storage_implementation" in observed.parent.spec.backupFlags, + }, + local vttabletFlags = vitess.serverFlags + + vitess.topoFlags(observed.parent.spec.cluster) + + defaultVttabletFlags + + observed.parent.spec.backupFlags + + (if "flags" in spec.vttablet then spec.vttablet.flags else {}), + + local mysqlctldFlags = vitess.baseFlags + { + tablet_uid: spec.uid, + socket_file: "$VTDATAROOT/mysqlctl.sock", + "db-config-dba-uname": "vt_dba", + "db-config-dba-charset": "utf8", + init_db_sql_file: "$VTROOT/config/init_db.sql", + } + (if "flags" in spec.mysql then spec.mysql.flags else {}), + + // TODO(enisoc): Allow customizing my.cnf somehow. + local extraMyCnf = [ + "/vt/config/mycnf/master_mysql56.cnf", + "/vt/vtdataroot/init/report-host.cnf", + ], + + local env = [ + {name: "EXTRA_MY_CNF", value: std.join(":", extraMyCnf)}, + ], + + apiVersion: "v1", + kind: "Pod", + metadata: { + name: podName, + labels: observed.parent.spec.template.metadata.labels + { + "vitess.io/component": "vttablet", + "vitess.io/tablet-uid": spec.uidString, + }, + }, + spec: { + hostname: spec.alias, + subdomain: spec.subdomain, + securityContext: {runAsUser: 999, fsGroup: 999}, + initContainers: [ + { + name: "init-vtdataroot", + image: spec.image, + command: ["bash", "-c", + "set -ex; mkdir -p $VTDATAROOT/init; + echo report-host=%s > $VTDATAROOT/init/report-host.cnf" + % hostnameExpr], + volumeMounts: [ + {name: "vtdataroot", mountPath: "/vt/vtdataroot"}, + ], + }, + ], + containers: [ + { + name: "vttablet", + image: spec.image, + livenessProbe: { + httpGet: {path: "/debug/vars", port: 15000}, + initialDelaySeconds: 60, + timeoutSeconds: 10, + }, + volumeMounts: [ + {name: "vtdataroot", mountPath: "/vt/vtdataroot"}, + // TODO(enisoc): Remove certs volume after switching to new vitess image. + {name: "certs", readOnly: true, mountPath: "/etc/ssl/certs/ca-certificates.crt"}, + ], + resources: spec.vttablet.resources, + ports: [ + {name: "web", containerPort: 15000}, + {name: "grpc", containerPort: 15999}, + ], + command: ["bash", "-c", + "set -ex; exec /vt/bin/vttablet " + + vitess.formatFlags(vttabletFlags)], + env: env, + }, + { + name: "mysql", + image: spec.image, + volumeMounts: [ + {name: "vtdataroot", mountPath: "/vt/vtdataroot"}, + ], + resources: spec.mysql.resources, + command: ["bash", "-c", + "set -ex; exec /vt/bin/mysqlctld " + + vitess.formatFlags(mysqlctldFlags)], + env: env, + }, + ], + volumes: [ + {name: "vtdataroot", persistentVolumeClaim: {claimName: podName}}, + // TODO(enisoc): Remove certs volume after switching to new vitess image. + {name: "certs", hostPath: {path: "/etc/ssl/certs/ca-certificates.crt"}}, + ], + }, + }, + + // Create/update a PVC for a tablet spec. + volume(observed, spec):: { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: observed.parent.spec.cluster + "-vttablet-" + spec.alias, + labels: observed.parent.spec.template.metadata.labels + { + "vitess.io/component": "vttablet", + "vitess.io/tablet-uid": spec.uidString, + }, + }, + spec: spec.volumeClaim, + }, + + // Create/update a vttablet headless Service for a VitessCell spec. + service(observed, spec):: { + apiVersion: "v1", + kind: "Service", + metadata: { + name: observed.parent.spec.cluster + "-vttablet-" + spec.name, + labels: observed.parent.spec.template.metadata.labels + { + "vitess.io/component": "vttablet", + }, + annotations: { + "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true", + }, + }, + spec: { + selector: observed.parent.spec.selector.matchLabels + { + "vitess.io/component": "vttablet", + }, + ports: [ + {name: "web", port: 15000}, + {name: "grpc", port: 15999}, + ], + clusterIP: "None", + publishNotReadyAddresses: true, + }, + }, + + getUid(tablet):: + // Unlike the other CRDs, we can't put our own fields in the Pod spec, + // so we put the UID in a label, which can also be used to select + // individual tablets (if combined with cluster and cell labels). + k8s.getLabel(tablet, "vitess.io/tablet-uid"), +} diff --git a/examples/vitess/my-vitess.yaml b/examples/vitess/my-vitess.yaml new file mode 100644 index 0000000..668824f --- /dev/null +++ b/examples/vitess/my-vitess.yaml @@ -0,0 +1,68 @@ +apiVersion: vitess.io/v1alpha1 +kind: VitessCluster +metadata: + name: vitess +spec: + selector: + matchLabels: + app: vitess + template: + metadata: + labels: + app: vitess + cells: + - name: global + etcd: + version: 3.2.11 + size: 3 + vtctld: + image: vitess/lite:v2.1.1 + replicas: 1 + resources: + limits: {cpu: "100m", memory: "128Mi"} + orchestrator: + image: vitess/orchestrator:latest + replicas: 1 + resources: + limits: {cpu: "100m", memory: "128Mi"} + - name: zone1 + vtgate: + image: vitess/lite:v2.1.1 + replicas: 3 + resources: + limits: {cpu: "100m", memory: "128Mi"} + keyspaces: + - name: main + cells: ["zone1"] + shards: + - keyRange: {start: "", end: "80"} + - keyRange: {start: "80", end: ""} + tablets: + defaults: + image: vitess/lite:v2.1.1 + volumeClaim: + accessModes: ["ReadWriteOnce"] + resources: + requests: {storage: "4Gi"} + masterEligible: + replicas: 2 + vttablet: + resources: + limits: {cpu: "500m", memory: "512Mi"} + mysql: + resources: + limits: {cpu: "500m", memory: "512Mi"} + batch: + replicas: 1 + vttablet: + resources: + limits: {cpu: "100m", memory: "256Mi"} + mysql: + resources: + limits: {cpu: "100m", memory: "256Mi"} +# These flags are added to any Vitess binaries that need access to backups. +# For example, if you're in GCE and gave your Kubernetes nodes the "storage" +# scope, you can specify a pre-existing GCS bucket like this: +# backupFlags: +# backup_storage_implementation: gcs +# gcs_backup_storage_bucket: your-bucket-name diff --git a/examples/vitess/test/k8s_test.jsonnet b/examples/vitess/test/k8s_test.jsonnet new file mode 100644 index 0000000..19d38bb --- /dev/null +++ b/examples/vitess/test/k8s_test.jsonnet @@ -0,0 +1,108 @@ +// Just run this file with the jsonnet CLI and check the summary at the bottom. + +local k8s = import "../hooks/k8s.libsonnet"; + +local test = function(name, got, want) + local passed = got == want; + { + name: name, + passed: passed, + [if !passed then "got"]: got, + [if !passed then "want"]: want, + }; + +local tests = [ + test("mergeListMap", + local lhs = [ + // Should preserve original list order. + {name: "z", value: 2}, + {name: "a", value: "a"}, + ]; + local rhs = [ + {name: "a", extra: 3, value+: "a"}, + {name: "b", extra: 4}, + ]; + k8s.mergeListMap(lhs, rhs) + , + want = [ + {name: "z", value: 2}, + {name: "a", value: "aa", extra: 3}, + {name: "b", extra: 4}, + ] + ), + + test("makePatch", + { + obj: { + changed: {initial: 1}, + unchanged: {initial: 2}, + list: [1,2,3], + // A Kubernetes "list map" keyed by the "name" field. + listMap: [ + {name: "item2", value: 2}, + {name: "item1", value: 1}, + ], + }, + } + + + k8s.makePatch({ + // This should behave as if it were "obj+: { ... }". + obj: { + // makePatch() should recursively apply to fields inside "obj", + // so this should behave as if it were "changed+: { ... }". + changed: {extra: 4}, + // This should replace the whole list. + list: [5,6,7], + // This should behave as if it used k8s.mergeListMap(). + listMap: [ + {name: "item1", value: 10, extra: 3}, + ], + // A listMap that doesn't exist in the original object. + newListMap: [ + {name: "item1", value: 11}, + ], + }, + }) + , + want = { + obj: { + changed: {initial: 1, extra: 4}, + unchanged: {initial: 2}, + list: [5,6,7], + listMap: [ + {name: "item2", value: 2}, + {name: "item1", value: 10, extra: 3}, + ], + newListMap: [ + {name: "item1", value: 11}, + ], + }, + } + ), + + local obj = {metadata:{labels:{a:1,b:2}}}; + local table = [ + {labels: {a:1}, match: true}, + {labels: {a:2}, match: false}, + {labels: {b:1}, match: false}, + {labels: {b:2}, match: true}, + {labels: {c:1}, match: false}, + {labels: {a:1,b:2}, match: true}, + {labels: {a:2,b:2}, match: false}, + {labels: {a:1,b:3}, match: false}, + ]; + test("matchLabels", + [k8s.matchLabels(obj, t.labels) for t in table] + , + want = [t.match for t in table] + ), +]; + +{ + all: tests, + summary: { + total: std.length(tests), + passed: std.length(std.filter(function(t) t.passed, tests)), + failed: self.total - self.passed, + }, +} diff --git a/examples/vitess/test/vitess_test.jsonnet b/examples/vitess/test/vitess_test.jsonnet new file mode 100644 index 0000000..f35a9c9 --- /dev/null +++ b/examples/vitess/test/vitess_test.jsonnet @@ -0,0 +1,51 @@ +// Just run this file with the jsonnet CLI and check the summary at the bottom. + +local vitess = import "../hooks/vitess.libsonnet"; + +local test = function(name, got, want) + local passed = got == want; + { + name: name, + passed: passed, + [if !passed then "got"]: got, + [if !passed then "want"]: want, + }; + +local tests = [ + local table = [ + {flags: {}, want: ""}, + { + flags: { + port: 15000, + mysqlctl_socket: "$VTDATAROOT/mysqlctl.sock", + enable_semi_sync: true, + "db-config-app-uname": "vt_app", + }, + want: "-db-config-app-uname=\"vt_app\" -enable_semi_sync=\"true\" -mysqlctl_socket=\"$VTDATAROOT/mysqlctl.sock\" -port=\"15000\"" + }, + ]; + test("formatFlags", + [vitess.formatFlags(t.flags) for t in table] + , + want = [t.want for t in table] + ), + + local table = [ + {cell: "zone1", keyspace: "main", shard: "-80", type: "replica", index: 0, uid: 214747900}, + {cell: "zone1", keyspace: "main", shard: "-80", type: "replica", index: 15, uid: 214747915}, + ]; + test("tabletUid", + [vitess.tabletUid(t) for t in table] + , + want = [t.uid for t in table] + ), +]; + +{ + all: tests, + summary: { + total: std.length(tests), + passed: std.length(std.filter(function(t) t.passed, tests)), + failed: self.total - self.passed, + }, +} diff --git a/examples/vitess/update-hooks.sh b/examples/vitess/update-hooks.sh new file mode 100755 index 0000000..590f8d3 --- /dev/null +++ b/examples/vitess/update-hooks.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +kubectl_output="$(kubectl create configmap vitess-operator-hooks -n metacontroller --from-file=hooks --append-hash)" +echo "${kubectl_output}" +expr='configmap "(.+)" created' +if [[ "${kubectl_output}" =~ $expr ]]; then + configmap="${BASH_REMATCH[1]}" + patch="{\"spec\":{\"template\":{\"spec\":{\"volumes\":[{\"name\":\"hooks\",\"configMap\":{\"name\":\"${configmap}\"}}]}}}}" + kubectl patch deployment -n metacontroller vitess-operator -p "${patch}" +fi diff --git a/examples/vitess/vitess-operator.yaml b/examples/vitess/vitess-operator.yaml new file mode 100644 index 0000000..f29451d --- /dev/null +++ b/examples/vitess/vitess-operator.yaml @@ -0,0 +1,183 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: vitessclusters.vitess.io +spec: + group: vitess.io + version: v1alpha1 + scope: Namespaced + names: + plural: vitessclusters + singular: vitesscluster + kind: VitessCluster + shortNames: ["vt"] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: vitesscells.vitess.io +spec: + group: vitess.io + version: v1alpha1 + scope: Namespaced + names: + plural: vitesscells + singular: vitesscell + kind: VitessCell + shortNames: ["vtc"] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: vitesskeyspaces.vitess.io +spec: + group: vitess.io + version: v1alpha1 + scope: Namespaced + names: + plural: vitesskeyspaces + singular: vitesskeyspace + kind: VitessKeyspace + shortNames: ["vtk"] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: vitessshards.vitess.io +spec: + group: vitess.io + version: v1alpha1 + scope: Namespaced + names: + plural: vitessshards + singular: vitessshard + kind: VitessShard + shortNames: ["vts"] +--- +apiVersion: metacontroller.k8s.io/v1alpha1 +kind: CompositeController +metadata: + name: vitesscluster-controller +spec: + updateStrategy: Apply + parentResource: + apiVersion: vitess.io/v1alpha1 + resource: vitessclusters + childResources: + - apiVersion: vitess.io/v1alpha1 + resources: ["vitesscells"] + clientConfig: + service: + name: vitess-operator + namespace: metacontroller + hooks: + sync: + path: /sync-cluster +--- +apiVersion: metacontroller.k8s.io/v1alpha1 +kind: CompositeController +metadata: + name: vitesscell-controller +spec: + updateStrategy: Apply + parentResource: + apiVersion: vitess.io/v1alpha1 + resource: vitesscells + childResources: + - apiVersion: v1 + resources: ["services", "configmaps"] + - apiVersion: apps/v1beta2 + resources: ["deployments"] + - apiVersion: batch/v1 + resources: ["jobs"] + - apiVersion: etcd.database.coreos.com/v1beta2 + resources: ["etcdclusters"] + - apiVersion: vitess.io/v1alpha1 + resources: ["vitesskeyspaces"] + clientConfig: + service: + name: vitess-operator + namespace: metacontroller + hooks: + sync: + path: /sync-cell +--- +apiVersion: metacontroller.k8s.io/v1alpha1 +kind: CompositeController +metadata: + name: vitesskeyspace-controller +spec: + updateStrategy: Apply + parentResource: + apiVersion: vitess.io/v1alpha1 + resource: vitesskeyspaces + childResources: + - apiVersion: vitess.io/v1alpha1 + resources: ["vitessshards"] + clientConfig: + service: + name: vitess-operator + namespace: metacontroller + hooks: + sync: + path: /sync-keyspace +--- +apiVersion: metacontroller.k8s.io/v1alpha1 +kind: CompositeController +metadata: + name: vitessshard-controller +spec: + updateStrategy: Apply + parentResource: + apiVersion: vitess.io/v1alpha1 + resource: vitessshards + childResources: + - apiVersion: v1 + resources: ["pods", "persistentvolumeclaims"] + clientConfig: + service: + name: vitess-operator + namespace: metacontroller + hooks: + sync: + path: /sync-shard +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: vitess-operator + namespace: metacontroller +spec: + replicas: 1 + selector: + matchLabels: + app: vitess-operator + template: + metadata: + labels: + app: vitess-operator + spec: + containers: + - name: hooks + image: gcr.io/enisoc-kubernetes/jsonnetd:0.1 + imagePullPolicy: Always + workingDir: /vt/operator/hooks + volumeMounts: + - name: hooks + mountPath: /vt/operator/hooks + volumes: + - name: hooks + configMap: + name: vitess-operator-hooks +--- +apiVersion: v1 +kind: Service +metadata: + name: vitess-operator + namespace: metacontroller +spec: + selector: + app: vitess-operator + ports: + - port: 80 + targetPort: 8080