diff --git a/.github/workflows/create-release.yaml b/.github/workflows/create-release.yaml index 72e6a9c..6618f13 100644 --- a/.github/workflows/create-release.yaml +++ b/.github/workflows/create-release.yaml @@ -10,7 +10,16 @@ jobs: runs-on: ubuntu-18.04 timeout-minutes: 5 steps: - - uses: actions/checkout@v2 + - name: Get milestone title + id: title + run: | + cat $GITHUB_EVENT_PATH | jq -r .milestone.title >> $MILESTONE_TITLE_FILE + echo "::set-output name=title::$(cat $MILESTONE_TITLE_FILE)" + + - name: Ensure tag is already present + uses: actions/checkout@v2 + with: + ref: ${{ steps.title.outputs.title }} - name: Create Release Notes uses: docker://decathlon/release-notes-generator-action:2.0.1 @@ -26,19 +35,7 @@ jobs: notes="${notes//$'\r'/'%0D'}" echo ::set-output name=notes::$notes - - name: Get milestone title - id: title - run: | - cat $GITHUB_EVENT_PATH | jq -r .milestone.title >> $MILESTONE_TITLE_FILE - echo "::set-output name=title::$(cat $MILESTONE_TITLE_FILE)" - - - name: Tag version - uses: anothrNick/github-tag-action@1.17.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CUSTOM_TAG: ${{ steps.title.outputs.title }} - - - name: Create Release + - name: Create draft release uses: actions/create-release@latest env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tag-docker-image-version.yaml b/.github/workflows/tag-docker-image-version.yaml new file mode 100644 index 0000000..5efa722 --- /dev/null +++ b/.github/workflows/tag-docker-image-version.yaml @@ -0,0 +1,28 @@ +name: Tag Docker image version +on: + push: + tags: + - v* +env: + MILESTONE_TITLE_FILE: milestone-title.txt + MILESTONE_RELEASE_NOTES_FILE: release_file.md + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_SOURCE_IMAGE: docker.pkg.github.com/${{ github.repository }}/build:${{ github.sha }} + DOCKER_TARGET_IMAGE_BASE: docker.pkg.github.com/${{ github.repository }}/elasticsearch-gist-operator +jobs: + tag-and-push: + runs-on: ubuntu-18.04 + timeout-minutes: 10 + steps: + - name: Tag and push Docker image with version + run: | + export VERSION=$(echo ${GITHUB_REF//v} | cut -d / -f 3) + docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD docker.pkg.github.com + docker pull $DOCKER_SOURCE_IMAGE + docker tag $DOCKER_SOURCE_IMAGE $DOCKER_TARGET_IMAGE_BASE:latest + docker push $DOCKER_TARGET_IMAGE_BASE:latest + docker tag $DOCKER_SOURCE_IMAGE $DOCKER_TARGET_IMAGE_BASE:${VERSION%.*.*} + docker push $DOCKER_TARGET_IMAGE_BASE:${VERSION%.*.*} + docker tag $DOCKER_SOURCE_IMAGE $DOCKER_TARGET_IMAGE_BASE:${VERSION%.*} + docker push $DOCKER_TARGET_IMAGE_BASE:${VERSION%.*} diff --git a/Makefile b/Makefile index 1064fa0..a829c90 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,7 @@ reset: # Forcefully removes all and any traces of existing custom resources kubectl delete elasticsearch-indices.frankkoornstra.nl,elasticsearch-templates.frankkoornstra.nl --all --grace-period=0 --force || true definitionUpdate: # Updates the custom resource definitions - kubectl apply -f ${current_dir}/crd/crd-template.yaml - kubectl apply -f ${current_dir}/crd/crd-index.yaml + for f in ${current_dir}/crd/crd-*.yaml; do kubectl apply -f $${f}; done templateUpdate: # Updates the example template custom resource kubectl apply -f ${current_dir}/crd/template.yaml diff --git a/README.md b/README.md index e71142f..e8cbf7f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,25 @@ Operator for Kubernetes that will take care of the [gist](https://www.dictionary This operator is the glue between the state of your Elasticsearch cluster and Kubernetes. By using Elasticsearch's excellent [Java client](https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/index.html), it wil try to create, update and delete the custom resources defined in [this directory](crd/). +## Run it + +**Always make sure only 1 instance of the Operator is running at the same time to prevent concurrency issues** + +The Docker image for this repository is available at `docker.pkg.github.com/frankkoornstra/elasticsearch-gist-operator/elasticsearch-gist-operator:{version tag}`. To use Github's Docker repository, you will [need to authenticate](https://help.github.com/en/packages/using-github-packages-with-your-projects-ecosystem/configuring-docker-for-use-with-github-packages#authenticating-to-github-packages). + +You'll probably want to deploy the Operator inside Kubernetes. If you want to see a simplistic example of how to do that, clone this repository and - _only when connected to your development cluster!_ - run: +```bash +cd kubernetes-resources +make GITHUB_USERNAME= GITHUB_TOKEN= +``` + +It will: +1. Create a secret, needed to pull the docker image from Github's Docker repository. You'll need to [create a token](https://github.com/settings/tokens) with `read:packages` capabilities to do this. +1. Update the CRDs that the operator defines +1. Create a deployment for the Operator + +By using the [`Recreate`](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#recreate-deployment) deployment strategy in Kubernetes, you make sure only one instance of the Operator is running at the same time. + ## Compatibility and versioning The Operator is semantically versioned. It obviously has a huge dependency on Elasticsearch so versioning of the Operator is tightly bound to the Elasticsearch version. @@ -27,7 +46,13 @@ In the table below you can find the compatibility between versions of this opera |-------------|---------------| | 0.1 (alpha) | 6.8 | -## Templates +## Resources + +The Operator controls the following resources: +* Templates +* Indices + +### Templates Usually one of the first steps is to setup [templates](https://www.elastic.co/guide/en/elasticsearch/reference/6.8/indices-templates.html#indices-templates). They contain mappings and settings as well as a pattern that can match to-be created indices. @@ -35,7 +60,7 @@ Once an index gets created that matches one or more template patterns, all the c You can create and update templates with this operator by sticking to [the definition of the custom resource](crd/crd-template.yaml), an example is [also availabe](crd/template.yaml) -## Indices +### Indices The bread and butter of Elasticsearch; this will contain all your documents. Indices have three core components: settings, a mapping and aliases. @@ -43,21 +68,21 @@ All three of them can be managed by this operator, although there are some thing You can create and update indices with this operator by sticking to [the definition of the custom resource](crd/crd-index.yaml), an example is [also availabe](crd/index.yaml) -### Settings +#### Settings Settings [come in two kinds](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#index-modules-settings): static or dynamic. When you _create_ an index, all settings in your CRD will be applied. When you _update_ and index, only the dynamic settings will be applied. -### Mapping +#### Mapping Mappings adhere to the concept of backwards compatibility (BC), for details browse the Elasticsearch documentation on what is and is not BC. Whenever you update a mapping in an index in a non-BC manner, the update will fail. This will be reflected by a `FAILED` status object in the CRD, the reason why it failed will also be in the status object. -### Aliases +#### Aliases Simple: it'll remove existing aliases that aren't in the CRD anymore, it will ad the ones that don't exist yet. @@ -70,4 +95,5 @@ Simple: it'll remove existing aliases that aren't in the CRD anymore, it will ad - [x] Upsert index aliases (without filters or routing) - [x] Integration tests with Elasticsearch - [x] Github Actions to run tests for PRs +- [ ] Release process - [ ] Status subresource diff --git a/kubernetes-resources/Makefile b/kubernetes-resources/Makefile new file mode 100644 index 0000000..dd84442 --- /dev/null +++ b/kubernetes-resources/Makefile @@ -0,0 +1,23 @@ +# set all to phony +SHELL=bash + +.PHONY: * + +mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) +current_dir := $(abspath $(patsubst %/,%,$(dir $(mkfile_path)))) + +install: upsert-github-credentials-secret update-crds deploy + +upsert-github-credentials-secret: # Creates a secret inside Kubernetes to be able to pull the operator image + kubectl get secret github-docker-secret > /dev/null || $(MAKE) create-github-credentials-secret + +GITHUB_USERNAME?=$(error GITHUB_USERNAME variable must be set) +GITHUB_TOKEN?=$(error GITHUB_TOKEN variable must be set. The token must have read:packages capabilities) +create-github-credentials-secret: #--namespace=kube-system + @kubectl create secret docker-registry github-docker-secret --docker-server=docker.pkg.github.com --docker-username=${GITHUB_USERNAME} --docker-password=${GITHUB_TOKEN} + +update-crds: # Updates the Custom Resource Definitions of the operator + for f in ${current_dir}/crd/crd-*.yaml; do kubectl apply -f $${f}; done + +deploy: # Deploys the operator + kubectl apply -f ${current_dir}/deployment.yaml diff --git a/kubernetes-resources/crd b/kubernetes-resources/crd new file mode 120000 index 0000000..c49a2e0 --- /dev/null +++ b/kubernetes-resources/crd @@ -0,0 +1 @@ +../crd \ No newline at end of file diff --git a/kubernetes-resources/deployment.yaml b/kubernetes-resources/deployment.yaml new file mode 100644 index 0000000..f8256c1 --- /dev/null +++ b/kubernetes-resources/deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch-gist-operator-app + labels: + app: elasticsearch-gist-operator +spec: + replicas: 1 + selector: + matchLabels: + app: elasticsearch-gist-operator + strategy: + type: Recreate + template: + metadata: + labels: + app: elasticsearch-gist-operator + spec: + imagePullSecrets: + - name: github-docker-secret + containers: + - name: operator + image: docker.pkg.github.com/frankkoornstra/elasticsearch-gist-operator/elasticsearch-gist-operator:latest # Using a version tag is recommended for production use + imagePullPolicy: Always