Skip to content

Commit

Permalink
add read-only replica
Browse files Browse the repository at this point in the history
  • Loading branch information
jp-gouin committed Sep 27, 2024
2 parents d1c380c + b3c79d4 commit f3fa7ea
Show file tree
Hide file tree
Showing 11 changed files with 309 additions and 9 deletions.
4 changes: 2 additions & 2 deletions .bin/kind-conf.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ nodes:
hostPort: 30636
- containerPort: 30389
hostPort: 30389
- containerPort: 31636
hostPort: 31636
- containerPort: 31389
hostPort: 31389
- containerPort: 31636
hostPort: 31636
- role: worker
- role: worker
113 changes: 113 additions & 0 deletions .bin/readonly.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
customSchemaFiles:
00-memberof.ldif: |-
# Load memberof module
dn: cn=module,cn=config
cn: module
objectClass: olcModuleList
olcModuleLoad: memberof
olcModulePath: /opt/bitnami/openldap/lib/openldap
dn: olcOverlay=memberof,olcDatabase={2}mdb,cn=config
changetype: add
objectClass: olcOverlayConfig
objectClass: olcMemberOf
olcOverlay: memberof
olcMemberOfRefint: TRUE
10_owncloud_schema.ldif: |-
# This LDIF files describes the ownCloud schema and can be used to
# add two optional attributes: ownCloudQuota and ownCloudUUID
# The ownCloudUUID is used to store a unique, non-reassignable, persistent identifier for users and groups
dn: cn=owncloud,cn=schema,cn=config
objectClass: olcSchemaConfig
cn: owncloud
olcObjectIdentifier: ownCloudOid 1.3.6.1.4.1.39430
olcAttributeTypes: ( ownCloudOid:1.1.1 NAME 'ownCloudQuota'
DESC 'User Quota (e.g. 2 GB)'
EQUALITY caseExactMatch
SUBSTR caseIgnoreSubstringsMatch
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE )
olcAttributeTypes: ( ownCloudOid:1.1.2 NAME 'ownCloudUUID'
DESC 'A non-reassignable and persistent account ID)'
EQUALITY uuidMatch
SUBSTR caseIgnoreSubstringsMatch
SYNTAX 1.3.6.1.1.16.1 SINGLE-VALUE )
olcObjectClasses: ( ownCloudOid:1.2.1 NAME 'ownCloud'
DESC 'ownCloud LDAP Schema'
AUXILIARY
MAY ( ownCloudQuota $ ownCloudUUID ) )
customLdifFiles:
00-root.ldif: |-
# Root creation
dn: dc=example,dc=org
objectClass: dcObject
objectClass: organization
o: Example, Inc
01-default-group.ldif: |-
dn: cn=myGroup,dc=example,dc=org
cn: myGroup
gidnumber: 500
objectclass: posixGroup
objectclass: top
02-default-user.ldif: |-
dn: cn=Jean Dupond,dc=example,dc=org
cn: Jean Dupond
gidnumber: 500
givenname: Jean
homedirectory: /home/users/jdupond
objectclass: inetOrgPerson
objectclass: posixAccount
objectClass: ownCloud
objectclass: top
sn: Dupond
uid: jdupond
uidnumber: 1000
userpassword: {MD5}KOULhzfBhPTq9k7a9XfCGw==
03-test-memberof.ldif: |-
dn: ou=Group,dc=example,dc=org
objectclass: organizationalUnit
ou: Group
dn: ou=People,dc=example,dc=org
objectclass: organizationalUnit
ou: People
dn: uid=test1,ou=People,dc=example,dc=org
objectclass: account
uid: test1
dn: cn=testgroup,ou=Group,dc=example,dc=org
objectclass: groupOfNames
cn: testgroup
member: uid=test1,ou=People,dc=example,dc=org
persistence:
accessModes:
- ReadWriteOnce
enabled: true
size: 1Gi
ltb-passwd:
enabled : false
phpldapadmin:
enabled: false
replicaCount: 3
readOnlyReplicaCount: 1
replication:
clusterName: cluster.local
enabled: true
interval: "00:00:00:10"
retry: 60
starttls: critical
timeout: 1
tls_reqcert: never
initTLSSecret:
tls_enabled: true
secret: "custom-cert"
service:
ldapPortNodePort: 30389
sslLdapPortNodePort: 30636
type: NodePort
serviceReadOnly:
ldapPortNodePort: 31389
sslLdapPortNodePort: 31636
type: NodePort
17 changes: 17 additions & 0 deletions .bin/user2.ldif
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
dn: uid=ro,ou=users,dc=example,dc=org
objectClass: inetOrgPerson
objectClass: organizationalPerson
objectClass: person
objectClass: posixAccount
objectClass: top
uid: ro
givenName: u
sn: u
cn: u
displayName: U
description: User to test that readonly cluster cannot be used to add more users.
mail: [email protected]
uidNumber: 21000
gidNumber: 31000
homeDirectory: /home/u
userPassword:: p1NTSEF9TXJEcXpFNGdKbXZxbVRVTGhvWEZ1VzJBbkV3NWFLK3J3WTIvbHc9PQ==
4 changes: 2 additions & 2 deletions .github/actions/setup/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@ runs:
- name: setup cluster
shell: bash
run: |
curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64
curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64
chmod +x /tmp/kind
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
rm -f kubectl
sudo apt update
sudo apt-get install -y ldap-utils
/tmp/kind create cluster --config=$GITHUB_WORKSPACE/.bin/kind-conf.yml --image=kindest/node:v1.29.4@sha256:3abb816a5b1061fb15c6e9e60856ec40d56b7b52bcea5f5f1350bc6e2320b6f8
/tmp/kind create cluster --config=$GITHUB_WORKSPACE/.bin/kind-conf.yml --image=kindest/node:v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865
kubectl apply -f https://projectcontour.io/quickstart/contour.yaml
kubectl patch daemonsets -n projectcontour envoy -p '{"spec":{"template":{"spec":{"nodeSelector":{"ingress-ready":"true"}}}}}'
- name: setup chaos mesh
Expand Down
62 changes: 62 additions & 0 deletions .github/workflows/ci-readonly.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# WIP!
name: Test readonly replica
on:
workflow_call:
jobs:
qualif:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v1
- name: Run custom action
# Use the location in the repository (without action.yml)
uses: ./.github/actions/setup
with:
install-chaos: false
- name: setup certs
shell: bash
run: |
openssl req -x509 -newkey rsa:4096 -nodes -subj '/CN=example.com' -keyout tls.key -out tls.crt -days 365
cp tls.crt ca.crt
kubectl create secret generic custom-cert --from-file=./tls.crt --from-file=./tls.key --from-file=./ca.crt
- name: deploy openldap-stack-ha
shell: bash
run: |
cd "$GITHUB_WORKSPACE"
helm install openldap-stack-ha -f .bin/readonly.yaml .
kubectl rollout status sts openldap-stack-ha
- name: verify deployment
shell: bash
run: |
echo "test access to openldap database"
sleep 10
LDAPTLS_REQCERT=never ldapsearch -x -D 'cn=admin,dc=example,dc=org' -w Not@SecurePassw0rd -H ldaps://localhost:30636 -b 'dc=example,dc=org'
- name: verify certs
shell: bash
run: |
echo "verify certificate"
echo | openssl s_client -showcerts -servername example.com -connect localhost:30636 2>/dev/null | openssl x509 -inform pem -noout -text > /tmp/test-cert.txt
if ! grep -q "CN = example.com" /tmp/test-cert.txt; then echo exit 1; fi
- name: test write on main cluster
shell: bash
run: |
echo "Write test to openldap database"
LDAPTLS_REQCERT=never ldapadd -x -D 'cn=admin,dc=example,dc=org' -w Not@SecurePassw0rd -H ldaps://localhost:30636 -f .bin/user.ldif
LDAPTLS_REQCERT=never ldapsearch -o nettimeout=20 -x -D 'cn=admin,dc=example,dc=org' -w Not@SecurePassw0rd -H ldaps://localhost:30636 -b 'dc=example,dc=org' > /tmp/test-write.txt
if ! grep "Einstein" /tmp/test-write.txt; then echo 'no Einstein entry found' ; fi
if ! grep "objectClass: ownCloud" /tmp/test-write.txt; then echo 'no ownCloud entry found'; fi
- name: test memberOf on main cluster
shell: bash
run: |
echo "MemberOf test to openldap database"
LDAPTLS_REQCERT=never ldapsearch -o nettimeout=20 -x -D 'cn=admin,dc=example,dc=org' -w Not@SecurePassw0rd -H ldaps://localhost:30636 -b 'dc=example,dc=org' "(memberOf=cn=testgroup,ou=Group,dc=example,dc=org)" > /tmp/test-write.txt
if [ $(grep "numResponses" /tmp/test-write.txt | cut -d ":" -f 2 | tr -d ' ') -ne 2 ]; then exit 1 ; fi
if ! grep -q "uid=test1,ou=People,dc=example,dc=org" /tmp/test-write.txt; then echo exit 1; fi
- name: test write on readonly replica
shell: bash
run: |
echo "Ensure readonly replica does not have mirror mode enabled"
kubectl exec openldap-stack-ha-readonly-0 -- bash -c "/opt/bitnami/openldap/bin/ldapmodify -Y EXTERNAL -H ldapi:/// -f /opt/bitnami/openldap/etc/schema/readonlyremovemirror.ldif"
echo "Write test to openldap readonly replica"
if LDAPTLS_REQCERT=never ldapadd -x -D 'cn=admin,dc=example,dc=org' -w Not@SecurePassw0rd -H ldaps://localhost:31636 -f .bin/user2.ldif; then echo exit 1; fi
4 changes: 3 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,6 @@ jobs:
call-ci-other:
uses: ./.github/workflows/ci-other.yml
call-ci-ha:
uses: ./.github/workflows/ci-ha.yml
uses: ./.github/workflows/ci-ha.yml
call-ci-readonly:
uses: ./.github/workflows/ci-readonly.yml
86 changes: 84 additions & 2 deletions templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,30 @@ Generate olcSyncRepl list
{{- $nodeCount := .Values.replicaCount | int }}
{{- range $index0 := until $nodeCount }}
{{- $index1 := $index0 | add1 }}
<<<<<<< HEAD
olcSyncRepl: rid=00{{ $index1 }} provider=ldap://{{ $name }}-{{ $index0 }}.{{ $name }}-headless.{{ $namespace }}.svc.{{ $cluster }}:1389 binddn="cn={{ $bindDNUser }},cn=config" bindmethod=simple credentials={{ $configPassword }} searchbase="cn=config" type=refreshAndPersist retry="{{ $retry }} +" timeout={{ $timeout }} starttls={{ $starttls }} tls_reqcert={{ $tls_reqcert }}
{{- end -}}
=======
olcSyncRepl: rid=00{{ $index1 }} provider=ldap://{{ $name }}-{{ $index0 }}.{{ $name }}-headless.{{ $namespace }}.svc.{{ $cluster }}:1389 binddn="cn={{ $bindDNUser }},cn=config" bindmethod=simple credentials={{ $configPassword }} searchbase="cn=config" type=refreshOnly retry="{{ $retry }} +" timeout={{ $timeout }} starttls={{ $starttls }} tls_reqcert={{ $tls_reqcert }}
{{- end -}}
{{- end -}}

{{- define "olcSyncReplsReadOnly" -}}
{{- $name := (include "openldap.fullname" .) }}
{{- $namespace := .Release.Namespace }}
{{- $bindDNUser := .Values.global.adminUser }}
{{- $cluster := .Values.replication.clusterName }}
{{- $configPassword := ternary .Values.global.configPassword "%%CONFIG_PASSWORD%%" (empty .Values.global.existingSecret) }}
{{- $retry := .Values.replication.retry }}
{{- $timeout := .Values.replication.timeout }}
{{- $starttls := .Values.replication.starttls }}
{{- $tls_reqcert := .Values.replication.tls_reqcert }}
{{- $nodeCount := .Values.replicaCount | int }}
{{- range $index0 := until $nodeCount }}
{{- $index1 := $index0 | add1 }}
olcSyncRepl: rid=00{{ $index1 }} provider=ldap://{{ $name }}-{{ $index0 }}.{{ $name }}-headless.{{ $namespace }}.svc.{{ $cluster }}:1389 binddn="cn={{ $bindDNUser }},cn=config" bindmethod=simple credentials={{ $configPassword }} searchbase="cn=config" type=refreshOnly retry="{{ $retry }} +" timeout={{ $timeout }} starttls={{ $starttls }} tls_reqcert={{ $tls_reqcert }}
{{- end -}}
>>>>>>> b3c79d4c96d71ea8aba375df8339aff84a5b6864
{{- end -}}

{{/*
Expand Down Expand Up @@ -108,14 +130,51 @@ Generate olcSyncRepl list
bindmethod=simple
credentials={{ $adminPassword }}
searchbase={{ $domain }}
type=refreshAndPersist
type=refreshOnly
interval={{ $interval }}
network-timeout=0
retry="{{ $retry }} +"
timeout={{ $timeout }}
starttls={{ $starttls }}
tls_reqcert={{ $tls_reqcert }}
exattrs=olcMirrorMode,olcMultiProvider
{{- end -}}
<<<<<<< HEAD
=======
{{- end -}}

{{- define "olcSyncRepls2ReadOnly" -}}
{{- $name := (include "openldap.fullname" .) }}
{{- $domain := (include "global.baseDomain" .) }}
{{- $bindDNUser := .Values.global.adminUser }}
{{- $namespace := .Release.Namespace }}
{{- $cluster := .Values.replication.clusterName }}
{{- $adminPassword := ternary .Values.global.adminPassword "%%ADMIN_PASSWORD%%" (empty .Values.global.existingSecret) }}
{{- $retry := .Values.replication.retry }}
{{- $timeout := .Values.replication.timeout }}
{{- $starttls := .Values.replication.starttls }}
{{- $tls_reqcert := .Values.replication.tls_reqcert }}
{{- $interval := .Values.replication.interval }}
{{- $nodeCount := .Values.replicaCount | int }}
{{- range $index0 := until $nodeCount }}
{{- $index1 := $index0 | add1 }}
olcSyncrepl:
rid=10{{ $index1 }}
provider=ldap://{{ $name }}-{{ $index0 }}.{{ $name }}-headless.{{ $namespace }}.svc.{{ $cluster }}:1389
binddn={{ printf "cn=%s,%s" $bindDNUser $domain }}
bindmethod=simple
credentials={{ $adminPassword }}
searchbase={{ $domain }}
type=refreshOnly
interval={{ $interval }}
network-timeout=0
retry="{{ $retry }} +"
timeout={{ $timeout }}
starttls={{ $starttls }}
tls_reqcert={{ $tls_reqcert }}
exattrs=olcMirrorMode,olcMultiProvider
{{- end -}}
>>>>>>> b3c79d4c96d71ea8aba375df8339aff84a5b6864
{{- end -}}

{{/*
Expand Down Expand Up @@ -174,6 +233,7 @@ Cannot return list => return string comma separated
*/}}
{{- define "openldap.builtinSchemaFiles" -}}
{{- $schemas := "" -}}
<<<<<<< HEAD
{{- $context := index . "context" -}}
{{- $mode := index . "mode" -}}
{{- if $context.Values.replication.enabled -}}
Expand All @@ -182,12 +242,24 @@ Cannot return list => return string comma separated
{{- else -}}
{{- $schemas = "syncprov,serverid,csyncprov,rep,bsyncprov,brep,acls" -}}
{{- end -}}
=======
{{- if .Values.replication.enabled -}}
{{- $schemas = "syncprov,serverid,csyncprov,rep,mirrorcfg,bsyncprov,brep,mirrormdb,acls" -}}
>>>>>>> b3c79d4c96d71ea8aba375df8339aff84a5b6864
{{- else -}}
{{- $schemas = "acls" -}}
{{- end -}}
{{- print $schemas -}}
{{- end -}}

{{- define "openldap.builtinSchemaFilesReadOnly" -}}
{{- $schemas := "" -}}
{{- if .Values.replication.enabled -}}
{{- $schemas = "serverid,readonlybrep,readonlyrep,readonlyremovemirror" -}}
{{- else -}}
{{- $schemas = "" -}}
{{- end -}}
{{- print $schemas -}}
{{- end -}}
{{/*
Return the list of custom schema files to use
Cannot return list => return string comma separated
Expand All @@ -214,6 +286,16 @@ Cannot return list => return string comma separated
{{- print $schemas -}}
{{- end -}}

{{- define "openldap.schemaFilesReadOnly" -}}
{{- $schemas := (include "openldap.builtinSchemaFilesReadOnly" .) -}}
{{- $custom_schemas := (include "openldap.customSchemaFiles" .) -}}
{{- if gt (len $custom_schemas) 0 -}}
{{- $schemas = print $schemas "," $custom_schemas -}}
{{- end -}}
{{- print $schemas -}}
{{- end -}}


{{/*
Return the proper base domain
*/}}
Expand Down
21 changes: 21 additions & 0 deletions templates/configmap-replication-acls.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,27 @@ data:
changetype: modify
add: olcMirrorMode
olcMirrorMode: TRUE
readonlybrep.ldif: |
dn: olcDatabase={2}mdb,cn=config
changetype: modify
add: olcSyncrepl
{{- include "olcSyncRepls2ReadOnly" . }}
# TODO - currently this needs to be executed manually e.g. via kubectl exec in each read only replica
# need to see if there is a way to make olcMultiProvider not be propagated from main cluster to readonly replicas.
readonlyremovemirror.ldif: |
dn: olcDatabase={2}mdb,cn=config
changetype: modify
delete: olcMultiProvider
dn: olcDatabase={0}config,cn=config
changetype: modify
delete: olcMultiProvider
readonlyrep.ldif: |
dn: olcDatabase={0}config,cn=config
changetype: modify
add: olcSyncRepl
{{- include "olcSyncReplsReadOnly" . }}
# acls
acls.ldif: |
{{- if .Values.customAcls }}
Expand Down
Loading

0 comments on commit f3fa7ea

Please sign in to comment.