From a1807f7baf47b26e61eb5208a92256ee0b2e93f8 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Tue, 11 Jul 2023 11:02:24 -0400 Subject: [PATCH 01/30] CI From 80c4eff8bdb9e303bcf997354ca93670637238e3 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Wed, 19 Jul 2023 12:05:09 -0400 Subject: [PATCH 02/30] Only test saml_login for now --- .github/workflows/integration_test_with_secrets.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration_test_with_secrets.yaml b/.github/workflows/integration_test_with_secrets.yaml index 04ed4fb1..50ae8bf9 100644 --- a/.github/workflows/integration_test_with_secrets.yaml +++ b/.github/workflows/integration_test_with_secrets.yaml @@ -38,6 +38,7 @@ jobs: run: | tox -e integration -- \ -m requires_secrets \ + -k test_saml_login \ --saml-email ${{ secrets.TEST_SAML_EMAIL }} \ --saml-password ${{ secrets.TEST_SAML_PASSWORD }} \ --discourse-image=localhost:32000/discourse:test From aec8f231f643a31896c51722972a32a6ce1995b1 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Wed, 19 Jul 2023 16:12:50 -0400 Subject: [PATCH 03/30] Re-run CI From a841b7aa43fca77fcccdee279c53320031b6cc06 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Wed, 19 Jul 2023 17:50:06 -0400 Subject: [PATCH 04/30] Re-run CI From 2b1db4f7fd66c793afd72ce007eac325ba1b97c2 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Thu, 20 Jul 2023 08:05:04 -0400 Subject: [PATCH 05/30] Fixes --- .github/workflows/integration_test_with_secrets.yaml | 1 - tests/integration/conftest.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test_with_secrets.yaml b/.github/workflows/integration_test_with_secrets.yaml index 50ae8bf9..04ed4fb1 100644 --- a/.github/workflows/integration_test_with_secrets.yaml +++ b/.github/workflows/integration_test_with_secrets.yaml @@ -38,7 +38,6 @@ jobs: run: | tox -e integration -- \ -m requires_secrets \ - -k test_saml_login \ --saml-email ${{ secrets.TEST_SAML_EMAIL }} \ --saml-password ${{ secrets.TEST_SAML_PASSWORD }} \ --discourse-image=localhost:32000/discourse:test diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 66a2e97a..784a1873 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -143,7 +143,8 @@ async def app_fixture( "discourse-image": pytestconfig.getoption("--discourse-image"), } - if charm := pytestconfig.getoption("--charm-file"): + charm = pytestconfig.getoption("--charm-file") + if charm: application = await model.deploy( f"./{charm}", resources=resources, From 901d6585066d8bbed702a742cae4e9c938b055d8 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Thu, 20 Jul 2023 08:13:50 -0400 Subject: [PATCH 06/30] Fix --- tests/integration/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 784a1873..4f4dfa66 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -123,7 +123,6 @@ async def discourse_address_fixture(model: Model, app: Application): @pytest_asyncio.fixture(scope="module", name="app") async def app_fixture( - ops_test: OpsTest, app_name: str, app_config: Dict[str, str], pytestconfig: Config, From 6cbf2d0424c6d29a95712802695243df1756c1f7 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Thu, 20 Jul 2023 08:18:03 -0400 Subject: [PATCH 07/30] Allow for not provided charm-file --- tests/integration/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 4f4dfa66..784a1873 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -123,6 +123,7 @@ async def discourse_address_fixture(model: Model, app: Application): @pytest_asyncio.fixture(scope="module", name="app") async def app_fixture( + ops_test: OpsTest, app_name: str, app_config: Dict[str, str], pytestconfig: Config, From 3739b1d7f0b9e771aa99bb8f4a8bf036c26913dc Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Thu, 20 Jul 2023 17:39:22 -0400 Subject: [PATCH 08/30] Fix usage of --charm-file --- tests/integration/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 784a1873..66a2e97a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -143,8 +143,7 @@ async def app_fixture( "discourse-image": pytestconfig.getoption("--discourse-image"), } - charm = pytestconfig.getoption("--charm-file") - if charm: + if charm := pytestconfig.getoption("--charm-file"): application = await model.deploy( f"./{charm}", resources=resources, From 2e2bcdde7ae25f673ddab67b61a4aea49b956c83 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Fri, 21 Jul 2023 11:41:12 -0400 Subject: [PATCH 09/30] Re-run CI From 8f04af68620544dc4956556c6b1047de0345fba6 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Fri, 21 Jul 2023 16:13:30 -0400 Subject: [PATCH 10/30] Try an old version of operator-wf --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index b1a4998a..798ac7d5 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -5,7 +5,7 @@ on: jobs: integration-tests: - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@a044ef084db982e6b0050aa96e27baec2326cc89 secrets: inherit with: chaos-app-label: app.kubernetes.io/name=discourse-k8s From c123a5d1b6d6b30bcafe7434ccadffb2f97fbaf0 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Fri, 21 Jul 2023 21:18:13 -0400 Subject: [PATCH 11/30] Skip test_prom_exporter test --- tests/integration/test_charm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index d8befa43..a9243c0c 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -56,6 +56,7 @@ async def test_discourse_up(requests_timeout: float, discourse_address: str): @pytest.mark.asyncio @pytest.mark.abort_on_fail +@pytest.mark.skip(reason="This test will need some rework") async def test_prom_exporter_is_up(app: Application): """ arrange: given charm in its initial state From 846a787cba9c31d5eced246208e3e037dda8c977 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Fri, 21 Jul 2023 22:30:34 -0400 Subject: [PATCH 12/30] Rerun CI From 77315de2f9d934f9a2c9b525e521cfcabf6cc868 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Sat, 22 Jul 2023 08:40:33 -0400 Subject: [PATCH 13/30] Skip s3 test --- tests/integration/test_charm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index a9243c0c..1b6cd9ae 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -99,6 +99,7 @@ async def test_setup_discourse( @pytest.mark.asyncio @pytest.mark.abort_on_fail +@pytest.mark.skip(reason="This test will need some rework") async def test_s3_conf(app: Application, localstack_address: str, model: Model): """Check that the bootstrap page is reachable with the charm configured with an S3 target From 8caff2993b912977448524c323e7d94bb55e589f Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Sat, 22 Jul 2023 09:06:57 -0400 Subject: [PATCH 14/30] Add one xfail --- tests/integration/test_charm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 1b6cd9ae..351359e4 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -303,6 +303,7 @@ def patched_getaddrinfo(*args): @pytest.mark.asyncio +@pytest.mark.xfail(reason="This test will need some rework") async def test_create_category( discourse_address: str, admin_credentials: types.Credentials, From 05e953f2835128fb07cfaeb9f80d00901a6d3dda Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Sat, 22 Jul 2023 13:34:42 -0400 Subject: [PATCH 15/30] Add some asserts for the category test --- tests/integration/test_charm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 351359e4..1b6cd9ae 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -303,7 +303,6 @@ def patched_getaddrinfo(*args): @pytest.mark.asyncio -@pytest.mark.xfail(reason="This test will need some rework") async def test_create_category( discourse_address: str, admin_credentials: types.Credentials, From 4eb1334ee29926d1ba8ca2f3c938b0b2fb31e36e Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Sun, 23 Jul 2023 08:30:39 -0400 Subject: [PATCH 16/30] Switch operator workflow to main --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 798ac7d5..b1a4998a 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -5,7 +5,7 @@ on: jobs: integration-tests: - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@a044ef084db982e6b0050aa96e27baec2326cc89 + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: chaos-app-label: app.kubernetes.io/name=discourse-k8s From 599fcbae7267b83f2cef9a4577bac1a99e4c7181 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Mon, 24 Jul 2023 09:47:21 -0400 Subject: [PATCH 17/30] Fix skipped tests --- tests/integration/test_charm.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 1b6cd9ae..d8befa43 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -56,7 +56,6 @@ async def test_discourse_up(requests_timeout: float, discourse_address: str): @pytest.mark.asyncio @pytest.mark.abort_on_fail -@pytest.mark.skip(reason="This test will need some rework") async def test_prom_exporter_is_up(app: Application): """ arrange: given charm in its initial state @@ -99,7 +98,6 @@ async def test_setup_discourse( @pytest.mark.asyncio @pytest.mark.abort_on_fail -@pytest.mark.skip(reason="This test will need some rework") async def test_s3_conf(app: Application, localstack_address: str, model: Model): """Check that the bootstrap page is reachable with the charm configured with an S3 target From 6236f02a590c6e484a8a0d8f40555847a59d1fa3 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Mon, 24 Jul 2023 10:15:22 -0400 Subject: [PATCH 18/30] Add support for the new database interface --- .../data_platform_libs/v0/data_interfaces.py | 1407 +++++++++++++++++ metadata.yaml | 6 +- src/charm.py | 116 +- src/database.py | 54 + tests/integration/conftest.py | 17 +- tests/unit/test_charm.py | 155 +- 6 files changed, 1605 insertions(+), 150 deletions(-) create mode 100644 lib/charms/data_platform_libs/v0/data_interfaces.py create mode 100644 src/database.py diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py new file mode 100644 index 00000000..10bda6db --- /dev/null +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -0,0 +1,1407 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library to manage the relation for the data-platform products. + +This library contains the Requires and Provides classes for handling the relation +between an application and multiple managed application supported by the data-team: +MySQL, Postgresql, MongoDB, Redis, and Kafka. + +### Database (MySQL, Postgresql, MongoDB, and Redis) + +#### Requires Charm +This library is a uniform interface to a selection of common database +metadata, with added custom events that add convenience to database management, +and methods to consume the application related data. + + +Following an example of using the DatabaseCreatedEvent, in the context of the +application charm code: + +```python + +from charms.data_platform_libs.v0.data_interfaces import ( + DatabaseCreatedEvent, + DatabaseRequires, +) + +class ApplicationCharm(CharmBase): + # Application charm that connects to database charms. + + def __init__(self, *args): + super().__init__(*args) + + # Charm events defined in the database requires charm library. + self.database = DatabaseRequires(self, relation_name="database", database_name="database") + self.framework.observe(self.database.on.database_created, self._on_database_created) + + def _on_database_created(self, event: DatabaseCreatedEvent) -> None: + # Handle the created database + + # Create configuration file for app + config_file = self._render_app_config_file( + event.username, + event.password, + event.endpoints, + ) + + # Start application with rendered configuration + self._start_application(config_file) + + # Set active status + self.unit.status = ActiveStatus("received database credentials") +``` + +As shown above, the library provides some custom events to handle specific situations, +which are listed below: + +- database_created: event emitted when the requested database is created. +- endpoints_changed: event emitted when the read/write endpoints of the database have changed. +- read_only_endpoints_changed: event emitted when the read-only endpoints of the database + have changed. Event is not triggered if read/write endpoints changed too. + +If it is needed to connect multiple database clusters to the same relation endpoint +the application charm can implement the same code as if it would connect to only +one database cluster (like the above code example). + +To differentiate multiple clusters connected to the same relation endpoint +the application charm can use the name of the remote application: + +```python + +def _on_database_created(self, event: DatabaseCreatedEvent) -> None: + # Get the remote app name of the cluster that triggered this event + cluster = event.relation.app.name +``` + +It is also possible to provide an alias for each different database cluster/relation. + +So, it is possible to differentiate the clusters in two ways. +The first is to use the remote application name, i.e., `event.relation.app.name`, as above. + +The second way is to use different event handlers to handle each cluster events. +The implementation would be something like the following code: + +```python + +from charms.data_platform_libs.v0.data_interfaces import ( + DatabaseCreatedEvent, + DatabaseRequires, +) + +class ApplicationCharm(CharmBase): + # Application charm that connects to database charms. + + def __init__(self, *args): + super().__init__(*args) + + # Define the cluster aliases and one handler for each cluster database created event. + self.database = DatabaseRequires( + self, + relation_name="database", + database_name="database", + relations_aliases = ["cluster1", "cluster2"], + ) + self.framework.observe( + self.database.on.cluster1_database_created, self._on_cluster1_database_created + ) + self.framework.observe( + self.database.on.cluster2_database_created, self._on_cluster2_database_created + ) + + def _on_cluster1_database_created(self, event: DatabaseCreatedEvent) -> None: + # Handle the created database on the cluster named cluster1 + + # Create configuration file for app + config_file = self._render_app_config_file( + event.username, + event.password, + event.endpoints, + ) + ... + + def _on_cluster2_database_created(self, event: DatabaseCreatedEvent) -> None: + # Handle the created database on the cluster named cluster2 + + # Create configuration file for app + config_file = self._render_app_config_file( + event.username, + event.password, + event.endpoints, + ) + ... + +``` + +When it's needed to check whether a plugin (extension) is enabled on the PostgreSQL +charm, you can use the is_postgresql_plugin_enabled method. To use that, you need to +add the following dependency to your charmcraft.yaml file: + +```yaml + +parts: + charm: + charm-binary-python-packages: + - psycopg[binary] + +``` + +### Provider Charm + +Following an example of using the DatabaseRequestedEvent, in the context of the +database charm code: + +```python +from charms.data_platform_libs.v0.data_interfaces import DatabaseProvides + +class SampleCharm(CharmBase): + + def __init__(self, *args): + super().__init__(*args) + # Charm events defined in the database provides charm library. + self.provided_database = DatabaseProvides(self, relation_name="database") + self.framework.observe(self.provided_database.on.database_requested, + self._on_database_requested) + # Database generic helper + self.database = DatabaseHelper() + + def _on_database_requested(self, event: DatabaseRequestedEvent) -> None: + # Handle the event triggered by a new database requested in the relation + # Retrieve the database name using the charm library. + db_name = event.database + # generate a new user credential + username = self.database.generate_user() + password = self.database.generate_password() + # set the credentials for the relation + self.provided_database.set_credentials(event.relation.id, username, password) + # set other variables for the relation event.set_tls("False") +``` +As shown above, the library provides a custom event (database_requested) to handle +the situation when an application charm requests a new database to be created. +It's preferred to subscribe to this event instead of relation changed event to avoid +creating a new database when other information other than a database name is +exchanged in the relation databag. + +### Kafka + +This library is the interface to use and interact with the Kafka charm. This library contains +custom events that add convenience to manage Kafka, and provides methods to consume the +application related data. + +#### Requirer Charm + +```python + +from charms.data_platform_libs.v0.data_interfaces import ( + BootstrapServerChangedEvent, + KafkaRequires, + TopicCreatedEvent, +) + +class ApplicationCharm(CharmBase): + + def __init__(self, *args): + super().__init__(*args) + self.kafka = KafkaRequires(self, "kafka_client", "test-topic") + self.framework.observe( + self.kafka.on.bootstrap_server_changed, self._on_kafka_bootstrap_server_changed + ) + self.framework.observe( + self.kafka.on.topic_created, self._on_kafka_topic_created + ) + + def _on_kafka_bootstrap_server_changed(self, event: BootstrapServerChangedEvent): + # Event triggered when a bootstrap server was changed for this application + + new_bootstrap_server = event.bootstrap_server + ... + + def _on_kafka_topic_created(self, event: TopicCreatedEvent): + # Event triggered when a topic was created for this application + username = event.username + password = event.password + tls = event.tls + tls_ca= event.tls_ca + bootstrap_server event.bootstrap_server + consumer_group_prefic = event.consumer_group_prefix + zookeeper_uris = event.zookeeper_uris + ... + +``` + +As shown above, the library provides some custom events to handle specific situations, +which are listed below: + +- topic_created: event emitted when the requested topic is created. +- bootstrap_server_changed: event emitted when the bootstrap server have changed. +- credential_changed: event emitted when the credentials of Kafka changed. + +### Provider Charm + +Following the previous example, this is an example of the provider charm. + +```python +class SampleCharm(CharmBase): + +from charms.data_platform_libs.v0.data_interfaces import ( + KafkaProvides, + TopicRequestedEvent, +) + + def __init__(self, *args): + super().__init__(*args) + + # Default charm events. + self.framework.observe(self.on.start, self._on_start) + + # Charm events defined in the Kafka Provides charm library. + self.kafka_provider = KafkaProvides(self, relation_name="kafka_client") + self.framework.observe(self.kafka_provider.on.topic_requested, self._on_topic_requested) + # Kafka generic helper + self.kafka = KafkaHelper() + + def _on_topic_requested(self, event: TopicRequestedEvent): + # Handle the on_topic_requested event. + + topic = event.topic + relation_id = event.relation.id + # set connection info in the databag relation + self.kafka_provider.set_bootstrap_server(relation_id, self.kafka.get_bootstrap_server()) + self.kafka_provider.set_credentials(relation_id, username=username, password=password) + self.kafka_provider.set_consumer_group_prefix(relation_id, ...) + self.kafka_provider.set_tls(relation_id, "False") + self.kafka_provider.set_zookeeper_uris(relation_id, ...) + +``` +As shown above, the library provides a custom event (topic_requested) to handle +the situation when an application charm requests a new topic to be created. +It is preferred to subscribe to this event instead of relation changed event to avoid +creating a new topic when other information other than a topic name is +exchanged in the relation databag. +""" + +import json +import logging +from abc import ABC, abstractmethod +from collections import namedtuple +from datetime import datetime +from typing import List, Optional + +from ops.charm import ( + CharmBase, + CharmEvents, + RelationChangedEvent, + RelationEvent, + RelationJoinedEvent, +) +from ops.framework import EventSource, Object +from ops.model import Relation + +# The unique Charmhub library identifier, never change it +LIBID = "6c3e6b6680d64e9c89e611d1a15f65be" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 13 + +PYDEPS = ["ops>=2.0.0"] + +logger = logging.getLogger(__name__) + +Diff = namedtuple("Diff", "added changed deleted") +Diff.__doc__ = """ +A tuple for storing the diff between two data mappings. + +added - keys that were added +changed - keys that still exist but have new values +deleted - key that were deleted""" + + +def diff(event: RelationChangedEvent, bucket: str) -> Diff: + """Retrieves the diff of the data in the relation changed databag. + + Args: + event: relation changed event. + bucket: bucket of the databag (app or unit) + + Returns: + a Diff instance containing the added, deleted and changed + keys from the event relation databag. + """ + # Retrieve the old data from the data key in the application relation databag. + old_data = json.loads(event.relation.data[bucket].get("data", "{}")) + # Retrieve the new data from the event relation databag. + new_data = { + key: value for key, value in event.relation.data[event.app].items() if key != "data" + } + + # These are the keys that were added to the databag and triggered this event. + added = new_data.keys() - old_data.keys() + # These are the keys that were removed from the databag and triggered this event. + deleted = old_data.keys() - new_data.keys() + # These are the keys that already existed in the databag, + # but had their values changed. + changed = {key for key in old_data.keys() & new_data.keys() if old_data[key] != new_data[key]} + # Convert the new_data to a serializable format and save it for a next diff check. + event.relation.data[bucket].update({"data": json.dumps(new_data)}) + + # Return the diff with all possible changes. + return Diff(added, changed, deleted) + + +# Base DataProvides and DataRequires + + +class DataProvides(Object, ABC): + """Base provides-side of the data products relation.""" + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + super().__init__(charm, relation_name) + self.charm = charm + self.local_app = self.charm.model.app + self.local_unit = self.charm.unit + self.relation_name = relation_name + self.framework.observe( + charm.on[relation_name].relation_changed, + self._on_relation_changed, + ) + + def _diff(self, event: RelationChangedEvent) -> Diff: + """Retrieves the diff of the data in the relation changed databag. + + Args: + event: relation changed event. + + Returns: + a Diff instance containing the added, deleted and changed + keys from the event relation databag. + """ + return diff(event, self.local_app) + + @abstractmethod + def _on_relation_changed(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation data has changed.""" + raise NotImplementedError + + def fetch_relation_data(self) -> dict: + """Retrieves data from relation. + + This function can be used to retrieve data from a relation + in the charm code when outside an event callback. + + Returns: + a dict of the values stored in the relation data bag + for all relation instances (indexed by the relation id). + """ + data = {} + for relation in self.relations: + data[relation.id] = { + key: value for key, value in relation.data[relation.app].items() if key != "data" + } + return data + + def _update_relation_data(self, relation_id: int, data: dict) -> None: + """Updates a set of key-value pairs in the relation. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + data: dict containing the key-value pairs + that should be updated in the relation. + """ + if self.local_unit.is_leader(): + relation = self.charm.model.get_relation(self.relation_name, relation_id) + relation.data[self.local_app].update(data) + + @property + def relations(self) -> List[Relation]: + """The list of Relation instances associated with this relation_name.""" + return list(self.charm.model.relations[self.relation_name]) + + def set_credentials(self, relation_id: int, username: str, password: str) -> None: + """Set credentials. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + username: user that was created. + password: password of the created user. + """ + self._update_relation_data( + relation_id, + { + "username": username, + "password": password, + }, + ) + + def set_tls(self, relation_id: int, tls: str) -> None: + """Set whether TLS is enabled. + + Args: + relation_id: the identifier for a particular relation. + tls: whether tls is enabled (True or False). + """ + self._update_relation_data(relation_id, {"tls": tls}) + + def set_tls_ca(self, relation_id: int, tls_ca: str) -> None: + """Set the TLS CA in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + tls_ca: TLS certification authority. + """ + self._update_relation_data(relation_id, {"tls-ca": tls_ca}) + + +class DataRequires(Object, ABC): + """Requires-side of the relation.""" + + def __init__( + self, + charm, + relation_name: str, + extra_user_roles: str = None, + ): + """Manager of base client relations.""" + super().__init__(charm, relation_name) + self.charm = charm + self.extra_user_roles = extra_user_roles + self.local_app = self.charm.model.app + self.local_unit = self.charm.unit + self.relation_name = relation_name + self.framework.observe( + self.charm.on[relation_name].relation_joined, self._on_relation_joined_event + ) + self.framework.observe( + self.charm.on[relation_name].relation_changed, self._on_relation_changed_event + ) + + @abstractmethod + def _on_relation_joined_event(self, event: RelationJoinedEvent) -> None: + """Event emitted when the application joins the relation.""" + raise NotImplementedError + + @abstractmethod + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + raise NotImplementedError + + def fetch_relation_data(self) -> dict: + """Retrieves data from relation. + + This function can be used to retrieve data from a relation + in the charm code when outside an event callback. + Function cannot be used in `*-relation-broken` events and will raise an exception. + + Returns: + a dict of the values stored in the relation data bag + for all relation instances (indexed by the relation ID). + """ + data = {} + for relation in self.relations: + data[relation.id] = { + key: value for key, value in relation.data[relation.app].items() if key != "data" + } + return data + + def _update_relation_data(self, relation_id: int, data: dict) -> None: + """Updates a set of key-value pairs in the relation. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + data: dict containing the key-value pairs + that should be updated in the relation. + """ + if self.local_unit.is_leader(): + relation = self.charm.model.get_relation(self.relation_name, relation_id) + relation.data[self.local_app].update(data) + + def _diff(self, event: RelationChangedEvent) -> Diff: + """Retrieves the diff of the data in the relation changed databag. + + Args: + event: relation changed event. + + Returns: + a Diff instance containing the added, deleted and changed + keys from the event relation databag. + """ + return diff(event, self.local_unit) + + @property + def relations(self) -> List[Relation]: + """The list of Relation instances associated with this relation_name.""" + return [ + relation + for relation in self.charm.model.relations[self.relation_name] + if self._is_relation_active(relation) + ] + + @staticmethod + def _is_relation_active(relation: Relation): + try: + _ = repr(relation.data) + return True + except RuntimeError: + return False + + @staticmethod + def _is_resource_created_for_relation(relation: Relation): + return ( + "username" in relation.data[relation.app] and "password" in relation.data[relation.app] + ) + + def is_resource_created(self, relation_id: Optional[int] = None) -> bool: + """Check if the resource has been created. + + This function can be used to check if the Provider answered with data in the charm code + when outside an event callback. + + Args: + relation_id (int, optional): When provided the check is done only for the relation id + provided, otherwise the check is done for all relations + + Returns: + True or False + + Raises: + IndexError: If relation_id is provided but that relation does not exist + """ + if relation_id is not None: + try: + relation = [relation for relation in self.relations if relation.id == relation_id][ + 0 + ] + return self._is_resource_created_for_relation(relation) + except IndexError: + raise IndexError(f"relation id {relation_id} cannot be accessed") + else: + return ( + all( + [ + self._is_resource_created_for_relation(relation) + for relation in self.relations + ] + ) + if self.relations + else False + ) + + +# General events + + +class ExtraRoleEvent(RelationEvent): + """Base class for data events.""" + + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + return self.relation.data[self.relation.app].get("extra-user-roles") + + +class AuthenticationEvent(RelationEvent): + """Base class for authentication fields for events.""" + + @property + def username(self) -> Optional[str]: + """Returns the created username.""" + return self.relation.data[self.relation.app].get("username") + + @property + def password(self) -> Optional[str]: + """Returns the password for the created user.""" + return self.relation.data[self.relation.app].get("password") + + @property + def tls(self) -> Optional[str]: + """Returns whether TLS is configured.""" + return self.relation.data[self.relation.app].get("tls") + + @property + def tls_ca(self) -> Optional[str]: + """Returns TLS CA.""" + return self.relation.data[self.relation.app].get("tls-ca") + + +# Database related events and fields + + +class DatabaseProvidesEvent(RelationEvent): + """Base class for database events.""" + + @property + def database(self) -> Optional[str]: + """Returns the database that was requested.""" + return self.relation.data[self.relation.app].get("database") + + +class DatabaseRequestedEvent(DatabaseProvidesEvent, ExtraRoleEvent): + """Event emitted when a new database is requested for use on this relation.""" + + +class DatabaseProvidesEvents(CharmEvents): + """Database events. + + This class defines the events that the database can emit. + """ + + database_requested = EventSource(DatabaseRequestedEvent) + + +class DatabaseRequiresEvent(RelationEvent): + """Base class for database events.""" + + @property + def database(self) -> Optional[str]: + """Returns the database name.""" + return self.relation.data[self.relation.app].get("database") + + @property + def endpoints(self) -> Optional[str]: + """Returns a comma separated list of read/write endpoints. + + In VM charms, this is the primary's address. + In kubernetes charms, this is the service to the primary pod. + """ + return self.relation.data[self.relation.app].get("endpoints") + + @property + def read_only_endpoints(self) -> Optional[str]: + """Returns a comma separated list of read only endpoints. + + In VM charms, this is the address of all the secondary instances. + In kubernetes charms, this is the service to all replica pod instances. + """ + return self.relation.data[self.relation.app].get("read-only-endpoints") + + @property + def replset(self) -> Optional[str]: + """Returns the replicaset name. + + MongoDB only. + """ + return self.relation.data[self.relation.app].get("replset") + + @property + def uris(self) -> Optional[str]: + """Returns the connection URIs. + + MongoDB, Redis, OpenSearch. + """ + return self.relation.data[self.relation.app].get("uris") + + @property + def version(self) -> Optional[str]: + """Returns the version of the database. + + Version as informed by the database daemon. + """ + return self.relation.data[self.relation.app].get("version") + + +class DatabaseCreatedEvent(AuthenticationEvent, DatabaseRequiresEvent): + """Event emitted when a new database is created for use on this relation.""" + + +class DatabaseEndpointsChangedEvent(AuthenticationEvent, DatabaseRequiresEvent): + """Event emitted when the read/write endpoints are changed.""" + + +class DatabaseReadOnlyEndpointsChangedEvent(AuthenticationEvent, DatabaseRequiresEvent): + """Event emitted when the read only endpoints are changed.""" + + +class DatabaseRequiresEvents(CharmEvents): + """Database events. + + This class defines the events that the database can emit. + """ + + database_created = EventSource(DatabaseCreatedEvent) + endpoints_changed = EventSource(DatabaseEndpointsChangedEvent) + read_only_endpoints_changed = EventSource(DatabaseReadOnlyEndpointsChangedEvent) + + +# Database Provider and Requires + + +class DatabaseProvides(DataProvides): + """Provider-side of the database relations.""" + + on = DatabaseProvidesEvents() + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + super().__init__(charm, relation_name) + + def _on_relation_changed(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Only the leader should handle this event. + if not self.local_unit.is_leader(): + return + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Emit a database requested event if the setup key (database name and optional + # extra user roles) was added to the relation databag by the application. + if "database" in diff.added: + self.on.database_requested.emit(event.relation, app=event.app, unit=event.unit) + + def set_database(self, relation_id: int, database_name: str) -> None: + """Set database name. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + database_name: database name. + """ + self._update_relation_data(relation_id, {"database": database_name}) + + def set_endpoints(self, relation_id: int, connection_strings: str) -> None: + """Set database primary connections. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + In VM charms, only the primary's address should be passed as an endpoint. + In kubernetes charms, the service endpoint to the primary pod should be + passed as an endpoint. + + Args: + relation_id: the identifier for a particular relation. + connection_strings: database hosts and ports comma separated list. + """ + self._update_relation_data(relation_id, {"endpoints": connection_strings}) + + def set_read_only_endpoints(self, relation_id: int, connection_strings: str) -> None: + """Set database replicas connection strings. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + connection_strings: database hosts and ports comma separated list. + """ + self._update_relation_data(relation_id, {"read-only-endpoints": connection_strings}) + + def set_replset(self, relation_id: int, replset: str) -> None: + """Set replica set name in the application relation databag. + + MongoDB only. + + Args: + relation_id: the identifier for a particular relation. + replset: replica set name. + """ + self._update_relation_data(relation_id, {"replset": replset}) + + def set_uris(self, relation_id: int, uris: str) -> None: + """Set the database connection URIs in the application relation databag. + + MongoDB, Redis, and OpenSearch only. + + Args: + relation_id: the identifier for a particular relation. + uris: connection URIs. + """ + self._update_relation_data(relation_id, {"uris": uris}) + + def set_version(self, relation_id: int, version: str) -> None: + """Set the database version in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + version: database version. + """ + self._update_relation_data(relation_id, {"version": version}) + + +class DatabaseRequires(DataRequires): + """Requires-side of the database relation.""" + + on = DatabaseRequiresEvents() + + def __init__( + self, + charm, + relation_name: str, + database_name: str, + extra_user_roles: str = None, + relations_aliases: List[str] = None, + ): + """Manager of database client relations.""" + super().__init__(charm, relation_name, extra_user_roles) + self.database = database_name + self.relations_aliases = relations_aliases + + # Define custom event names for each alias. + if relations_aliases: + # Ensure the number of aliases does not exceed the maximum + # of connections allowed in the specific relation. + relation_connection_limit = self.charm.meta.requires[relation_name].limit + if len(relations_aliases) != relation_connection_limit: + raise ValueError( + f"The number of aliases must match the maximum number of connections allowed in the relation. " + f"Expected {relation_connection_limit}, got {len(relations_aliases)}" + ) + + for relation_alias in relations_aliases: + self.on.define_event(f"{relation_alias}_database_created", DatabaseCreatedEvent) + self.on.define_event( + f"{relation_alias}_endpoints_changed", DatabaseEndpointsChangedEvent + ) + self.on.define_event( + f"{relation_alias}_read_only_endpoints_changed", + DatabaseReadOnlyEndpointsChangedEvent, + ) + + def _assign_relation_alias(self, relation_id: int) -> None: + """Assigns an alias to a relation. + + This function writes in the unit data bag. + + Args: + relation_id: the identifier for a particular relation. + """ + # If no aliases were provided, return immediately. + if not self.relations_aliases: + return + + # Return if an alias was already assigned to this relation + # (like when there are more than one unit joining the relation). + if ( + self.charm.model.get_relation(self.relation_name, relation_id) + .data[self.local_unit] + .get("alias") + ): + return + + # Retrieve the available aliases (the ones that weren't assigned to any relation). + available_aliases = self.relations_aliases[:] + for relation in self.charm.model.relations[self.relation_name]: + alias = relation.data[self.local_unit].get("alias") + if alias: + logger.debug("Alias %s was already assigned to relation %d", alias, relation.id) + available_aliases.remove(alias) + + # Set the alias in the unit relation databag of the specific relation. + relation = self.charm.model.get_relation(self.relation_name, relation_id) + relation.data[self.local_unit].update({"alias": available_aliases[0]}) + + def _emit_aliased_event(self, event: RelationChangedEvent, event_name: str) -> None: + """Emit an aliased event to a particular relation if it has an alias. + + Args: + event: the relation changed event that was received. + event_name: the name of the event to emit. + """ + alias = self._get_relation_alias(event.relation.id) + if alias: + getattr(self.on, f"{alias}_{event_name}").emit( + event.relation, app=event.app, unit=event.unit + ) + + def _get_relation_alias(self, relation_id: int) -> Optional[str]: + """Returns the relation alias. + + Args: + relation_id: the identifier for a particular relation. + + Returns: + the relation alias or None if the relation was not found. + """ + for relation in self.charm.model.relations[self.relation_name]: + if relation.id == relation_id: + return relation.data[self.local_unit].get("alias") + return None + + def is_postgresql_plugin_enabled(self, plugin: str, relation_index: int = 0) -> bool: + """Returns whether a plugin is enabled in the database. + + Args: + plugin: name of the plugin to check. + relation_index: optional relation index to check the database + (default: 0 - first relation). + + PostgreSQL only. + """ + # Psycopg 3 is imported locally to avoid the need of its package installation + # when relating to a database charm other than PostgreSQL. + import psycopg + + # Return False if no relation is established. + if len(self.relations) == 0: + return False + + relation_data = self.fetch_relation_data()[self.relations[relation_index].id] + host = relation_data.get("endpoints") + + # Return False if there is no endpoint available. + if host is None: + return False + + host = host.split(":")[0] + user = relation_data.get("username") + password = relation_data.get("password") + connection_string = ( + f"host='{host}' dbname='{self.database}' user='{user}' password='{password}'" + ) + try: + with psycopg.connect(connection_string) as connection: + with connection.cursor() as cursor: + cursor.execute(f"SELECT TRUE FROM pg_extension WHERE extname='{plugin}';") + return cursor.fetchone() is not None + except psycopg.Error as e: + logger.exception( + f"failed to check whether {plugin} plugin is enabled in the database: %s", str(e) + ) + return False + + def _on_relation_joined_event(self, event: RelationJoinedEvent) -> None: + """Event emitted when the application joins the database relation.""" + # If relations aliases were provided, assign one to the relation. + self._assign_relation_alias(event.relation.id) + + # Sets both database and extra user roles in the relation + # if the roles are provided. Otherwise, sets only the database. + if self.extra_user_roles: + self._update_relation_data( + event.relation.id, + { + "database": self.database, + "extra-user-roles": self.extra_user_roles, + }, + ) + else: + self._update_relation_data(event.relation.id, {"database": self.database}) + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the database relation has changed.""" + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Check if the database is created + # (the database charm shared the credentials). + if "username" in diff.added and "password" in diff.added: + # Emit the default event (the one without an alias). + logger.info("database created at %s", datetime.now()) + self.on.database_created.emit(event.relation, app=event.app, unit=event.unit) + + # Emit the aliased event (if any). + self._emit_aliased_event(event, "database_created") + + # To avoid unnecessary application restarts do not trigger + # “endpoints_changed“ event if “database_created“ is triggered. + return + + # Emit an endpoints changed event if the database + # added or changed this info in the relation databag. + if "endpoints" in diff.added or "endpoints" in diff.changed: + # Emit the default event (the one without an alias). + logger.info("endpoints changed on %s", datetime.now()) + self.on.endpoints_changed.emit(event.relation, app=event.app, unit=event.unit) + + # Emit the aliased event (if any). + self._emit_aliased_event(event, "endpoints_changed") + + # To avoid unnecessary application restarts do not trigger + # “read_only_endpoints_changed“ event if “endpoints_changed“ is triggered. + return + + # Emit a read only endpoints changed event if the database + # added or changed this info in the relation databag. + if "read-only-endpoints" in diff.added or "read-only-endpoints" in diff.changed: + # Emit the default event (the one without an alias). + logger.info("read-only-endpoints changed on %s", datetime.now()) + self.on.read_only_endpoints_changed.emit( + event.relation, app=event.app, unit=event.unit + ) + + # Emit the aliased event (if any). + self._emit_aliased_event(event, "read_only_endpoints_changed") + + +# Kafka related events + + +class KafkaProvidesEvent(RelationEvent): + """Base class for Kafka events.""" + + @property + def topic(self) -> Optional[str]: + """Returns the topic that was requested.""" + return self.relation.data[self.relation.app].get("topic") + + @property + def consumer_group_prefix(self) -> Optional[str]: + """Returns the consumer-group-prefix that was requested.""" + return self.relation.data[self.relation.app].get("consumer-group-prefix") + + +class TopicRequestedEvent(KafkaProvidesEvent, ExtraRoleEvent): + """Event emitted when a new topic is requested for use on this relation.""" + + +class KafkaProvidesEvents(CharmEvents): + """Kafka events. + + This class defines the events that the Kafka can emit. + """ + + topic_requested = EventSource(TopicRequestedEvent) + + +class KafkaRequiresEvent(RelationEvent): + """Base class for Kafka events.""" + + @property + def topic(self) -> Optional[str]: + """Returns the topic.""" + return self.relation.data[self.relation.app].get("topic") + + @property + def bootstrap_server(self) -> Optional[str]: + """Returns a comma-separated list of broker uris.""" + return self.relation.data[self.relation.app].get("endpoints") + + @property + def consumer_group_prefix(self) -> Optional[str]: + """Returns the consumer-group-prefix.""" + return self.relation.data[self.relation.app].get("consumer-group-prefix") + + @property + def zookeeper_uris(self) -> Optional[str]: + """Returns a comma separated list of Zookeeper uris.""" + return self.relation.data[self.relation.app].get("zookeeper-uris") + + +class TopicCreatedEvent(AuthenticationEvent, KafkaRequiresEvent): + """Event emitted when a new topic is created for use on this relation.""" + + +class BootstrapServerChangedEvent(AuthenticationEvent, KafkaRequiresEvent): + """Event emitted when the bootstrap server is changed.""" + + +class KafkaRequiresEvents(CharmEvents): + """Kafka events. + + This class defines the events that the Kafka can emit. + """ + + topic_created = EventSource(TopicCreatedEvent) + bootstrap_server_changed = EventSource(BootstrapServerChangedEvent) + + +# Kafka Provides and Requires + + +class KafkaProvides(DataProvides): + """Provider-side of the Kafka relation.""" + + on = KafkaProvidesEvents() + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + super().__init__(charm, relation_name) + + def _on_relation_changed(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Only the leader should handle this event. + if not self.local_unit.is_leader(): + return + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Emit a topic requested event if the setup key (topic name and optional + # extra user roles) was added to the relation databag by the application. + if "topic" in diff.added: + self.on.topic_requested.emit(event.relation, app=event.app, unit=event.unit) + + def set_topic(self, relation_id: int, topic: str) -> None: + """Set topic name in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + topic: the topic name. + """ + self._update_relation_data(relation_id, {"topic": topic}) + + def set_bootstrap_server(self, relation_id: int, bootstrap_server: str) -> None: + """Set the bootstrap server in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + bootstrap_server: the bootstrap server address. + """ + self._update_relation_data(relation_id, {"endpoints": bootstrap_server}) + + def set_consumer_group_prefix(self, relation_id: int, consumer_group_prefix: str) -> None: + """Set the consumer group prefix in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + consumer_group_prefix: the consumer group prefix string. + """ + self._update_relation_data(relation_id, {"consumer-group-prefix": consumer_group_prefix}) + + def set_zookeeper_uris(self, relation_id: int, zookeeper_uris: str) -> None: + """Set the zookeeper uris in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + zookeeper_uris: comma-separated list of ZooKeeper server uris. + """ + self._update_relation_data(relation_id, {"zookeeper-uris": zookeeper_uris}) + + +class KafkaRequires(DataRequires): + """Requires-side of the Kafka relation.""" + + on = KafkaRequiresEvents() + + def __init__( + self, + charm, + relation_name: str, + topic: str, + extra_user_roles: Optional[str] = None, + consumer_group_prefix: Optional[str] = None, + ): + """Manager of Kafka client relations.""" + # super().__init__(charm, relation_name) + super().__init__(charm, relation_name, extra_user_roles) + self.charm = charm + self.topic = topic + self.consumer_group_prefix = consumer_group_prefix or "" + + @property + def topic(self): + """Topic to use in Kafka.""" + return self._topic + + @topic.setter + def topic(self, value): + # Avoid wildcards + if value == "*": + raise ValueError(f"Error on topic '{value}', cannot be a wildcard.") + self._topic = value + + def _on_relation_joined_event(self, event: RelationJoinedEvent) -> None: + """Event emitted when the application joins the Kafka relation.""" + # Sets topic, extra user roles, and "consumer-group-prefix" in the relation + relation_data = { + f: getattr(self, f.replace("-", "_"), "") + for f in ["consumer-group-prefix", "extra-user-roles", "topic"] + } + + self._update_relation_data(event.relation.id, relation_data) + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the Kafka relation has changed.""" + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Check if the topic is created + # (the Kafka charm shared the credentials). + if "username" in diff.added and "password" in diff.added: + # Emit the default event (the one without an alias). + logger.info("topic created at %s", datetime.now()) + self.on.topic_created.emit(event.relation, app=event.app, unit=event.unit) + + # To avoid unnecessary application restarts do not trigger + # “endpoints_changed“ event if “topic_created“ is triggered. + return + + # Emit an endpoints (bootstrap-server) changed event if the Kafka endpoints + # added or changed this info in the relation databag. + if "endpoints" in diff.added or "endpoints" in diff.changed: + # Emit the default event (the one without an alias). + logger.info("endpoints changed on %s", datetime.now()) + self.on.bootstrap_server_changed.emit( + event.relation, app=event.app, unit=event.unit + ) # here check if this is the right design + return + + +# Opensearch related events + + +class OpenSearchProvidesEvent(RelationEvent): + """Base class for OpenSearch events.""" + + @property + def index(self) -> Optional[str]: + """Returns the index that was requested.""" + return self.relation.data[self.relation.app].get("index") + + +class IndexRequestedEvent(OpenSearchProvidesEvent, ExtraRoleEvent): + """Event emitted when a new index is requested for use on this relation.""" + + +class OpenSearchProvidesEvents(CharmEvents): + """OpenSearch events. + + This class defines the events that OpenSearch can emit. + """ + + index_requested = EventSource(IndexRequestedEvent) + + +class OpenSearchRequiresEvent(DatabaseRequiresEvent): + """Base class for OpenSearch requirer events.""" + + +class IndexCreatedEvent(AuthenticationEvent, OpenSearchRequiresEvent): + """Event emitted when a new index is created for use on this relation.""" + + +class OpenSearchRequiresEvents(CharmEvents): + """OpenSearch events. + + This class defines the events that the opensearch requirer can emit. + """ + + index_created = EventSource(IndexCreatedEvent) + endpoints_changed = EventSource(DatabaseEndpointsChangedEvent) + authentication_updated = EventSource(AuthenticationEvent) + + +# OpenSearch Provides and Requires Objects + + +class OpenSearchProvides(DataProvides): + """Provider-side of the OpenSearch relation.""" + + on = OpenSearchProvidesEvents() + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + super().__init__(charm, relation_name) + + def _on_relation_changed(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Only the leader should handle this event. + if not self.local_unit.is_leader(): + return + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Emit an index requested event if the setup key (index name and optional extra user roles) + # have been added to the relation databag by the application. + if "index" in diff.added: + self.on.index_requested.emit(event.relation, app=event.app, unit=event.unit) + + def set_index(self, relation_id: int, index: str) -> None: + """Set the index in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + index: the index as it is _created_ on the provider charm. This needn't match the + requested index, and can be used to present a different index name if, for example, + the requested index is invalid. + """ + self._update_relation_data(relation_id, {"index": index}) + + def set_endpoints(self, relation_id: int, endpoints: str) -> None: + """Set the endpoints in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + endpoints: the endpoint addresses for opensearch nodes. + """ + self._update_relation_data(relation_id, {"endpoints": endpoints}) + + def set_version(self, relation_id: int, version: str) -> None: + """Set the opensearch version in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + version: database version. + """ + self._update_relation_data(relation_id, {"version": version}) + + +class OpenSearchRequires(DataRequires): + """Requires-side of the OpenSearch relation.""" + + on = OpenSearchRequiresEvents() + + def __init__( + self, charm, relation_name: str, index: str, extra_user_roles: Optional[str] = None + ): + """Manager of OpenSearch client relations.""" + super().__init__(charm, relation_name, extra_user_roles) + self.charm = charm + self.index = index + + def _on_relation_joined_event(self, event: RelationJoinedEvent) -> None: + """Event emitted when the application joins the OpenSearch relation.""" + # Sets both index and extra user roles in the relation if the roles are provided. + # Otherwise, sets only the index. + data = {"index": self.index} + if self.extra_user_roles: + data["extra-user-roles"] = self.extra_user_roles + + self._update_relation_data(event.relation.id, data) + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the OpenSearch relation has changed. + + This event triggers individual custom events depending on the changing relation. + """ + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Check if authentication has updated, emit event if so + updates = {"username", "password", "tls", "tls-ca"} + if len(set(diff._asdict().keys()) - updates) < len(diff): + logger.info("authentication updated at: %s", datetime.now()) + self.on.authentication_updated.emit(event.relation, app=event.app, unit=event.unit) + + # Check if the index is created + # (the OpenSearch charm shares the credentials). + if "username" in diff.added and "password" in diff.added: + # Emit the default event (the one without an alias). + logger.info("index created at: %s", datetime.now()) + self.on.index_created.emit(event.relation, app=event.app, unit=event.unit) + + # To avoid unnecessary application restarts do not trigger + # “endpoints_changed“ event if “index_created“ is triggered. + return + + # Emit a endpoints changed event if the OpenSearch application added or changed this info + # in the relation databag. + if "endpoints" in diff.added or "endpoints" in diff.changed: + # Emit the default event (the one without an alias). + logger.info("endpoints changed on %s", datetime.now()) + self.on.endpoints_changed.emit( + event.relation, app=event.app, unit=event.unit + ) # here check if this is the right design + return diff --git a/metadata.yaml b/metadata.yaml index 3f4f3317..04510db8 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -43,9 +43,9 @@ requires: redis: interface: redis limit: 1 - db: - interface: pgsql - limit: 1 + database: + interface: postgresql_client + limit: 1 nginx-route: interface: nginx-route limit: 1 diff --git a/src/charm.py b/src/charm.py index 3280145e..f3354be4 100755 --- a/src/charm.py +++ b/src/charm.py @@ -21,6 +21,8 @@ from ops.model import ActiveStatus, BlockedStatus, MaintenanceStatus, WaitingStatus from ops.pebble import ExecError, ExecProcess, Plan +from database import DatabaseObserver + logger = logging.getLogger(__name__) pgsql = ops.lib.use("pgsql", 1, "postgresql-charmers@lists.launchpad.net") @@ -72,11 +74,20 @@ def __init__(self, *args): """Initialize defaults and event handlers.""" super().__init__(*args) + self._database = DatabaseObserver(self) + + self.framework.observe( + self._database.database.on.database_created, self._database_relation_changed + ) + self.framework.observe( + self._database.database.on.endpoints_changed, self._database_relation_changed + ) + self.framework.observe( + self.on[self._database._RELATION_NAME].relation_broken, + self._reload_configuration, + ) + self._stored.set_default( - db_name=None, - db_user=None, - db_password=None, - db_host=None, redis_relation={}, ) self._require_nginx_route() @@ -85,11 +96,6 @@ def __init__(self, *args): self.framework.observe(self.on.discourse_pebble_ready, self._config_changed) self.framework.observe(self.on.config_changed, self._config_changed) - self.db_client = pgsql.PostgreSQLClient(self, "db") - self.framework.observe( - self.db_client.on.database_relation_joined, self._on_database_relation_joined - ) - self.framework.observe(self.db_client.on.master_changed, self._on_database_changed) self.framework.observe(self.on.add_admin_user_action, self._on_add_admin_user_action) self.framework.observe(self.on.anonymize_user_action, self._on_anonymize_user_action) @@ -245,22 +251,26 @@ def _get_s3_env(self) -> typing.Dict[str, typing.Any]: return s3_env + def _get_redis_relation_data(self) -> typing.Tuple[typing.Any, typing.Any]: + # This is the current recommended way of accessing the relation data. + for redis_unit in self._stored.redis_relation: # type: ignore + # mypy fails to see that this is indexable + redis_unit_data = self._stored.redis_relation[redis_unit] # type: ignore + redis_hostname = redis_unit_data.get("hostname", "") # type: ignore + redis_port = redis_unit_data.get("port", 6379) # type: ignore + logger.debug( + "Got redis connection details from relation of %s:%s", redis_hostname, redis_port + ) + return (redis_hostname, redis_port) + def _create_discourse_environment_settings(self) -> typing.Dict[str, typing.Any]: """Create a layer config based on our current configuration. Returns: Dictionary with all the environment settings. """ - # Get redis connection information from the relation. - redis_hostname = None - redis_port = 6379 - # This is the current recommended way of accessing the relation data. - for redis_unit in self._stored.redis_relation: # type: ignore - redis_hostname = self._stored.redis_relation[redis_unit].get("hostname") # type: ignore - redis_port = self._stored.redis_relation[redis_unit].get("port") # type: ignore - logger.debug( - "Got redis connection details from relation of %s:%s", redis_hostname, redis_port - ) + database_relation_data = self._database.get_relation_data() + redis_relation_data = self._get_redis_relation_data() pod_config = { # Since pebble exec command doesn't copy the container env (envVars set in Dockerfile), @@ -269,15 +279,15 @@ def _create_discourse_environment_settings(self) -> typing.Dict[str, typing.Any] "CONTAINER_APP_ROOT": "/srv/discourse", "CONTAINER_APP_USERNAME": "discourse", "DISCOURSE_CORS_ORIGIN": self.config["cors_origin"], - "DISCOURSE_DB_HOST": self._stored.db_host, - "DISCOURSE_DB_NAME": self._stored.db_name, - "DISCOURSE_DB_PASSWORD": self._stored.db_password, - "DISCOURSE_DB_USERNAME": self._stored.db_user, + "DISCOURSE_DB_HOST": database_relation_data["POSTGRES_HOST"], + "DISCOURSE_DB_NAME": database_relation_data["POSTGRES_DB"], + "DISCOURSE_DB_PASSWORD": database_relation_data["POSTGRES_PASSWORD"], + "DISCOURSE_DB_USERNAME": database_relation_data["POSTGRES_USER"], "DISCOURSE_DEVELOPER_EMAILS": self.config["developer_emails"], "DISCOURSE_ENABLE_CORS": str(self.config["enable_cors"]).lower(), "DISCOURSE_HOSTNAME": self._get_external_hostname(), - "DISCOURSE_REDIS_HOST": redis_hostname, - "DISCOURSE_REDIS_PORT": str(redis_port), + "DISCOURSE_REDIS_HOST": redis_relation_data[0], + "DISCOURSE_REDIS_PORT": str(redis_relation_data[1]), "DISCOURSE_REFRESH_MAXMIND_DB_DURING_PRECOMPILE_DAYS": "0", "DISCOURSE_SERVE_STATIC_ASSETS": "true", "DISCOURSE_SMTP_ADDRESS": self.config["smtp_address"], @@ -362,14 +372,25 @@ def _are_db_relations_ready(self) -> bool: Returns: If the needed relations have been established. """ - # mypy fails do detect this stored value can be False - if not self._stored.db_name: # type: ignore + db_rel = self._database.get_relation_data() + if db_rel is None: self.model.unit.status = WaitingStatus("Waiting for database relation") return False + if None in ( + db_rel["POSTGRES_USER"], + db_rel["POSTGRES_PASSWORD"], + db_rel["POSTGRES_DB"], + db_rel["POSTGRES_HOST"], + ): + self.model.unit.status = WaitingStatus("Waiting for database relation to initialize") + return False # mypy fails do detect this stored value can be False if not self._stored.redis_relation: # type: ignore self.model.unit.status = WaitingStatus("Waiting for redis relation") return False + if self._get_redis_relation_data()[0] == "": + self.model.unit.status = WaitingStatus("Waiting for redis relation to initialize") + return False return True def _set_up_discourse(self, event: HookEvent) -> None: @@ -464,7 +485,7 @@ def _config_changed(self, event: HookEvent) -> None: self._config_force_https() self.model.unit.status = ActiveStatus() - def _reload_configuration(self) -> None: + def _reload_configuration(self, _=None) -> None: # mypy has some trouble with dynamic attributes if not self._is_setup_completed(): logger.info("Defer starting the discourse server until discourse setup completed") @@ -475,46 +496,11 @@ def _reload_configuration(self) -> None: container.add_layer(SERVICE_NAME, layer_config, combine=True) container.pebble.replan_services() - def _redis_relation_changed(self, _: HookEvent) -> None: + def _database_relation_changed(self, _: HookEvent) -> None: if self._are_db_relations_ready(): self._reload_configuration() - # pgsql.DatabaseRelationJoinedEvent is actually defined - def _on_database_relation_joined( - self, event: pgsql.DatabaseRelationJoinedEvent # type: ignore - ) -> None: - """Handle db-relation-joined. - - Args: - event: Event triggering the database relation joined handler. - """ - if self.model.unit.is_leader(): - event.database = DATABASE_NAME - event.extensions = ["hstore:public", "pg_trgm:public"] - elif event.database != DATABASE_NAME: - # Leader has not yet set requirements. Defer, in case this unit - # becomes leader and needs to perform that operation. - event.defer() - return - - # pgsql.DatabaseChangedEvent is actually defined - def _on_database_changed(self, event: pgsql.DatabaseChangedEvent) -> None: # type: ignore - """Handle changes in the primary database unit. - - Args: - event: Event triggering the database master changed handler. - """ - if event.master is None: - self._stored.db_name = None - self._stored.db_user = None - self._stored.db_password = None - self._stored.db_host = None - else: - self._stored.db_name = event.master.dbname - self._stored.db_user = event.master.user - self._stored.db_password = event.master.password - self._stored.db_host = event.master.host - + def _redis_relation_changed(self, _: HookEvent) -> None: if self._are_db_relations_ready(): self._reload_configuration() diff --git a/src/database.py b/src/database.py new file mode 100644 index 00000000..a27692f6 --- /dev/null +++ b/src/database.py @@ -0,0 +1,54 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Provide the DatabaseObserver class to handle database relation and state.""" + +import typing + +from charms.data_platform_libs.v0.data_interfaces import DatabaseRequires +from ops.charm import CharmBase +from ops.framework import Object + +DATABASE_NAME = "discourse" + + +class DatabaseObserver(Object): + """The Database relation observer.""" + + _RELATION_NAME = "database" + + def __init__(self, charm: CharmBase): + """Initialize the observer and register event handlers. + + Args: + charm: The parent charm to attach the observer to. + """ + super().__init__(charm, "database-observer") + self._charm = charm + self.database = DatabaseRequires( + self._charm, + relation_name=self._RELATION_NAME, + database_name=DATABASE_NAME, + ) + + def get_relation_data(self) -> typing.Optional[typing.Dict]: + """Get database data from relation. + + Returns: + Dict: Information needed for setting environment variables. + """ + if self.model.get_relation(self._RELATION_NAME) is None: + return None + + relation_id = self.database.relations[0].id + relation_data = self.database.fetch_relation_data()[relation_id] + + endpoint = relation_data.get("endpoints", ":") + + return { + "POSTGRES_USER": relation_data.get("username"), + "POSTGRES_PASSWORD": relation_data.get("password"), + "POSTGRES_HOST": endpoint.split(":")[0], + "POSTGRES_PORT": endpoint.split(":")[1], + "POSTGRES_DB": relation_data.get("database"), + } diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 66a2e97a..b8a750a5 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -133,11 +133,12 @@ async def app_fixture( Builds the charm and deploys it and the relations it depends on. """ # Deploy relations to speed up overall execution - await asyncio.gather( - model.deploy("postgresql-k8s", channel="latest/stable", series="focal"), + related_apps = await asyncio.gather( + model.deploy("postgresql-k8s", channel="14/edge", series="jammy", trust=True), model.deploy("redis-k8s", series="focal"), model.deploy("nginx-ingress-integrator", series="focal", trust=True), ) + postgres_app = related_apps[0] resources = { "discourse-image": pytestconfig.getoption("--discourse-image"), @@ -160,13 +161,23 @@ async def app_fixture( config=app_config, series="focal", ) + + await model.wait_for_idle() + + # configure postgres + await postgres_app.set_config( + { + "plugin_hstore_enable": "true", + "plugin_pg_trgm_enable": "true", + } + ) await model.wait_for_idle() # Add required relations unit = model.applications[app_name].units[0] assert unit.workload_status == WaitingStatus.name # type: ignore await asyncio.gather( - model.add_relation(app_name, "postgresql-k8s:db-admin"), + model.add_relation(app_name, "postgresql-k8s:database"), model.add_relation(app_name, "redis-k8s"), model.add_relation(app_name, "nginx-ingress-integrator"), ) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 935aa79e..459d818a 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -21,13 +21,41 @@ BLOCKED_STATUS = BlockedStatus.name # type: ignore +DATABASE_NAME = "discourse" + +# pylint: disable=too-many-public-methods class TestDiscourseK8sCharm(unittest.TestCase): """Unit tests for Discourse charm.""" + def start_harness(self, with_postgres: bool, with_redis: bool): + """Start a harness discourse charm. + + Args: + - with_postgres: should a postgres relation be added + - with_redis: should a redis relation be added + """ + self.harness = Harness(DiscourseCharm) + self.harness.disable_hooks() + self.harness._framework = ops.framework.Framework( + self.harness._storage, self.harness._charm_dir, self.harness._meta, self.harness._model + ) + if with_postgres: + self._add_postgres_relation() + self.harness.enable_hooks() + self.harness.begin_with_initial_hooks() + if with_redis: + self._add_redis_relation() + with self._patch_exec(): + self.harness.framework.reemit() + + with self._patch_exec(), self._patch_setup_completed(): + charm: DiscourseCharm = typing.cast(DiscourseCharm, self.harness.charm) + charm._set_setup_completed() + def setUp(self): pgsql_patch.start() - self.harness = Harness(DiscourseCharm) + self.start_harness(with_postgres=True, with_redis=True) self.addCleanup(self.harness.cleanup) def tearDown(self): @@ -73,9 +101,7 @@ def test_relations_not_ready(self): act: when pebble ready event is triggered assert: it will wait for the db relation. """ - self.harness.begin_with_initial_hooks() - self.harness.container_pebble_ready("discourse") - + self.start_harness(with_postgres=False, with_redis=False) self.assertEqual( self.harness.model.unit.status, WaitingStatus("Waiting for database relation"), @@ -87,10 +113,7 @@ def test_db_relation_not_ready(self): act: when pebble ready event is triggered assert: it will wait for the db relation. """ - self.harness.begin_with_initial_hooks() - self._add_redis_relation() - self.harness.container_pebble_ready("discourse") - + self.start_harness(with_postgres=False, with_redis=True) self.assertEqual( self.harness.model.unit.status, WaitingStatus("Waiting for database relation"), @@ -102,9 +125,7 @@ def test_redis_relation_not_ready(self): act: when pebble ready event is triggered assert: it will wait for the redis relation. """ - self.harness.begin_with_initial_hooks() - self._add_postgres_relation() - self.harness.container_pebble_ready("discourse") + self.start_harness(with_postgres=True, with_redis=False) self.assertEqual( self.harness.model.unit.status, @@ -117,9 +138,8 @@ def test_ingress_relation_not_ready(self): act: when pebble ready event is triggered assert: it will wait for the ingress relation. """ - self.harness.begin_with_initial_hooks() + self.start_harness(with_postgres=False, with_redis=False) self._add_ingress_relation() - self.harness.container_pebble_ready("discourse") self.assertEqual( self.harness.model.unit.status, @@ -132,8 +152,6 @@ def test_config_changed_when_no_saml_target(self): act: when force_saml_login configuration is True and there's no saml_target_url assert: it will get to blocked status waiting for the latter. """ - self.harness.begin() - self._add_database_relations() self.harness.update_config({"force_saml_login": True, "saml_target_url": ""}) with self._patch_exec(): self.harness.container_pebble_ready("discourse") @@ -149,8 +167,6 @@ def test_config_changed_when_saml_sync_groups_and_no_url_invalid(self): act: when saml_sync_groups configuration is provided and there's no saml_target_url assert: it will get to blocked status waiting for the latter. """ - self.harness.begin() - self._add_database_relations() self.harness.update_config({"saml_sync_groups": "group1", "saml_target_url": ""}) with self._patch_exec(): self.harness.container_pebble_ready("discourse") @@ -166,8 +182,6 @@ def test_config_changed_when_saml_target_url_and_force_https_disabled(self): act: when saml_target_url configuration is provided and force_https is False assert: it will get to blocked status waiting for the latter. """ - self.harness.begin() - self._add_database_relations() self.harness.update_config({"saml_target_url": "group1", "force_https": False}) with self._patch_exec(): self.harness.container_pebble_ready("discourse") @@ -185,15 +199,20 @@ def test_config_changed_when_no_cors(self): act: when cors_origin configuration is empty assert: it will get to blocked status waiting for it. """ - self.harness.begin() - self._add_database_relations() self.harness.update_config({"cors_origin": ""}) with self._patch_exec(): self.harness.container_pebble_ready("discourse") + self.assertNotEqual( + self.harness.charm._database.get_relation_data(), + None, + "database name should be set after relation joined", + ) + self.assertEqual( - self.harness.model.unit.status, - BlockedStatus("Required configuration missing: cors_origin"), + self.harness.charm._database.get_relation_data().get("POSTGRES_DB"), + "discourse", + "database name should be set after relation joined", ) def test_config_changed_when_throttle_mode_invalid(self): @@ -202,10 +221,7 @@ def test_config_changed_when_throttle_mode_invalid(self): act: when throttle_level configuration is invalid assert: it will get to blocked status waiting for a valid value to be provided. """ - self.harness.begin() - self._add_database_relations() self.harness.update_config({"throttle_level": "Scream"}) - self.harness.container_pebble_ready("discourse") self.assertEqual(self.harness.model.unit.status.name, BLOCKED_STATUS) self.assertTrue("none permissive strict" in self.harness.model.unit.status.message) @@ -216,8 +232,6 @@ def test_config_changed_when_s3_and_no_bucket_invalid(self): act: when s3_enabled configuration is True and there's no s3_bucket assert: it will get to blocked status waiting for the latter. """ - self.harness.begin() - self._add_database_relations() self.harness.update_config( { "s3_access_key_id": "3|33+", @@ -227,7 +241,6 @@ def test_config_changed_when_s3_and_no_bucket_invalid(self): "s3_secret_access_key": "s|kI0ure_k3Y", } ) - self.harness.container_pebble_ready("discourse") self.assertEqual( self.harness.model.unit.status, @@ -241,11 +254,8 @@ def test_config_changed_when_valid_no_s3_backup_nor_cdn(self): assert: the appropriate configuration values are passed to the pod and the unit reaches Active status. """ - with self._patch_exec() as mock_exec, self._patch_setup_completed(): - self.harness.begin_with_initial_hooks() - self.harness.disable_hooks() + with self._patch_exec() as mock_exec: self.harness.set_leader(True) - self._add_database_relations() self.harness.update_config( { "s3_access_key_id": "3|33+", @@ -259,18 +269,19 @@ def test_config_changed_when_valid_no_s3_backup_nor_cdn(self): self.harness.container_pebble_ready("discourse") self.harness.framework.reemit() - updated_plan = self.harness.get_container_pebble_plan("discourse").to_dict() - updated_plan_env = updated_plan["services"]["discourse"]["environment"] + assert self.harness._charm mock_exec.assert_any_call( [f"{DISCOURSE_PATH}/bin/bundle", "exec", "rake", "s3:upload_assets"], - environment=updated_plan_env, + environment=self.harness._charm._create_discourse_environment_settings(), working_dir=DISCOURSE_PATH, user="discourse", ) + updated_plan = self.harness.get_container_pebble_plan("discourse").to_dict() + updated_plan_env = updated_plan["services"]["discourse"]["environment"] self.assertNotIn("DISCOURSE_BACKUP_LOCATION", updated_plan_env) self.assertEqual("*", updated_plan_env["DISCOURSE_CORS_ORIGIN"]) self.assertEqual("dbhost", updated_plan_env["DISCOURSE_DB_HOST"]) - self.assertEqual("discourse-k8s", updated_plan_env["DISCOURSE_DB_NAME"]) + self.assertEqual(DATABASE_NAME, updated_plan_env["DISCOURSE_DB_NAME"]) self.assertEqual("somepasswd", updated_plan_env["DISCOURSE_DB_PASSWORD"]) self.assertEqual("someuser", updated_plan_env["DISCOURSE_DB_USERNAME"]) self.assertTrue(updated_plan_env["DISCOURSE_ENABLE_CORS"]) @@ -295,10 +306,7 @@ def test_config_changed_when_valid_no_fingerprint(self): assert: the appropriate configuration values are passed to the pod and the unit reaches Active status. """ - self.harness.begin_with_initial_hooks() - self.harness.disable_hooks() - self._add_database_relations() - with self._patch_exec(), self._patch_setup_completed(): + with self._patch_exec(): self.harness.update_config( { "force_saml_login": True, @@ -315,7 +323,7 @@ def test_config_changed_when_valid_no_fingerprint(self): updated_plan_env = updated_plan["services"]["discourse"]["environment"] self.assertEqual("*", updated_plan_env["DISCOURSE_CORS_ORIGIN"]) self.assertEqual("dbhost", updated_plan_env["DISCOURSE_DB_HOST"]) - self.assertEqual("discourse-k8s", updated_plan_env["DISCOURSE_DB_NAME"]) + self.assertEqual(DATABASE_NAME, updated_plan_env["DISCOURSE_DB_NAME"]) self.assertEqual("somepasswd", updated_plan_env["DISCOURSE_DB_PASSWORD"]) self.assertEqual("someuser", updated_plan_env["DISCOURSE_DB_USERNAME"]) self.assertTrue(updated_plan_env["DISCOURSE_ENABLE_CORS"]) @@ -343,10 +351,7 @@ def test_config_changed_when_valid(self): assert: the appropriate configuration values are passed to the pod and the unit reaches Active status. """ - self.harness.begin_with_initial_hooks() - self.harness.disable_hooks() - self._add_database_relations() - with self._patch_exec(), self._patch_setup_completed(): + with self._patch_exec(): self.harness.update_config( { "developer_emails": "user@foo.internal", @@ -378,7 +383,7 @@ def test_config_changed_when_valid(self): self.assertEqual("s3", updated_plan_env["DISCOURSE_BACKUP_LOCATION"]) self.assertEqual("*", updated_plan_env["DISCOURSE_CORS_ORIGIN"]) self.assertEqual("dbhost", updated_plan_env["DISCOURSE_DB_HOST"]) - self.assertEqual("discourse-k8s", updated_plan_env["DISCOURSE_DB_NAME"]) + self.assertEqual(DATABASE_NAME, updated_plan_env["DISCOURSE_DB_NAME"]) self.assertEqual("somepasswd", updated_plan_env["DISCOURSE_DB_PASSWORD"]) self.assertEqual("someuser", updated_plan_env["DISCOURSE_DB_USERNAME"]) self.assertEqual("user@foo.internal", updated_plan_env["DISCOURSE_DEVELOPER_EMAILS"]) @@ -418,14 +423,11 @@ def test_db_relation(self): act: when the database relation is added assert: the appropriate database name is set. """ - self.harness.begin() - self._add_database_relations() self.harness.set_leader(True) - # testing harness not re-emits deferred events, manually trigger that - self.harness.framework.reemit() db_relation_data = self.harness.get_relation_data( - self.db_relation_id, self.harness.charm.app.name + self.db_relation_id, + "postgresql", ) self.assertEqual( @@ -434,6 +436,12 @@ def test_db_relation(self): "database name should be set after relation joined", ) + self.assertEqual( + self.harness.charm._database.get_relation_data().get("POSTGRES_DB"), + "discourse", + "database name should be set after relation joined", + ) + @patch.object(Container, "exec") def test_add_admin_user(self, mock_exec): """ @@ -442,12 +450,7 @@ def test_add_admin_user(self, mock_exec): assert: the underlying rake command to add the user is executed with the appropriate parameters. """ - self.harness.begin() - self.harness.disable_hooks() - self._add_database_relations() - charm: DiscourseCharm = typing.cast(DiscourseCharm, self.harness.charm) - self.harness.container_pebble_ready("discourse") email = "sample@email.com" password = "somepassword" # nosec @@ -480,13 +483,7 @@ def test_anonymize_user(self, mock_exec): assert: the underlying rake command to anonymize the user is executed with the appropriate parameters. """ - self.harness.begin() - self.harness.disable_hooks() - self._add_database_relations() - charm: DiscourseCharm = typing.cast(DiscourseCharm, self.harness.charm) - self.harness.container_pebble_ready("discourse") - username = "someusername" event = MagicMock(spec=ActionEvent) event.params = {"username": username} @@ -509,10 +506,8 @@ def test_install_when_leader(self): act: trigger the install event on a leader unit assert: migrations are executed and assets are precompiled. """ - self.harness.begin_with_initial_hooks() self.harness.set_leader(True) - self._add_database_relations() - with self._patch_exec() as mock_exec, self._patch_setup_completed(): + with self._patch_exec() as mock_exec: self.harness.container_pebble_ready("discourse") self.harness.charm.on.install.emit() self.harness.framework.reemit() @@ -544,10 +539,8 @@ def test_install_when_not_leader(self): act: trigger the install event on a leader unit assert: migrations are executed and assets are precompiled. """ - self.harness.begin_with_initial_hooks() self.harness.set_leader(False) - self._add_database_relations() - with self._patch_exec() as mock_exec, self._patch_setup_completed(): + with self._patch_exec() as mock_exec: self.harness.container_pebble_ready("discourse") self.harness.charm.on.install.emit() self.harness.framework.reemit() @@ -569,15 +562,24 @@ def test_install_when_not_leader(self): def _add_postgres_relation(self): "Add postgresql relation and relation data to the charm." - self.harness.charm._stored.db_name = "discourse-k8s" - self.harness.charm._stored.db_user = "someuser" - self.harness.charm._stored.db_password = "somepasswd" # nosec - self.harness.charm._stored.db_host = "dbhost" + + relation_data = { + "database": DATABASE_NAME, + "endpoints": "dbhost:5432,dbhost-2:5432", + "password": "somepasswd", # nosec + "username": "someuser", + } + # get a relation ID for the test outside of __init__ (note pylint disable) self.db_relation_id = ( # pylint: disable=attribute-defined-outside-init - self.harness.add_relation("db", "postgresql") + self.harness.add_relation("database", "postgresql") ) self.harness.add_relation_unit(self.db_relation_id, "postgresql/0") + self.harness.update_relation_data( + self.db_relation_id, + "postgresql", + relation_data, + ) def _add_redis_relation(self): "Add redis relation and relation data to the charm." @@ -591,8 +593,3 @@ def _add_ingress_relation(self): "Add ingress relation and relation data to the charm." nginx_route_relation_id = self.harness.add_relation("nginx-route", "ingress") self.harness.add_relation_unit(nginx_route_relation_id, "ingress/0") - - def _add_database_relations(self): - "Add postgresql and redis relations and relation data to the charm." - self._add_postgres_relation() - self._add_redis_relation() From 4a72b51ad7716fae70afed6fb8650fb10796ddf7 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Mon, 24 Jul 2023 11:40:52 -0400 Subject: [PATCH 19/30] Fix excessive spacing (covid's not an excuse) --- metadata.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata.yaml b/metadata.yaml index 04510db8..a32e387c 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -44,8 +44,8 @@ requires: interface: redis limit: 1 database: - interface: postgresql_client - limit: 1 + interface: postgresql_client + limit: 1 nginx-route: interface: nginx-route limit: 1 From b1dd08cea24e2e7148d2e140b86edad17f92c4ea Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Mon, 24 Jul 2023 11:53:55 -0400 Subject: [PATCH 20/30] Add missing docstring --- src/charm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/charm.py b/src/charm.py index f3354be4..4ac353a9 100755 --- a/src/charm.py +++ b/src/charm.py @@ -252,6 +252,11 @@ def _get_s3_env(self) -> typing.Dict[str, typing.Any]: return s3_env def _get_redis_relation_data(self) -> typing.Tuple[typing.Any, typing.Any]: + """Get the hostname and port from the redis relation data. + + Returns: + Tuple with the hostname and port of the related redis + """ # This is the current recommended way of accessing the relation data. for redis_unit in self._stored.redis_relation: # type: ignore # mypy fails to see that this is indexable From f2b41d2fc4f7d471b96c8f2e3148ea48d161198c Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Mon, 24 Jul 2023 15:17:30 -0400 Subject: [PATCH 21/30] Move is_relation_ready() for the db to the database module --- src/charm.py | 11 +---------- src/database.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/charm.py b/src/charm.py index 4ac353a9..9d57d1c1 100755 --- a/src/charm.py +++ b/src/charm.py @@ -377,18 +377,9 @@ def _are_db_relations_ready(self) -> bool: Returns: If the needed relations have been established. """ - db_rel = self._database.get_relation_data() - if db_rel is None: + if not self._database.is_relation_ready(): self.model.unit.status = WaitingStatus("Waiting for database relation") return False - if None in ( - db_rel["POSTGRES_USER"], - db_rel["POSTGRES_PASSWORD"], - db_rel["POSTGRES_DB"], - db_rel["POSTGRES_HOST"], - ): - self.model.unit.status = WaitingStatus("Waiting for database relation to initialize") - return False # mypy fails do detect this stored value can be False if not self._stored.redis_relation: # type: ignore self.model.unit.status = WaitingStatus("Waiting for redis relation") diff --git a/src/database.py b/src/database.py index a27692f6..3dcad68c 100644 --- a/src/database.py +++ b/src/database.py @@ -36,6 +36,7 @@ def get_relation_data(self) -> typing.Optional[typing.Dict]: Returns: Dict: Information needed for setting environment variables. + Returns None if the relation data is not correctly initialized. """ if self.model.get_relation(self._RELATION_NAME) is None: return None @@ -43,12 +44,35 @@ def get_relation_data(self) -> typing.Optional[typing.Dict]: relation_id = self.database.relations[0].id relation_data = self.database.fetch_relation_data()[relation_id] - endpoint = relation_data.get("endpoints", ":") + endpoints = relation_data.get("endpoints", "").split(",") + if len(endpoints) < 1: + return None + + primary_endpoint = endpoints[0].split(":") + if len(primary_endpoint) < 2: + return None - return { + data = { "POSTGRES_USER": relation_data.get("username"), "POSTGRES_PASSWORD": relation_data.get("password"), - "POSTGRES_HOST": endpoint.split(":")[0], - "POSTGRES_PORT": endpoint.split(":")[1], + "POSTGRES_HOST": primary_endpoint[0], + "POSTGRES_PORT": primary_endpoint[1], "POSTGRES_DB": relation_data.get("database"), } + + if None in ( + data["POSTGRES_USER"], + data["POSTGRES_PASSWORD"], + data["POSTGRES_DB"], + ): + return None + + return data + + def is_relation_ready(self) -> bool: + """Check if the relation is ready. + + Returns: + bool: returns True if the relation is ready. + """ + return self.get_relation_data() is not None From a39dda9cf38ccea4973f0f11c9e2d4c8e3ca404c Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Mon, 24 Jul 2023 15:33:14 -0400 Subject: [PATCH 22/30] Add a test for is_relation_ready() --- tests/unit/test_charm.py | 52 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 459d818a..27206ddc 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -593,3 +593,55 @@ def _add_ingress_relation(self): "Add ingress relation and relation data to the charm." nginx_route_relation_id = self.harness.add_relation("nginx-route", "ingress") self.harness.add_relation_unit(nginx_route_relation_id, "ingress/0") + + def test_postgres_relation_data(self): + test_cases = [ + ( + { + "database": DATABASE_NAME, + "endpoints": "dbhost:5432,dbhost-2:5432", + "password": "somepasswd", # nosec + "username": "someuser", + }, True + ), + ( + { + "database": DATABASE_NAME, + "endpoints": "foo", + "password": "somepasswd", # nosec + "username": "someuser", + }, False + ), + ( + { + "database": DATABASE_NAME, + "endpoints": "dbhost:5432,dbhost-2:5432", + "password": "", + "username": "someuser", + }, False + ), + ] + + for relation_data, should_be_ready in test_cases: + with self.subTest(relation_data=relation_data, should_be_ready=should_be_ready): + self.start_harness(with_postgres=False, with_redis=False) + # get a relation ID for the test outside of __init__ (note pylint disable) + self.db_relation_id = ( # pylint: disable=attribute-defined-outside-init + self.harness.add_relation("database", "postgresql") + ) + self.harness.add_relation_unit(self.db_relation_id, "postgresql/0") + self.harness.update_relation_data( + self.db_relation_id, + "postgresql", + relation_data, + ) + if should_be_ready: + self.assertEqual( + self.harness.model.unit.status, + WaitingStatus("Waiting for redis relation"), + ) + else: + self.assertEqual( + self.harness.model.unit.status, + WaitingStatus("Waiting for database relation"), + ) From 2cc17432db3914091175351220e56ad5ee867e55 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Mon, 24 Jul 2023 18:24:45 -0400 Subject: [PATCH 23/30] Fix linting --- tests/unit/test_charm.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 27206ddc..8095b21c 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -595,6 +595,11 @@ def _add_ingress_relation(self): self.harness.add_relation_unit(nginx_route_relation_id, "ingress/0") def test_postgres_relation_data(self): + """ + arrange: given a deployed discourse charm and some relation data + act: add the postgresql relation to the charm + assert: the charm should wait for some correct relation data + """ test_cases = [ ( { @@ -602,7 +607,8 @@ def test_postgres_relation_data(self): "endpoints": "dbhost:5432,dbhost-2:5432", "password": "somepasswd", # nosec "username": "someuser", - }, True + }, + True, ), ( { @@ -610,7 +616,8 @@ def test_postgres_relation_data(self): "endpoints": "foo", "password": "somepasswd", # nosec "username": "someuser", - }, False + }, + False, ), ( { @@ -618,7 +625,8 @@ def test_postgres_relation_data(self): "endpoints": "dbhost:5432,dbhost-2:5432", "password": "", "username": "someuser", - }, False + }, + False, ), ] From b81cfdb2399fe831cf135e47a2efe5917687c130 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Mon, 24 Jul 2023 18:32:28 -0400 Subject: [PATCH 24/30] Remove ops from requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fd6adcd0..ca8415da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -ops ops-lib-pgsql From d04f750362809eb0656747d865a15e1671d653bc Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Tue, 25 Jul 2023 15:53:07 -0400 Subject: [PATCH 25/30] Add redis default port as a constant --- src/charm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 9d57d1c1..f5b5105c 100755 --- a/src/charm.py +++ b/src/charm.py @@ -62,6 +62,7 @@ SERVICE_NAME = "discourse" SERVICE_PORT = 3000 SETUP_COMPLETED_FLAG_FILE = "/run/discourse-k8s-operator/setup_completed" +DEFAULT_REDIS_PORT = 6379 class DiscourseCharm(CharmBase): @@ -262,7 +263,7 @@ def _get_redis_relation_data(self) -> typing.Tuple[typing.Any, typing.Any]: # mypy fails to see that this is indexable redis_unit_data = self._stored.redis_relation[redis_unit] # type: ignore redis_hostname = redis_unit_data.get("hostname", "") # type: ignore - redis_port = redis_unit_data.get("port", 6379) # type: ignore + redis_port = redis_unit_data.get("port", DEFAULT_REDIS_PORT) # type: ignore logger.debug( "Got redis connection details from relation of %s:%s", redis_hostname, redis_port ) From b0392f0620345d3142a5681b5a7e20ac248d78f1 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Tue, 25 Jul 2023 16:00:51 -0400 Subject: [PATCH 26/30] Comply to ISD014 for event handlers --- src/charm.py | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/charm.py b/src/charm.py index f5b5105c..4b8f5fcf 100755 --- a/src/charm.py +++ b/src/charm.py @@ -10,12 +10,16 @@ import ops import ops.lib +from charms.data_platform_libs.v0.data_interfaces import ( + DatabaseCreatedEvent, + DatabaseEndpointsChangedEvent, +) from charms.grafana_k8s.v0.grafana_dashboard import GrafanaDashboardProvider from charms.loki_k8s.v0.loki_push_api import LogProxyConsumer from charms.nginx_ingress_integrator.v0.nginx_route import require_nginx_route from charms.prometheus_k8s.v0.prometheus_scrape import MetricsEndpointProvider from charms.redis_k8s.v0.redis import RedisRelationCharmEvents, RedisRequires -from ops.charm import ActionEvent, CharmBase, HookEvent +from ops.charm import ActionEvent, CharmBase, HookEvent, RelationBrokenEvent from ops.framework import StoredState from ops.main import main from ops.model import ActiveStatus, BlockedStatus, MaintenanceStatus, WaitingStatus @@ -78,14 +82,14 @@ def __init__(self, *args): self._database = DatabaseObserver(self) self.framework.observe( - self._database.database.on.database_created, self._database_relation_changed + self._database.database.on.database_created, self._on_database_created ) self.framework.observe( - self._database.database.on.endpoints_changed, self._database_relation_changed + self._database.database.on.endpoints_changed, self._on_database_endpoints_changed ) self.framework.observe( self.on[self._database._RELATION_NAME].relation_broken, - self._reload_configuration, + self._on_database_relation_broken, ) self._stored.set_default( @@ -111,6 +115,32 @@ def __init__(self, *args): ) self._grafana_dashboards = GrafanaDashboardProvider(self) + def _on_database_created(self, _: DatabaseCreatedEvent) -> None: + """Handle database created. + + Args: + event: Event triggering the database created handler. + """ + if self._are_db_relations_ready(): + self._reload_configuration() + + def _on_database_endpoints_changed(self, _: DatabaseEndpointsChangedEvent) -> None: + """Handle endpoints change. + + Args: + event: Event triggering the endpoints changed handler. + """ + if self._are_db_relations_ready(): + self._reload_configuration() + + def _on_database_relation_broken(self, _: RelationBrokenEvent) -> None: + """Handle broken relation. + + Args: + event: Event triggering the broken relation handler. + """ + self._reload_configuration() + def _require_nginx_route(self) -> None: """Create minimal ingress configuration.""" require_nginx_route( From dbaad684c9595098d2267068294a520d115f048f Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Tue, 25 Jul 2023 16:10:25 -0400 Subject: [PATCH 27/30] Remove unecessary arg --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 4b8f5fcf..ff8c2bde 100755 --- a/src/charm.py +++ b/src/charm.py @@ -512,7 +512,7 @@ def _config_changed(self, event: HookEvent) -> None: self._config_force_https() self.model.unit.status = ActiveStatus() - def _reload_configuration(self, _=None) -> None: + def _reload_configuration(self) -> None: # mypy has some trouble with dynamic attributes if not self._is_setup_completed(): logger.info("Defer starting the discourse server until discourse setup completed") From 1749143a68cffc1b622ae58d824763b39cae0b17 Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Wed, 26 Jul 2023 07:29:00 -0400 Subject: [PATCH 28/30] Rename observer to handler --- src/charm.py | 4 ++-- src/database.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/charm.py b/src/charm.py index ff8c2bde..4361be2f 100755 --- a/src/charm.py +++ b/src/charm.py @@ -25,7 +25,7 @@ from ops.model import ActiveStatus, BlockedStatus, MaintenanceStatus, WaitingStatus from ops.pebble import ExecError, ExecProcess, Plan -from database import DatabaseObserver +from database import DatabaseHandler logger = logging.getLogger(__name__) pgsql = ops.lib.use("pgsql", 1, "postgresql-charmers@lists.launchpad.net") @@ -79,7 +79,7 @@ def __init__(self, *args): """Initialize defaults and event handlers.""" super().__init__(*args) - self._database = DatabaseObserver(self) + self._database = DatabaseHandler(self) self.framework.observe( self._database.database.on.database_created, self._on_database_created diff --git a/src/database.py b/src/database.py index 3dcad68c..fa26afc8 100644 --- a/src/database.py +++ b/src/database.py @@ -12,7 +12,7 @@ DATABASE_NAME = "discourse" -class DatabaseObserver(Object): +class DatabaseHandler(Object): """The Database relation observer.""" _RELATION_NAME = "database" From 94f1b27431d8065d5531c96112428c2a25fd5c4a Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Wed, 26 Jul 2023 10:45:28 -0400 Subject: [PATCH 29/30] Change the contract to get_relation_data() --- src/charm.py | 4 ---- src/database.py | 22 +++++++++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/charm.py b/src/charm.py index 4361be2f..369def1e 100755 --- a/src/charm.py +++ b/src/charm.py @@ -523,10 +523,6 @@ def _reload_configuration(self) -> None: container.add_layer(SERVICE_NAME, layer_config, combine=True) container.pebble.replan_services() - def _database_relation_changed(self, _: HookEvent) -> None: - if self._are_db_relations_ready(): - self._reload_configuration() - def _redis_relation_changed(self, _: HookEvent) -> None: if self._are_db_relations_ready(): self._reload_configuration() diff --git a/src/database.py b/src/database.py index fa26afc8..26fd27f0 100644 --- a/src/database.py +++ b/src/database.py @@ -31,26 +31,34 @@ def __init__(self, charm: CharmBase): database_name=DATABASE_NAME, ) - def get_relation_data(self) -> typing.Optional[typing.Dict]: + def get_relation_data(self) -> typing.Dict[str, str]: """Get database data from relation. Returns: Dict: Information needed for setting environment variables. - Returns None if the relation data is not correctly initialized. + Returns default if the relation data is not correctly initialized. """ + default = { + "POSTGRES_USER": "", + "POSTGRES_PASSWORD": "", + "POSTGRES_HOST": "", + "POSTGRES_PORT": "", + "POSTGRES_DB": "", + } + if self.model.get_relation(self._RELATION_NAME) is None: - return None + return default relation_id = self.database.relations[0].id relation_data = self.database.fetch_relation_data()[relation_id] endpoints = relation_data.get("endpoints", "").split(",") if len(endpoints) < 1: - return None + return default primary_endpoint = endpoints[0].split(":") if len(primary_endpoint) < 2: - return None + return default data = { "POSTGRES_USER": relation_data.get("username"), @@ -65,7 +73,7 @@ def get_relation_data(self) -> typing.Optional[typing.Dict]: data["POSTGRES_PASSWORD"], data["POSTGRES_DB"], ): - return None + return default return data @@ -75,4 +83,4 @@ def is_relation_ready(self) -> bool: Returns: bool: returns True if the relation is ready. """ - return self.get_relation_data() is not None + return self.get_relation_data()["POSTGRES_HOST"] != "" From 92ed5bee150420f72af848e2ce9888b3f16dbcde Mon Sep 17 00:00:00 2001 From: Niels Robin-Aubertin Date: Fri, 28 Jul 2023 09:48:35 -0400 Subject: [PATCH 30/30] Address comments --- src/charm.py | 9 +++++---- src/database.py | 9 ++++----- tests/integration/conftest.py | 9 +++++---- tests/unit/test_charm.py | 5 +++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/charm.py b/src/charm.py index 369def1e..81c49022 100755 --- a/src/charm.py +++ b/src/charm.py @@ -67,6 +67,7 @@ SERVICE_PORT = 3000 SETUP_COMPLETED_FLAG_FILE = "/run/discourse-k8s-operator/setup_completed" DEFAULT_REDIS_PORT = 6379 +DATABASE_RELATION_NAME = "database" class DiscourseCharm(CharmBase): @@ -79,7 +80,7 @@ def __init__(self, *args): """Initialize defaults and event handlers.""" super().__init__(*args) - self._database = DatabaseHandler(self) + self._database = DatabaseHandler(self, DATABASE_RELATION_NAME) self.framework.observe( self._database.database.on.database_created, self._on_database_created @@ -88,7 +89,7 @@ def __init__(self, *args): self._database.database.on.endpoints_changed, self._on_database_endpoints_changed ) self.framework.observe( - self.on[self._database._RELATION_NAME].relation_broken, + self.on[DATABASE_RELATION_NAME].relation_broken, self._on_database_relation_broken, ) @@ -282,7 +283,7 @@ def _get_s3_env(self) -> typing.Dict[str, typing.Any]: return s3_env - def _get_redis_relation_data(self) -> typing.Tuple[typing.Any, typing.Any]: + def _get_redis_relation_data(self) -> typing.Tuple[str, int]: """Get the hostname and port from the redis relation data. Returns: @@ -299,7 +300,7 @@ def _get_redis_relation_data(self) -> typing.Tuple[typing.Any, typing.Any]: ) return (redis_hostname, redis_port) - def _create_discourse_environment_settings(self) -> typing.Dict[str, typing.Any]: + def _create_discourse_environment_settings(self) -> typing.Dict[str, str]: """Create a layer config based on our current configuration. Returns: diff --git a/src/database.py b/src/database.py index 26fd27f0..b416e238 100644 --- a/src/database.py +++ b/src/database.py @@ -15,9 +15,7 @@ class DatabaseHandler(Object): """The Database relation observer.""" - _RELATION_NAME = "database" - - def __init__(self, charm: CharmBase): + def __init__(self, charm: CharmBase, relation_name): """Initialize the observer and register event handlers. Args: @@ -25,9 +23,10 @@ def __init__(self, charm: CharmBase): """ super().__init__(charm, "database-observer") self._charm = charm + self.relation_name = relation_name self.database = DatabaseRequires( self._charm, - relation_name=self._RELATION_NAME, + relation_name=self.relation_name, database_name=DATABASE_NAME, ) @@ -46,7 +45,7 @@ def get_relation_data(self) -> typing.Dict[str, str]: "POSTGRES_DB": "", } - if self.model.get_relation(self._RELATION_NAME) is None: + if self.model.get_relation(self.relation_name) is None: return default relation_id = self.database.relations[0].id diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b8a750a5..fd992da8 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -135,7 +135,7 @@ async def app_fixture( # Deploy relations to speed up overall execution related_apps = await asyncio.gather( model.deploy("postgresql-k8s", channel="14/edge", series="jammy", trust=True), - model.deploy("redis-k8s", series="focal"), + model.deploy("redis-k8s", series="jammy", channel="latest/edge"), model.deploy("nginx-ingress-integrator", series="focal", trust=True), ) postgres_app = related_apps[0] @@ -162,7 +162,8 @@ async def app_fixture( series="focal", ) - await model.wait_for_idle() + await model.wait_for_idle(apps=[application.name], status="waiting") + await model.wait_for_idle(apps=(app.name for app in related_apps), status="active") # configure postgres await postgres_app.set_config( @@ -171,7 +172,7 @@ async def app_fixture( "plugin_pg_trgm_enable": "true", } ) - await model.wait_for_idle() + await model.wait_for_idle(apps=[postgres_app.name], status="active") # Add required relations unit = model.applications[app_name].units[0] @@ -181,7 +182,7 @@ async def app_fixture( model.add_relation(app_name, "redis-k8s"), model.add_relation(app_name, "nginx-ingress-integrator"), ) - await model.wait_for_idle(status="active", raise_on_error=False) + await model.wait_for_idle(status="active") yield application diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 8095b21c..a56e5366 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -7,6 +7,7 @@ # Protected access check is disabled in tests as we're injecting test data import contextlib +import secrets import typing import unittest from unittest.mock import MagicMock, patch @@ -605,7 +606,7 @@ def test_postgres_relation_data(self): { "database": DATABASE_NAME, "endpoints": "dbhost:5432,dbhost-2:5432", - "password": "somepasswd", # nosec + "password": secrets.token_hex(16), "username": "someuser", }, True, @@ -614,7 +615,7 @@ def test_postgres_relation_data(self): { "database": DATABASE_NAME, "endpoints": "foo", - "password": "somepasswd", # nosec + "password": secrets.token_hex(16), "username": "someuser", }, False,