From eca23d5333882a4c8503c9ec606fa1553404badb Mon Sep 17 00:00:00 2001 From: Christian Assing Date: Thu, 2 Mar 2023 10:39:59 +0100 Subject: [PATCH] [glitchtip-project-dsn] new integration (#3257) The glitchtip-project-dsn integration populates glitchtip project dsn's as Kubernetes secret into the tenant's namespaces. Ticket: APPSRE-6631 Depends on: qontract-schemas#398 A .gitleaks.toml config file has been added because the word secret is in one of the fixture files. For more information, see https://source.redhat.com/departments/it/it-information-security/wiki/details_about_rover_github_information_security_and_scanning#how-can-i-tell-the-scanner-to-allow-certain-things-in-my-repo- --- .gitleaks.toml | 10 + README.md | 43 +++- reconcile/cli.py | 21 ++ reconcile/glitchtip_project_dsn/__init__.py | 0 .../glitchtip_project_dsn/integration.py | 241 ++++++++++++++++++ .../glitchtip/glitchtip_project.gql | 28 ++ .../glitchtip/glitchtip_project.py | 98 +++++++ .../apollo-11-flight-control/keys/get.json | 14 + .../glitchtip/desire_state_projects.yml | 4 + .../test/fixtures/glitchtip/dsn_projects.yml | 148 +++++++++++ reconcile/test/glitchtip/conftest.py | 33 +++ .../glitchtip/test_glitchtip_project_dsn.py | 69 +++++ .../glitchtip/test_utils_glitchtip_client.py | 9 + reconcile/typed_queries/glitchtip_settings.py | 18 ++ reconcile/utils/glitchtip/client.py | 12 + reconcile/utils/glitchtip/models.py | 5 + 16 files changed, 748 insertions(+), 5 deletions(-) create mode 100644 .gitleaks.toml create mode 100644 reconcile/glitchtip_project_dsn/__init__.py create mode 100644 reconcile/glitchtip_project_dsn/integration.py create mode 100644 reconcile/test/fixtures/glitchtip/api/0/projects/nasa/apollo-11-flight-control/keys/get.json create mode 100644 reconcile/test/fixtures/glitchtip/dsn_projects.yml create mode 100644 reconcile/test/glitchtip/test_glitchtip_project_dsn.py create mode 100644 reconcile/typed_queries/glitchtip_settings.py diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000000..6487ad6b21 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,10 @@ + +# Configure the Red Hat InfoSec security scanner to ignore certain files +# See https://source.redhat.com/departments/it/it-information-security/wiki/details_about_rover_github_information_security_and_scanning#how-can-i-tell-the-scanner-to-allow-certain-things-in-my-repo- +[allowlist] +description = "Global Allowlist" + +# Ignore based on any subset of the file path +paths = [ + '''reconcile\/test\/fixtures\/glitchtip\/api\/0\/projects\/nasa\/apollo-11-flight-control\/keys\/get.json$''', +] diff --git a/README.md b/README.md index 4af9afbdc4..0900659bfb 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,12 @@ Additional tools that use the libraries created by the reconciliations are also blackbox-exporter-endpoint-monitoring Manages Prometheus Probe resources for blackbox-exporter + change-owners Detects owners for changes in app-interface + PRs and allows them to self-service merge. cluster-deployment-mapper Maps ClusterDeployment resources to Cluster IDs. + cna-resources Manage Cloud Resources using Cloud Native + Assets (CNA). dashdotdb-cso Collects the ImageManifestVuln CRs from all the clusters and posts them to Dashdotdb. dashdotdb-dvo Collects the DeploymentValidations from all @@ -64,6 +68,10 @@ Additional tools that use the libraries created by the reconciliations are also based on OWNERS files schema. gitlab-permissions Manage permissions on GitLab projects. gitlab-projects Create GitLab projects. + glitchtip Configure and enforce glitchtip instance + configuration. + glitchtip-project-dsn Glitchtip project dsn as openshift secret. + integrations-manager Manages Qontract Reconcile integrations. integrations-validator Ensures all integrations are defined in App- Interface. jenkins-job-builder Manage Jenkins jobs configurations using @@ -78,6 +86,7 @@ Additional tools that use the libraries created by the reconciliations are also jenkins-webhooks Manage web hooks to Jenkins jobs. jenkins-webhooks-cleaner Remove webhooks to previous Jenkins instances. + jenkins-worker-fleets Manage Jenkins worker fleets via JCasC. jira-watcher Watch for changes in Jira boards and notify on Slack. kafka-clusters Manages Kafka clusters via OCM. @@ -85,8 +94,15 @@ Additional tools that use the libraries created by the reconciliations are also search. ocm-additional-routers Manage additional routers in OCM. ocm-addons Manages cluster Addons in OCM. + ocm-addons-upgrade-scheduler-org + Manage Addons Upgrade Policy schedules in + OCM organizations. + ocm-addons-upgrade-tests-trigger + Trigger jenkins jobs following Addon + upgrades. ocm-aws-infrastructure-access Grants AWS infrastructure access to members in AWS groups via OCM. + ocm-cluster-admin Manage Cluster Admin in OCM. ocm-clusters Manages clusters via OCM. ocm-external-configuration-labels Manage External Configuration labels in OCM. @@ -94,10 +110,14 @@ Additional tools that use the libraries created by the reconciliations are also ocm-groups Manage membership in OpenShift groups via OCM. ocm-machine-pools Manage Machine Pools in OCM. + ocm-oidc-idp Manage OIDC Identity Providers in OCM. + ocm-update-recommended-version Update recommended version for OCM orgs ocm-upgrade-scheduler Manage Upgrade Policy schedules in OCM. - ocm-upgrade-scheduler-org Manage Upgrade Policy schedules in OCM organizations. + ocm-upgrade-scheduler-org Manage Upgrade Policy schedules in OCM + organizations. ocm-upgrade-scheduler-org-updater - Update Upgrade Policy schedules in OCM organizations. + Update Upgrade Policy schedules in OCM + organizations. ocp-release-mirror Mirrors OCP release images. openshift-clusterrolebindings Configures ClusterRolebindings in OpenShift clusters. @@ -113,11 +133,16 @@ Additional tools that use the libraries created by the reconciliations are also openshift-routes Manages OpenShift Routes. openshift-saas-deploy Manage OpenShift resources defined in Saas files. + openshift-saas-deploy-change-tester + Runs openshift-saas-deploy for each saas- + file that changed within a bundle. openshift-saas-deploy-trigger-cleaner Clean up deployment related resources. openshift-saas-deploy-trigger-configs Trigger deployments when configuration changes. + openshift-saas-deploy-trigger-images + Trigger deployments when images are pushed. openshift-saas-deploy-trigger-moving-commits Trigger deployments when a commit changed for a ref. @@ -143,6 +168,8 @@ Additional tools that use the libraries created by the reconciliations are also compatibility. requests-sender Send emails to users based on requests submitted to app-interface. + resource-scraper Get resources from clusters and store in + Vault. saas-file-owners Manages labels on merge requests based on approver schema for saas files. saas-file-validator Validates Saas files. @@ -157,15 +184,18 @@ Additional tools that use the libraries created by the reconciliations are also signalfx-prometheus-endpoint-monitoring Manages Prometheus Probe resources for signalfx exporter + skupper-network Manages Skupper Networks. slack-usergroups Manage Slack User Groups (channels and - users) and Slack Cluster User Groups - for OpenShift users notifications. + users). sql-query Runs SQL Queries against app-interface RDS resources. status-page-components Manages components on statuspage.io hosted status pages. + template-tester Tests templating of resources. terraform-aws-route53 Manage AWS Route53 resources using Terraform. + terraform-cloudflare-dns Manage Cloudflare DNS using Terraform. + terraform-cloudflare-resources Manage Cloudflare Resources using Terraform. terraform-resources Manage AWS Resources using Terraform. terraform-tgw-attachments Manages Transit Gateway attachments. terraform-users Manage AWS users using Terraform. @@ -174,7 +204,10 @@ Additional tools that use the libraries created by the reconciliations are also terraform-cloudflare-users Manage user access to Cloudflare accounts. unleash-watcher Watch for changes in Unleah feature toggles and notify on Slack. - user-validator Validate user files. + vault-replication Allow vault to replicate secrets to other + instances. + vpc-peerings-validator Validates that VPC peerings do not exist + between public and internal clusters. ``` ### e2e-tests diff --git a/reconcile/cli.py b/reconcile/cli.py index 5e02a8a621..45ab01623d 100644 --- a/reconcile/cli.py +++ b/reconcile/cli.py @@ -2473,6 +2473,27 @@ def glitchtip(ctx, instance): run_integration(reconcile.glitchtip.integration, ctx.obj, instance) +@integration.command(short_help="Glitchtip project dsn as openshift secret.") +@threaded() +@binary(["oc", "ssh"]) +@binary_version("oc", ["version", "--client"], OC_VERSION_REGEX, OC_VERSION) +@internal() +@use_jump_host() +@click.option("--instance", help="Reconcile just this instance.", default=None) +@click.pass_context +def glitchtip_project_dsn(ctx, thread_pool_size, internal, use_jump_host, instance): + import reconcile.glitchtip_project_dsn.integration + + run_integration( + reconcile.glitchtip_project_dsn.integration, + ctx.obj, + thread_pool_size, + internal, + use_jump_host, + instance, + ) + + @integration.command(short_help="Manages Skupper Networks.") @threaded() @binary(["oc", "ssh"]) diff --git a/reconcile/glitchtip_project_dsn/__init__.py b/reconcile/glitchtip_project_dsn/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/reconcile/glitchtip_project_dsn/integration.py b/reconcile/glitchtip_project_dsn/integration.py new file mode 100644 index 0000000000..126fca1eb0 --- /dev/null +++ b/reconcile/glitchtip_project_dsn/integration.py @@ -0,0 +1,241 @@ +import logging +from collections.abc import ( + Callable, + Iterable, +) +from typing import ( + Any, + Optional, +) + +from sretoolbox.utils import threaded + +import reconcile.openshift_base as ob +from reconcile.gql_definitions.glitchtip.glitchtip_instance import ( + DEFINITION as GLITCHTIP_INSTANCE_DEFINITION, +) +from reconcile.gql_definitions.glitchtip.glitchtip_instance import ( + query as glitchtip_instance_query, +) +from reconcile.gql_definitions.glitchtip.glitchtip_project import ( + DEFINITION as GLITCHTIP_PROJECT_DEFINITION, +) +from reconcile.gql_definitions.glitchtip.glitchtip_project import GlitchtipProjectsV1 +from reconcile.gql_definitions.glitchtip.glitchtip_project import ( + query as glitchtip_project_query, +) +from reconcile.typed_queries.app_interface_vault_settings import ( + get_app_interface_vault_settings, +) +from reconcile.typed_queries.glitchtip_settings import get_glitchtip_settings +from reconcile.utils import gql +from reconcile.utils.defer import defer +from reconcile.utils.disabled_integrations import integration_is_enabled +from reconcile.utils.glitchtip import GlitchtipClient +from reconcile.utils.glitchtip.models import ( + Organization, + Project, + ProjectKey, +) +from reconcile.utils.oc_map import ( + OCMap, + init_oc_map_from_namespaces, +) +from reconcile.utils.openshift_resource import OpenshiftResource as OR +from reconcile.utils.openshift_resource import ResourceInventory +from reconcile.utils.secret_reader import create_secret_reader +from reconcile.utils.semver_helper import make_semver + +QONTRACT_INTEGRATION = "glitchtip-project-dsn" +QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0) + +LABELS = { + "app.kubernetes.io/name": "glitchtip-project-dsn", + "app.kubernetes.io/part-of": "glitchtip", + "app.kubernetes.io/managed-by": "qontract-reconcile", +} + + +def glitchtip_project_dsn_secret(project: Project, key: ProjectKey) -> dict[str, Any]: + return { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": f"{project.slug}-dsn", + "labels": LABELS, + }, + "type": "Opaque", + "stringData": { + "dsn": key.dsn, + "security_endpoint": key.security_endpoint, + }, + } + + +def fetch_current_state( + project: GlitchtipProjectsV1, + oc_map: OCMap, + ri: ResourceInventory, +) -> None: + for namespace in project.namespaces: + oc = oc_map.get_cluster(namespace.cluster.name) + if not oc.project_exists(namespace.name): + logging.info( + f"{namespace.cluster.name}/{namespace.name}: Namespace does not exist (yet). Skipping for now!" + ) + continue + + for item in oc.get_items( + kind="Secret", + namespace=namespace.name, + labels=LABELS, + ): + openshift_resource = OR( + body=item, + integration=QONTRACT_INTEGRATION, + integration_version=QONTRACT_INTEGRATION_VERSION, + ) + ri.initialize_resource_type( + cluster=namespace.cluster.name, + namespace=namespace.name, + resource_type=openshift_resource.kind, + ) + ri.add_current( + cluster=namespace.cluster.name, + namespace=namespace.name, + resource_type="Secret", + name=openshift_resource.name, + value=openshift_resource, + ) + + +def fetch_desired_state( + glitchtip_projects: Iterable[GlitchtipProjectsV1], + ri: ResourceInventory, + glitchtip_client: GlitchtipClient, +) -> None: + for glitchtip_project in glitchtip_projects: + org = Organization(name=glitchtip_project.organization.name) + project = Project(name=glitchtip_project.name) + key = glitchtip_client.project_key( + organization_slug=org.slug, project_slug=project.slug + ) + secret = glitchtip_project_dsn_secret(project, key) + openshift_resource = OR( + body=secret, + integration=QONTRACT_INTEGRATION, + integration_version=QONTRACT_INTEGRATION_VERSION, + ) + for namespace in glitchtip_project.namespaces: + ri.initialize_resource_type( + cluster=namespace.cluster.name, + namespace=namespace.name, + resource_type=openshift_resource.kind, + ) + ri.add_desired( + cluster=namespace.cluster.name, + namespace=namespace.name, + resource_type="Secret", + name=openshift_resource.name, + value=openshift_resource, + ) + + +def projects_query(query_func: Callable) -> list[GlitchtipProjectsV1]: + glitchtip_projects = [] + for project in ( + glitchtip_project_query(query_func=query_func).glitchtip_projects or [] + ): + # remove namespaces marked for deletion or where the integration is disabled + project.namespaces = [ + ns + for ns in project.namespaces + if not ns.delete + and integration_is_enabled(QONTRACT_INTEGRATION, ns.cluster) + ] + if not project.namespaces: + # skip projects with no namespaces + continue + glitchtip_projects.append(project) + + return glitchtip_projects + + +@defer +def run( + dry_run: bool, + thread_pool_size: int = 10, + internal: Optional[bool] = None, + use_jump_host: bool = True, + instance: Optional[str] = None, + defer: Optional[Callable] = None, +) -> None: + # settings + vault_settings = get_app_interface_vault_settings() + read_timeout, max_retries, _ = get_glitchtip_settings() + + # data + gqlapi = gql.get_api() + glitchtip_instances = glitchtip_instance_query(query_func=gqlapi.query).instances + glitchtip_projects = projects_query(query_func=gqlapi.query) + + # APIs + secret_reader = create_secret_reader(use_vault=vault_settings.vault) + oc_map = init_oc_map_from_namespaces( + namespaces=[ + namespace + for project in glitchtip_projects + for namespace in project.namespaces + ], + secret_reader=secret_reader, + integration=QONTRACT_INTEGRATION, + use_jump_host=use_jump_host, + thread_pool_size=thread_pool_size, + internal=internal, + ) + if defer: + defer(oc_map.cleanup) + + ri = ResourceInventory() + + for glitchtip_instance in glitchtip_instances: + if instance and glitchtip_instance.name != instance: + continue + + glitchtip_client = GlitchtipClient( + host=glitchtip_instance.console_url, + token=secret_reader.read_secret(glitchtip_instance.automation_token), + read_timeout=read_timeout, + max_retries=max_retries, + ) + threaded.run( + fetch_current_state, + [ + p + for p in glitchtip_projects + if p.organization.instance.name == glitchtip_instance.name + ], + thread_pool_size, + oc_map=oc_map, + ri=ri, + ) + fetch_desired_state( + glitchtip_projects=[ + p + for p in glitchtip_projects + if p.organization.instance.name == glitchtip_instance.name + ], + ri=ri, + glitchtip_client=glitchtip_client, + ) + + # create/update/delete all secrets + ob.realize_data(dry_run, oc_map, ri, thread_pool_size) + + +def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]: + gqlapi = gql.get_api() + return { + "projects": gqlapi.query(GLITCHTIP_PROJECT_DEFINITION)["glitchtip_projects"], + "instances": gqlapi.query(GLITCHTIP_INSTANCE_DEFINITION)["instances"], + } diff --git a/reconcile/gql_definitions/glitchtip/glitchtip_project.gql b/reconcile/gql_definitions/glitchtip/glitchtip_project.gql index 69956d1543..e85ef41517 100644 --- a/reconcile/gql_definitions/glitchtip/glitchtip_project.gql +++ b/reconcile/gql_definitions/glitchtip/glitchtip_project.gql @@ -24,5 +24,33 @@ query Projects { name } } + # for glitchtip-project-dsn + namespaces { + name + delete + clusterAdmin + cluster { + name + serverUrl + insecureSkipTLSVerify + jumpHost { + ...CommonJumphostFields + } + spec { + private + } + automationToken { + ...VaultSecret + } + clusterAdminAutomationToken { + ...VaultSecret + } + internal + disable { + integrations + e2eTests + } + } + } } } diff --git a/reconcile/gql_definitions/glitchtip/glitchtip_project.py b/reconcile/gql_definitions/glitchtip/glitchtip_project.py index c4c1005537..ee920e490c 100644 --- a/reconcile/gql_definitions/glitchtip/glitchtip_project.py +++ b/reconcile/gql_definitions/glitchtip/glitchtip_project.py @@ -16,8 +16,31 @@ Json, ) +from reconcile.gql_definitions.fragments.jumphost_common_fields import ( + CommonJumphostFields, +) +from reconcile.gql_definitions.fragments.vault_secret import VaultSecret + DEFINITION = """ +fragment CommonJumphostFields on ClusterJumpHost_v1 { + hostname + knownHosts + user + port + remotePort + identity { + ... VaultSecret + } +} + +fragment VaultSecret on VaultSecret_v1 { + path + field + version + format +} + query Projects { glitchtip_projects: glitchtip_projects_v1 { name @@ -42,6 +65,34 @@ name } } + # for glitchtip-project-dsn + namespaces { + name + delete + clusterAdmin + cluster { + name + serverUrl + insecureSkipTLSVerify + jumpHost { + ...CommonJumphostFields + } + spec { + private + } + automationToken { + ...VaultSecret + } + clusterAdminAutomationToken { + ...VaultSecret + } + internal + disable { + integrations + e2eTests + } + } + } } } """ @@ -109,6 +160,52 @@ class Config: extra = Extra.forbid +class ClusterSpecV1(BaseModel): + private: bool = Field(..., alias="private") + + class Config: + smart_union = True + extra = Extra.forbid + + +class DisableClusterAutomationsV1(BaseModel): + integrations: Optional[list[str]] = Field(..., alias="integrations") + e2e_tests: Optional[list[str]] = Field(..., alias="e2eTests") + + class Config: + smart_union = True + extra = Extra.forbid + + +class ClusterV1(BaseModel): + name: str = Field(..., alias="name") + server_url: str = Field(..., alias="serverUrl") + insecure_skip_tls_verify: Optional[bool] = Field(..., alias="insecureSkipTLSVerify") + jump_host: Optional[CommonJumphostFields] = Field(..., alias="jumpHost") + spec: Optional[ClusterSpecV1] = Field(..., alias="spec") + automation_token: Optional[VaultSecret] = Field(..., alias="automationToken") + cluster_admin_automation_token: Optional[VaultSecret] = Field( + ..., alias="clusterAdminAutomationToken" + ) + internal: Optional[bool] = Field(..., alias="internal") + disable: Optional[DisableClusterAutomationsV1] = Field(..., alias="disable") + + class Config: + smart_union = True + extra = Extra.forbid + + +class NamespaceV1(BaseModel): + name: str = Field(..., alias="name") + delete: Optional[bool] = Field(..., alias="delete") + cluster_admin: Optional[bool] = Field(..., alias="clusterAdmin") + cluster: ClusterV1 = Field(..., alias="cluster") + + class Config: + smart_union = True + extra = Extra.forbid + + class GlitchtipProjectsV1(BaseModel): name: str = Field(..., alias="name") platform: str = Field(..., alias="platform") @@ -116,6 +213,7 @@ class GlitchtipProjectsV1(BaseModel): organization: GlitchtipProjectsV1_GlitchtipOrganizationV1 = Field( ..., alias="organization" ) + namespaces: list[NamespaceV1] = Field(..., alias="namespaces") class Config: smart_union = True diff --git a/reconcile/test/fixtures/glitchtip/api/0/projects/nasa/apollo-11-flight-control/keys/get.json b/reconcile/test/fixtures/glitchtip/api/0/projects/nasa/apollo-11-flight-control/keys/get.json new file mode 100644 index 0000000000..bb22bd068c --- /dev/null +++ b/reconcile/test/fixtures/glitchtip/api/0/projects/nasa/apollo-11-flight-control/keys/get.json @@ -0,0 +1,14 @@ +[ + { + "dateCreated": "2023-03-01T11:09:50.223950Z", + "dsn": { + "public": "http://public_dsn", + "secret": "http://secret_dsn", + "security": "http://security_endpoint" + }, + "id": "idid", + "label": "", + "public": "publicpublic", + "projectId": 2 + } +] diff --git a/reconcile/test/fixtures/glitchtip/desire_state_projects.yml b/reconcile/test/fixtures/glitchtip/desire_state_projects.yml index 0c7465710b..50cc07dad1 100644 --- a/reconcile/test/fixtures/glitchtip/desire_state_projects.yml +++ b/reconcile/test/fixtures/glitchtip/desire_state_projects.yml @@ -1,6 +1,7 @@ --- - name: rosetta-spacecraft platform: python + namespaces: [] teams: - name: esa-pilots roles: @@ -34,6 +35,7 @@ name: glitchtip-dev - name: rosetta-flight-control platform: python + namespaces: [] teams: - name: esa-flight-control roles: @@ -59,6 +61,7 @@ name: glitchtip-dev - name: apollo-11-spacecraft platform: python + namespaces: [] teams: - name: nasa-pilots roles: @@ -92,6 +95,7 @@ name: glitchtip-dev - name: apollo-11-flight-control platform: python + namespaces: [] teams: - name: nasa-flight-control roles: diff --git a/reconcile/test/fixtures/glitchtip/dsn_projects.yml b/reconcile/test/fixtures/glitchtip/dsn_projects.yml new file mode 100644 index 0000000000..97ef723452 --- /dev/null +++ b/reconcile/test/fixtures/glitchtip/dsn_projects.yml @@ -0,0 +1,148 @@ +--- +glitchtip_projects: +- name: no-namespaces + platform: python + namespaces: [] + teams: [] + organization: + name: NASA + instance: + name: glitchtip-dev +- name: integration-disabled + platform: python + namespaces: + - name: namespace-1 + delete: null + clusterAdmin: null + cluster: + name: cluster-1 + serverUrl: "https://api.cluster-1" + insecureSkipTLSVerify: null + jumpHost: null + spec: + private: false + automationToken: + path: creds/kube-configs/small-1 + field: token + version: null + format: null + clusterAdminAutomationToken: null + internal: false + disable: + integrations: + - glitchtip-project-dsn + e2eTests: null + teams: [] + organization: + name: NASA + instance: + name: glitchtip-dev +- name: namespace_delete + platform: python + namespaces: + - name: namespace-1 + delete: true + clusterAdmin: null + cluster: + name: cluster-1 + serverUrl: "https://api.cluster-1" + insecureSkipTLSVerify: null + jumpHost: null + spec: + private: false + automationToken: + path: creds/kube-configs/small-1 + field: token + version: null + format: null + clusterAdminAutomationToken: null + internal: false + disable: null + teams: [] + organization: + name: NASA + instance: + name: glitchtip-dev +- name: apollo-11-flight-control + platform: python + teams: [] + organization: + name: NASA + instance: + name: glitchtip-dev + namespaces: + - name: namespace-1 + delete: null + clusterAdmin: null + cluster: + name: cluster-1 + serverUrl: "https://api.cluster-1" + insecureSkipTLSVerify: null + jumpHost: null + spec: + private: false + automationToken: + path: creds/kube-configs/small-1 + field: token + version: null + format: null + clusterAdminAutomationToken: null + internal: false + disable: null + - name: namespace-1 + delete: null + clusterAdmin: null + cluster: + name: cluster-2 + serverUrl: "https://api.cluster-2" + insecureSkipTLSVerify: null + jumpHost: null + spec: + private: false + automationToken: + path: creds/kube-configs/small-1 + field: token + version: null + format: null + clusterAdminAutomationToken: null + internal: false + disable: null + - name: namespace-delete + delete: true + clusterAdmin: null + cluster: + name: cluster-2 + serverUrl: "https://api.cluster-2" + insecureSkipTLSVerify: null + jumpHost: null + spec: + private: false + automationToken: + path: creds/kube-configs/small-1 + field: token + version: null + format: null + clusterAdminAutomationToken: null + internal: false + disable: null + - name: cluster-integration-disabled + delete: null + clusterAdmin: null + cluster: + name: cluster-1 + serverUrl: "https://api.cluster-1" + insecureSkipTLSVerify: null + jumpHost: null + spec: + private: false + automationToken: + path: creds/kube-configs/small-1 + field: token + version: null + format: null + clusterAdminAutomationToken: null + internal: false + disable: + integrations: + - glitchtip-project-dsn + e2eTests: null diff --git a/reconcile/test/glitchtip/conftest.py b/reconcile/test/glitchtip/conftest.py index 9738895a3b..c8eb238ca8 100644 --- a/reconcile/test/glitchtip/conftest.py +++ b/reconcile/test/glitchtip/conftest.py @@ -1,10 +1,14 @@ from pathlib import Path +from typing import Any import httpretty as httpretty_module import pytest +from pytest_mock import MockerFixture from reconcile.test.fixtures import Fixtures from reconcile.utils.glitchtip import GlitchtipClient +from reconcile.utils.oc import OCNative +from reconcile.utils.oc_map import OCMap @pytest.fixture @@ -64,6 +68,8 @@ def glitchtip_server_full_api_response( "api/0/teams/nasa/nasa-pilots/projects/", "api/0/teams/nasa/nasa-pilots/projects/science-tools/", "api/0/teams/nasa/nasa-flight-control/members/", + # glitchtip-project-dsn + "api/0/projects/nasa/apollo-11-flight-control/keys/", ]: get_file = Path(fx.path(path)) / "get.json" if get_file.exists(): @@ -97,3 +103,30 @@ def glitchtip_server_full_api_response( body=delete_file.read_text(), content_type="text/json", ) + + +@pytest.fixture +def fake_secret() -> dict[str, Any]: + return { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "fake-secret"}, + "data": { + "dsn": "fake", + "security_endpoint": "fake", + }, + } + + +@pytest.fixture +def oc(mocker: MockerFixture, fake_secret: dict[str, Any]) -> OCNative: + oc = mocker.patch("reconcile.utils.oc.OCNative", autospec=True) + oc.get_items.return_value = [fake_secret] + return oc + + +@pytest.fixture +def oc_map(mocker: MockerFixture, oc: OCNative) -> OCMap: + oc_map = mocker.patch("reconcile.utils.oc_map.OCMap", autospec=True) + oc_map.get_cluster.return_value = oc + return oc_map diff --git a/reconcile/test/glitchtip/test_glitchtip_project_dsn.py b/reconcile/test/glitchtip/test_glitchtip_project_dsn.py new file mode 100644 index 0000000000..84c760503c --- /dev/null +++ b/reconcile/test/glitchtip/test_glitchtip_project_dsn.py @@ -0,0 +1,69 @@ +from collections.abc import Sequence +from typing import Any + +import pytest + +from reconcile.glitchtip_project_dsn.integration import ( + LABELS, + fetch_current_state, + fetch_desired_state, + projects_query, +) +from reconcile.gql_definitions.glitchtip.glitchtip_project import GlitchtipProjectsV1 +from reconcile.test.fixtures import Fixtures +from reconcile.utils.glitchtip import GlitchtipClient +from reconcile.utils.oc_map import OCMap +from reconcile.utils.openshift_resource import ResourceInventory + + +@pytest.fixture +def projects(fx: Fixtures) -> list[GlitchtipProjectsV1]: + def q(*args: Any, **kwargs: Any) -> dict: + return fx.get_anymarkup("dsn_projects.yml") + + return projects_query(q) + + +def test_project_query(projects: Sequence[GlitchtipProjectsV1]) -> None: + assert len(projects) == 1 + assert len(projects[0].namespaces) == 2 + assert projects[0].namespaces[0].name == "namespace-1" + assert projects[0].namespaces[0].cluster.name == "cluster-1" + assert projects[0].namespaces[1].name == "namespace-1" + assert projects[0].namespaces[1].cluster.name == "cluster-2" + + +def test_fetch_current_state( + oc_map: OCMap, projects: Sequence[GlitchtipProjectsV1] +) -> None: + ri = ResourceInventory() + fetch_current_state(projects[0], oc_map, ri) + # see oc_map fixture for the mocked data + assert ri.get_current("cluster-1", "namespace-1", "Secret", "fake-secret") + + +def test_desire_state( + glitchtip_client: GlitchtipClient, + glitchtip_server_full_api_response: None, + projects: Sequence[GlitchtipProjectsV1], +) -> None: + ri = ResourceInventory() + fetch_desired_state( + glitchtip_projects=projects, ri=ri, glitchtip_client=glitchtip_client + ) + secret = ri.get_desired( + "cluster-1", "namespace-1", "Secret", "apollo-11-flight-control-dsn" + ) + assert secret.body == { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "apollo-11-flight-control-dsn", + "labels": LABELS, + }, + "type": "Opaque", + "stringData": { + "dsn": "http://public_dsn", + "security_endpoint": "http://security_endpoint", + }, + } diff --git a/reconcile/test/glitchtip/test_utils_glitchtip_client.py b/reconcile/test/glitchtip/test_utils_glitchtip_client.py index 2cf293f4e3..25125d5380 100644 --- a/reconcile/test/glitchtip/test_utils_glitchtip_client.py +++ b/reconcile/test/glitchtip/test_utils_glitchtip_client.py @@ -16,6 +16,7 @@ User, ) from reconcile.utils.glitchtip.client import get_next_url +from reconcile.utils.glitchtip.models import ProjectKey @pytest.mark.parametrize( @@ -225,6 +226,14 @@ def test_glitchtip_projects(glitchtip_client: GlitchtipClient) -> None: ] +def test_glitchtip_project_key(glitchtip_client: GlitchtipClient) -> None: + assert glitchtip_client.project_key( + organization_slug="nasa", project_slug="apollo-11-flight-control" + ) == ProjectKey( + dsn="http://public_dsn", security_endpoint="http://security_endpoint" + ) + + def test_glitchtip_create_project(glitchtip_client: GlitchtipClient) -> None: project = glitchtip_client.create_project( organization_slug="nasa", diff --git a/reconcile/typed_queries/glitchtip_settings.py b/reconcile/typed_queries/glitchtip_settings.py new file mode 100644 index 0000000000..51d70a6b9e --- /dev/null +++ b/reconcile/typed_queries/glitchtip_settings.py @@ -0,0 +1,18 @@ +from reconcile.gql_definitions.glitchtip.glitchtip_settings import query +from reconcile.utils import gql + + +def get_glitchtip_settings( + read_timeout: int = 30, max_retries: int = 3, mail_domain: str = "redhat.com" +) -> tuple[int, int, str]: + """Returns Glitchtip Settings.""" + gqlapi = gql.get_api() + if _s := query(query_func=gqlapi.query).settings: + if _gs := _s[0].glitchtip: + if _gs.read_timeout is not None: + read_timeout = _gs.read_timeout + if _gs.max_retries is not None: + max_retries = _gs.max_retries + if _gs.mail_domain is not None: + mail_domain = _gs.mail_domain + return read_timeout, max_retries, mail_domain diff --git a/reconcile/utils/glitchtip/client.py b/reconcile/utils/glitchtip/client.py index 8484e0474c..ce214d44b1 100644 --- a/reconcile/utils/glitchtip/client.py +++ b/reconcile/utils/glitchtip/client.py @@ -10,6 +10,7 @@ from reconcile.utils.glitchtip.models import ( Organization, Project, + ProjectKey, Team, User, ) @@ -150,6 +151,17 @@ def delete_project(self, organization_slug: str, team_slug: str, slug: str) -> N f"/api/0/teams/{organization_slug}/{team_slug}/projects/{slug}/", ) + def project_key(self, organization_slug: str, project_slug: str) -> ProjectKey: + """Retrieve project key (DSN).""" + keys = self._list(f"/api/0/projects/{organization_slug}/{project_slug}/keys/") + if not keys: + # only happens if org_slug/project_slug does not exist + raise ValueError(f"No keys found for project {project_slug}") + # always return the first key + return ProjectKey( + dsn=keys[0]["dsn"]["public"], security_endpoint=keys[0]["dsn"]["security"] + ) + def add_project_to_team( self, organization_slug: str, team_slug: str, slug: str ) -> Project: diff --git a/reconcile/utils/glitchtip/models.py b/reconcile/utils/glitchtip/models.py index 3c74919df3..c38d18fafa 100644 --- a/reconcile/utils/glitchtip/models.py +++ b/reconcile/utils/glitchtip/models.py @@ -76,6 +76,11 @@ def __hash__(self) -> int: return hash(self.slug) +class ProjectKey(BaseModel): + dsn: str + security_endpoint: str + + class Project(BaseModel): pk: Optional[int] = Field(None, alias="id") name: str