From 1b7f8dc453c9d8615986dccf81b6829f6f5a9488 Mon Sep 17 00:00:00 2001 From: Dragomir Penev <6687393+dragomirp@users.noreply.github.com> Date: Thu, 6 Apr 2023 12:21:43 +0300 Subject: [PATCH] Migrating to Rock OCI image (#118) * Initial switch * Use stable tls-certificates-operator * Temporarily manually installing ca-certs * Revert "Temporarily manually installing ca-certs" This reverts commit 2fd47dbb94da1b130585a8ec7bb7016ff061f6d9. * Read version from the Rock image * Switch to GHCR Rock --- metadata.yaml | 6 +++--- src/charm.py | 3 ++- src/patroni.py | 14 ++++++++++++++ templates/patroni.yml.j2 | 3 ++- tests/integration/test_backups.py | 2 +- tests/integration/test_tls.py | 2 +- tests/unit/test_charm.py | 2 +- tests/unit/test_patroni.py | 22 +++++++++++++++++++--- 8 files changed, 43 insertions(+), 11 deletions(-) diff --git a/metadata.yaml b/metadata.yaml index f64113fca1..9694ab9d60 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -19,8 +19,8 @@ containers: resources: postgresql-image: type: oci-image - description: OCI image for PostgreSQL (dataplatformoci/postgres-patroni) - upstream-source: dataplatformoci/postgres-patroni + description: OCI image for PostgreSQL + upstream-source: ghcr.io/canonical/charmed-postgresql:14.7-22.04_edge peers: database-peers: @@ -47,4 +47,4 @@ requires: storage: pgdata: type: filesystem - location: /var/lib/postgresql/data \ No newline at end of file + location: /var/lib/postgresql/data diff --git a/src/charm.py b/src/charm.py index 91b7045fb8..b880eb18c9 100755 --- a/src/charm.py +++ b/src/charm.py @@ -729,6 +729,7 @@ def _set_primary_status_message(self) -> None: def _patroni(self): """Returns an instance of the Patroni object.""" return Patroni( + self, self._endpoint, self._endpoints, self.primary_endpoint, @@ -814,7 +815,7 @@ def _postgresql_layer(self) -> Layer: self._postgresql_service: { "override": "replace", "summary": "entrypoint of the postgresql + patroni image", - "command": f"/usr/bin/python3 /usr/local/bin/patroni {self._storage_path}/patroni.yml", + "command": f"patroni {self._storage_path}/patroni.yml", "startup": "enabled", "user": WORKLOAD_OS_USER, "group": WORKLOAD_OS_GROUP, diff --git a/src/patroni.py b/src/patroni.py index e4961a5558..1708cdc9e4 100644 --- a/src/patroni.py +++ b/src/patroni.py @@ -10,6 +10,7 @@ from typing import List, Optional import requests +import yaml from jinja2 import Template from tenacity import ( AttemptManager, @@ -40,6 +41,7 @@ class Patroni: def __init__( self, + charm, endpoint: str, endpoints: List[str], primary_endpoint: str, @@ -50,6 +52,7 @@ def __init__( rewind_password: str, tls_enabled: bool, ): + self._charm = charm self._endpoint = endpoint self._endpoints = endpoints self._primary_endpoint = primary_endpoint @@ -70,6 +73,16 @@ def _patroni_url(self) -> str: """Patroni REST API URL.""" return f"{'https' if self._tls_enabled else 'http'}://{self._endpoint}:8008" + @property + def rock_postgresql_version(self) -> Optional[str]: + """Version of Postgresql installed in the Rock image.""" + container = self._charm.unit.get_container("postgresql") + if not container.can_connect(): + logger.debug("Cannot get Postgresql version from Rock. Container inaccessible") + return + snap_meta = container.pull("/meta.charmed-postgresql/snap.yaml") + return yaml.safe_load(snap_meta)["version"] + def _get_alternative_patroni_url(self, attempt: AttemptManager) -> str: """Get an alternative REST API URL from another member each time. @@ -231,6 +244,7 @@ def render_patroni_yml_file( restoring_backup=backup_id is not None, backup_id=backup_id, stanza=stanza, + version=self.rock_postgresql_version.split(".")[0], ) self._render_file(f"{self._storage_path}/patroni.yml", rendered, 0o644) diff --git a/templates/patroni.yml.j2 b/templates/patroni.yml.j2 index 52b4526a83..f6f27fee4f 100644 --- a/templates/patroni.yml.j2 +++ b/templates/patroni.yml.j2 @@ -4,6 +4,7 @@ bootstrap: use_pg_rewind: true remove_data_directory_on_rewind_failure: true remove_data_directory_on_diverged_timelines: true + bin_dir: /usr/lib/postgresql/{{ version }}/bin parameters: {%- if enable_pgbackrest %} archive_command: 'pgbackrest --stanza={{ stanza }} archive-push %p' @@ -24,7 +25,6 @@ bootstrap: - auth-host: md5 - auth-local: trust - encoding: UTF8 - - locale: en_US.UTF-8 - data-checksums {%- endif %} pg_hba: @@ -52,6 +52,7 @@ postgresql: connect_address: '{{ endpoint }}:5432' custom_conf: {{ storage_path }}/postgresql-k8s-operator.conf data_dir: {{ storage_path }}/pgdata + bin_dir: /usr/lib/postgresql/{{ version }}/bin listen: 0.0.0.0:5432 parameters: {%- if enable_pgbackrest %} diff --git a/tests/integration/test_backups.py b/tests/integration/test_backups.py index 9b533c261d..a311909013 100644 --- a/tests/integration/test_backups.py +++ b/tests/integration/test_backups.py @@ -30,7 +30,7 @@ async def test_backup_and_restore(ops_test: OpsTest, cloud_configs: Tuple[Dict, # Deploy S3 Integrator and TLS Certificates Operator. await ops_test.model.deploy(S3_INTEGRATOR_APP_NAME, channel="edge") config = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} - await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="beta", config=config) + await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, config=config) for cloud, config in cloud_configs[0].items(): # Deploy and relate PostgreSQL to S3 integrator (one database app for each cloud for now diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index f40debebc2..018a6fc5d4 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -45,7 +45,7 @@ async def test_mattermost_db(ops_test: OpsTest) -> None: async with ops_test.fast_forward(): # Deploy TLS Certificates operator. config = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} - await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="beta", config=config) + await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, config=config) # Relate it to the PostgreSQL to enable TLS. await ops_test.model.relate(DATABASE_APP_NAME, TLS_CERTIFICATES_APP_NAME) await ops_test.model.wait_for_idle(status="active", timeout=1000) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 39b94ae6f6..7282f8d53a 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -423,7 +423,7 @@ def test_postgresql_layer(self, _, __, ___, ____): self._postgresql_service: { "override": "replace", "summary": "entrypoint of the postgresql + patroni image", - "command": "/usr/bin/python3 /usr/local/bin/patroni /var/lib/postgresql/data/patroni.yml", + "command": "patroni /var/lib/postgresql/data/patroni.yml", "startup": "enabled", "user": "postgres", "group": "postgres", diff --git a/tests/unit/test_patroni.py b/tests/unit/test_patroni.py index fe4ede2102..3d57c58db1 100644 --- a/tests/unit/test_patroni.py +++ b/tests/unit/test_patroni.py @@ -3,20 +3,30 @@ # See LICENSE file for licensing details. import unittest -from unittest.mock import mock_open, patch +from unittest.mock import PropertyMock, mock_open, patch from jinja2 import Template +from ops.testing import Harness from tenacity import RetryError +from charm import PostgresqlOperatorCharm from constants import REWIND_USER from patroni import Patroni -from tests.helpers import STORAGE_PATH +from tests.helpers import STORAGE_PATH, patch_network_get class TestPatroni(unittest.TestCase): + @patch("charm.KubernetesServicePatch", lambda x, y: None) + @patch_network_get(private_address="1.1.1.1") def setUp(self): + self.harness = Harness(PostgresqlOperatorCharm) + self.addCleanup(self.harness.cleanup) + self.harness.begin() + self.charm = self.harness.charm + # Setup Patroni wrapper. self.patroni = Patroni( + self.charm, "postgresql-k8s-0", ["postgresql-k8s-0"], "postgresql-k8s-primary.dev.svc.cluster.local", @@ -77,8 +87,11 @@ def test_render_file(self, _temp_file, _pwnam, _chown, _chmod): # Ensure the file is chown'd correctly. _chown.assert_called_with(filename, uid=35, gid=35) + @patch("charm.Patroni.rock_postgresql_version", new_callable=PropertyMock) @patch("charm.Patroni._render_file") - def test_render_patroni_yml_file(self, _render_file): + def test_render_patroni_yml_file(self, _render_file, _rock_postgresql_version): + _rock_postgresql_version.return_value = "14.7" + # Get the expected content from a file. with open("templates/patroni.yml.j2") as file: template = Template(file.read()) @@ -92,6 +105,7 @@ def test_render_patroni_yml_file(self, _render_file): replication_password=self.patroni._replication_password, rewind_user=REWIND_USER, rewind_password=self.patroni._rewind_password, + version="14", ) # Setup a mock for the `open` method, set returned data to postgresql.conf template. @@ -125,6 +139,7 @@ def test_render_patroni_yml_file(self, _render_file): replication_password=self.patroni._replication_password, rewind_user=REWIND_USER, rewind_password=self.patroni._rewind_password, + version="14", ) self.assertNotEqual(expected_content_with_tls, expected_content) @@ -180,6 +195,7 @@ def test_render_postgresql_conf_file(self, _render_file): # Also test with multiple planned units (synchronous_commit is turned on). self.patroni = Patroni( + self.charm, "postgresql-k8s-0", ["postgresql-k8s-0", "postgresql-k8s-1"], "postgresql-k8s-primary.dev.svc.cluster.local",