[DPE-1999] Fixed TLS race condition (#196)

* Fixed TLS race condition

* Changed Redis channel in test

* Added additional logic

Author: marceloneppel

lib/charms/postgresql_k8s/v0/postgresql_tls.py

@@ -33,7 +33,7 @@
 )
 from cryptography import x509
 from cryptography.x509.extensions import ExtensionType
-from ops.charm import ActionEvent
+from ops.charm import ActionEvent, RelationBrokenEvent
 from ops.framework import Object
 from ops.pebble import ConnectionError, PathError, ProtocolError
 
@@ -45,7 +45,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version.
-LIBPATCH = 6
+LIBPATCH = 7
 
 logger = logging.getLogger(__name__)
 SCOPE = "unit"
@@ -116,12 +116,14 @@ def _on_tls_relation_joined(self, _) -> None:
         """Request certificate when TLS relation joined."""
         self._request_certificate(None)
 
-    def _on_tls_relation_broken(self, _) -> None:
+    def _on_tls_relation_broken(self, event: RelationBrokenEvent) -> None:
         """Disable TLS when TLS relation broken."""
         self.charm.set_secret(SCOPE, "ca", None)
         self.charm.set_secret(SCOPE, "cert", None)
         self.charm.set_secret(SCOPE, "chain", None)
-        self.charm.update_config()
+        if not self.charm.update_config():
+            logger.debug("Cannot update config at this moment")
+            event.defer()
 
     def _on_certificate_available(self, event: CertificateAvailableEvent) -> None:
         """Enable TLS when TLS certificate available."""
@@ -139,7 +141,10 @@ def _on_certificate_available(self, event: CertificateAvailableEvent) -> None:
         self.charm.set_secret(SCOPE, "ca", event.ca)
 
         try:
-            self.charm.push_tls_files_to_workload()
+            if not self.charm.push_tls_files_to_workload():
+                logger.debug("Cannot push TLS certificates at this moment")
+                event.defer()
+                return
         except (ConnectionError, PathError, ProtocolError) as e:
             logger.error("Cannot push TLS certificates: %r", e)
             event.defer()

src/charm.py

@@ -1042,7 +1042,7 @@ def _peers(self) -> Relation:
         """
         return self.model.get_relation(PEER)
 
-    def push_tls_files_to_workload(self, container: Container = None) -> None:
+    def push_tls_files_to_workload(self, container: Container = None) -> bool:
         """Uploads TLS files to the workload container."""
         if container is None:
             container = self.unit.get_container("postgresql")
@@ -1086,7 +1086,7 @@ def push_tls_files_to_workload(self, container: Container = None) -> None:
                 group=WORKLOAD_OS_GROUP,
             )
 
-        self.update_config()
+        return self.update_config()
 
     def _restart(self, event: RunWithLock) -> None:
         """Restart PostgreSQL."""
@@ -1109,7 +1109,20 @@ def _restart(self, event: RunWithLock) -> None:
         # Start or stop the pgBackRest TLS server service when TLS certificate change.
         self.backup.start_stop_pgbackrest_service()
 
-    def update_config(self) -> None:
+    @property
+    def _is_workload_running(self) -> bool:
+        """Returns whether the workload is running (in an active state)."""
+        container = self.unit.get_container("postgresql")
+        if not container.can_connect():
+            return False
+
+        services = container.pebble.get_services(names=[self._postgresql_service])
+        if len(services) == 0:
+            return False
+
+        return services[0].current == ServiceStatus.ACTIVE
+
+    def update_config(self) -> bool:
         """Updates Patroni config file based on the existence of the TLS files."""
         # Update and reload configuration based on TLS files availability.
         self._patroni.render_patroni_yml_file(
@@ -1119,14 +1132,18 @@ def update_config(self) -> None:
             stanza=self.app_peer_data.get("stanza"),
             restore_stanza=self.app_peer_data.get("restore-stanza"),
         )
-        if not self._patroni.member_started:
+        if not self._is_workload_running:
             # If Patroni/PostgreSQL has not started yet and TLS relations was initialised,
             # then mark TLS as enabled. This commonly happens when the charm is deployed
             # in a bundle together with the TLS certificates operator. This flag is used to
             # know when to call the Patroni API using HTTP or HTTPS.
             self.unit_peer_data.update({"tls": "enabled" if self.is_tls_enabled else ""})
+            logger.debug("Early exit update_config: Workload not started yet")
+            return True
+
+        if not self._patroni.member_started:
             logger.debug("Early exit update_config: Patroni not started yet")
-            return
+            return False
 
         restart_postgresql = self.is_tls_enabled != self.postgresql.is_tls_enabled()
         self._patroni.reload_patroni_configuration()
@@ -1140,6 +1157,8 @@ def update_config(self) -> None:
             )
             self.on[self.restart_manager.name].acquire_lock.emit()
 
+        return True
+
     def _update_pebble_layers(self) -> None:
         """Update the pebble layers to keep the health check URL up-to-date."""
         container = self.unit.get_container("postgresql")