Add ability to configure s3 path for instances being restored from (#783

)

Co-authored-by: Nick Hudson <nick.hudson@gmail.com>

charts/tembo-operator/Chart.yaml

@@ -1,9 +1,9 @@
 apiVersion: v2
 name: tembo-operator
-description: 'Helm chart to deploy the tembo-operator'
+description: "Helm chart to deploy the tembo-operator"
 type: application
 icon: https://cloud.tembo.io/images/TemboElephant.png
-version: 0.5.2
+version: 0.5.3
 home: https://tembo.io
 sources:
   - https://github.com/tembo-io/tembo

charts/tembo-operator/templates/crd.yaml

@@ -2320,6 +2320,13 @@ spec:
                   **Default**: disabled
                 nullable: true
                 properties:
+                  backupsPath:
+                    description: |-
+                      The object storage path and bucket name of the instance you wish to restore from.  This maps to the `Backup` `destinationPath` field for the original instance.
+
+                      **Example**: If you have an instance with `spec.backup.destinationPath` set to `s3://my-bucket/v2/test-db` then you would set `backupsPath` to `s3://my-bucket/v2/test-db`. And backups are saved in that bucket under `s3://my-bucket/v2/test-db/server_name`
+                    nullable: true
+                    type: string
                   endpointURL:
                     description: endpointURL is the S3 compatable endpoint URL
                     nullable: true

tembo-operator/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "controller"
 description = "Tembo Operator for Postgres"
-version = "0.47.2"
+version = "0.47.3"
 edition = "2021"
 default-run = "controller"
 license = "Apache-2.0"

tembo-operator/src/apis/coredb_types.rs

@@ -269,6 +269,15 @@ pub struct Restore {
     #[serde(rename = "serverName")]
     pub server_name: String,
 
+    /// The object storage path and bucket name of the instance you wish to restore from.  This maps to the `Backup`
+    /// `destinationPath` field for the original instance.
+    ///
+    /// **Example**: If you have an instance with `spec.backup.destinationPath`
+    /// set to `s3://my-bucket/v2/test-db` then you would set `backupsPath` to `s3://my-bucket/v2/test-db`.
+    /// And backups are saved in that bucket under `s3://my-bucket/v2/test-db/server_name`
+    #[serde(rename = "backupsPath")]
+    pub backups_path: Option<String>,
+
     /// recovery_target_time is the time base target for point-in-time recovery.
     #[serde(rename = "recoveryTargetTime")]
     pub recovery_target_time: Option<String>,

tembo-operator/src/cloudnativepg/cnpg.rs

@@ -1,3 +1,5 @@
+use crate::apis::coredb_types;
+use crate::apis::coredb_types::Restore;
 use crate::ingress_route_crd::{
     IngressRoute, IngressRouteRoutes, IngressRouteRoutesKind, IngressRouteRoutesServices,
     IngressRouteRoutesServicesKind, IngressRouteSpec, IngressRouteTls,
@@ -59,7 +61,6 @@ use crate::{
     is_postgres_ready,
     postgres_exporter::EXPORTER_CONFIGMAP_PREFIX,
     psql::PsqlOutput,
-    // snapshots::volumesnapshots::reconcile_volume_snapshot_restore,
     trunk::extensions_that_require_load,
     Context,
 };
@@ -413,14 +414,11 @@ pub fn cnpg_cluster_bootstrap_from_cdb(
     let coredb_cluster = if let Some(restore) = &cdb.spec.restore {
         let s3_credentials = generate_s3_restore_credentials(restore.s3_credentials.as_ref());
         // Find destination_path from Backup to generate the restore destination path
-        let restore_destination_path = match &cdb.spec.backup.destinationPath {
-            Some(path) => generate_restore_destination_path(path),
-            None => "".to_string(),
-        };
+        let restore_destination_path = generate_restore_destination_path(restore, &cdb.spec.backup);
         ClusterExternalClusters {
             name: "tembo-recovery".to_string(),
             barman_object_store: Some(ClusterExternalClustersBarmanObjectStore {
-                destination_path: format!("{}/{}", restore_destination_path, restore.server_name),
+                destination_path: restore_destination_path,
                 endpoint_url: restore.endpoint_url.clone(),
                 s3_credentials: Some(s3_credentials),
                 wal: Some(ClusterExternalClustersBarmanObjectStoreWal {
@@ -2011,10 +2009,22 @@ pub async fn unfence_pod(cdb: &CoreDB, ctx: Arc<Context>, pod_name: &str) -> Res
 // generate_restore_destination_path function will generate the restore destination path from the backup
 // object and return a string
 #[instrument(fields(trace_id, path))]
-fn generate_restore_destination_path(path: &str) -> String {
-    let mut parts: Vec<&str> = path.split('/').collect();
-    parts.pop();
-    parts.join("/")
+fn generate_restore_destination_path(restore: &Restore, backup: &coredb_types::Backup) -> String {
+    match restore.backups_path.clone() {
+        Some(path) => return path.clone(),
+        None => {
+            let this_instance_destination_path = match backup.destinationPath.clone() {
+                Some(path) => path.clone(),
+                None => "".to_string(),
+            };
+            let mut path_prefix_from_this_instance: Vec<&str> =
+                this_instance_destination_path.split('/').collect();
+            path_prefix_from_this_instance.pop();
+            let prefix = path_prefix_from_this_instance.join("/");
+            let destination_path = format!("{}/{}", prefix, restore.server_name.clone());
+            destination_path
+        }
+    }
 }
 
 // generate_s3_backup_credentials function will generate the s3 backup credentials from
@@ -2803,23 +2813,85 @@ mod tests {
         let _result: Cluster =
             serde_json::from_str(json_str).expect("Should be able to deserialize");
     }
+    #[test]
+    fn test_generate_restore_destination_path_null() {
+        let backup = coredb_types::Backup {
+            destinationPath: Some(
+                "s3://cdb-plat-use1-dev-instance-backups/v2/homely-musical-bullsnake".to_string(),
+            ),
+            ..Default::default()
+        };
+        let restore = Restore {
+            server_name: "org-foobar-inst-test".to_string(),
+            backups_path: None,
+            ..Default::default()
+        };
+        assert_eq!(
+            generate_restore_destination_path(&restore, &backup),
+            "s3://cdb-plat-use1-dev-instance-backups/v2/org-foobar-inst-test".to_string()
+        );
+    }
 
     #[test]
-    fn test_generate_restore_destination_path() {
-        // Define test cases
-        let test_cases = [
-            (
-                "s3://cdb-plat-use1-dev-instance-backups/coredb/coredb/org-coredb-inst-test-testing-test-1",
-                "s3://cdb-plat-use1-dev-instance-backups/coredb/coredb",
+    fn test_generate_restore_destination_path_null_old_format() {
+        let backup = coredb_types::Backup {
+            destinationPath: Some(
+                "s3://cdb-plat-use1-dev-instance-backups/coredb/coredb/org-foobar-inst-test"
+                    .to_string(),
             ),
-            ("s3://path/with/multiple/segments", "s3://path/with/multiple"),
-            ("s3://short/path", "s3://short"),
-            ("single_segment", ""),
-        ];
+            ..Default::default()
+        };
+        let restore = Restore {
+            server_name: "org-coredb-inst-pgtrunkio-dev".to_string(),
+            backups_path: None,
+            ..Default::default()
+        };
+        assert_eq!(
+            generate_restore_destination_path(&restore, &backup),
+            "s3://cdb-plat-use1-dev-instance-backups/coredb/coredb/org-coredb-inst-pgtrunkio-dev"
+                .to_string()
+        );
+    }
 
-        for (input, expected) in test_cases.iter() {
-            assert_eq!(generate_restore_destination_path(input), *expected);
-        }
+    #[test]
+    fn test_generate_restore_destination_path_from_old_to_new() {
+        let backup = coredb_types::Backup {
+            destinationPath: Some(
+                "s3://cdb-plat-use1-dev-instance-backups/v2/org-foobar-inst-test".to_string(),
+            ),
+            ..Default::default()
+        };
+        let restore = Restore {
+            server_name: "org-coredb-inst-pgtrunkio-dev".to_string(),
+            backups_path: Some("s3://cdb-plat-use1-dev-instance-backups/coredb/coredb/org-coredb-inst-pgtrunkio-dev".to_string()),
+            ..Default::default()
+        };
+        assert_eq!(
+            generate_restore_destination_path(&restore, &backup),
+            "s3://cdb-plat-use1-dev-instance-backups/coredb/coredb/org-coredb-inst-pgtrunkio-dev"
+                .to_string()
+        );
+    }
+    #[test]
+    fn test_generate_restore_destination_path_from_new_to_new() {
+        let backup = coredb_types::Backup {
+            destinationPath: Some(
+                "s3://cdb-plat-use1-dev-instance-backups/v2/org-foobar-inst-test".to_string(),
+            ),
+            ..Default::default()
+        };
+        let restore = Restore {
+            server_name: "org-coredb-inst-pgtrunkio-dev".to_string(),
+            backups_path: Some(
+                "s3://cdb-plat-use1-dev-instance-backups/v2/org-coredb-inst-pgtrunkio-dev"
+                    .to_string(),
+            ),
+            ..Default::default()
+        };
+        assert_eq!(
+            generate_restore_destination_path(&restore, &backup),
+            "s3://cdb-plat-use1-dev-instance-backups/v2/org-coredb-inst-pgtrunkio-dev".to_string()
+        );
     }
 
     #[test]