Merge branch 'main' into nhudson/update_tembo_chart

conductor/src/azure/uami_builder.rs

@@ -5,6 +5,7 @@ use azure_error::AzureError;
 use azure_identity::TokenCredentialOptions;
 use azure_identity::WorkloadIdentityCredential;
 use azure_mgmt_authorization;
+use azure_mgmt_authorization::models::role_assignment_properties::PrincipalType;
 use azure_mgmt_authorization::models::RoleAssignmentProperties;
 use azure_mgmt_msi::models::{
     FederatedIdentityCredential, FederatedIdentityCredentialProperties, Identity, TrackedResource,
@@ -79,11 +80,10 @@ pub async fn get_role_definition_id(
 // Get storage account ID
 pub async fn get_storage_account_id(
     subscription_id: &str,
-    resource_group_prefix: &str,
+    resource_group: &str,
     storage_account_name: &str,
     credentials: Arc<dyn TokenCredential>,
 ) -> Result<String, AzureError> {
-    let resource_group = format!("{resource_group_prefix}-instances");
     let storage_client = azure_mgmt_storage::Client::builder(credentials).build()?;
     let storage_account_list = storage_client
         .storage_accounts_client()
@@ -186,17 +186,18 @@ pub async fn create_role_assignment(
     )
     .await?
     {
-        info!("Role assignment already exists, skipping creation");
+        info!("Role assignment already exists for {namespace}, skipping creation");
         return Ok(());
     }
 
+    info!("Role assignment does not exist for {namespace}, creating");
     // Set parameters for Role Assignment
     let role_assignment_params = azure_mgmt_authorization::models::RoleAssignmentCreateParameters {
         properties: RoleAssignmentProperties {
             scope: None,
             role_definition_id: role_definition,
             principal_id: uami_principal_id.to_string(),
-            principal_type: None,
+            principal_type: Some(PrincipalType::ServicePrincipal),
             description: None,
             condition: None,
             condition_version: None,
@@ -250,7 +251,10 @@ pub async fn get_cluster_issuer(
     let response_json = response.json::<serde_json::Value>().await?;
     let issuer_url = response_json["properties"]["oidcIssuerProfile"]["issuerURL"]
         .as_str()
-        .unwrap();
+        .ok_or(AzureError::from(AzureSDKError::new(
+            azure_core::error::ErrorKind::Other,
+            "Issuer URL not found in response".to_string(),
+        )))?;
     Ok(issuer_url.to_string())
 }
 
@@ -266,7 +270,7 @@ pub async fn create_federated_identity_credentials(
     let federated_identity_client = azure_mgmt_msi::Client::builder(credentials.clone()).build()?;
     let cluster_issuer = get_cluster_issuer(
         subscription_id,
-        &resource_group,
+        &resource_group_prefix,
         &format!("aks-{resource_group_prefix}"),
         credentials.clone(),
     )

conductor/src/cloud.rs

@@ -68,6 +68,17 @@ impl CloudProvider {
         }
     }
 
+    // If azure, generate storage_account_url for Azure restore scenarios
+    pub fn storage_account_url(&self, azure_storage_account: Option<&str>) -> String {
+        match self {
+            CloudProvider::Azure => format!(
+                "{}.blob.core.windows.net/",
+                azure_storage_account.unwrap_or("")
+            ),
+            _ => "".to_string(),
+        }
+    }
+
     pub fn builder() -> CloudProviderBuilder {
         CloudProviderBuilder::new()
     }

conductor/src/lib.rs

@@ -45,6 +45,7 @@ pub async fn generate_spec(
     data_plane_id: &str,
     namespace: &str,
     backups_bucket: &str,
+    azure_storage_account: Option<&str>,
     spec: &CoreDBSpec,
     cloud_provider: &CloudProvider,
 ) -> Result<Value, ConductorError> {
@@ -54,16 +55,24 @@ pub async fn generate_spec(
         CloudProvider::AWS | CloudProvider::GCP | CloudProvider::Azure => {
             let prefix = cloud_provider.prefix();
 
+            // Generate the storage_account_url for Azure restore scenarios
+            let storage_account_url = cloud_provider.storage_account_url(azure_storage_account);
+
             // Format the backups_path with the correct prefix
             if let Some(restore) = &mut spec.restore {
                 if let Some(backups_path) = &mut restore.backups_path {
                     let clean_path = remove_known_prefixes(backups_path);
                     if clean_path.starts_with(backups_bucket) {
-                        // If the path already includes the bucket, just add the prefix
-                        *backups_path = format!("{}{}", prefix, clean_path);
+                        // If the path already includes the bucket, just add the prefix. The
+                        // storage_account_url is specific to Azure and will be empty for AWS and GCP
+                        *backups_path = format!("{}{}{}", prefix, storage_account_url, clean_path);
                     } else {
-                        // If the path doesn't include the bucket, add both prefix and bucket
-                        *backups_path = format!("{}{}/{}", prefix, backups_bucket, clean_path);
+                        // If the path doesn't include the bucket, add both prefix and bucket. The
+                        // storage_account_url is specific to Azure and will be empty for AWS and GCP
+                        *backups_path = format!(
+                            "{}{}{}/{}",
+                            prefix, storage_account_url, backups_bucket, clean_path
+                        );
                     }
                 }
             }
@@ -745,6 +754,7 @@ mod tests {
             "aws_data_1_use1",
             "namespace",
             "my-bucket",
+            None,
             &spec,
             &cloud_provider,
         )
@@ -776,6 +786,7 @@ mod tests {
             "aws_data_1_use1",
             "namespace",
             "my-bucket",
+            None,
             &spec,
             &cloud_provider,
         )
@@ -805,6 +816,7 @@ mod tests {
             "aws_data_1_use1",
             "namespace",
             "my-bucket",
+            None,
             &spec,
             &cloud_provider,
         )
@@ -827,6 +839,7 @@ mod tests {
             "aws_data_1_use1",
             "namespace",
             "my-bucket",
+            None,
             &spec,
             &cloud_provider,
         )
@@ -852,6 +865,7 @@ mod tests {
             "gcp_data_1_usc1",
             "namespace",
             "my-bucket",
+            None,
             &spec,
             &cloud_provider,
         )
@@ -881,6 +895,7 @@ mod tests {
             "gcp_data_1_usc1",
             "namespace",
             "my-bucket",
+            None,
             &spec,
             &cloud_provider,
         )
@@ -892,4 +907,66 @@ mod tests {
             expected_backups_path
         );
     }
+
+    #[tokio::test]
+    async fn test_generate_spec_with_non_matching_azure_bucket() {
+        let spec = CoreDBSpec {
+            restore: Some(Restore {
+                backups_path: Some("https://v2/test-instance".to_string()),
+                ..Restore::default()
+            }),
+            ..CoreDBSpec::default()
+        };
+        let cloud_provider = CloudProvider::Azure;
+        let result = generate_spec(
+            "org-id",
+            "entity-name",
+            "instance-id",
+            "azure_data_1_eus1",
+            "namespace",
+            "my-blob",
+            Some("eusdevsg"),
+            &spec,
+            &cloud_provider,
+        )
+        .await
+        .expect("Failed to generate spec");
+        let expected_backups_path =
+            "https://eusdevsg.blob.core.windows.net/my-blob/v2/test-instance";
+        assert_eq!(
+            result["spec"]["restore"]["backupsPath"].as_str().unwrap(),
+            expected_backups_path
+        );
+    }
+
+    #[tokio::test]
+    async fn test_generate_spec_with_azure_bucket() {
+        let spec = CoreDBSpec {
+            restore: Some(Restore {
+                backups_path: Some("https://my-blob/v2/test-instance".to_string()),
+                ..Restore::default()
+            }),
+            ..CoreDBSpec::default()
+        };
+        let cloud_provider = CloudProvider::Azure;
+        let result = generate_spec(
+            "org-id",
+            "entity-name",
+            "instance-id",
+            "azure_data_1_eus1",
+            "namespace",
+            "my-blob",
+            Some("eusdevsg"),
+            &spec,
+            &cloud_provider,
+        )
+        .await
+        .expect("Failed to generate spec");
+        let expected_backups_path =
+            "https://eusdevsg.blob.core.windows.net/my-blob/v2/test-instance";
+        assert_eq!(
+            result["spec"]["restore"]["backupsPath"].as_str().unwrap(),
+            expected_backups_path
+        );
+    }
 }

conductor/src/main.rs

@@ -189,7 +189,7 @@ async fn run(metrics: CustomMetrics) -> Result<(), ConductorError> {
 
     loop {
         // Read from queue (check for new message)
-        // messages that dont fit a CRUDevent will error
+        // messages that don't fit a CRUDevent will error
         // set visibility timeout to 90 seconds
         let read_msg = queue
             .read::<CRUDevent>(&control_plane_events_queue, 90_i32)
@@ -411,13 +411,21 @@ async fn run(metrics: CustomMetrics) -> Result<(), ConductorError> {
                     &mut coredb_spec,
                 );
 
+                // If cloud provider is Azure, we need to pass the storage account name to generate_spec
+                // so that the storage account URL can be generated for Azure restore scenarios
+                let azure_storage_account = match cloud_provider {
+                    CloudProvider::Azure => Some(azure_storage_account.as_str()),
+                    _ => None,
+                };
+
                 let spec = generate_spec(
                     org_id,
                     &stack_type,
                     instance_id,
                     &read_msg.message.data_plane_id,
                     &namespace,
                     &backup_archive_bucket,
+                    azure_storage_account,
                     &coredb_spec,
                     &cloud_provider,
                 )

tembo-stacks/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tembo-stacks"
 description = "Tembo Stacks for Postgres"
-version = "0.19.0"
+version = "0.20.0"
 authors = ["tembo.io"]
 edition = "2021"
 license = "Apache-2.0"
@@ -18,15 +18,18 @@ path = "src/main.rs"
 anyhow = "1.0.71"
 clap = { version = "4.5.4", features = ["derive"] }
 futures = "0.3.28"
-k8s-openapi = { version = "0.18.0", features = ["v1_25", "schemars"], default-features = false } # This version has to be in line with the same version we use in the controller
+k8s-openapi = { version = "0.18.0", features = [
+  "v1_25",
+  "schemars",
+], default-features = false } # This version has to be in line with the same version we use in the controller
 lazy_static = "1.4.0"
-schemars = {version = "0.8.12", features = ["chrono"]}
+schemars = { version = "0.8.12", features = ["chrono"] }
 serde = "1.0.152"
 serde_json = "1.0.114"
 serde_yaml = "0.9.21"
 strum = "0.26.2"
 strum_macros = "0.26.2"
-tembo-controller = { package = "controller", version = "0.51.0" }
+tembo-controller = { package = "controller", version = "0.52.0" }
 tracing = "0.1"
 utoipa = { version = "3", features = ["actix_extras", "chrono"] }
 

tembo-stacks/src/stacks/specs/analytics.yaml

@@ -3,9 +3,10 @@ description: A Postgres instance equipped ParadeDB's pg_analytics extension.
 repository: "quay.io/tembo"
 organization: tembo
 images:
-  14: "standard-cnpg:14-5120dd1"
-  15: "standard-cnpg:15-5120dd1"
-  16: "standard-cnpg:16-5120dd1"
+  14: "standard-cnpg:14-30219f2"
+  15: "standard-cnpg:15-30219f2"
+  16: "standard-cnpg:16-30219f2"
+  17: "standard-cnpg:17-30219f2"
 stack_version: 0.1.0
 postgres_config_engine: olap
 postgres_config:

tembo-stacks/src/stacks/specs/api.yaml

@@ -3,9 +3,10 @@ description: Tembo Stack with REST and graphQL interfaces.
 repository: "quay.io/tembo"
 organization: tembo
 images:
-  14: "standard-cnpg:14-5120dd1"
-  15: "standard-cnpg:15-5120dd1"
-  16: "standard-cnpg:16-5120dd1"
+  14: "standard-cnpg:14-30219f2"
+  15: "standard-cnpg:15-30219f2"
+  16: "standard-cnpg:16-30219f2"
+  17: "standard-cnpg:17-30219f2"
 stack_version: 0.1.0
 appServices:
   - image: postgrest/postgrest:v12.0.0

tembo-stacks/src/stacks/specs/data_warehouse.yaml

@@ -3,9 +3,9 @@ description: A Postgres instance equipped with configuration and extensions for
 repository: "quay.io/tembo"
 organization: tembo
 images:
-  14: "dw-cnpg:14-5120dd1"
-  15: "dw-cnpg:15-5120dd1"
-  16: "dw-cnpg:16-5120dd1"
+  14: "dw-cnpg:14-30219f2"
+  15: "dw-cnpg:15-30219f2"
+  16: "dw-cnpg:16-30219f2" 
 stack_version: 0.1.0
 postgres_config_engine: olap
 postgres_config:

tembo-stacks/src/stacks/specs/gis.yaml

@@ -3,9 +3,10 @@ description: Postgres for geospatial workloads.
 repository: "quay.io/tembo"
 organization: tembo
 images:
-  14: "geo-cnpg:14-5120dd1"
-  15: "geo-cnpg:15-5120dd1"
-  16: "geo-cnpg:16-5120dd1"
+  14: "geo-cnpg:14-30219f2"
+  15: "geo-cnpg:15-30219f2"
+  16: "geo-cnpg:16-30219f2"
+  17: "geo-cnpg:17-30219f2"
 stack_version: 0.1.0
 postgres_config_engine: standard
 postgres_config:

tembo-stacks/src/stacks/specs/message_queue.yaml

@@ -3,9 +3,10 @@ description: A Tembo Postgres Stack optimized for Message Queue workloads.
 repository: "quay.io/tembo"
 organization: tembo
 images:
-  14: "standard-cnpg:14-5120dd1"
-  15: "standard-cnpg:15-5120dd1"
-  16: "standard-cnpg:16-5120dd1"
+  14: "standard-cnpg:14-30219f2"
+  15: "standard-cnpg:15-30219f2"
+  16: "standard-cnpg:16-30219f2"
+  17: "standard-cnpg:17-30219f2"
 stack_version: 0.3.0
 appServices:
   - name: mq-api

tembo-stacks/src/stacks/specs/mongo_alternative.yaml

@@ -3,9 +3,10 @@ description: Document-facing workloads on Postgres.
 repository: "quay.io/tembo"
 organization: tembo
 images:
-  14: "standard-cnpg:14-5120dd1"
-  15: "standard-cnpg:15-5120dd1"
-  16: "standard-cnpg:16-5120dd1"
+  14: "standard-cnpg:14-30219f2"
+  15: "standard-cnpg:15-30219f2"
+  16: "standard-cnpg:16-30219f2"
+  17: "standard-cnpg:17-30219f2"
 stack_version: 0.1.0
 appServices:
   - name: fdb-api