-
Notifications
You must be signed in to change notification settings - Fork 117
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add core support for database_schema block
- Loading branch information
1 parent
d888aa0
commit 9adbdc4
Showing
6 changed files
with
241 additions
and
96 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| use crate::blocks::block::{Block, BlockResult, BlockType, Env}; | ||
| use crate::Rule; | ||
| use anyhow::{anyhow, Result}; | ||
| use async_trait::async_trait; | ||
| use futures::future::try_join_all; | ||
| use pest::iterators::Pair; | ||
| use serde_json::Value; | ||
| use tokio::sync::mpsc::UnboundedSender; | ||
|
|
||
| use super::helpers::get_data_source_project; | ||
|
|
||
| #[derive(Clone)] | ||
| pub struct DatabaseSchema {} | ||
|
|
||
| impl DatabaseSchema { | ||
| pub fn parse(_block_pair: Pair<Rule>) -> Result<Self> { | ||
| Ok(DatabaseSchema {}) | ||
| } | ||
| } | ||
|
|
||
| #[async_trait] | ||
| impl Block for DatabaseSchema { | ||
| fn block_type(&self) -> BlockType { | ||
| BlockType::DatabaseSchema | ||
| } | ||
|
|
||
| fn inner_hash(&self) -> String { | ||
| let mut hasher = blake3::Hasher::new(); | ||
| hasher.update("database_schema".as_bytes()); | ||
| format!("{}", hasher.finalize().to_hex()) | ||
| } | ||
|
|
||
| async fn execute( | ||
| &self, | ||
| name: &str, | ||
| env: &Env, | ||
| _event_sender: Option<UnboundedSender<Value>>, | ||
| ) -> Result<BlockResult> { | ||
| let config = env.config.config_for_block(name); | ||
|
|
||
| // TODO: finish error message | ||
| let err_msg = format!( | ||
| "Invalid or missing `databases` in configuration for \ | ||
| `database_schema` block `{}` expecting `{{ \"databases\": \ | ||
| [ {{ \"workspace_id\": ..., \"data_source_id\": ..., \"database_id\": ... }}, ... ] }}`", | ||
| name | ||
| ); | ||
|
|
||
| let databases = match config { | ||
| Some(v) => match v.get("databases") { | ||
| Some(Value::Array(a)) => a | ||
| .iter() | ||
| .map(|v| { | ||
| let workspace_id = match v.get("workspace_id") { | ||
| Some(Value::String(s)) => s, | ||
| _ => Err(anyhow!(err_msg.clone()))?, | ||
| }; | ||
| let data_source_id = match v.get("data_source_id") { | ||
| Some(Value::String(s)) => s, | ||
| _ => Err(anyhow!(err_msg.clone()))?, | ||
| }; | ||
| let database_id = match v.get("database_id") { | ||
| Some(Value::String(s)) => s, | ||
| _ => Err(anyhow!(err_msg.clone()))?, | ||
| }; | ||
|
|
||
| Ok((workspace_id, data_source_id, database_id)) | ||
| }) | ||
| .collect::<Result<Vec<_>>>(), | ||
| _ => Err(anyhow!(err_msg)), | ||
| }, | ||
| None => Err(anyhow!(err_msg)), | ||
| }?; | ||
|
|
||
| let schemas = try_join_all(databases.iter().map( | ||
| |(workspace_id, data_source_id, database_id)| { | ||
| get_database_schema(workspace_id, data_source_id, database_id, env) | ||
| }, | ||
| )) | ||
| .await?; | ||
|
|
||
| Ok(BlockResult { | ||
| value: serde_json::to_value(schemas)?, | ||
| meta: None, | ||
| }) | ||
| } | ||
|
|
||
| fn clone_box(&self) -> Box<dyn Block + Sync + Send> { | ||
| Box::new(self.clone()) | ||
| } | ||
|
|
||
| fn as_any(&self) -> &dyn std::any::Any { | ||
| self | ||
| } | ||
| } | ||
|
|
||
| async fn get_database_schema( | ||
| workspace_id: &String, | ||
| data_source_id: &String, | ||
| database_id: &String, | ||
| env: &Env, | ||
| ) -> Result<crate::databases::database::DatabaseSchema> { | ||
| let project = get_data_source_project(workspace_id, data_source_id, env).await?; | ||
| let database = match env | ||
| .store | ||
| .load_database(&project, data_source_id, database_id) | ||
| .await? | ||
| { | ||
| Some(d) => d, | ||
| None => Err(anyhow!( | ||
| "Database `{}` not found in data source `{}`", | ||
| database_id, | ||
| data_source_id | ||
| ))?, | ||
| }; | ||
|
|
||
| match database.get_schema(&project, env.store.clone()).await { | ||
| Ok(s) => Ok(s), | ||
| Err(e) => Err(anyhow!( | ||
| "Error getting schema for database `{}` in data source `{}`: {}", | ||
| database_id, | ||
| data_source_id, | ||
| e | ||
| )), | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| use super::block::Env; | ||
| use crate::project::Project; | ||
| use anyhow::{anyhow, Result}; | ||
| use hyper::header; | ||
| use hyper::{body::Buf, http::StatusCode, Body, Client, Method, Request}; | ||
| use hyper_tls::HttpsConnector; | ||
| use serde_json::Value; | ||
| use std::io::prelude::*; | ||
| use url::Url; | ||
| use urlencoding::encode; | ||
|
|
||
| pub async fn get_data_source_project( | ||
| workspace_id: &String, | ||
| data_source_id: &String, | ||
| env: &Env, | ||
| ) -> Result<Project> { | ||
| let dust_workspace_id = match env.credentials.get("DUST_WORKSPACE_ID") { | ||
| None => Err(anyhow!( | ||
| "DUST_WORKSPACE_ID credentials missing, but `workspace_id` \ | ||
| is set in `data_source` block config" | ||
| ))?, | ||
| Some(v) => v.clone(), | ||
| }; | ||
| let registry_secret = match std::env::var("DUST_REGISTRY_SECRET") { | ||
| Ok(key) => key, | ||
| Err(_) => Err(anyhow!( | ||
| "Environment variable `DUST_REGISTRY_SECRET` is not set." | ||
| ))?, | ||
| }; | ||
| let front_api = match std::env::var("DUST_FRONT_API") { | ||
| Ok(key) => key, | ||
| Err(_) => Err(anyhow!("Environment variable `DUST_FRONT_API` is not set."))?, | ||
| }; | ||
|
|
||
| let url = format!( | ||
| "{}/api/registry/data_sources/lookup?workspace_id={}&data_source_id={}", | ||
| front_api.as_str(), | ||
| encode(&workspace_id), | ||
| encode(&data_source_id), | ||
| ); | ||
| let parsed_url = Url::parse(url.as_str())?; | ||
|
|
||
| let mut req = Request::builder().method(Method::GET).uri(url.as_str()); | ||
|
|
||
| { | ||
| let headers = match req.headers_mut() { | ||
| Some(h) => h, | ||
| None => Err(anyhow!("Invalid URL: {}", url.as_str()))?, | ||
| }; | ||
| headers.insert( | ||
| header::AUTHORIZATION, | ||
| header::HeaderValue::from_bytes( | ||
| format!("Bearer {}", registry_secret.as_str()).as_bytes(), | ||
| )?, | ||
| ); | ||
| headers.insert( | ||
| header::HeaderName::from_bytes("X-Dust-Workspace-Id".as_bytes())?, | ||
| header::HeaderValue::from_bytes(dust_workspace_id.as_bytes())?, | ||
| ); | ||
| } | ||
| let req = req.body(Body::empty())?; | ||
|
|
||
| let res = match parsed_url.scheme() { | ||
| "https" => { | ||
| let https = HttpsConnector::new(); | ||
| let cli = Client::builder().build::<_, hyper::Body>(https); | ||
| cli.request(req).await? | ||
| } | ||
| "http" => { | ||
| let cli = Client::new(); | ||
| cli.request(req).await? | ||
| } | ||
| _ => Err(anyhow!( | ||
| "Only the `http` and `https` schemes are authorized." | ||
| ))?, | ||
| }; | ||
|
|
||
| let status = res.status(); | ||
| if status != StatusCode::OK { | ||
| Err(anyhow!( | ||
| "Failed to retrieve DataSource `{} > {}`", | ||
| workspace_id, | ||
| data_source_id, | ||
| ))?; | ||
| } | ||
|
|
||
| let body = hyper::body::aggregate(res).await?; | ||
| let mut b: Vec<u8> = vec![]; | ||
| body.reader().read_to_end(&mut b)?; | ||
|
|
||
| let response_body = String::from_utf8_lossy(&b).into_owned(); | ||
|
|
||
| let body = match serde_json::from_str::<serde_json::Value>(&response_body) { | ||
| Ok(body) => body, | ||
| Err(_) => Err(anyhow!("Failed to parse registry response"))?, | ||
| }; | ||
|
|
||
| match body.get("project_id") { | ||
| Some(Value::Number(p)) => match p.as_i64() { | ||
| Some(p) => Ok(Project::new_from_id(p)), | ||
| None => Err(anyhow!("Failed to parse registry response")), | ||
| }, | ||
| _ => Err(anyhow!("Failed to parse registry response")), | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.