From 6aff4edfe30ff7f11f2a1c6d12d89015429bc498 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 17 Sep 2024 16:33:35 +0200 Subject: [PATCH] refactor code into seperate modules --- Cargo.toml | 9 +- crates/pixi-build-python/Cargo.toml | 2 + crates/pixi-build-python/src/main.rs | 592 ++---------------- crates/pixi-build-python/src/protocol.rs | 34 + crates/pixi-build-python/src/python.rs | 446 +++++++++++++ crates/pixi-build-python/src/server.rs | 136 ++++ .../pixi-build-python/src/temporary_recipe.rs | 53 ++ 7 files changed, 719 insertions(+), 553 deletions(-) create mode 100644 crates/pixi-build-python/src/protocol.rs create mode 100644 crates/pixi-build-python/src/python.rs create mode 100644 crates/pixi-build-python/src/server.rs create mode 100644 crates/pixi-build-python/src/temporary_recipe.rs diff --git a/Cargo.toml b/Cargo.toml index d42ee75..1d65446 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ resolver = "2" edition = "2021" [workspace.dependencies] +async-trait = "0.1.82" clap = "4.5.11" chrono = "0.4.38" @@ -28,7 +29,7 @@ rattler-build = { git = "https://github.com/baszalmstra/rattler-build", branch = rattler_conda_types = "0.27.5" rattler_package_streaming = "0.22.6" -pixi_build_types = { path="../pixi-build-branch/crates/pixi_build_types" } -pixi_consts = { path="../pixi-build-branch/crates/pixi_consts" } -pixi_manifest = { path="../pixi-build-branch/crates/pixi_manifest" } -pixi_spec = { path="../pixi-build-branch/crates/pixi_spec" } +pixi_build_types = { path = "../pixi-build-branch/crates/pixi_build_types" } +pixi_consts = { path = "../pixi-build-branch/crates/pixi_consts" } +pixi_manifest = { path = "../pixi-build-branch/crates/pixi_manifest" } +pixi_spec = { path = "../pixi-build-branch/crates/pixi_spec" } diff --git a/crates/pixi-build-python/Cargo.toml b/crates/pixi-build-python/Cargo.toml index 34e3314..ea22f8f 100644 --- a/crates/pixi-build-python/Cargo.toml +++ b/crates/pixi-build-python/Cargo.toml @@ -4,6 +4,8 @@ version = "0.1.0" edition.workspace = true [dependencies] +async-trait = { workspace = true } + pixi_build_types = { workspace = true } pixi_consts = { workspace = true } pixi_manifest = { workspace = true } diff --git a/crates/pixi-build-python/src/main.rs b/crates/pixi-build-python/src/main.rs index c421c4e..76d9494 100644 --- a/crates/pixi-build-python/src/main.rs +++ b/crates/pixi-build-python/src/main.rs @@ -1,53 +1,32 @@ mod consts; +mod logging; +mod protocol; +mod python; +mod server; +mod temporary_recipe; -use std::{ - collections::BTreeMap, - future::Future, - io::BufWriter, - net::SocketAddr, - path::{Path, PathBuf}, - str::FromStr, - sync::Arc, -}; +use std::path::{Path, PathBuf}; -use chrono::Utc; use clap::{Parser, Subcommand}; use clap_verbosity_flag::{InfoLevel, Verbosity}; -use jsonrpc_core::{serde_json, to_value, Error, IoHandler}; -use jsonrpc_http_server::jsonrpc_core::Params; -use miette::{Context, IntoDiagnostic, JSONReportHandler}; -use parking_lot::Mutex; +use miette::IntoDiagnostic; use pixi_build_types::{ - procedures, procedures::{ + conda_build::{CondaBuildParams, CondaOutputIdentifier}, conda_metadata::{CondaMetadataParams, CondaMetadataResult}, - initialize::{InitializeParams, InitializeResult}, - }, - BackendCapabilities, CondaPackageMetadata, -}; -use pixi_manifest::{Dependencies, Manifest, SpecType}; -use pixi_spec::PixiSpec; -use rattler_build::{ - build::run_build, - console_utils::{get_default_env_filter, LoggingOutputHandler}, - hash::HashInfo, - metadata::{BuildConfiguration, Directories, Output, PackagingSettings}, - recipe::{ - parser::{Build, Dependency, Package, PathSource, Requirements, ScriptContent, Source}, - Recipe, }, - render::resolved_dependencies::DependencyInfo, - tool_configuration::Configuration, -}; -use rattler_conda_types::{ - package::ArchiveType, ChannelConfig, MatchSpec, NoArchType, PackageName, Platform, Version, - VersionWithSource, + ChannelConfiguration, }; -use rattler_package_streaming::write::CompressionLevel; -use reqwest::Url; -use tempfile::tempdir; +use rattler_build::console_utils::{get_default_env_filter, LoggingOutputHandler}; +use rattler_conda_types::ChannelConfig; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +use crate::{ + protocol::{Protocol, ProtocolFactory}, + python::PythonBuildBackend, + server::Server, +}; + #[allow(missing_docs)] #[derive(Parser)] pub struct App { @@ -85,132 +64,13 @@ pub async fn main() { } } -fn convert_error(err: miette::Report) -> jsonrpc_core::Error { - let rendered = JSONReportHandler::new(); - let mut json_str = String::new(); - rendered - .render_report(&mut json_str, err.as_ref()) - .expect("failed to convert error to json"); - let data = serde_json::from_str(&json_str).expect("failed to parse json error"); - jsonrpc_core::Error { - code: jsonrpc_core::ErrorCode::ServerError(-32000), - message: err.to_string(), - data: Some(data), - } -} - -struct BuildServer { - manifest: Manifest, - logging_output_handler: LoggingOutputHandler, -} - -impl BuildServer { - pub fn new( - manifest_path: &Path, - logging_output_handler: LoggingOutputHandler, - ) -> miette::Result { - // Load the manifest from the source directory - let manifest = Manifest::from_path(&manifest_path).with_context(|| { - format!("failed to parse manifest from {}", manifest_path.display()) - })?; - - Ok(Self { - manifest, - logging_output_handler, - }) - } - - pub fn manifest_root(&self) -> &Path { - self.manifest - .path - .parent() - .expect("manifest should always reside in a directory") - } - - pub async fn get_conda_metadata( - &self, - params: CondaMetadataParams, - ) -> miette::Result { - let channel_config = ChannelConfig { - channel_alias: params.channel_configuration.base_url, - root_dir: self.manifest_root().to_path_buf(), - }; - let channels = params - .channel_base_urls - .unwrap_or_else(|| channels_from_manifest(&self.manifest, &channel_config)); - - get_conda_metadata_from_manifest( - &self.manifest, - &channel_config, - channels, - self.logging_output_handler.clone(), - ) - .await - } -} - -async fn run_server( - port: Option, - logging_output_handler: LoggingOutputHandler, -) -> miette::Result<()> { - let build_server = Arc::new(Mutex::new(None)); - - // Construct a server - let mut io = IoHandler::new(); - - let initialize_build_server = build_server.clone(); - io.add_sync_method( - procedures::initialize::METHOD_NAME, - move |params: Params| { - let params: InitializeParams = params.parse()?; - - let mut build_server = initialize_build_server.lock(); - if build_server.is_some() { - return Err(Error::invalid_request()); - } - - *build_server = Some(Arc::new( - BuildServer::new(¶ms.manifest_path, logging_output_handler.clone()) - .map_err(convert_error)?, - )); - - Ok(to_value(InitializeResult { - capabilities: BackendCapabilities { - provides_conda_metadata: Some(true), - }, - }) - .expect("failed to convert to json")) - }, - ); - - io.add_method( - procedures::conda_metadata::METHOD_NAME, - move |params: Params| { - let build_server = build_server.clone(); - let build_server = build_server.lock().as_ref().cloned(); - async move { - let params: CondaMetadataParams = params.parse()?; - - build_server - .ok_or_else(|| Error::invalid_request())? - .get_conda_metadata(params) - .await - .map(|value| to_value(value).expect("failed to convert to json")) - .map_err(convert_error) - } - }, - ); - +async fn run_server(port: Option, protocol: T) -> miette::Result<()> { + let server = Server::new(protocol); if let Some(port) = port { - jsonrpc_http_server::ServerBuilder::new(io) - .start_http(&SocketAddr::from(([127, 0, 0, 1], port))) - .into_diagnostic()? - .wait() + server.run_over_http(port) } else { - jsonrpc_stdio_server::ServerBuilder::new(io).build().await; + server.run().await } - - Ok(()) } async fn actual_main() -> miette::Result<()> { @@ -223,7 +83,7 @@ async fn actual_main() -> miette::Result<()> { registry.with(log_handler.clone()).init(); match args.command { - None => run_server(args.http_port, log_handler).await, + None => run_server(args.http_port, PythonBuildBackend::factory(log_handler)).await, Some(Commands::Build { manifest_path }) => build(log_handler, &manifest_path).await, Some(Commands::GetCondaMetadata { manifest_path }) => { let metadata = get_conda_metadata(log_handler, &manifest_path).await?; @@ -237,415 +97,49 @@ async fn get_conda_metadata( logging_output_handler: LoggingOutputHandler, manifest_path: &Path, ) -> miette::Result { - let manifest = Manifest::from_path(&manifest_path) - .with_context(|| format!("failed to parse manifest from {}", manifest_path.display()))?; let channel_config = ChannelConfig::default_with_root_dir( manifest_path .parent() .expect("manifest should always reside in a directory") .to_path_buf(), ); - let channels = channels_from_manifest(&manifest, &channel_config); - - get_conda_metadata_from_manifest(&manifest, &channel_config, channels, logging_output_handler) - .await -} -async fn get_conda_metadata_from_manifest( - manifest: &Manifest, - channel_config: &ChannelConfig, - channels: Vec, - logging_output_handler: LoggingOutputHandler, -) -> miette::Result { - // TODO: Determine how and if we can determine this from the manifest. - let recipe = manifest_to_recipe(&manifest, &channel_config)?; - let output = Output { - build_configuration: manifest_to_build_configuration(&manifest, &recipe, channels).await?, - recipe, - finalized_dependencies: None, - finalized_cache_dependencies: None, - finalized_sources: None, - build_summary: Arc::default(), - system_tools: Default::default(), - extra_meta: None, - }; - let tool_config = get_tool_configuration(logging_output_handler, &channel_config)?; - - let temp_recipe = TemporaryRenderedRecipe::from_output(&output)?; - let output = temp_recipe - .within_context_async(move || async move { - output - .resolve_dependencies(&tool_config) - .await - .into_diagnostic() + let backend = PythonBuildBackend::new(manifest_path, logging_output_handler)?; + backend + .get_conda_metadata(CondaMetadataParams { + target_platform: None, + channel_base_urls: None, + channel_configuration: ChannelConfiguration { + base_url: channel_config.channel_alias, + }, }) - .await?; - - let finalized_deps = &output - .finalized_dependencies - .as_ref() - .expect("dependencies should be resolved at this point") - .run; - - Ok(CondaMetadataResult { - packages: vec![CondaPackageMetadata { - name: output.name().clone(), - version: output.version().clone().into(), - build: output.build_string().into_owned(), - build_number: output.recipe.build.number, - subdir: output.build_configuration.target_platform, - depends: finalized_deps - .depends - .iter() - .map(DependencyInfo::spec) - .cloned() - .collect(), - constraints: finalized_deps - .constraints - .iter() - .map(DependencyInfo::spec) - .cloned() - .collect(), - license: output.recipe.about.license.map(|l| l.to_string()), - license_family: output.recipe.about.license_family, - noarch: output.recipe.build.noarch, - }], - }) + .await } async fn build( logging_output_handler: LoggingOutputHandler, manifest_path: &Path, ) -> miette::Result<()> { - let manifest = Manifest::from_path(&manifest_path) - .with_context(|| format!("failed to parse manifest from {}", manifest_path.display()))?; let channel_config = ChannelConfig::default_with_root_dir( manifest_path .parent() .expect("manifest should always reside in a directory") .to_path_buf(), ); - let channels = channels_from_manifest(&manifest, &channel_config); - - build_manifest(&manifest, &channel_config, channels, logging_output_handler).await -} -async fn build_manifest( - manifest: &Manifest, - channel_config: &ChannelConfig, - channels: Vec, - logging_output_handler: LoggingOutputHandler, -) -> miette::Result<()> { - let recipe = manifest_to_recipe(&manifest, &channel_config)?; - let output = Output { - build_configuration: manifest_to_build_configuration(&manifest, &recipe, channels).await?, - recipe, - finalized_dependencies: None, - finalized_cache_dependencies: None, - finalized_sources: None, - build_summary: Arc::default(), - system_tools: Default::default(), - extra_meta: None, - }; - let tool_config = get_tool_configuration(logging_output_handler, &channel_config)?; - - let temp_recipe = TemporaryRenderedRecipe::from_output(&output)?; - let (_output, package) = temp_recipe - .within_context_async(move || async move { run_build(output, &tool_config).await }) - .await?; - eprintln!("Successfully build '{}'", package.display()); - - Ok(()) -} - -fn get_tool_configuration( - logging_output_handler: LoggingOutputHandler, - channel_config: &ChannelConfig, -) -> miette::Result { - Ok(Configuration::builder() - .with_logging_output_handler(logging_output_handler) - .with_channel_config(channel_config.clone()) - .finish()) -} - -async fn manifest_to_build_configuration( - manifest: &Manifest, - recipe: &Recipe, - channels: Vec, -) -> miette::Result { - // Parse the package name from the manifest - let Some(name) = manifest.parsed.project.name.clone() else { - miette::bail!("a 'name' field is required in the project manifest"); - }; - let name = PackageName::from_str(&name).into_diagnostic()?; - - // TODO: Setup defaults - let output_dir = tempdir() - .into_diagnostic() - .context("failed to create temporary directory")?; - std::fs::create_dir_all(&output_dir) - .into_diagnostic() - .context("failed to create output directory")?; - let directories = Directories::setup( - name.as_normalized(), - manifest.path.as_path(), - output_dir.path(), - false, - &Utc::now(), - ) - .into_diagnostic() - .context("failed to setup build directories")?; - - let host_platform = Platform::current(); - let build_platform = Platform::current(); - - let variant = BTreeMap::new(); - - Ok(BuildConfiguration { - // TODO: NoArch?? - target_platform: Platform::NoArch, - host_platform, - build_platform, - hash: HashInfo::from_variant(&variant, &recipe.build.noarch), - variant, - directories, - channels, - channel_priority: Default::default(), - solve_strategy: Default::default(), - timestamp: chrono::Utc::now(), - subpackages: Default::default(), // TODO: ??? - packaging_settings: PackagingSettings::from_args( - ArchiveType::Conda, - CompressionLevel::default(), - ), - store_recipe: true, - force_colors: true, - }) -} - -fn manifest_to_recipe( - manifest: &Manifest, - channel_config: &ChannelConfig, -) -> miette::Result { - let manifest_root = manifest - .path - .parent() - .expect("the project manifest must reside in a directory"); - - // Parse the package name from the manifest - let Some(name) = manifest.parsed.project.name.clone() else { - miette::bail!("a 'name' field is required in the project manifest"); - }; - let name = PackageName::from_str(&name).into_diagnostic()?; - - // Parse the package version from the manifest. The version is optional, so we - // default to "0dev0" if it is not present. - let version = manifest - .parsed - .project - .version - .clone() - .unwrap_or_else(|| Version::from_str("0dev0").unwrap()); - - // TODO: NoArchType??? - let noarch_type = NoArchType::python(); - - // TODO: Read from config / project. - let requirements = requirements_from_manifest(&manifest, &channel_config); - let build_platform = Platform::current(); - let build_number = 0; - - Ok(Recipe { - schema_version: 1, - package: Package { - version: VersionWithSource::from(version), - name, - }, - cache: None, - source: vec![Source::Path(PathSource { - // TODO: How can we use a git source? - path: manifest_root.to_path_buf(), - sha256: None, - md5: None, - patches: vec![], - target_directory: None, - file_name: None, - use_gitignore: true, - })], - build: Build { - number: build_number, - string: Default::default(), - - // skip: Default::default(), - script: ScriptContent::Commands( - if build_platform.is_windows() { - vec![ - "%PYTHON% -m pip install --ignore-installed --no-deps --no-build-isolation . -vv".to_string(), - "if errorlevel 1 exit 1".to_string()] - } else { - vec!["$PYTHON -m pip install --ignore-installed --no-deps --no-build-isolation . -vv".to_string()] - }) - .into(), - noarch: noarch_type, - - // TODO: Python is not exposed properly - //python: Default::default(), - // dynamic_linking: Default::default(), - // always_copy_files: Default::default(), - // always_include_files: Default::default(), - // merge_build_and_host_envs: false, - // variant: Default::default(), - // prefix_detection: Default::default(), - // post_process: vec![], - // files: Default::default(), - ..Build::default() - }, - // TODO read from manifest - requirements, - tests: vec![], - about: Default::default(), - extra: Default::default(), - }) -} - -/// Get the requirements for a default feature -fn requirements_from_manifest(manifest: &Manifest, channel_config: &ChannelConfig) -> Requirements { - let mut requirements = Requirements::default(); - let default_features = vec![manifest.default_feature()]; - - // Get all different feature types - let run_dependencies = Dependencies::from( - default_features - .iter() - .filter_map(|f| f.dependencies(Some(SpecType::Run), None)), - ); - let mut host_dependencies = Dependencies::from( - default_features - .iter() - .filter_map(|f| f.dependencies(Some(SpecType::Host), None)), - ); - let build_dependencies = Dependencies::from( - default_features - .iter() - .filter_map(|f| f.dependencies(Some(SpecType::Build), None)), - ); - - // Ensure python and pip are available in the host dependencies section. - for pkg_name in ["pip", "python"] { - if host_dependencies.contains_key(pkg_name) { - // If the host dependencies already contain the package, we don't need to add it - // again. - continue; - } - - if let Some(run_requirements) = run_dependencies.get(pkg_name) { - // Copy the run requirements to the host requirements. - for req in run_requirements { - host_dependencies.insert(PackageName::from_str(pkg_name).unwrap(), req.clone()); - } - } else { - host_dependencies.insert( - PackageName::from_str(pkg_name).unwrap(), - PixiSpec::default(), - ); - } - } - - requirements.build = dependencies_into_matchspecs(build_dependencies, channel_config) - .into_iter() - .map(Dependency::Spec) - .collect(); - requirements.host = dependencies_into_matchspecs(host_dependencies, channel_config) - .into_iter() - .map(Dependency::Spec) - .collect(); - requirements.run = dependencies_into_matchspecs(run_dependencies, channel_config) - .into_iter() - .map(Dependency::Spec) - .collect(); - - requirements -} - -fn channels_from_manifest(manifest: &Manifest, channel_config: &ChannelConfig) -> Vec { - // TODO: Improve - manifest - .parsed - .project - .channels - .iter() - .map(|c| c.channel.clone().into_base_url(channel_config)) - .collect() -} - -fn dependencies_into_matchspecs( - deps: Dependencies, - channel_config: &ChannelConfig, -) -> Vec { - deps.into_specs() - .filter_map(|(name, spec)| { - spec.try_into_nameless_match_spec(channel_config) - .expect("failed to convert spec into match spec") - .map(|spec| MatchSpec::from_nameless(spec, Some(name))) + let backend = PythonBuildBackend::new(manifest_path, logging_output_handler)?; + let result = backend + .build_conda(CondaBuildParams { + target_platform: None, + channel_base_urls: None, + channel_configuration: ChannelConfiguration { + base_url: channel_config.channel_alias, + }, + output: CondaOutputIdentifier::default(), }) - .collect() -} - -/// A helper struct that owns a temporary file containing a rendered recipe. -/// If `finish` is not called, the temporary file will stay on disk for -/// debugging purposes. -struct TemporaryRenderedRecipe { - file: PathBuf, -} - -impl TemporaryRenderedRecipe { - pub fn from_output(output: &Output) -> miette::Result { - // Ensure that the output directory exists - std::fs::create_dir_all(&output.build_configuration.directories.output_dir) - .into_diagnostic() - .context("failed to create output directory")?; - - let (recipe_file, recipe_path) = tempfile::Builder::new() - .prefix(".rendered-recipe") - .suffix(".yaml") - .tempfile_in(&output.build_configuration.directories.output_dir) - .into_diagnostic() - .context("failed to create temporary file for recipe")? - .into_parts(); - - // Write the recipe back to a file - serde_yaml::to_writer(BufWriter::new(recipe_file), &output.recipe) - .into_diagnostic() - .context("failed to write recipe to temporary file")?; - - Ok(Self { - file: recipe_path.keep().unwrap(), - }) - } + .await?; - pub fn within_context miette::Result>( - self, - operation: F, - ) -> miette::Result { - let result = operation()?; - std::fs::remove_file(self.file) - .into_diagnostic() - .context("failed to remove temporary recipe file")?; - Ok(result) - } + eprintln!("Successfully build '{}'", result.path.display()); - pub async fn within_context_async< - R, - Fut: Future>, - F: FnOnce() -> Fut, - >( - self, - operation: F, - ) -> miette::Result { - let result = operation().await?; - std::fs::remove_file(self.file) - .into_diagnostic() - .context("failed to remove temporary recipe file")?; - Ok(result) - } + Ok(()) } diff --git a/crates/pixi-build-python/src/protocol.rs b/crates/pixi-build-python/src/protocol.rs new file mode 100644 index 0000000..c06341f --- /dev/null +++ b/crates/pixi-build-python/src/protocol.rs @@ -0,0 +1,34 @@ +use pixi_build_types::procedures::{ + conda_build::{CondaBuildParams, CondaBuildResult}, + conda_metadata::{CondaMetadataParams, CondaMetadataResult}, + initialize::{InitializeParams, InitializeResult}, +}; + +/// A trait that is used to initialize a new protocol connection. +#[async_trait::async_trait] +pub trait ProtocolFactory: Send + Sync + 'static { + type Protocol: Protocol + Send + Sync + 'static; + + /// Called when the client requests initialization. + async fn initialize( + &self, + params: InitializeParams, + ) -> miette::Result<(Self::Protocol, InitializeResult)>; +} + +/// A trait that defines the protocol for a pixi build backend. +#[async_trait::async_trait] +pub trait Protocol { + /// Called when the client requests metadata for a Conda package. + async fn get_conda_metadata( + &self, + _params: CondaMetadataParams, + ) -> miette::Result { + unimplemented!("get_conda_metadata not implemented"); + } + + /// Called when the client requests to build a Conda package. + async fn build_conda(&self, _params: CondaBuildParams) -> miette::Result { + unimplemented!("build_conda not implemented"); + } +} diff --git a/crates/pixi-build-python/src/python.rs b/crates/pixi-build-python/src/python.rs new file mode 100644 index 0000000..b01a0de --- /dev/null +++ b/crates/pixi-build-python/src/python.rs @@ -0,0 +1,446 @@ +use std::{collections::BTreeMap, path::Path, str::FromStr, sync::Arc}; + +use chrono::Utc; +use miette::{Context, IntoDiagnostic}; +use pixi_build_types::procedures::conda_build::CondaBuildParams; +use pixi_build_types::procedures::conda_metadata::{CondaMetadataParams, CondaMetadataResult}; +use pixi_build_types::{ + procedures::{ + conda_build::CondaBuildResult, + initialize::{InitializeParams, InitializeResult}, + }, + BackendCapabilities, CondaPackageMetadata, FrontendCapabilities, +}; +use pixi_manifest::{Dependencies, Manifest, SpecType}; +use pixi_spec::PixiSpec; +use rattler_build::render::resolved_dependencies::DependencyInfo; +use rattler_build::{ + build::run_build, + console_utils::LoggingOutputHandler, + hash::HashInfo, + metadata::{BuildConfiguration, Directories, Output, PackagingSettings}, + recipe::{ + parser::{Build, Dependency, Package, PathSource, Requirements, ScriptContent, Source}, + Recipe, + }, + tool_configuration::Configuration, +}; +use rattler_conda_types::{ + package::ArchiveType, ChannelConfig, MatchSpec, NoArchType, PackageName, Platform, Version, +}; +use rattler_package_streaming::write::CompressionLevel; +use reqwest::Url; +use tempfile::tempdir; + +use crate::{ + protocol::{Protocol, ProtocolFactory}, + temporary_recipe::TemporaryRenderedRecipe, +}; + +pub struct PythonBuildBackend { + logging_output_handler: LoggingOutputHandler, + manifest: Manifest, +} + +impl PythonBuildBackend { + /// Returns a new instance of [`PythonBuildBackendFactory`]. + /// + /// This type implements [`ProtocolFactory`] and can be used to initialize a + /// new [`PythonBuildBackend`]. + pub fn factory(logging_output_handler: LoggingOutputHandler) -> PythonBuildBackendFactory { + PythonBuildBackendFactory { + logging_output_handler, + } + } + + /// Returns a new instance of [`PythonBuildBackend`] by reading the manifest + /// at the given path. + pub fn new( + manifest_path: &Path, + logging_output_handler: LoggingOutputHandler, + ) -> miette::Result { + // Load the manifest from the source directory + let manifest = Manifest::from_path(manifest_path).with_context(|| { + format!("failed to parse manifest from {}", manifest_path.display()) + })?; + + Ok(Self { + manifest, + logging_output_handler, + }) + } + + /// Returns the capabilities of this backend based on the capabilities of + /// the frontend. + pub fn capabilites( + &self, + _frontend_capabilities: &FrontendCapabilities, + ) -> BackendCapabilities { + BackendCapabilities { + provides_conda_metadata: Some(true), + provides_conda_build: Some(true), + } + } + + /// Returns the root directory of the pixi project. + pub fn manifest_root(&self) -> &Path { + self.manifest + .path + .parent() + .expect("manifest should always reside in a directory") + } + + /// Returns the channels from the manifest. + fn channels(&self, channel_config: &ChannelConfig) -> Vec { + self.manifest + .parsed + .project + .channels + .iter() + .map(|c| c.channel.clone().into_base_url(channel_config)) + .collect() + } + + /// Returns the requirements of the project that should be used for a + /// recipe. + fn requirements(&self, channel_config: &ChannelConfig) -> Requirements { + fn dependencies_into_matchspecs( + deps: Dependencies, + channel_config: &ChannelConfig, + ) -> Vec { + deps.into_specs() + .filter_map(|(name, spec)| { + spec.try_into_nameless_match_spec(channel_config) + .expect("failed to convert spec into match spec") + .map(|spec| MatchSpec::from_nameless(spec, Some(name))) + }) + .collect() + } + + let mut requirements = Requirements::default(); + let default_features = [self.manifest.default_feature()]; + + // Get all different feature types + let run_dependencies = Dependencies::from( + default_features + .iter() + .filter_map(|f| f.dependencies(Some(SpecType::Run), None)), + ); + let mut host_dependencies = Dependencies::from( + default_features + .iter() + .filter_map(|f| f.dependencies(Some(SpecType::Host), None)), + ); + let build_dependencies = Dependencies::from( + default_features + .iter() + .filter_map(|f| f.dependencies(Some(SpecType::Build), None)), + ); + + // Ensure python and pip are available in the host dependencies section. + for pkg_name in ["pip", "python"] { + if host_dependencies.contains_key(pkg_name) { + // If the host dependencies already contain the package, we don't need to add it + // again. + continue; + } + + if let Some(run_requirements) = run_dependencies.get(pkg_name) { + // Copy the run requirements to the host requirements. + for req in run_requirements { + host_dependencies.insert(PackageName::from_str(pkg_name).unwrap(), req.clone()); + } + } else { + host_dependencies.insert( + PackageName::from_str(pkg_name).unwrap(), + PixiSpec::default(), + ); + } + } + + requirements.build = dependencies_into_matchspecs(build_dependencies, channel_config) + .into_iter() + .map(Dependency::Spec) + .collect(); + requirements.host = dependencies_into_matchspecs(host_dependencies, channel_config) + .into_iter() + .map(Dependency::Spec) + .collect(); + requirements.run = dependencies_into_matchspecs(run_dependencies, channel_config) + .into_iter() + .map(Dependency::Spec) + .collect(); + + requirements + } + + /// Constructs a [`Recipe`] from the current manifest. + fn recipe(&self, channel_config: &ChannelConfig) -> miette::Result { + let manifest_root = self + .manifest + .path + .parent() + .expect("the project manifest must reside in a directory"); + + // Parse the package name from the manifest + let Some(name) = self.manifest.parsed.project.name.clone() else { + miette::bail!("a 'name' field is required in the project manifest"); + }; + let name = PackageName::from_str(&name).into_diagnostic()?; + + // Parse the package version from the manifest. The version is optional, so we + // default to "0dev0" if it is not present. + let version = self + .manifest + .parsed + .project + .version + .clone() + .unwrap_or_else(|| Version::from_str("0dev0").unwrap()); + + // TODO: NoArchType??? + let noarch_type = NoArchType::python(); + + // TODO: Read from config / project. + let requirements = self.requirements(channel_config); + let build_platform = Platform::current(); + let build_number = 0; + + Ok(Recipe { + schema_version: 1, + package: Package { + version: version.into(), + name, + }, + cache: None, + source: vec![Source::Path(PathSource { + // TODO: How can we use a git source? + path: manifest_root.to_path_buf(), + sha256: None, + md5: None, + patches: vec![], + target_directory: None, + file_name: None, + use_gitignore: true, + })], + build: Build { + number: build_number, + string: Default::default(), + + // skip: Default::default(), + script: ScriptContent::Commands( + if build_platform.is_windows() { + vec![ + "%PYTHON% -m pip install --ignore-installed --no-deps --no-build-isolation . -vv".to_string(), + "if errorlevel 1 exit 1".to_string()] + } else { + vec!["$PYTHON -m pip install --ignore-installed --no-deps --no-build-isolation . -vv".to_string()] + }) + .into(), + noarch: noarch_type, + + // TODO: Python is not exposed properly + //python: Default::default(), + // dynamic_linking: Default::default(), + // always_copy_files: Default::default(), + // always_include_files: Default::default(), + // merge_build_and_host_envs: false, + // variant: Default::default(), + // prefix_detection: Default::default(), + // post_process: vec![], + // files: Default::default(), + ..Build::default() + }, + // TODO read from manifest + requirements, + tests: vec![], + about: Default::default(), + extra: Default::default(), + }) + } + + /// Returns the build configuration for a recipe + pub async fn build_configuration( + &self, + recipe: &Recipe, + channels: Vec, + ) -> miette::Result { + // Parse the package name from the manifest + let Some(name) = self.manifest.parsed.project.name.clone() else { + miette::bail!("a 'name' field is required in the project manifest"); + }; + let name = PackageName::from_str(&name).into_diagnostic()?; + + // TODO: Setup defaults + let output_dir = tempdir() + .into_diagnostic() + .context("failed to create temporary directory")?; + std::fs::create_dir_all(&output_dir) + .into_diagnostic() + .context("failed to create output directory")?; + let directories = Directories::setup( + name.as_normalized(), + self.manifest.path.as_path(), + output_dir.path(), + false, + &Utc::now(), + ) + .into_diagnostic() + .context("failed to setup build directories")?; + + let host_platform = Platform::current(); + let build_platform = Platform::current(); + + let variant = BTreeMap::new(); + + Ok(BuildConfiguration { + // TODO: NoArch?? + target_platform: Platform::NoArch, + host_platform, + build_platform, + hash: HashInfo::from_variant(&variant, &recipe.build.noarch), + variant, + directories, + channels, + channel_priority: Default::default(), + solve_strategy: Default::default(), + timestamp: chrono::Utc::now(), + subpackages: Default::default(), // TODO: ??? + packaging_settings: PackagingSettings::from_args( + ArchiveType::Conda, + CompressionLevel::default(), + ), + store_recipe: true, + force_colors: true, + }) + } +} + +#[async_trait::async_trait] +impl Protocol for PythonBuildBackend { + async fn get_conda_metadata( + &self, + params: CondaMetadataParams, + ) -> miette::Result { + let channel_config = ChannelConfig { + channel_alias: params.channel_configuration.base_url, + root_dir: self.manifest_root().to_path_buf(), + }; + let channels = params + .channel_base_urls + .unwrap_or_else(|| self.channels(&channel_config)); + + // TODO: Determine how and if we can determine this from the manifest. + let recipe = self.recipe(&channel_config)?; + let output = Output { + build_configuration: self.build_configuration(&recipe, channels).await?, + recipe, + finalized_dependencies: None, + finalized_cache_dependencies: None, + finalized_sources: None, + build_summary: Arc::default(), + system_tools: Default::default(), + extra_meta: None, + }; + let tool_config = Configuration::builder() + .with_logging_output_handler(self.logging_output_handler.clone()) + .with_channel_config(channel_config.clone()) + .finish(); + + let temp_recipe = TemporaryRenderedRecipe::from_output(&output)?; + let output = temp_recipe + .within_context_async(move || async move { + output + .resolve_dependencies(&tool_config) + .await + .into_diagnostic() + }) + .await?; + + let finalized_deps = &output + .finalized_dependencies + .as_ref() + .expect("dependencies should be resolved at this point") + .run; + + Ok(CondaMetadataResult { + packages: vec![CondaPackageMetadata { + name: output.name().clone(), + version: output.version().clone().into(), + build: output.build_string().into_owned(), + build_number: output.recipe.build.number, + subdir: output.build_configuration.target_platform, + depends: finalized_deps + .depends + .iter() + .map(DependencyInfo::spec) + .cloned() + .collect(), + constraints: finalized_deps + .constraints + .iter() + .map(DependencyInfo::spec) + .cloned() + .collect(), + license: output.recipe.about.license.map(|l| l.to_string()), + license_family: output.recipe.about.license_family, + noarch: output.recipe.build.noarch, + }], + }) + } + + async fn build_conda(&self, params: CondaBuildParams) -> miette::Result { + let channel_config = ChannelConfig { + channel_alias: params.channel_configuration.base_url, + root_dir: self.manifest_root().to_path_buf(), + }; + let channels = params + .channel_base_urls + .unwrap_or_else(|| self.channels(&channel_config)); + + let recipe = self.recipe(&channel_config)?; + let output = Output { + build_configuration: self.build_configuration(&recipe, channels).await?, + recipe, + finalized_dependencies: None, + finalized_cache_dependencies: None, + finalized_sources: None, + build_summary: Arc::default(), + system_tools: Default::default(), + extra_meta: None, + }; + let tool_config = Configuration::builder() + .with_logging_output_handler(self.logging_output_handler.clone()) + .with_channel_config(channel_config.clone()) + .finish(); + + let temp_recipe = TemporaryRenderedRecipe::from_output(&output)?; + let (_output, package) = temp_recipe + .within_context_async(move || async move { run_build(output, &tool_config).await }) + .await?; + + Ok(CondaBuildResult { path: package }) + } +} + +pub struct PythonBuildBackendFactory { + logging_output_handler: LoggingOutputHandler, +} + +#[async_trait::async_trait] +impl ProtocolFactory for PythonBuildBackendFactory { + type Protocol = PythonBuildBackend; + + async fn initialize( + &self, + params: InitializeParams, + ) -> miette::Result<(Self::Protocol, InitializeResult)> { + let instance = PythonBuildBackend::new( + params.manifest_path.as_path(), + self.logging_output_handler.clone(), + )?; + + let capabilities = instance.capabilites(¶ms.capabilities); + Ok((instance, InitializeResult { capabilities })) + } +} diff --git a/crates/pixi-build-python/src/server.rs b/crates/pixi-build-python/src/server.rs new file mode 100644 index 0000000..c357600 --- /dev/null +++ b/crates/pixi-build-python/src/server.rs @@ -0,0 +1,136 @@ +use std::{net::SocketAddr, sync::Arc}; + +use jsonrpc_core::{serde_json, to_value, Error, IoHandler, Params}; +use miette::{IntoDiagnostic, JSONReportHandler}; +use pixi_build_types::{ + procedures, + procedures::{ + conda_build::CondaBuildParams, conda_metadata::CondaMetadataParams, + initialize::InitializeParams, + }, +}; +use tokio::sync::RwLock; + +use crate::protocol::{Protocol, ProtocolFactory}; + +/// A JSONRPC server that can be used to communicate with a client. +pub struct Server { + factory: T, +} + +enum ServerState { + Uninitialized(T), + Initialized(T::Protocol), +} + +impl ServerState { + pub fn as_protocol(&self) -> Result<&T::Protocol, jsonrpc_core::Error> { + match self { + Self::Initialized(protocol) => Ok(protocol), + _ => Err(Error::invalid_request()), + } + } +} + +impl Server { + pub fn new(factory: T) -> Self { + Self { factory } + } + + pub async fn run(self) -> miette::Result<()> { + let io = self.setup_io(); + jsonrpc_stdio_server::ServerBuilder::new(io).build().await; + Ok(()) + } + + pub fn run_over_http(self, port: u16) -> miette::Result<()> { + let io = self.setup_io(); + jsonrpc_http_server::ServerBuilder::new(io) + .start_http(&SocketAddr::from(([127, 0, 0, 1], port))) + .into_diagnostic()? + .wait(); + Ok(()) + } + + fn setup_io(self) -> IoHandler { + // Construct a server + let mut io = IoHandler::new(); + let state = Arc::new(RwLock::new(ServerState::Uninitialized(self.factory))); + + let initialize_state = state.clone(); + io.add_method( + procedures::initialize::METHOD_NAME, + move |params: Params| { + let state = initialize_state.clone(); + + async move { + let params: InitializeParams = params.parse()?; + let mut state = state.write().await; + let ServerState::Uninitialized(factory) = &mut *state else { + return Err(Error::invalid_request()); + }; + + let (protocol, result) = + factory.initialize(params).await.map_err(convert_error)?; + *state = ServerState::Initialized(protocol); + + Ok(to_value(result).expect("failed to convert to json")) + } + }, + ); + + let conda_get_metadata = state.clone(); + io.add_method( + procedures::conda_metadata::METHOD_NAME, + move |params: Params| { + let state = conda_get_metadata.clone(); + + async move { + let params: CondaMetadataParams = params.parse()?; + let state = state.read().await; + state + .as_protocol()? + .get_conda_metadata(params) + .await + .map(|value| to_value(value).expect("failed to convert to json")) + .map_err(convert_error) + } + }, + ); + + let conda_build = state.clone(); + io.add_method( + procedures::conda_metadata::METHOD_NAME, + move |params: Params| { + let state = conda_build.clone(); + + async move { + let params: CondaBuildParams = params.parse()?; + let state = state.read().await; + state + .as_protocol()? + .build_conda(params) + .await + .map(|value| to_value(value).expect("failed to convert to json")) + .map_err(convert_error) + } + }, + ); + + io + } +} + +fn convert_error(err: miette::Report) -> jsonrpc_core::Error { + let rendered = JSONReportHandler::new(); + let mut json_str = String::new(); + rendered + .render_report(&mut json_str, err.as_ref()) + .expect("failed to convert error to json"); + let data = serde_json::from_str(&json_str).expect("failed to parse json error"); + jsonrpc_core::Error { + code: jsonrpc_core::ErrorCode::ServerError(-32000), + message: err.to_string(), + data: Some(data), + } +} diff --git a/crates/pixi-build-python/src/temporary_recipe.rs b/crates/pixi-build-python/src/temporary_recipe.rs new file mode 100644 index 0000000..cd2dba6 --- /dev/null +++ b/crates/pixi-build-python/src/temporary_recipe.rs @@ -0,0 +1,53 @@ +use miette::{IntoDiagnostic, WrapErr}; +use rattler_build::metadata::Output; +use std::future::Future; +use std::io::BufWriter; +use std::path::PathBuf; + +/// A helper struct that owns a temporary file containing a rendered recipe. +/// If `finish` is not called, the temporary file will stay on disk for +/// debugging purposes. +pub struct TemporaryRenderedRecipe { + file: PathBuf, +} + +impl TemporaryRenderedRecipe { + pub fn from_output(output: &Output) -> miette::Result { + // Ensure that the output directory exists + std::fs::create_dir_all(&output.build_configuration.directories.output_dir) + .into_diagnostic() + .context("failed to create output directory")?; + + let (recipe_file, recipe_path) = tempfile::Builder::new() + .prefix(".rendered-recipe") + .suffix(".yaml") + .tempfile_in(&output.build_configuration.directories.output_dir) + .into_diagnostic() + .context("failed to create temporary file for recipe")? + .into_parts(); + + // Write the recipe back to a file + serde_yaml::to_writer(BufWriter::new(recipe_file), &output.recipe) + .into_diagnostic() + .context("failed to write recipe to temporary file")?; + + Ok(Self { + file: recipe_path.keep().unwrap(), + }) + } + + pub async fn within_context_async< + R, + Fut: Future>, + F: FnOnce() -> Fut, + >( + self, + operation: F, + ) -> miette::Result { + let result = operation().await?; + std::fs::remove_file(self.file) + .into_diagnostic() + .context("failed to remove temporary recipe file")?; + Ok(result) + } +}