From bce8106bd3c96b83f46c5032d532799efa375726 Mon Sep 17 00:00:00 2001 From: sougata-progress Date: Mon, 9 Sep 2024 21:44:49 +0000 Subject: [PATCH 1/2] Enables native packages by default Signed-off-by: sougata-progress --- components/builder-api/src/config.rs | 47 +++++++++---------- components/builder-api/src/server/mod.rs | 8 ++-- .../builder-api/src/server/resources/pkgs.rs | 34 -------------- test/builder-api/src/packages.js | 13 ----- 4 files changed, 24 insertions(+), 78 deletions(-) diff --git a/components/builder-api/src/config.rs b/components/builder-api/src/config.rs index b9a7b6ecb..8c71e3673 100644 --- a/components/builder-api/src/config.rs +++ b/components/builder-api/src/config.rs @@ -97,19 +97,18 @@ impl Default for S3Cfg { #[derive(Debug, Clone, Deserialize)] #[serde(default)] pub struct ApiCfg { - pub data_path: PathBuf, - pub log_path: PathBuf, + pub data_path: PathBuf, + pub log_path: PathBuf, /// Location of Builder encryption keys - pub key_path: KeyCache, - pub targets: Vec, - pub build_targets: Vec, + pub key_path: KeyCache, + pub targets: Vec, + pub build_targets: Vec, #[serde(with = "deserialize_into_vec")] - pub features_enabled: Vec, - pub build_on_upload: bool, - pub private_max_age: usize, - pub saas_bldr_url: String, + pub features_enabled: Vec, + pub build_on_upload: bool, + pub private_max_age: usize, + pub saas_bldr_url: String, pub suppress_autobuild_origins: Vec, - pub allowed_native_package_origins: Vec, } mod deserialize_into_vec { @@ -127,19 +126,18 @@ mod deserialize_into_vec { impl Default for ApiCfg { fn default() -> Self { - ApiCfg { data_path: PathBuf::from("/hab/svc/builder-api/data"), - log_path: env::temp_dir(), - key_path: KeyCache::new("/hab/svc/builder-api/files"), - targets: vec![target::X86_64_LINUX, - target::X86_64_LINUX_KERNEL2, - target::X86_64_WINDOWS,], - build_targets: vec![target::X86_64_LINUX, target::X86_64_WINDOWS], - features_enabled: vec!["jobsrv".to_string()], - build_on_upload: true, - private_max_age: 300, - saas_bldr_url: "https://bldr.habitat.sh".to_string(), - suppress_autobuild_origins: vec![], - allowed_native_package_origins: vec![], } + ApiCfg { data_path: PathBuf::from("/hab/svc/builder-api/data"), + log_path: env::temp_dir(), + key_path: KeyCache::new("/hab/svc/builder-api/files"), + targets: vec![target::X86_64_LINUX, + target::X86_64_LINUX_KERNEL2, + target::X86_64_WINDOWS,], + build_targets: vec![target::X86_64_LINUX, target::X86_64_WINDOWS], + features_enabled: vec!["jobsrv".to_string()], + build_on_upload: true, + private_max_age: 300, + saas_bldr_url: "https://bldr.habitat.sh".to_string(), + suppress_autobuild_origins: vec![], } } } @@ -343,7 +341,6 @@ mod tests { build_on_upload = false private_max_age = 400 suppress_autobuild_origins = ["origin1", "origin2"] - allowed_native_package_origins = [] [http] listen = "0:0:0:0:0:0:0:1" @@ -433,8 +430,6 @@ mod tests { assert_eq!(&config.api.suppress_autobuild_origins, &["origin1".to_string(), "origin2".to_string()]); - assert_eq!(config.api.allowed_native_package_origins.len(), 0); - assert_eq!(&format!("{}", config.jobsrv), "http://1.2.3.4:1234"); assert_eq!(config.http.port, 9636); diff --git a/components/builder-api/src/server/mod.rs b/components/builder-api/src/server/mod.rs index 821872182..045f3853f 100644 --- a/components/builder-api/src/server/mod.rs +++ b/components/builder-api/src/server/mod.rs @@ -68,8 +68,7 @@ features! { const Jobsrv = 0b0000_0010, const LegacyProject = 0b0000_0011, const Artifactory = 0b0000_0100, - const BuildDeps = 0b0000_1000, - const NativePackages = 0b0001_0000 + const BuildDeps = 0b0000_1000 } } @@ -106,9 +105,8 @@ fn enable_features(config: &Config) { ("JOBSRV", feat::Jobsrv), ("LEGACYPROJECT", feat::LegacyProject), ("ARTIFACTORY", feat::Artifactory), - ("BUILDDEPS", feat::BuildDeps), - ("NATIVEPACKAGES", - feat::NativePackages),]); + ("BUILDDEPS", feat::BuildDeps),]); + for key in &config.api.features_enabled { if features.contains_key(key.as_str()) { info!("Enabling feature: {}", key); diff --git a/components/builder-api/src/server/resources/pkgs.rs b/components/builder-api/src/server/resources/pkgs.rs index fcf574114..5e3a8caae 100644 --- a/components/builder-api/src/server/resources/pkgs.rs +++ b/components/builder-api/src/server/resources/pkgs.rs @@ -14,7 +14,6 @@ use crate::{bldr_core::{error::Error::RpcError, metrics::CounterMetric}, - config::Config, db::models::{channel::{Channel, ChannelWithPromotion}, origin::*, @@ -128,18 +127,6 @@ pub struct OriginScheduleStatus { limit: String, } -fn enabled_native_upload() -> bool { feat::is_enabled(feat::NativePackages) } - -fn is_origin_allowed(origin: &str, config: &Config) -> bool { - if config.api - .allowed_native_package_origins - .contains(&origin.to_string()) - { - return true; - } - false -} - pub struct Packages {} impl Packages { @@ -1126,27 +1113,6 @@ async fn do_upload_package_finish(req: &HttpRequest, } }; - if package_type == PackageType::Native && !enabled_native_upload() { - debug!("Unsupported package type {}.", package_type); - let body = Bytes::from( - format!("Uploading '{}' package is not supported", package_type).into_bytes(), - ); - let body = BoxBody::new(body); - return HttpResponse::with_body(StatusCode::UNPROCESSABLE_ENTITY, body); - }; - - if package_type == PackageType::Native - && !is_origin_allowed(&ident.origin, &req_state(req).config) - { - debug!("Native packages not allowed for the origin {:?}", - ident.origin.clone()); - let body = Bytes::from(format!("Native Package upload for the origin '{}' is not \ - supported", - ident.origin.clone()).into_bytes()); - let body = BoxBody::new(body); - return HttpResponse::with_body(StatusCode::UNPROCESSABLE_ENTITY, body); - }; - let target_from_artifact = match archive.target() { Ok(target) => target, Err(e) => { diff --git a/test/builder-api/src/packages.js b/test/builder-api/src/packages.js index a47eb95a8..7836a489e 100644 --- a/test/builder-api/src/packages.js +++ b/test/builder-api/src/packages.js @@ -230,19 +230,6 @@ describe('Working with packages', function () { }); }); - it('upload does not allow native packages until feature is set', function (done) { - request.post(`/depot/pkgs/neurosis/native-testapp/0.1.0/${release13}`) - .set('Authorization', global.boboBearer) - .set('Content-Length', file13.length) - .query({ checksum: '31d158dad181b3370f00efe39ed9fca5dcbef53d5666657f87c814812cd59f09' }) - .send(file13) - .expect(422) - .end(function (err, res) { - expect(res.text).to.equal(`Uploading \'native\' package is not supported`); - done(err); - }); - }); - // Test weird versions it('uploads a unusual versioned package five', function (done) { From dd9cb42319e5091b81201e70aa0dbdaf2056e952 Mon Sep 17 00:00:00 2001 From: sougata-progress Date: Thu, 12 Sep 2024 14:42:10 +0000 Subject: [PATCH 2/2] Updated tests Signed-off-by: sougata-progress --- test/builder-api/src/channels.js | 22 ++--- test/builder-api/src/packages.js | 149 ++++++++++++++++++------------- 2 files changed, 100 insertions(+), 71 deletions(-) diff --git a/test/builder-api/src/channels.js b/test/builder-api/src/channels.js index 9b19288bf..072f27ad2 100644 --- a/test/builder-api/src/channels.js +++ b/test/builder-api/src/channels.js @@ -686,19 +686,19 @@ describe('Channels API', function () { .end(function (err, res) { expect(res.body.channel).to.equal('unstable'); expect(res.body.target).to.equal('x86_64-linux'); - expect(res.body.data.length).to.equal(12); + expect(res.body.data.length).to.equal(13); expect(res.body.data[0].origin).to.equal('neurosis'); expect(res.body.data[0].name).to.equal('abracadabra'); expect(res.body.data[0].version).to.equal('3.0'); expect(res.body.data[0].release).to.equal('20190618175235'); - expect(res.body.data[10].origin).to.equal('neurosis'); - expect(res.body.data[10].name).to.equal('testapp'); - expect(res.body.data[10].version).to.equal('0.1.13'); - expect(res.body.data[10].release).to.equal('20190511004436'); expect(res.body.data[11].origin).to.equal('neurosis'); - expect(res.body.data[11].name).to.equal('testapp2'); - expect(res.body.data[11].version).to.equal('v1.2.3-master'); - expect(res.body.data[11].release).to.equal('20181018162212'); + expect(res.body.data[11].name).to.equal('testapp'); + expect(res.body.data[11].version).to.equal('0.1.13'); + expect(res.body.data[11].release).to.equal('20190511004436'); + expect(res.body.data[12].origin).to.equal('neurosis'); + expect(res.body.data[12].name).to.equal('testapp2'); + expect(res.body.data[12].version).to.equal('v1.2.3-master'); + expect(res.body.data[12].release).to.equal('20181018162212'); done(err); }); }); @@ -904,9 +904,9 @@ describe('Channels API', function () { .expect(200) .end(function (err, res) { expect(res.body.range_start).to.equal(0); - expect(res.body.range_end).to.equal(30); - expect(res.body.total_count).to.equal(31); - expect(res.body.data.length).to.equal(31); + expect(res.body.range_end).to.equal(31); + expect(res.body.total_count).to.equal(32); + expect(res.body.data.length).to.equal(32); done(err); }); }); diff --git a/test/builder-api/src/packages.js b/test/builder-api/src/packages.js index 7836a489e..977fe0068 100644 --- a/test/builder-api/src/packages.js +++ b/test/builder-api/src/packages.js @@ -230,6 +230,19 @@ describe('Working with packages', function () { }); }); + it('allows native packages to be uploaded', function (done) { + request.post(`/depot/pkgs/neurosis/native-testapp/0.1.0/${release13}`) + .set('Authorization', global.boboBearer) + .set('Content-Length', file13.length) + .query({ checksum: '31d158dad181b3370f00efe39ed9fca5dcbef53d5666657f87c814812cd59f09' }) + .send(file13) + .expect(201) + .end(function (err, res) { + if (err) return done(err); + expect(res.text).to.include(`/pkgs/neurosis/native-testapp/0.1.0/${release13}/download`); + done(); + }); + }); // Test weird versions it('uploads a unusual versioned package five', function (done) { @@ -379,27 +392,31 @@ describe('Working with packages', function () { .expect(200) .end(function (err, res) { expect(res.body.range_start).to.equal(0); - expect(res.body.range_end).to.equal(8); - expect(res.body.total_count).to.equal(9); - expect(res.body.data.length).to.equal(9); + expect(res.body.range_end).to.equal(9); + expect(res.body.total_count).to.equal(10); + expect(res.body.data.length).to.equal(10); expect(res.body.data[0].origin).to.equal('neurosis'); - expect(res.body.data[0].name).to.equal('testapp'); - expect(res.body.data[0].version).to.equal('0.1.13'); - expect(res.body.data[0].release).to.equal(release10); - expect(res.body.data[1].version).to.equal('0.1.3'); - expect(res.body.data[1].release).to.equal(release1); - expect(res.body.data[2].origin).to.equal('neurosis'); - expect(res.body.data[2].name).to.equal('testapp'); + expect(res.body.data[0].name).to.equal('native-testapp'); + expect(res.body.data[0].version).to.equal('0.1.0'); + expect(res.body.data[0].release).to.equal(release13); + expect(res.body.data[1].origin).to.equal('neurosis'); + expect(res.body.data[1].name).to.equal('testapp'); + expect(res.body.data[1].version).to.equal('0.1.13'); + expect(res.body.data[1].release).to.equal(release10); expect(res.body.data[2].version).to.equal('0.1.3'); - expect(res.body.data[2].release).to.equal(release2); + expect(res.body.data[2].release).to.equal(release1); expect(res.body.data[3].origin).to.equal('neurosis'); expect(res.body.data[3].name).to.equal('testapp'); expect(res.body.data[3].version).to.equal('0.1.3'); - expect(res.body.data[3].release).to.equal(release8); - expect(res.body.data[8].origin).to.equal('xmen'); - expect(res.body.data[8].name).to.equal('testapp'); - expect(res.body.data[8].version).to.equal('0.1.4'); - expect(res.body.data[8].release).to.equal(release4); + expect(res.body.data[3].release).to.equal(release2); + expect(res.body.data[4].origin).to.equal('neurosis'); + expect(res.body.data[4].name).to.equal('testapp'); + expect(res.body.data[4].version).to.equal('0.1.3'); + expect(res.body.data[4].release).to.equal(release8); + expect(res.body.data[9].origin).to.equal('xmen'); + expect(res.body.data[9].name).to.equal('testapp'); + expect(res.body.data[9].version).to.equal('0.1.4'); + expect(res.body.data[9].release).to.equal(release4); done(err); }); }); @@ -411,27 +428,31 @@ describe('Working with packages', function () { .expect(200) .end(function (err, res) { expect(res.body.range_start).to.equal(0); - expect(res.body.range_end).to.equal(7); - expect(res.body.total_count).to.equal(8); - expect(res.body.data.length).to.equal(8); + expect(res.body.range_end).to.equal(8); + expect(res.body.total_count).to.equal(9); + expect(res.body.data.length).to.equal(9); expect(res.body.data[0].origin).to.equal('neurosis'); - expect(res.body.data[0].name).to.equal('testapp'); - expect(res.body.data[0].version).to.equal('0.1.13'); - expect(res.body.data[0].release).to.equal(release10); - expect(res.body.data[1].version).to.equal('0.1.3'); - expect(res.body.data[1].release).to.equal(release1); - expect(res.body.data[2].origin).to.equal('neurosis'); - expect(res.body.data[2].name).to.equal('testapp'); + expect(res.body.data[0].name).to.equal('native-testapp'); + expect(res.body.data[0].version).to.equal('0.1.0'); + expect(res.body.data[0].release).to.equal(release13); + expect(res.body.data[1].origin).to.equal('neurosis'); + expect(res.body.data[1].name).to.equal('testapp'); + expect(res.body.data[1].version).to.equal('0.1.13'); + expect(res.body.data[1].release).to.equal(release10); expect(res.body.data[2].version).to.equal('0.1.3'); - expect(res.body.data[2].release).to.equal(release2); - expect(res.body.data[6].origin).to.equal('neurosis'); - expect(res.body.data[6].name).to.equal('testapp2'); - expect(res.body.data[6].version).to.equal('v1.2.3-aaster'); - expect(res.body.data[6].release).to.equal(release6); + expect(res.body.data[2].release).to.equal(release1); + expect(res.body.data[3].origin).to.equal('neurosis'); + expect(res.body.data[3].name).to.equal('testapp'); + expect(res.body.data[3].version).to.equal('0.1.3'); + expect(res.body.data[3].release).to.equal(release2); expect(res.body.data[7].origin).to.equal('neurosis'); expect(res.body.data[7].name).to.equal('testapp2'); - expect(res.body.data[7].version).to.equal('v1.2.3-master'); - expect(res.body.data[7].release).to.equal(release5); + expect(res.body.data[7].version).to.equal('v1.2.3-aaster'); + expect(res.body.data[7].release).to.equal(release6); + expect(res.body.data[8].origin).to.equal('neurosis'); + expect(res.body.data[8].name).to.equal('testapp2'); + expect(res.body.data[8].version).to.equal('v1.2.3-master'); + expect(res.body.data[8].release).to.equal(release5); done(err); }); }); @@ -459,15 +480,17 @@ describe('Working with packages', function () { .expect(200) .end(function (err, res) { expect(res.body.range_start).to.equal(0); - expect(res.body.range_end).to.equal(2); - expect(res.body.total_count).to.equal(3); - expect(res.body.data.length).to.equal(3); + expect(res.body.range_end).to.equal(3); + expect(res.body.total_count).to.equal(4); + expect(res.body.data.length).to.equal(4); expect(res.body.data[0].origin).to.equal('neurosis'); - expect(res.body.data[0].name).to.equal('testapp'); - expect(res.body.data[1].origin).to.equal('xmen'); + expect(res.body.data[0].name).to.equal('native-testapp'); + expect(res.body.data[1].origin).to.equal('neurosis'); expect(res.body.data[1].name).to.equal('testapp'); - expect(res.body.data[2].origin).to.equal('neurosis'); - expect(res.body.data[2].name).to.equal('testapp2'); + expect(res.body.data[2].origin).to.equal('xmen'); + expect(res.body.data[2].name).to.equal('testapp'); + expect(res.body.data[3].origin).to.equal('neurosis'); + expect(res.body.data[3].name).to.equal('testapp2'); done(err); }); }); @@ -479,13 +502,15 @@ describe('Working with packages', function () { .expect(200) .end(function (err, res) { expect(res.body.range_start).to.equal(0); - expect(res.body.range_end).to.equal(1); - expect(res.body.total_count).to.equal(2); - expect(res.body.data.length).to.equal(2); + expect(res.body.range_end).to.equal(2); + expect(res.body.total_count).to.equal(3); + expect(res.body.data.length).to.equal(3); expect(res.body.data[0].origin).to.equal('neurosis'); - expect(res.body.data[0].name).to.equal('testapp'); + expect(res.body.data[0].name).to.equal('native-testapp'); expect(res.body.data[1].origin).to.equal('neurosis'); - expect(res.body.data[1].name).to.equal('testapp2'); + expect(res.body.data[1].name).to.equal('testapp'); + expect(res.body.data[2].origin).to.equal('neurosis'); + expect(res.body.data[2].name).to.equal('testapp2'); done(err); }); }); @@ -497,9 +522,9 @@ describe('Working with packages', function () { .expect(200) .end(function (err, res) { expect(res.body.range_start).to.equal(0); - expect(res.body.range_end).to.equal(9); - expect(res.body.total_count).to.equal(10); - expect(res.body.data.length).to.equal(10); + expect(res.body.range_end).to.equal(10); + expect(res.body.total_count).to.equal(11); + expect(res.body.data.length).to.equal(11); expect(res.body.data[2].origin).to.equal('neurosis'); expect(res.body.data[2].name).to.equal('testapp'); expect(res.body.data[2].version).to.equal('0.1.4'); @@ -523,17 +548,19 @@ describe('Working with packages', function () { .expect(200) .end(function (err, res) { expect(res.body.range_start).to.equal(0); - expect(res.body.range_end).to.equal(3); - expect(res.body.total_count).to.equal(4); - expect(res.body.data.length).to.equal(4); + expect(res.body.range_end).to.equal(4); + expect(res.body.total_count).to.equal(5); + expect(res.body.data.length).to.equal(5); expect(res.body.data[0].origin).to.equal('neurosis'); expect(res.body.data[0].name).to.equal('abracadabra'); expect(res.body.data[1].origin).to.equal('neurosis'); - expect(res.body.data[1].name).to.equal('neurosis'); + expect(res.body.data[1].name).to.equal('native-testapp'); expect(res.body.data[2].origin).to.equal('neurosis'); - expect(res.body.data[2].name).to.equal('testapp'); + expect(res.body.data[2].name).to.equal('neurosis'); expect(res.body.data[3].origin).to.equal('neurosis'); - expect(res.body.data[3].name).to.equal('testapp2'); + expect(res.body.data[3].name).to.equal('testapp'); + expect(res.body.data[4].origin).to.equal('neurosis'); + expect(res.body.data[4].name).to.equal('testapp2'); done(err); }); }); @@ -560,17 +587,19 @@ describe('Working with packages', function () { .expect(200) .end(function (err, res) { expect(res.body.range_start).to.equal(0); - expect(res.body.range_end).to.equal(3); - expect(res.body.total_count).to.equal(4); - expect(res.body.data.length).to.equal(4); + expect(res.body.range_end).to.equal(4); + expect(res.body.total_count).to.equal(5); + expect(res.body.data.length).to.equal(5); expect(res.body.data[0].origin).to.equal('neurosis'); expect(res.body.data[0].name).to.equal('abracadabra'); expect(res.body.data[1].origin).to.equal('neurosis'); - expect(res.body.data[1].name).to.equal('neurosis'); + expect(res.body.data[1].name).to.equal('native-testapp'); expect(res.body.data[2].origin).to.equal('neurosis'); - expect(res.body.data[2].name).to.equal('testapp'); + expect(res.body.data[2].name).to.equal('neurosis'); expect(res.body.data[3].origin).to.equal('neurosis'); - expect(res.body.data[3].name).to.equal('testapp2'); + expect(res.body.data[3].name).to.equal('testapp'); + expect(res.body.data[4].origin).to.equal('neurosis'); + expect(res.body.data[4].name).to.equal('testapp2'); done(err); }); });