From ee06b42392887d9d9e47354f06f1e4167295e947 Mon Sep 17 00:00:00 2001 From: Sebastian Riedel Date: Fri, 25 Oct 2024 17:29:45 +0200 Subject: [PATCH] Rewrite IBS embargo feature to rely on SMASH --- cavil.conf | 5 +- lib/Cavil/OBS.pm | 90 +++++++++----- t/embargo.t | 37 +++--- t/obs.t | 314 +++++++++++++++++++---------------------------- 4 files changed, 215 insertions(+), 231 deletions(-) diff --git a/cavil.conf b/cavil.conf index f13711246..036217e92 100644 --- a/cavil.conf +++ b/cavil.conf @@ -26,8 +26,9 @@ password => 'OBS_PASSWORD' }, 'api.suse.de' => { - user => 'IBS_USER', - ssh_key => 'SSH_PRIVATE_KEY_PATH' + user => 'IBS_USER', + ssh_key => 'SSH_PRIVATE_KEY_PATH', + embargoed_bugs => 'https://smash.suse.de/api/embargoed-bugs/' } }, diff --git a/lib/Cavil/OBS.pm b/lib/Cavil/OBS.pm index d9d800339..ddbc488f8 100644 --- a/lib/Cavil/OBS.pm +++ b/lib/Cavil/OBS.pm @@ -40,26 +40,53 @@ has ua => sub { }; sub check_for_embargo ($self, $api, $request) { - my $url = _url($api, 'request', $request); - my $res = $self->_get($url); + my $bugrefs = $self->get_bugrefs_for_request($api, $request); + my $embargoed_bugrefs = $self->get_embargoed_bugrefs($api); + + my %enbargoed = map { $_ => 1 } @$embargoed_bugrefs; + for my $bugref (@$bugrefs) { + return 1 if $enbargoed{$bugref}; + } + + return 0; +} + +sub get_bugrefs_for_request ($self, $api, $request) { + my $url = _url($api, 'request', $request) + ->query({cmd => 'diff', view => 'xml', withissues => 1, withdescriptionissues => 1}); + my $res = $self->_request($url, 'POST'); croak "$url: " . $res->code unless $res->is_success; - for my $project ($res->dom->find('action [project]')->map('attr', 'project')->uniq->each) { - my $url = _url($api, 'source', $project, '_attribute'); - my $res = $self->_get($url); - my $code = $res->code; - next if $code == 404; + my $dom = $res->dom; + my $bugrefs = {}; + for my $issue ($dom->find('issues > issue[label]')->each) { + my $label = $issue->{label}; + $bugrefs->{$label}++; + + # Labels in SMASH and IBS can be inconsistent with "bnc#" and "bsc#" (so we generate both) + $bugrefs->{"bnc#$1"}++ if $label =~ /^bsc#(\d+)$/; + } - # Looks like OBS has repo types that do not support attributes yet and produces a bad 400 response in such cases - # (501 is supposed to treplace it in the future) - next if $code == 400 && $res->dom->at('status[code=remote_project]'); - next if $code == 501; + return [sort keys %$bugrefs]; +} - croak "$url: " . $res->code unless $res->is_success; - return 1 if $res->dom->at('attributes attribute[name=EmbargoDate]'); +sub get_embargoed_bugrefs ($self, $api) { + my $ua = $self->ua; + + my $host = _url($api)->host; + die "Missing configuration for OBS instance: $host" unless my $config = $self->config->{$host}; + return [] unless my $embargoed_bugs = $config->{embargoed_bugs}; + + my $bugs = $ua->get($embargoed_bugs)->result->json; + my $bugrefs = {}; + for my $bug (@$bugs) { + $bugrefs->{$bug->{bug}{name}}++; + for my $cve (@{$bug->{cves}}) { + $bugrefs->{$cve->{name}}++; + } } - return 0; + return [sort keys %$bugrefs]; } sub download_source ($self, $api, $project, $pkg, $dir, $options = {}) { @@ -68,7 +95,7 @@ sub download_source ($self, $api, $project, $pkg, $dir, $options = {}) { # List files my $url = _url($api, 'source', $project, $pkg)->query(expand => 1); $url->query([rev => $options->{rev}]) if defined $options->{rev}; - my $res = $self->_get($url); + my $res = $self->_request($url); croak "$url: " . $res->code unless $res->is_success; my $dom = $res->dom; my $srcmd5 = $dom->at('directory')->{srcmd5}; @@ -82,7 +109,7 @@ sub download_source ($self, $api, $project, $pkg, $dir, $options = {}) { my $url = _url($api, 'source', $project, $pkg, $file->{name}); $url->query([expand => 1, rev => $srcmd5]); - my $res = $self->_get($url); + my $res = $self->_request($url); croak "$url: " . $res->code unless $res->is_success; my $target = $dir->child($file->{name}); $res->content->asset->move_to($target); @@ -94,7 +121,7 @@ sub download_source ($self, $api, $project, $pkg, $dir, $options = {}) { sub package_info ($self, $api, $project, $pkg, $options = {}) { my $url = _url($api, 'source', $project, $pkg)->query(view => 'info'); $url->query([rev => $options->{rev}]) if defined $options->{rev}; - my $res = $self->_get($url); + my $res = $self->_request($url); croak "$url: " . $res->code unless $res->is_success; my $source = $res->dom->at('sourceinfo'); @@ -111,7 +138,7 @@ sub _find_link_target ($self, $api, $project, $pkg, $lrev) { my $query = {expand => 1}; $query->{rev} = $lrev if defined $lrev; $url->query($query); - my $res = $self->_get($url); + my $res = $self->_request($url); return undef unless $res->is_success; # Check if we're on track @@ -126,7 +153,7 @@ sub _find_link_target ($self, $api, $project, $pkg, $lrev) { } } $url = _url($api, 'source', $project, $pkg, '_meta'); - $res = $self->_get($url); + $res = $self->_request($url); # This is severe as we already checked the sources croak "$url: " . $res->code unless $res->is_success; @@ -136,7 +163,13 @@ sub _find_link_target ($self, $api, $project, $pkg, $lrev) { return {%linfo, package => $rn->text}; } -sub _get ($self, $url) { +sub _md5 ($file) { + my $md5 = Digest::MD5->new; + $md5->addfile(path($file)->open('r')); + return $md5->hexdigest; +} + +sub _request ($self, $url, $method = 'GET') { my $ua = $self->ua; my $host = $url->host; @@ -146,27 +179,24 @@ sub _get ($self, $url) { # "api.suse.de" needs ssh authentication my $tx; if (my $ssh_key = $config->{ssh_key}) { - $tx = $ua->get($url); + $tx = $ua->start($ua->build_tx($method => $url)); my $res = $tx->res; - $tx = $ua->get($url, {Authorization => obs_ssh_auth($res->headers->www_authenticate, $user, $ssh_key)}) - if $res->code == 401; + $tx = $ua->start( + $ua->build_tx( + $method => $url => {Authorization => obs_ssh_auth($res->headers->www_authenticate, $user, $ssh_key)} + ) + ) if $res->code == 401; } # All other instances should use basic authentication else { die "Missing password for OBS instance: $host" unless my $password = $config->{password}; - $tx = $ua->get($url->userinfo("$user:$password")); + $tx = $ua->start($ua->build_tx($method => $url->userinfo("$user:$password"))); } return $tx->result; } -sub _md5 ($file) { - my $md5 = Digest::MD5->new; - $md5->addfile(path($file)->open('r')); - return $md5->hexdigest; -} - sub _url ($api, @path) { my $url = Mojo::URL->new($api); my $path = $url->path->leading_slash(1); diff --git a/t/embargo.t b/t/embargo.t index a204a3bc3..6ec553d62 100644 --- a/t/embargo.t +++ b/t/embargo.t @@ -84,23 +84,30 @@ get("/source/:project/perl-Mojolicious.SUSE_SLE-15-SP2_Update//$_" => [project = {data => path(__FILE__)->sibling('legal-bot', 'perl-Mojolicious', 'c7cfdab0e71b0bebfdf8b2dc3badfecd', $_)->slurp}) for @files; -get '/request/4321' => {text => <<'EOF'}; - - - - +post '/request/4321' => (query => {cmd => 'diff', view => 'xml', withissues => 1, withdescriptionissues => 1}) => + {status => 200, text => <<'EOF'}; + + + + + + + - requesting release EOF -get '/source/:project/_attribute' => [project => 'SUSE:Maintenance:4321'] => {text => <<'EOF'}; - - - 2024-03-27 07:00 UTC - - +get '/api/embargoed-bugs' => {format => 'json', text => <<'EOF'}; +[ + { + "bug": { + "name": "bnc#1196706", + "url": "https://bugzilla.suse.com/show_bug.cgi?id=1196706", + "source": "SUSE Bugzilla" + }, + "cves": [] + } +] EOF get '/*whatever' => {whatever => ''} => {text => '', status => 404}; @@ -113,7 +120,8 @@ my $dir = $cavil_test->checkout_dir; # Connect mock web service my $mock_app = app; my $api = 'http://127.0.0.1:' . $t->app->obs->ua->server->app($mock_app)->url->port; -$t->app->obs->config({'127.0.0.1' => {user => 'test', password => 'testing'}}); +$t->app->obs->config( + {'127.0.0.1' => {user => 'test', password => 'testing', embargoed_bugs => "$api/api/embargoed-bugs"}}); subtest 'Embargoed package does not existy yet' => sub { $t->get_ok('/package/3' => {Authorization => 'Token test_token'})->status_is(404)->content_like(qr/No such package/); @@ -144,6 +152,7 @@ subtest 'Embargoed packages' => sub { return unless $task eq 'obs_import'; $job->app->obs(Cavil::OBS->new(config => $job->app->obs->config)); my $api = 'http://127.0.0.1:' . $job->app->obs->ua->server->app($mock_app)->url->port; + $job->app->obs->config->{'127.0.0.1'}{embargoed_bugs} = "$api/api/embargoed-bugs"; $job->args->[1]{api} = $api; } ); diff --git a/t/obs.t b/t/obs.t index e634affe3..961ed33c0 100644 --- a/t/obs.t +++ b/t/obs.t @@ -389,200 +389,115 @@ get '/source/:project/postgresql96-plr/_meta' => [project => ['server:database:p EOF -get '/request/1234' => {text => <<'EOF'}; - - - - +post '/request/1235' => (query => {cmd => 'diff', view => 'xml', withissues => 1, withdescriptionissues => 1}) => + {status => 200, text => <<'EOF'}; +EOF + +post '/request/344036' => (query => {cmd => 'diff', view => 'xml', withissues => 1, withdescriptionissues => 1}) => + {status => 200, text => <<'EOF'}; + + + + + + + + + + + + @@ -1,4 +1,265 @@ + ------------------------------------------------------------------- + +Wed Sep 03 02:01:10 UTC 2023 - tester@suse.com + + + +- bsc#1225312 - CVE-2024-3654 - Stuff + + + + + + + + + + + + + + - requesting release - -EOF - -get '/source/:project/_attribute' => [project => 'SUSE:Maintenance:4321'] => {text => <<'EOF'}; - - - 2024-03-27 07:00 UTC - - -EOF - -get '/request/1235' => {text => <<'EOF'}; - - - - + + - requesting release + + + + + + EOF -get '/source/:project/_attribute' => [project => 'SUSE:Maintenance:5321'] => {text => <<'EOF'}; - -EOF - -get '/request/324874' => {text => <<'EOF'}; - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +post '/request/1236' => (query => {cmd => 'diff', view => 'xml', withissues => 1, withdescriptionissues => 1}) => + {status => 200, text => <<'EOF'}; + + + + + + + - - - - - - - - - - - - Auto accept - - - reviewed_okay - - Review got accepted - reviewed_okay - - - - OK - - Review got accepted - OK - - - - reviewers added: qam-openqa - - Review got accepted - reviewers added: qam-openqa - - - - Request accepted for 'qam-openqa' based on data in http://dashboard.qam.suse.de/ - - Review got accepted - Request accepted for 'qam-openqa' based on data in http://dashboard.qam.suse.de/ - - - requesting release EOF -get '/source/:project/_attribute' => [project => 'SUSE:Maintenance:33127'] => {text => <<'EOF'}; - -EOF - -get '/source/:project/_attribute' => [project => 'SUSE:Maintenance:34725'] => {text => <<'EOF'}; - - - 2024-07-16 12:00 UTC - - - 338592:admin - 341702:admin - - - 2024-09-24 12:00 UTC - - - -EOF - -get '/request/5678' => {text => <<'EOF'}; - - - - +post '/request/1237' => (query => {cmd => 'diff', view => 'xml', withissues => 1, withdescriptionissues => 1}) => + {status => 200, text => <<'EOF'}; + + + + - requesting release EOF -get '/source/:project/_attribute' => [project => 'SUSE:Maintenance:5678'] => {status => 400, text => <<'EOF'}; - - Attribute access to remote project is not yet supported - {text => <<'EOF'}; - - - - - - requesting release - +get '/api/embargoed-bugs' => {format => 'json', text => <<'EOF'}; +[ + { + "bug": { + "name": "bnc#1196706", + "url": "https://bugzilla.suse.com/show_bug.cgi?id=1196706", + "source": "SUSE Bugzilla" + }, + "cves": [] + }, + { + "bug": { + "name": "bnc#1202470", + "url": "https://bugzilla.suse.com/show_bug.cgi?id=1202470", + "source": "SUSE Bugzilla" + }, + "cves":[] + }, + { + "bug": { + "name": "bnc#1230469", + "url": "https://bugzilla.suse.com/show_bug.cgi?id=1230469", + "source":"SUSE Bugzilla" + }, + "cves": [ + { + "name": "CVE-2024-22038", + "url": "http://web.nvd.nist.gov/view/vuln/detail?vulnId=CVE-2024-22038", + "source": "NVD" + } + ] + } +] EOF -get '/source/:project/_attribute' => [project => 'SUSE:Maintenance:6789'] => {status => 501, text => <<'EOF'}; - - Attribute access to remote project is not yet supported - {format => 'json', text => <<'EOF'}; +[] EOF my $AUTHENTICATED = 0; @@ -610,7 +525,7 @@ get '/*whatever' => {whatever => ''} => {text => '', status => 404}; # Connect mock web service my $obs = Cavil::OBS->new; my $api = 'http://127.0.0.1:' . $obs->ua->server->app(app)->url->port; -$obs->config({'127.0.0.1' => {user => 'test', password => 'testing'}}); +$obs->config({'127.0.0.1' => {user => 'test', password => 'testing', embargoed_bugs => "$api/api/embargoed-bugs"}}); subtest 'Package info' => sub { my $info = { @@ -687,11 +602,40 @@ subtest 'Source download for missing packages' => sub { }; subtest 'Embargo' => sub { - is $obs->check_for_embargo($api, 1234), 1, 'embargoed'; - is $obs->check_for_embargo($api, 1235), 0, 'not embargoed'; - is $obs->check_for_embargo($api, 324874), 1, 'embargoed'; - is $obs->check_for_embargo($api, 5678), 0, 'not embargoed'; - is $obs->check_for_embargo($api, 6789), 0, 'not embargoed'; + subtest 'SMASH bugrefs' => sub { + subtest 'Realistic example' => sub { + is_deeply $obs->get_embargoed_bugrefs($api), ['CVE-2024-22038', 'bnc#1196706', 'bnc#1202470', 'bnc#1230469'], + 'right bugrefs'; + }; + + subtest 'No embargoed bugs' => sub { + local $obs->config->{'127.0.0.1'}{embargoed_bugs} = "$api/api/embargoed-bugs/none"; + is_deeply $obs->get_embargoed_bugrefs($api), [], 'right bugrefs'; + }; + + subtest 'Not configured' => sub { + local $obs->config->{'127.0.0.1'}{embargoed_bugs} = undef; + is_deeply $obs->get_embargoed_bugrefs($api), [], 'right bugrefs'; + }; + }; + + subtest 'OBS bugrefs' => sub { + is_deeply $obs->get_bugrefs_for_request($api, 1236), ['CVE-2024-22038'], 'right bugrefs'; + is_deeply $obs->get_bugrefs_for_request($api, 1237), [], 'no bugrefs'; + is_deeply $obs->get_bugrefs_for_request($api, 344036), + [ + 'CVE-2022-2850', 'CVE-2024-1062', 'CVE-2024-2199', 'CVE-2024-3657', 'CVE-2024-5953', 'bnc#1202470', + 'bnc#1219836', 'bnc#1225512', 'bsc#1202470', 'bsc#1219836', 'bsc#1225512' + ], + 'right bugrefs'; + }; + + subtest 'Check for embargo' => sub { + is $obs->check_for_embargo($api, 1235), 0, 'not embargoed'; + is $obs->check_for_embargo($api, 1236), 1, 'embargoed'; + is $obs->check_for_embargo($api, 1237), 0, 'not embargoed'; + is $obs->check_for_embargo($api, 344036), 1, 'embargoed'; + }; }; subtest 'Bot API (with Minion background jobs)' => sub {