From d510cd7d4efaee9cfbfafd501d486c1dc0f4241d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 9 Aug 2023 01:02:39 +0530 Subject: [PATCH] Change get_enhanced_package logic and add tests Signed-off-by: Tushar Goel --- packagedb/api.py | 26 +- packagedb/tests/test_api.py | 76 +++++- ...cksums-enhanced-package-data-expected.json | 227 ++++++++++++++++++ .../package-filter_by_checksums-expected.json | 49 +++- 4 files changed, 354 insertions(+), 24 deletions(-) create mode 100644 packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json diff --git a/packagedb/api.py b/packagedb/api.py index 14150c13..26dc2cd2 100644 --- a/packagedb/api.py +++ b/packagedb/api.py @@ -522,11 +522,16 @@ def filter_by_checksums(self, request, *args, **kwargs): qs = Package.objects.filter(q) paginated_qs = self.paginate_queryset(qs) - get_enhanced_package_data = request.query_params.get('get_enhanced_package_data', False) - if get_enhanced_package_data: - enhanced_package_data = [get_enhanced_package(package=package) for package in qs] - enhanced_package_data = [package for package in enhanced_package_data if package] - paginated_response = self.get_paginated_response(enhanced_package_data) + enhance_package_data = request.query_params.get('enhance_package_data', False) + if enhance_package_data: + package_data = [] + for package in qs: + enhanced_package = get_enhanced_package(package=package) + if enhanced_package: + package_data.append(enhanced_package) + else: + package_data.append(package.to_dict()) + paginated_response = self.get_paginated_response(package_data) return paginated_response serializer = PackageAPISerializer(paginated_qs, many=True, context={'request': request}) return self.get_paginated_response(serializer.data) @@ -615,16 +620,9 @@ def _get_enhanced_package(package, packages): Return a mapping of package data based on `package` and Packages in `packages`. """ - mixing = False - package_data = {} + package_data = package.to_dict() for peer in packages: - if peer == package: - mixing = True - package_data = package.to_dict() - continue - if not mixing: - continue - if peer.package_content == package.package_content: + if peer.package_content >= package.package_content: # We do not want to mix data with peers of the same package content continue enhanced = False diff --git a/packagedb/tests/test_api.py b/packagedb/tests/test_api.py index 531d8f2b..3389ec2d 100644 --- a/packagedb/tests/test_api.py +++ b/packagedb/tests/test_api.py @@ -277,11 +277,72 @@ def setUp(self): 'sha1': 'testsha1-4', 'md5': 'testmd5-3', 'size': 100, - 'package_content': 5, + 'package_content': PackageContentType.BINARY, } self.package4 = Package.objects.create(**self.package_data4) self.package4.refresh_from_db() + self.package_data5= { + 'type': 'maven', + 'namespace': 'foot', + 'name': 'baz', + 'version': '90.123', + 'qualifiers': 'classifier=source', + 'subpath': '', + 'download_url': 'http://test-maven.com', + 'filename': 'Baz.zip', + 'sha1': 'testsha1-5', + 'md5': 'testmd5-11', + 'size': 100, + 'package_content': PackageContentType.SOURCE_ARCHIVE, + 'declared_license_expression': 'MIT', + } + + self.package5 = Package.objects.create(**self.package_data5) + self.package5.refresh_from_db() + + self.package_data6= { + 'type': 'maven', + 'namespace': 'fooo', + 'name': 'baz', + 'version': '90.123', + 'qualifiers': '', + 'subpath': '', + 'download_url': 'http://test-maven-11.com', + 'filename': 'Baz.zip', + 'sha1': 'testsha1-6', + 'md5': 'testmd5-11', + 'size': 100, + 'package_content': PackageContentType.BINARY, + } + + self.package6 = Package.objects.create(**self.package_data6) + self.package6.refresh_from_db() + + self.package_data7= { + 'type': 'github', + 'namespace': 'glue', + 'name': 'cat', + 'version': '90.123', + 'qualifiers': '', + 'subpath': '', + 'download_url': 'http://test-maven-111.com', + 'filename': 'Baz.zip', + 'sha1': 'testsha1-7', + 'md5': 'testmd5-11', + 'size': 100, + 'copyright': 'BACC', + 'package_content': PackageContentType.SOURCE_REPO, + } + + self.package7 = Package.objects.create(**self.package_data7) + self.package7.refresh_from_db() + + self.packageset_1 = PackageSet.objects.create() + self.packageset_1.packages.add(self.package6) + self.packageset_1.packages.add(self.package5) + self.packageset_1.packages.add(self.package7) + self.test_url = 'http://testserver/api/packages/{}/' self.client = APIClient() @@ -289,7 +350,7 @@ def setUp(self): def test_package_api_list_endpoint(self): response = self.client.get('/api/packages/') self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(4, response.data.get('count')) + self.assertEqual(7, response.data.get('count')) def test_package_api_list_endpoint_filter(self): for key, value in self.package_data.items(): @@ -466,16 +527,19 @@ def test_package_api_filter_by_checksums(self): 'testsha1-2', 'testsha1-3', 'testsha1-4', + 'testsha1-6', ] data = { 'sha1': sha1s } - enhanced_response = self.client.post('/api/packages/filter_by_checksums/?get_enhanced_package_data=true', data=data) - self.assertEqual(1, len(enhanced_response.data['results'])) + enhanced_response = self.client.post('/api/packages/filter_by_checksums/?enhance_package_data=true', data=data) + self.assertEqual(5, len(enhanced_response.data['results'])) + expected = self.get_test_loc('api/package-filter_by_checksums-enhanced-package-data-expected.json') + self.check_expected_results(enhanced_response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets",], regen=True) response = self.client.post('/api/packages/filter_by_checksums/', data=data) - self.assertEqual(4, response.data['count']) + self.assertEqual(5, response.data['count']) expected = self.get_test_loc('api/package-filter_by_checksums-expected.json') - self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources"], regen=False) + self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets",], regen=True) class PackageApiReindexingTestCase(JsonBasedTesting, TestCase): diff --git a/packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json b/packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json new file mode 100644 index 00000000..01a65ab6 --- /dev/null +++ b/packagedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json @@ -0,0 +1,227 @@ +[ + { + "type":"generic", + "namespace":"generic", + "name":"foo", + "version":"12.34", + "qualifiers":"test_qual=qual", + "subpath":"test_subpath", + "package_content":null, + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://example.com", + "size":101, + "md5":"testmd5", + "sha1":"testsha1", + "sha256":null, + "sha512":null, + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "dependencies":[], + "package_uid":"pkg:generic/generic/foo@12.34?test_qual=qual&uuid=fixed-uid-done-for-testing-5642512d1758#test_subpath", + "datasource_id":null, + "purl":"pkg:generic/generic/foo@12.34?test_qual=qual#test_subpath", + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "file_references":[] + }, + { + "type":"npm", + "namespace":"example", + "name":"bar", + "version":"56.78", + "qualifiers":"", + "subpath":"", + "package_content":null, + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://somethingelse.org", + "size":100, + "md5":"testmd5-2", + "sha1":"testsha1-2", + "sha256":null, + "sha512":null, + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "dependencies":[], + "package_uid":"pkg:npm/example/bar@56.78?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "purl":"pkg:npm/example/bar@56.78", + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "file_references":[] + }, + { + "type":"jar", + "namespace":"sample", + "name":"baz", + "version":"90.12", + "qualifiers":"", + "subpath":"", + "package_content":null, + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://anotherexample.com", + "size":100, + "md5":"testmd5-3", + "sha1":"testsha1-3", + "sha256":null, + "sha512":null, + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "dependencies":[], + "package_uid":"pkg:jar/sample/baz@90.12?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "purl":"pkg:jar/sample/baz@90.12", + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "file_references":[] + }, + { + "type":"jar", + "namespace":"sample", + "name":"baz", + "version":"90.123", + "qualifiers":"", + "subpath":"", + "package_content":"binary", + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://anothersample.com", + "size":100, + "md5":"testmd5-3", + "sha1":"testsha1-4", + "sha256":null, + "sha512":null, + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "dependencies":[], + "package_uid":"pkg:jar/sample/baz@90.123?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "purl":"pkg:jar/sample/baz@90.123", + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "file_references":[] + }, + { + "type":"maven", + "namespace":"fooo", + "name":"baz", + "version":"90.123", + "qualifiers":"", + "subpath":"", + "package_content":"binary", + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://test-maven-11.com", + "size":100, + "md5":"testmd5-11", + "sha1":"testsha1-6", + "sha256":null, + "sha512":null, + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "copyright":"BACC", + "holder":null, + "declared_license_expression":"MIT", + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{ + "enhanced_by":[ + "pkg:github/glue/cat@90.123", + "pkg:maven/foot/baz@90.123?classifier=source" + ] + }, + "dependencies":[], + "package_uid":"pkg:maven/fooo/baz@90.123?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "purl":"pkg:maven/fooo/baz@90.123", + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "file_references":[] + } +] \ No newline at end of file diff --git a/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json index 7bd514ec..4fd07d81 100644 --- a/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json +++ b/packagedb/tests/testfiles/api/package-filter_by_checksums-expected.json @@ -1,7 +1,6 @@ [ { "filename":"Foo.zip", - "package_sets":[], "package_content":null, "purl":"pkg:generic/generic/foo@12.34?test_qual=qual#test_subpath", "type":"generic", @@ -47,7 +46,6 @@ }, { "filename":"Bar.zip", - "package_sets":[], "package_content":null, "purl":"pkg:npm/example/bar@56.78", "type":"npm", @@ -93,7 +91,6 @@ }, { "filename":"Baz.zip", - "package_sets":[], "package_content":null, "purl":"pkg:jar/sample/baz@90.12", "type":"jar", @@ -139,7 +136,6 @@ }, { "filename":"Baz.zip", - "package_sets":[], "package_content":"binary", "purl":"pkg:jar/sample/baz@90.123", "type":"jar", @@ -182,5 +178,50 @@ "datasource_id":null, "file_references":[], "dependencies":[] + }, + { + "filename":"Baz.zip", + "package_content":"binary", + "purl":"pkg:maven/fooo/baz@90.123", + "type":"maven", + "namespace":"fooo", + "name":"baz", + "version":"90.123", + "qualifiers":"", + "subpath":"", + "primary_language":null, + "description":null, + "release_date":null, + "parties":[], + "keywords":[], + "homepage_url":null, + "download_url":"http://test-maven-11.com", + "bug_tracking_url":null, + "code_view_url":null, + "vcs_url":null, + "repository_homepage_url":null, + "repository_download_url":null, + "api_data_url":null, + "size":100, + "md5":"testmd5-11", + "sha1":"testsha1-6", + "sha256":null, + "sha512":null, + "copyright":null, + "holder":null, + "declared_license_expression":null, + "declared_license_expression_spdx":null, + "license_detections":[], + "other_license_expression":null, + "other_license_expression_spdx":null, + "other_license_detections":[], + "extracted_license_statement":null, + "notice_text":null, + "source_packages":[], + "extra_data":{}, + "package_uid":"pkg:maven/fooo/baz@90.123?uuid=fixed-uid-done-for-testing-5642512d1758", + "datasource_id":null, + "file_references":[], + "dependencies":[] } ] \ No newline at end of file