Skip to content

Commit

Permalink
Change get_enhanced_package logic and add tests
Browse files Browse the repository at this point in the history 
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
  • Loading branch information
@TG1999
TG1999 committed Aug 8, 2023
1 parent d96a719 commit d510cd7
Show file tree
Hide file tree
Showing 4 changed files with 354 additions and 24 deletions.
26 changes: 12 additions & 14 deletions packagedb/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -522,11 +522,16 @@ def filter_by_checksums(self, request, *args, **kwargs):

qs = Package.objects.filter(q)
paginated_qs = self.paginate_queryset(qs)
get_enhanced_package_data = request.query_params.get('get_enhanced_package_data', False)
if get_enhanced_package_data:
enhanced_package_data = [get_enhanced_package(package=package) for package in qs]
enhanced_package_data = [package for package in enhanced_package_data if package]
paginated_response = self.get_paginated_response(enhanced_package_data)
enhance_package_data = request.query_params.get('enhance_package_data', False)
if enhance_package_data:
package_data = []
for package in qs:
enhanced_package = get_enhanced_package(package=package)
if enhanced_package:
package_data.append(enhanced_package)
else:
package_data.append(package.to_dict())
paginated_response = self.get_paginated_response(package_data)
return paginated_response
serializer = PackageAPISerializer(paginated_qs, many=True, context={'request': request})
return self.get_paginated_response(serializer.data)
Expand Down Expand Up @@ -615,16 +620,9 @@ def _get_enhanced_package(package, packages):
Return a mapping of package data based on `package` and Packages in
`packages`.
"""
mixing = False
package_data = {}
package_data = package.to_dict()
for peer in packages:
if peer == package:
mixing = True
package_data = package.to_dict()
continue
if not mixing:
continue
if peer.package_content == package.package_content:
if peer.package_content >= package.package_content:
# We do not want to mix data with peers of the same package content
continue
enhanced = False
Expand Down
76 changes: 70 additions & 6 deletions packagedb/tests/test_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,19 +277,80 @@ def setUp(self):
'sha1': 'testsha1-4',
'md5': 'testmd5-3',
'size': 100,
'package_content': 5,
'package_content': PackageContentType.BINARY,
}
self.package4 = Package.objects.create(**self.package_data4)
self.package4.refresh_from_db()

self.package_data5= {
'type': 'maven',
'namespace': 'foot',
'name': 'baz',
'version': '90.123',
'qualifiers': 'classifier=source',
'subpath': '',
'download_url': 'http://test-maven.com',
'filename': 'Baz.zip',
'sha1': 'testsha1-5',
'md5': 'testmd5-11',
'size': 100,
'package_content': PackageContentType.SOURCE_ARCHIVE,
'declared_license_expression': 'MIT',
}

self.package5 = Package.objects.create(**self.package_data5)
self.package5.refresh_from_db()

self.package_data6= {
'type': 'maven',
'namespace': 'fooo',
'name': 'baz',
'version': '90.123',
'qualifiers': '',
'subpath': '',
'download_url': 'http://test-maven-11.com',
'filename': 'Baz.zip',
'sha1': 'testsha1-6',
'md5': 'testmd5-11',
'size': 100,
'package_content': PackageContentType.BINARY,
}

self.package6 = Package.objects.create(**self.package_data6)
self.package6.refresh_from_db()

self.package_data7= {
'type': 'github',
'namespace': 'glue',
'name': 'cat',
'version': '90.123',
'qualifiers': '',
'subpath': '',
'download_url': 'http://test-maven-111.com',
'filename': 'Baz.zip',
'sha1': 'testsha1-7',
'md5': 'testmd5-11',
'size': 100,
'copyright': 'BACC',
'package_content': PackageContentType.SOURCE_REPO,
}

self.package7 = Package.objects.create(**self.package_data7)
self.package7.refresh_from_db()

self.packageset_1 = PackageSet.objects.create()
self.packageset_1.packages.add(self.package6)
self.packageset_1.packages.add(self.package5)
self.packageset_1.packages.add(self.package7)

self.test_url = 'http://testserver/api/packages/{}/'

self.client = APIClient()

def test_package_api_list_endpoint(self):
response = self.client.get('/api/packages/')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(4, response.data.get('count'))
self.assertEqual(7, response.data.get('count'))

def test_package_api_list_endpoint_filter(self):
for key, value in self.package_data.items():
Expand Down Expand Up @@ -466,16 +527,19 @@ def test_package_api_filter_by_checksums(self):
'testsha1-2',
'testsha1-3',
'testsha1-4',
'testsha1-6',
]
data = {
'sha1': sha1s
}
enhanced_response = self.client.post('/api/packages/filter_by_checksums/?get_enhanced_package_data=true', data=data)
self.assertEqual(1, len(enhanced_response.data['results']))
enhanced_response = self.client.post('/api/packages/filter_by_checksums/?enhance_package_data=true', data=data)
self.assertEqual(5, len(enhanced_response.data['results']))
expected = self.get_test_loc('api/package-filter_by_checksums-enhanced-package-data-expected.json')
self.check_expected_results(enhanced_response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets",], regen=True)
response = self.client.post('/api/packages/filter_by_checksums/', data=data)
self.assertEqual(4, response.data['count'])
self.assertEqual(5, response.data['count'])
expected = self.get_test_loc('api/package-filter_by_checksums-expected.json')
self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources"], regen=False)
self.check_expected_results(response.data['results'], expected, fields_to_remove=["url", "uuid", "resources", "package_sets",], regen=True)


class PackageApiReindexingTestCase(JsonBasedTesting, TestCase):
Expand Down
227 changes: 227 additions & 0 deletions ...agedb/tests/testfiles/api/package-filter_by_checksums-enhanced-package-data-expected.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,227 @@
[
{
"type":"generic",
"namespace":"generic",
"name":"foo",
"version":"12.34",
"qualifiers":"test_qual=qual",
"subpath":"test_subpath",
"package_content":null,
"primary_language":null,
"description":null,
"release_date":null,
"parties":[],
"keywords":[],
"homepage_url":null,
"download_url":"http://example.com",
"size":101,
"md5":"testmd5",
"sha1":"testsha1",
"sha256":null,
"sha512":null,
"bug_tracking_url":null,
"code_view_url":null,
"vcs_url":null,
"copyright":null,
"holder":null,
"declared_license_expression":null,
"declared_license_expression_spdx":null,
"license_detections":[],
"other_license_expression":null,
"other_license_expression_spdx":null,
"other_license_detections":[],
"extracted_license_statement":null,
"notice_text":null,
"source_packages":[],
"extra_data":{},
"dependencies":[],
"package_uid":"pkg:generic/generic/foo@12.34?test_qual=qual&uuid=fixed-uid-done-for-testing-5642512d1758#test_subpath",
"datasource_id":null,
"purl":"pkg:generic/generic/foo@12.34?test_qual=qual#test_subpath",
"repository_homepage_url":null,
"repository_download_url":null,
"api_data_url":null,
"file_references":[]
},
{
"type":"npm",
"namespace":"example",
"name":"bar",
"version":"56.78",
"qualifiers":"",
"subpath":"",
"package_content":null,
"primary_language":null,
"description":null,
"release_date":null,
"parties":[],
"keywords":[],
"homepage_url":null,
"download_url":"http://somethingelse.org",
"size":100,
"md5":"testmd5-2",
"sha1":"testsha1-2",
"sha256":null,
"sha512":null,
"bug_tracking_url":null,
"code_view_url":null,
"vcs_url":null,
"copyright":null,
"holder":null,
"declared_license_expression":null,
"declared_license_expression_spdx":null,
"license_detections":[],
"other_license_expression":null,
"other_license_expression_spdx":null,
"other_license_detections":[],
"extracted_license_statement":null,
"notice_text":null,
"source_packages":[],
"extra_data":{},
"dependencies":[],
"package_uid":"pkg:npm/example/bar@56.78?uuid=fixed-uid-done-for-testing-5642512d1758",
"datasource_id":null,
"purl":"pkg:npm/example/bar@56.78",
"repository_homepage_url":null,
"repository_download_url":null,
"api_data_url":null,
"file_references":[]
},
{
"type":"jar",
"namespace":"sample",
"name":"baz",
"version":"90.12",
"qualifiers":"",
"subpath":"",
"package_content":null,
"primary_language":null,
"description":null,
"release_date":null,
"parties":[],
"keywords":[],
"homepage_url":null,
"download_url":"http://anotherexample.com",
"size":100,
"md5":"testmd5-3",
"sha1":"testsha1-3",
"sha256":null,
"sha512":null,
"bug_tracking_url":null,
"code_view_url":null,
"vcs_url":null,
"copyright":null,
"holder":null,
"declared_license_expression":null,
"declared_license_expression_spdx":null,
"license_detections":[],
"other_license_expression":null,
"other_license_expression_spdx":null,
"other_license_detections":[],
"extracted_license_statement":null,
"notice_text":null,
"source_packages":[],
"extra_data":{},
"dependencies":[],
"package_uid":"pkg:jar/sample/baz@90.12?uuid=fixed-uid-done-for-testing-5642512d1758",
"datasource_id":null,
"purl":"pkg:jar/sample/baz@90.12",
"repository_homepage_url":null,
"repository_download_url":null,
"api_data_url":null,
"file_references":[]
},
{
"type":"jar",
"namespace":"sample",
"name":"baz",
"version":"90.123",
"qualifiers":"",
"subpath":"",
"package_content":"binary",
"primary_language":null,
"description":null,
"release_date":null,
"parties":[],
"keywords":[],
"homepage_url":null,
"download_url":"http://anothersample.com",
"size":100,
"md5":"testmd5-3",
"sha1":"testsha1-4",
"sha256":null,
"sha512":null,
"bug_tracking_url":null,
"code_view_url":null,
"vcs_url":null,
"copyright":null,
"holder":null,
"declared_license_expression":null,
"declared_license_expression_spdx":null,
"license_detections":[],
"other_license_expression":null,
"other_license_expression_spdx":null,
"other_license_detections":[],
"extracted_license_statement":null,
"notice_text":null,
"source_packages":[],
"extra_data":{},
"dependencies":[],
"package_uid":"pkg:jar/sample/baz@90.123?uuid=fixed-uid-done-for-testing-5642512d1758",
"datasource_id":null,
"purl":"pkg:jar/sample/baz@90.123",
"repository_homepage_url":null,
"repository_download_url":null,
"api_data_url":null,
"file_references":[]
},
{
"type":"maven",
"namespace":"fooo",
"name":"baz",
"version":"90.123",
"qualifiers":"",
"subpath":"",
"package_content":"binary",
"primary_language":null,
"description":null,
"release_date":null,
"parties":[],
"keywords":[],
"homepage_url":null,
"download_url":"http://test-maven-11.com",
"size":100,
"md5":"testmd5-11",
"sha1":"testsha1-6",
"sha256":null,
"sha512":null,
"bug_tracking_url":null,
"code_view_url":null,
"vcs_url":null,
"copyright":"BACC",
"holder":null,
"declared_license_expression":"MIT",
"declared_license_expression_spdx":null,
"license_detections":[],
"other_license_expression":null,
"other_license_expression_spdx":null,
"other_license_detections":[],
"extracted_license_statement":null,
"notice_text":null,
"source_packages":[],
"extra_data":{
"enhanced_by":[
"pkg:github/glue/cat@90.123",
"pkg:maven/foot/baz@90.123?classifier=source"
]
},
"dependencies":[],
"package_uid":"pkg:maven/fooo/baz@90.123?uuid=fixed-uid-done-for-testing-5642512d1758",
"datasource_id":null,
"purl":"pkg:maven/fooo/baz@90.123",
"repository_homepage_url":null,
"repository_download_url":null,
"api_data_url":null,
"file_references":[]
}
]
Loading

0 comments on commit d510cd7

Please sign in to comment.