Skip to content

Commit 8880942

Browse files
Wheel resolver (#149)
This introduces a new tool which was previously unreleased. It is based on work that @samwestmoreland began but became stagnant. It attempts to resolve the download url for the compiled wheel package (based on the system tags provided). --------- Co-authored-by: abrandt <abrandt@thoughtmachine.net> Co-authored-by: rgodden <rgodden@thoughtmachine.net>
1 parent f3c3f87 commit 8880942

File tree

11 files changed

+521
-4
lines changed

11 files changed

+521
-4
lines changed

third_party/python/BUILD

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,14 @@ python_wheel(
103103
)
104104

105105
_coverage_version = "7.5.0"
106-
_coverage_soabis = ["cp38", "cp39", "cp310", "cp311", "cp312"]
106+
107+
_coverage_soabis = [
108+
"cp38",
109+
"cp39",
110+
"cp310",
111+
"cp311",
112+
"cp312",
113+
]
107114

108115
if is_platform(
109116
arch = "amd64",
@@ -149,8 +156,8 @@ python_wheel(
149156

150157
python_wheel(
151158
name = "packaging",
152-
hashes = ["170748228214b70b672c581a3dd610ee51f733018650740e98c7df862a583f73"],
153-
version = "20.1",
159+
hashes = [],
160+
version = "24.1",
154161
)
155162

156163
pip_library(
@@ -430,6 +437,7 @@ pip_library(
430437

431438
pip_library(
432439
name = "distlib",
440+
licences = ["PSF-2.0"],
433441
version = "0.3.2",
434442
)
435443

@@ -505,3 +513,21 @@ filegroup(
505513
":test_bootstrap",
506514
],
507515
)
516+
517+
python_wheel(
518+
name = "click",
519+
hashes = [],
520+
version = "8.1.7",
521+
deps = [],
522+
)
523+
524+
python_wheel(
525+
name = "click-log",
526+
package_name = "click_log",
527+
outs = ["click_log"],
528+
hashes = [],
529+
name_scheme = "{url_base}/ae/5a/4f025bc751087833686892e17e7564828e409c43b632878afeae554870cd/{package_name}-{version}-py2.py3-none-any.whl",
530+
repo = "https://files.pythonhosted.org/packages",
531+
version = "0.4.0",
532+
deps = [],
533+
)

tools/ChangeLog

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
Version 1.3.0
2+
-------------
3+
* Added wheel_resolver tool; see [README](./wheel_resolver/README.md)
4+
(#149)
5+
16
Version 1.2.2
27
-------------
38
* Always clean up exploded pex when exiting the Python entry point (#142)

tools/VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.2.2
1+
1.3.0

tools/wheel_resolver/BUILD

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
subinclude("//build_defs:python")
2+
3+
python_binary(
4+
name = "wheel_resolver",
5+
main = "main.py",
6+
visibility = ["PUBLIC"],
7+
deps = [
8+
":wheel",
9+
],
10+
)
11+
12+
python_library(
13+
name = "wheel",
14+
srcs = [
15+
"__init__.py",
16+
"wheel.py",
17+
"output.py",
18+
],
19+
deps = [
20+
"//third_party/python:click",
21+
"//third_party/python:click-log",
22+
"//third_party/python:distlib",
23+
"//third_party/python:packaging",
24+
"//third_party/python:requests",
25+
],
26+
)
27+
28+
python_test(
29+
name = "test",
30+
timeout = 600,
31+
srcs = [
32+
"__init___test.py",
33+
"wheel_test.py",
34+
],
35+
test_runner = "pytest",
36+
deps = [
37+
":wheel",
38+
"//third_party/python:pytest",
39+
],
40+
)

tools/wheel_resolver/README.md

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# wheel_resolver
2+
3+
By Alex Brandt and José de la Puente of Thought Machine
4+
5+
## Description
6+
7+
You can use wheel_resolver to resolve a PyPI package name with optional version
8+
to it's download URL for a compatible wheel.
9+
10+
## Terms of use
11+
12+
You are free to use wheel_resolver for your own projects according to the
13+
[LICENSE]. See the [LICENSE] file for more details.
14+
15+
## How to use
16+
17+
1. `plz run //tools/wheel_resolver -- --help`
18+
1. `plz run //tools/wheel_resolver -- --package-name pyyaml`
19+
20+
## Documentation
21+
22+
- [please.build]: The please documentation
23+
- [python-rules]: The please Python plugin
24+
- [LICENSE]: The license governing use of wheel_resolver
25+
26+
## Getting help
27+
28+
- [GitHub Issues]: Support requests, bug reports, and feature requests
29+
30+
## How to help
31+
32+
- Submit [issues][GitHub Issues] for problems or questions
33+
- Submit [pull requests][GitHub Pull Requests] for proposed changes
34+
35+
[LICENSE]: https://github.com/please-build/python-rules/blob/master/LICENSE
36+
[please.build]: https://please.build/
37+
[python-rules]: https://github.com/please-build/python-rules
38+
[GitHub Issues]: https://github.com/please-build/python-rules/issues
39+
[GitHub Pull Requests]: https://github.com/please-build/python-rules/pulls

tools/wheel_resolver/__init__.py

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import click
2+
import typing
3+
import requests
4+
import logging
5+
import click_log
6+
import sys
7+
import tools.wheel_resolver.wheel as wheel
8+
import tools.wheel_resolver.output as output
9+
import packaging.tags as tags
10+
import distlib.locators
11+
12+
_LOGGER = logging.getLogger(__name__)
13+
14+
click_log.basic_config(_LOGGER)
15+
16+
@click.command()
17+
@click.option(
18+
"--url",
19+
"--urls",
20+
multiple=True,
21+
metavar="URL",
22+
default=[],
23+
help="URLs to check for package before looking in the wheel index",
24+
)
25+
@click.option(
26+
"--package-name",
27+
"--package",
28+
metavar="NAME",
29+
required=True,
30+
help="Name of Python package in PyPI",
31+
)
32+
@click.option(
33+
"--package-version",
34+
"--version",
35+
metavar="VERSION",
36+
help="Version of Python package in PyPI",
37+
)
38+
@click.option(
39+
"--interpreter",
40+
default={t.interpreter for t in tags.sys_tags()},
41+
multiple=True,
42+
metavar="INTERPRETER",
43+
show_default=True,
44+
help="The interpreter name or abbreviation code with version, for example py31 or cp310",
45+
)
46+
@click.option(
47+
"--platform",
48+
# Must cast to list so click knows we want multiple default values.
49+
default=set(tags.platform_tags()),
50+
metavar="PLATFORM",
51+
multiple=True,
52+
help="The platform identifier, for example linux_x86_64 or linux_i686",
53+
)
54+
@click.option(
55+
"--abi",
56+
default={t.abi for t in tags.sys_tags()},
57+
metavar="ABI",
58+
show_default=True,
59+
multiple=True,
60+
help="The ABI identifier, for example cp310 or abi3",
61+
)
62+
@click_log.simple_verbosity_option(_LOGGER)
63+
def main(
64+
url: typing.List[str],
65+
package_name: typing.Optional[str],
66+
package_version: typing.Optional[str],
67+
interpreter: typing.Tuple[str, ...],
68+
platform: typing.Tuple[str, ...],
69+
abi: typing.Tuple[str, ...],
70+
):
71+
"""Resolve a wheel by name and version to a URL.
72+
73+
If URLs are specified, they are checked literally before doing a lookup in
74+
PyPI for PACKAGE with VERSION.
75+
76+
"""
77+
for u in url:
78+
response = requests.head(u)
79+
if response.status_code != requests.codes.ok:
80+
_LOGGER.warning(
81+
"%s-%s is not available, tried %r", package_name, package_version, u
82+
)
83+
else:
84+
click.echo(u)
85+
return
86+
87+
u = wheel.url(
88+
package_name=package_name,
89+
package_version=package_version,
90+
tags=[
91+
str(x)
92+
for i in interpreter
93+
for x in tags.generic_tags(
94+
interpreter=i,
95+
abis=set(abi),
96+
platforms=set(platform).union({"any"}),
97+
)
98+
],
99+
# We're currently hardcoding PyPI but we should consider allowing other
100+
# repositories
101+
locator=distlib.locators.SimpleScrapingLocator(url="https://pypi.org/simple"),
102+
)
103+
response = requests.head(u)
104+
if response.status_code != requests.codes.ok:
105+
_LOGGER.error(
106+
"%s-%s is not available, tried %r", package_name, package_version, u
107+
)
108+
sys.exit(1)
109+
110+
if not output.try_download(u):
111+
_LOGGER.error("Could not download from %r", u)
112+
sys.exit(1)
113+
114+
click.echo(u)
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import click.testing
2+
import pytest
3+
import unittest.mock
4+
import requests
5+
6+
import tools.wheel_resolver as sut
7+
8+
9+
class TestMain:
10+
def test_help(self) -> None:
11+
runner = click.testing.CliRunner()
12+
result = runner.invoke(cli=sut.main, args=["--help"])
13+
assert result.exit_code == 0
14+
15+
@unittest.mock.patch.object(sut.output, "try_download")
16+
@unittest.mock.patch.object(sut.requests, "head")
17+
def test_any_in_platforms(
18+
self, _mock_requests_head: unittest.mock.MagicMock, _mock_try_download: unittest.mock.MagicMock
19+
) -> None:
20+
_mock_try_download.return_value = True
21+
_mock_requests_head.return_value = requests.Response()
22+
_mock_requests_head.return_value.status_code = requests.codes.ok
23+
24+
runner = click.testing.CliRunner()
25+
with unittest.mock.patch.object(sut.wheel, "url") as mock_url:
26+
result = runner.invoke(
27+
cli=sut.main,
28+
args=[
29+
"--package-name",
30+
"pytest-unordered",
31+
"--package-version",
32+
"0.6.0",
33+
"--interpreter",
34+
"py3",
35+
],
36+
)
37+
for _, _, kwargs in mock_url.mock_calls:
38+
# Due to tags being a required keyword argument
39+
if "tags" in kwargs:
40+
assert any([t for t in kwargs["tags"] if t.endswith("any")])
41+
assert result.exit_code == 0

tools/wheel_resolver/main.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
from tools.wheel_resolver import main
2+
3+
if __name__ == "__main__":
4+
main()

tools/wheel_resolver/output.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import os
2+
import sys
3+
import urllib.request
4+
import logging
5+
6+
_LOGGER = logging.getLogger(__name__)
7+
8+
class OutputNotSetError(RuntimeError):
9+
pass
10+
11+
def try_download(url):
12+
"""
13+
Try to download url to $OUTS. Returns false if
14+
it failed.
15+
"""
16+
output = os.environ.get("OUTS")
17+
if output is None:
18+
raise OutputNotSetError()
19+
20+
try:
21+
urllib.request.urlretrieve(url, output)
22+
except urllib.error.HTTPError:
23+
return False
24+
25+
return True
26+
27+

0 commit comments

Comments
 (0)