From 1ac31aa7db3eeac10487d3fc01866194ebf6f1b1 Mon Sep 17 00:00:00 2001 From: VJ Patel Date: Fri, 25 Apr 2025 16:21:00 +0100 Subject: [PATCH 1/2] feat: support overriding module-dir `python_binary`, `python_test` and `python_wheel` all currently only support using the repository-wide module-dir which forces us to share the same set of python dependencies for every binary. This commit allows overriding this so we can independently version dependencies. --- build_defs/python.build_defs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/build_defs/python.build_defs b/build_defs/python.build_defs index e872c3e..2d0ab7d 100644 --- a/build_defs/python.build_defs +++ b/build_defs/python.build_defs @@ -94,7 +94,8 @@ def python_library(name:str, srcs:list=[], resources:list=[], deps:list=[], visi def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=None, deps:list=[], data:list=None, visibility:list=None, test_only:bool=False, zip_safe:bool=None, site:bool=False, strip:bool=False, interpreter:str=CONFIG.PYTHON.DEFAULT_INTERPRETER, - shebang:str=CONFIG.PYTHON.DEFAULT_SHEBANG, labels:list&features&tags=[]): + shebang:str=CONFIG.PYTHON.DEFAULT_SHEBANG, module_dir:str=CONFIG.PYTHON.MODULE_DIR, + labels:list&features&tags=[]): """Generates a Python binary target. This compiles all source files together into a single .pex file which can @@ -127,6 +128,8 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N shebang (str): Exact shebang to apply to the generated file. Defaults to the value of the python.defaultshebang setting, if not set we will determine something appropriate for the given interpreter. + module_dir (str): The path to the third party python directory in python import path format. + Defaults to value of python.moduledir setting. labels (list): Labels to apply to this rule. """ assert main not in srcs, "main file should not be included in srcs" @@ -145,7 +148,7 @@ def python_binary(name:str, main:str, srcs:list=[], resources:list=[], out:str=N shebang = shebang or _interpreter_cmd(interpreter) zipsafe_flag = '' if zip_safe is False else '--zip_safe' cmd = '$TOOLS_PEX -s "%s" -m "%s" %s --interpreter_options="%s"' % ( - shebang, CONFIG.PYTHON.MODULE_DIR, zipsafe_flag, CONFIG.PYTHON.INTERPRETER_OPTIONS) + shebang, module_dir, zipsafe_flag, CONFIG.PYTHON.INTERPRETER_OPTIONS) # If content hashing feature flag is enabled, we use the hash of the built # dependencies instead of the RULE_HASH as the base of the pex extraction @@ -219,7 +222,8 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps: labels:list&features&tags=[], size:str=None, flags:str='', visibility:list=None, sandbox:bool=None, timeout:int=0, flaky:bool|int=0, env:dict=None, test_outputs:list=None, zip_safe:bool=None, interpreter:str=CONFIG.PYTHON.DEFAULT_INTERPRETER, - site:bool=False, test_runner:str=None, shebang:str=CONFIG.PYTHON.DEFAULT_SHEBANG): + site:bool=False, test_runner:str=None, shebang:str=CONFIG.PYTHON.DEFAULT_SHEBANG, + module_dir:str=CONFIG.PYTHON.MODULE_DIR): """Generates a Python test target. This works very similarly to python_binary; it is also a single .pex file @@ -257,6 +261,8 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps: shebang (str): Exact shebang to apply to the generated file. Defaults to the value of the python.defaultshebang setting, if not set we will determine something appropriate for the given interpreter. + module_dir (str): The path to the third party python directory in python import path format. + Defaults to value of python.moduledir setting. site (bool): Allows the Python interpreter to import site; conversely if False, it will be started with the -S flag to avoid importing site. test_runner (str): Specify which Python test runner to use for these tests. One of @@ -264,7 +270,7 @@ def python_test(name:str, srcs:list, data:list|dict=[], resources:list=[], deps: """ test_runner = test_runner or CONFIG.PYTHON.TEST_RUNNER shebang = shebang or _interpreter_cmd(interpreter) - cmd = f'$TOOLS_PEX -t -s "{shebang}" -m "{CONFIG.PYTHON.MODULE_DIR}" -r "{test_runner}" --zip_safe --add_test_runner_deps --interpreter_options="{CONFIG.PYTHON.INTERPRETER_OPTIONS}" --stamp="$RULE_HASH"' + cmd = f'$TOOLS_PEX -t -s "{shebang}" -m "{module_dir}" -r "{test_runner}" --zip_safe --add_test_runner_deps --interpreter_options="{CONFIG.PYTHON.INTERPRETER_OPTIONS}" --stamp="$RULE_HASH"' if site: cmd += ' -S' @@ -517,7 +523,8 @@ def python_wheel(name:str, version:str, labels:list=[], hashes:list=None, packag deps:list=[], name_scheme:str|list=CONFIG.PYTHON.WHEEL_NAME_SCHEME, strip:list=['*.pyc', 'tests'], binary = False, entry_points:dict|str={}, tool:str=CONFIG.PYTHON.WHEEL_TOOL, prereleases:bool=CONFIG.PYTHON.PRERELEASES, - tool_verbosity:str=CONFIG.PYTHON.VERBOSITY, interpreter:str=CONFIG.PYTHON.DEFAULT_INTERPRETER): + tool_verbosity:str=CONFIG.PYTHON.VERBOSITY, interpreter:str=CONFIG.PYTHON.DEFAULT_INTERPRETER, + module_dir:str=CONFIG.PYTHON.MODULE_DIR): """Downloads a Python wheel and extracts it. This is a lightweight pip-free alternative to pip_library which supports cross-compiling. @@ -561,6 +568,8 @@ def python_wheel(name:str, version:str, labels:list=[], hashes:list=None, packag This parameter can be a string, in which case the rule can be ran directly, or a dictionary, for example `{"protoc", "protoc.__main__:main"}. For the latter, each key can be ran using annotated labels, e.g. `plz run //third_party/python:protobuf|protoc` + module_dir (str): The path to the third party python directory in python import path format. + Defaults to value of python.moduledir setting. tool (str): Tool to locate python wheel file in an index """ binary = binary or entry_points @@ -689,6 +698,7 @@ def python_wheel(name:str, version:str, labels:list=[], hashes:list=None, packag lib_rule = lib_rule, visibility = visibility, test_only = test_only, + module_dir = module_dir, ) entry_point_binaries = [_wheel_entrypoint_binary( @@ -698,6 +708,7 @@ def python_wheel(name:str, version:str, labels:list=[], hashes:list=None, packag visibility = visibility, test_only = test_only, out = f"{alias}.pex", + module_dir = module_dir, ) for alias, ep in entry_points.items()] return filegroup( @@ -713,7 +724,7 @@ def python_wheel(name:str, version:str, labels:list=[], hashes:list=None, packag ) return lib_rule -def _wheel_entrypoint_binary(name:str, entrypoint:str, lib_rule, visibility, test_only, out=None): +def _wheel_entrypoint_binary(name:str, entrypoint:str, lib_rule, visibility, test_only, module_dir:str, out=None): module, _, func = entrypoint.rpartition(":") main = text_file( name = tag(name, "main"), @@ -727,6 +738,7 @@ def _wheel_entrypoint_binary(name:str, entrypoint:str, lib_rule, visibility, tes out = out, deps = [lib_rule], test_only = test_only, + module_dir = module_dir, visibility = visibility, ) From ee47ac9e66d4ded378fe1445ddde42449e808d3a Mon Sep 17 00:00:00 2001 From: VJ Patel Date: Fri, 25 Apr 2025 16:54:40 +0100 Subject: [PATCH 2/2] bump minor version and add changelog entry --- ChangeLog | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 4b739b4..f5a373e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +Version 1.13.0 +-------------- + * Allow overriding of repo-wide ModuleDir per `python_binary`, `python_test` and `python_wheel` (#240) + Version 1.12.2 -------------- * Bootstrap libraries are correctly included in pexes (#233) diff --git a/VERSION b/VERSION index 6b89d58..feaae22 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.12.2 +1.13.0