Added wheel building utilities (#2)

.gitignore

@@ -1,3 +1,6 @@
 .idea
 *.iml
-*.pyc
+*.pyc
+.pytest_cache
+*.egg-info
+tests/**/build

NOTICE

@@ -10,6 +10,10 @@ License - https://github.com/databricks/databricks-sdk-py/blob/main/LICENSE
 
 This software contains code from the following open source projects, licensed under the MIT License (MIT).
 
-pytest -https://pytest.org
+pytest - https://pytest.org
 Copyright (c) 2004 Holger Krekel and others
 License - https://github.com/pytest-dev/pytest/blob/main/LICENSE
+
+A simple, correct Python build frontend - https://github.com/pypa/build
+Copyright © 2019 Filipe Laíns <filipe.lains@gmail.com>
+License - https://github.com/pypa/build/blob/main/LICENSE

pyproject.toml

@@ -3,13 +3,13 @@ name = "databricks-labs-pytester"
 version = "0.0.1"
 description = "Python Testing for Databricks"
 authors = ["Serge Smertin <serge.smertin@databricks.com>"]
-readme = "README.md"
+packages = [{include = "databricks", from = "src"}]
 
 [tool.poetry.dependencies]
 python = "^3.9"
 databricks-sdk = "^0.7.0"
 pytest = "^7.4.0"
-
+build = "^0.10.0"
 
 [build-system]
 requires = ["poetry-core"]

src/databricks/labs/pytester/fixtures/__init__.py

@@ -3,11 +3,13 @@
 from .iam import make_group, make_user
 from .notebooks import make_notebook, make_directory, make_repo
 from .secrets import make_secret_scope, make_secret_scope_acl
+from .wheel import workspace_library
 
 __all__ = [
     'ws', 'make_random',
     'make_instance_pool', 'make_job', 'make_cluster', 'make_cluster_policy',
     'make_group', 'make_user',
     'make_notebook', 'make_directory', 'make_repo',
     'make_secret_scope', 'make_secret_scope_acl',
+    'workspace_library'
 ]

src/databricks/labs/pytester/fixtures/wheel.py

@@ -0,0 +1,78 @@
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+from databricks.sdk.service.workspace import ImportFormat
+
+from pathlib import Path
+from typing import Optional
+
+
+def find_dir_with_leaf(folder: Path, leaf: str) -> Optional[Path]:
+    root = folder.root
+    while str(folder.absolute()) != root:
+        if (folder / leaf).exists():
+            return folder
+        folder = folder.parent
+    return None
+
+
+def find_project_root(folder: Path) -> Optional[Path]:
+    for leaf in ['pyproject.toml', 'setup.py']:
+        root = find_dir_with_leaf(folder, leaf)
+        if root is not None:
+            return root
+    return None
+
+
+def build_wheel_in(project_path: Path, out_path: Path) -> Path:
+    try:
+        subprocess.run(
+            [sys.executable, "-m", "build", "--wheel", "--outdir",
+             out_path.absolute(), project_path.absolute()],
+            capture_output=True,
+            check=True,
+        )
+    except subprocess.CalledProcessError as e:
+        if e.stderr is not None:
+            sys.stderr.write(e.stderr.decode())
+        raise RuntimeError(e.output.decode().strip()) from None
+
+    found_wheels = list(out_path.glob(f"*.whl"))
+    if not found_wheels:
+        msg = f"cannot find *.whl in {out_path}"
+        raise RuntimeError(msg)
+    if len(found_wheels) > 1:
+        conflicts = ", ".join(str(whl) for whl in found_wheels)
+        msg = f"more than one wheel match: {conflicts}"
+        raise RuntimeError(msg)
+    wheel_file = found_wheels[0]
+
+    return wheel_file
+
+
+@pytest.fixture
+def fresh_local_wheel_file(tmp_path) -> Path:
+    project_root = find_project_root(Path(os.getcwd()))
+    build_root = tmp_path / fresh_local_wheel_file.__name__
+    shutil.copytree(project_root, build_root)
+
+    return build_wheel_in(build_root, tmp_path / 'dist')
+
+
+@pytest.fixture
+def workspace_library(ws, fresh_local_wheel_file, make_random):
+    my_user = ws.current_user.me().user_name
+    workspace_folder = f"/Users/{my_user}/wheels/{make_random(10)}"
+    ws.workspace.mkdirs(workspace_folder)
+
+    wheel_path = f"{workspace_folder}/{fresh_local_wheel_file.name}"
+    with fresh_local_wheel_file.open("rb") as f:
+        ws.workspace.upload(wheel_path, f, format=ImportFormat.AUTO)
+
+    yield f'/Workspace/{wheel_path}'
+
+    ws.workspace.delete(workspace_folder, recursive=True)

tests/test_fixtures.py → tests/integration/test_fixtures.py

@@ -3,7 +3,8 @@
 
 from databricks.sdk.service.workspace import AclPermission
 
-from databricks.labs.pytester.fixtures import * # noqa: F403
+from databricks.labs.pytester.fixtures import make_user
+
 from databricks.labs.pytester.environment import load_debug_env_if_runs_from_ide
 
 load_debug_env_if_runs_from_ide("ucws")  # noqa: F405

tests/resources/hatchling-whl/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "hatchling-whl"
+dynamic = ["version"]
+description = 'dummy'
+requires-python = ">=3.7"
+license = "MIT"
+keywords = []
+authors = [
+  { name = "Jonh Doe", email = "john@example.com" },
+]
+dependencies = []
+
+[tool.hatch.version]
+path = "src/hatchling_whl/__about__.py"
+

tests/resources/hatchling-whl/src/hatchling_whl/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

tests/resources/hatchling-whl/src/hatchling_whl/__init__.py

@@ -0,0 +1,5 @@
+def main():
+    print('it works')
+
+if __name__ == '__main__':
+    main()

tests/resources/poetry-whl/poetry_whl/__init__.py

@@ -0,0 +1,5 @@
+def main():
+    print('it works')
+
+if __name__ == '__main__':
+    main()

tests/resources/poetry-whl/pyproject.toml

@@ -0,0 +1,12 @@
+[tool.poetry]
+name = "poetry_whl"
+version = "0.0.1"
+description = "dummy project"
+authors = ["John Doe <john@example.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

tests/resources/setuppy-whl/setup.py

@@ -0,0 +1,11 @@
+import io
+import pathlib
+
+from setuptools import setup, find_packages
+
+setup(name="setuppy-whl",
+      version="0.0.1",
+      packages=find_packages(exclude=["tests", "*tests.*", "*tests"]),
+      python_requires=">=3.7",
+      author="John Doe",
+      author_email="john@example.com")

tests/resources/setuppy-whl/setuppy_whl/__init__.py

@@ -0,0 +1,5 @@
+def main():
+    print('it works')
+
+if __name__ == '__main__':
+    main()

tests/resources/setuptools-whl/pyproject.toml

@@ -0,0 +1,11 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "setuptools-whl"
+version = "0.0.1"
+description = "dummy"
+
+[tool.distutils.bdist_wheel]
+universal = true

tests/resources/setuptools-whl/setuptools_whl/__init__.py

@@ -0,0 +1,5 @@
+def main():
+    print('it works')
+
+if __name__ == '__main__':
+    main()

tests/test_wheels.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+import pytest
+
+from databricks.labs.pytester.fixtures.wheel import build_wheel_in, find_dir_with_leaf, fresh_local_wheel_file, workspace_library
+
+__folder__ = Path(__file__).parent
+
+
+def test_find_dir_with_leaf():
+    x = find_dir_with_leaf(Path(__file__), '.gitignore')
+    assert x is not None
+
+
+def test_find_dir_with_leaf_missing():
+    x = find_dir_with_leaf(Path(__file__), '.nothing')
+    assert x is None
+
+
+@pytest.mark.parametrize("build_system", ['hatchling', 'poetry', 'setuppy', 'setuptools'])
+def test_building_wheels(tmp_path, build_system):
+    whl = build_wheel_in(__folder__ / f'resources/{build_system}-whl', tmp_path)
+    assert whl.exists()
+
+
+def test_fresh_wheel_file(fresh_local_wheel_file):
+    assert fresh_local_wheel_file is not None