Fix requirements checking in pypi_deploy.sh and document scripts. (#1…

…9137)

* `set -e` was exiting before we could log helpful messages
* `check_exists` was assuming a command would exist in `PATH`, but some
of the requirements are python packages without scripts sharing their
name

build_tools/python_deploy/README.md

@@ -1,8 +1,61 @@
 # Python Deployment
 
+These scripts assist with building Python packages and pushing them to
+[PyPI (the Python Package Index)](https://pypi.org/). See also
+
+* The Python Packaging User Guide: <https://packaging.python.org/en/latest/>
+* Our release management documentation:
+  https://iree.dev/developers/general/release-management/
+
+## Overview
+
 See comments in scripts for canonical usage. This page includes additional
 notes.
 
+### Package building
+
+These scripts build all packages we maintain, for all Python versions and
+platforms that we support:
+
+* [`build_linux_packages.sh`](./build_linux_packages.sh)
+* [`build_macos_packages.sh`](./build_macos_packages.sh)
+* [`build_windows_packages.ps1`](./build_windows_packages.ps1)
+
+To assist with environment setup, we use a
+[manylinux Docker image](https://github.com/iree-org/base-docker-images/blob/main/dockerfiles/manylinux_x86_64.Dockerfile)
+for Linux builds and these scripts on other platforms:
+
+* [`install_macos_deps.sh`](./install_macos_deps.sh)
+* [`install_windows_deps.ps1`](./install_windows_deps.ps1)
+
+### Version management
+
+These scripts handle versioning across packages, including considerations like
+major, minor, and patch levels (`X.Y.Z`), as well as suffixes like
+`rc20241107` or `dev+{git hash}`:
+
+* [`compute_common_version.py`](./compute_common_version.py)
+* [`compute_local_version.py`](./compute_local_version.py)
+* [`promote_whl_from_rc_to_final.py`](./promote_whl_from_rc_to_final.py)
+
+### PyPI deployment
+
+These scripts handle promoting nightly releases packages to stable and pushing
+to PyPI:
+
+* [`promote_whl_from_rc_to_final.py`](./promote_whl_from_rc_to_final.py)
+* [`pypi_deploy.sh`](./pypi_deploy.sh)
+
+Both of these scripts expect to have the dependencies from
+[`pypi_deploy_requirements.txt`](./pypi_deploy_requirements.txt) installed.
+This can be easily managed by using a Python virtual environment:
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+python -m pip install -r ./pypi_deploy_requirements.txt
+```
+
 ## Debugging manylinux builds
 
 We build releases under a manylinux derived docker image. When all goes well,

build_tools/python_deploy/pypi_deploy.sh

@@ -19,7 +19,10 @@
 # You must have `gh` installed and authenticated (run `gh auth`).
 #
 # Usage:
-# ./pypi_deploy.sh candidate-20220930.282
+#   python -m venv .venv
+#   source .venv/bin/activate
+#   python -m pip install -r ./pypi_deploy_requirements.txt
+#   ./pypi_deploy.sh candidate-20220930.282
 
 set -euo pipefail
 
@@ -29,15 +32,22 @@ SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")";
 REQUIREMENTS_FILE="${SCRIPT_DIR}/pypi_deploy_requirements.txt"
 TMPDIR="$(mktemp --directory --tmpdir iree_pypi_wheels.XXXXX)"
 
-function check_exists() {
+function check_command_exists() {
   if ! command -v "$1" > /dev/null; then
     echo "$1 not found."
-    exit 1
+    return 1
   fi
+  return 0
+}
+
+function check_python_package_installed() {
+  if ! pip show "$1" > /dev/null; then
+    echo "$1 not installed."
+    return 1
+  fi
+  return 0
 }
 
-# It really *seems* like there should be a pip command to do this, but there's
-# not, apparently.
 function check_requirements() {
   while read line; do
     # Read in the package, ignoring everything after the first '='
@@ -48,7 +58,7 @@ function check_requirements() {
       echo "Reading requirements file '${REQUIREMENTS_FILE}' failed."
       exit "${ret}"
     fi
-    if ! check_exists "${package}"; then
+    if ! check_python_package_installed "${package}"; then
       echo "Recommend installing python dependencies in a venv using pypi_deploy_requirements.txt"
       exit 1
     fi
@@ -88,13 +98,16 @@ function upload_wheels() {
 function main() {
   echo "Changing into ${TMPDIR}"
   cd "${TMPDIR}"
+
+  set +e
   check_requirements
 
-  if ! check_exists gh; then
+  if ! check_command_exists gh; then
     echo "The GitHub CLI 'gh' is required. See https://github.com/cli/cli#installation."
     echo " Googlers, the PPA should already be on your linux machine."
     exit 1
   fi
+  set -e
 
   download_wheels
   edit_release_versions