Upload python package to PyPI (#433)

* Upload to PyPI Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * typo Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * typo Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * another typo Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * typo Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * typo Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * typo Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * review feedback Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * circleci config typo Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * clean up Makefile Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * do pypi upload in build job Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * remove UnimplementedException from harness Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * py_test asserting requirements are in sync Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * update comment Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * typos Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * fix circleci env variables Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * nit Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * experiment with vars Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * experiment with vars Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * experiment with vars Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * stable state Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com> * remove testpypi upload Signed-off-by: Dmitriy Kunitskiy <dkunitskiy@lyft.com>
bufbuild · Mar 15, 2021 · 4ff9878 · 4ff9878
1 parent 12afc6d
commit 4ff9878
Show file tree

Hide file tree

Showing 26 changed files with 370 additions and 278 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -7,7 +7,11 @@ jobs:
       - checkout
       - run: docker build -t protoc-gen-validate .
       - run: docker run --rm protoc-gen-validate ci
-
+      - run: |
+          if [ "${CIRCLE_BRANCH}" == "main" ]; then
+            docker run --rm --env PYPI_REPO=pypi --env PGV_PYPI_TOKEN="${PGV_PYPI_TOKEN}" protoc-gen-validate python-release
+          fi
+
   javabuild:
     machine: true
     working_directory: ~/.go_workspace/src/github.com/envoyproxy/protoc-gen-validate/java
@@ -19,13 +23,13 @@ jobs:
       - checkout:
           path: ~/.go_workspace/src/github.com/envoyproxy/protoc-gen-validate
       - run: sudo rm -rf /usr/local/go && curl -O https://dl.google.com/go/go${CI_GO_VERSION}.linux-amd64.tar.gz && sudo tar -C /usr/local -xzf go${CI_GO_VERSION}.linux-amd64.tar.gz && rm go${CI_GO_VERSION}.linux-amd64.tar.gz && go version
-      - run: mvn -B verify  
+      - run: mvn -B verify
       - add_ssh_keys:
           fingerprints:
             - "cd:a8:80:f3:5e:9a:37:30:ef:55:20:b5:1f:b9:e5:18"
       - deploy:
-          command: | # Deploy if on master branch. If the $RELEASE and $NEXT variables are set then prepare a full maven release.
-            if [ "${CIRCLE_BRANCH}" == "master" ]; then
+          command: | # Deploy if on main branch. If the $RELEASE and $NEXT variables are set then prepare a full maven release.
+            if [ "${CIRCLE_BRANCH}" == "main" ]; then
               echo $GPG_KEY | base64 --decode > signing-key
               gpg --passphrase $GPG_PASSPHRASE --import signing-key
               shred signing-key
@@ -44,7 +48,3 @@ workflows:
     jobs:
       - build
       - javabuild
-
-
-
-
diff --git a/.gitignore b/.gitignore
@@ -1,10 +1,17 @@
 bazel-*
-
+**/.DS_Store
 !vendor/*
 
 /bin
 /protoc-gen-validate
 
+/python/LICENSE
+/python/validate.proto
+/python/dist/
+*.egg-info/
+__pycache__/
+*.py[cod]
+
 /tests/harness/cases/go
 /tests/harness/cases/gogo
 /tests/harness/cases/other_package/go

diff --git a/Dockerfile b/Dockerfile
@@ -12,11 +12,17 @@ ENV INSTALL_DEPS \
   wget \
   maven \
   patch \
-  python
+  python3.8 \
+  python3.8-distutils \
+  python3-setuptools
 RUN apt-get update \
   && apt-get install -y -q --no-install-recommends curl openjdk-8-jdk \
   && echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list \
   && curl https://bazel.build/bazel-release.pub.gpg | apt-key add - \
+  # software-properties-common allows adding PPA (Personal Package Archive) repositories
+  && apt install -y -q --no-install-recommends software-properties-common \
+  # deadsnakes is a PPA with newer releases of python than default Ubuntu repositories
+  && add-apt-repository ppa:deadsnakes/ppa \
   && apt-get update \
   && apt-get install -y -q --no-install-recommends ${INSTALL_DEPS} \
   && apt-get clean \
@@ -62,6 +68,14 @@ RUN go get github.com/bazelbuild/buildtools/buildozer
 WORKDIR ${GOPATH}/src/github.com/envoyproxy/protoc-gen-validate
 COPY . .
 
+# python must be on PATH for the execution of py_binary bazel targets, but
+# the distribution we installed doesn't provide this alias
+RUN ln -s /usr/bin/python3.8 /usr/bin/python
+
+# python tooling for linting and uploading to PyPI
+RUN python3.8 -m easy_install pip \
+  && python3.8 -m pip install -r requirements.txt
+
 RUN make build
 
 ENTRYPOINT ["make"]

diff --git a/Makefile b/Makefile
@@ -40,6 +40,9 @@ lint: bin/golint bin/shadow
 	# golint -set_exit_status
 	# check for variable shadowing
 	go vet -vettool=$(shell pwd)/bin/shadow ./...
+	# lints the python code for style enforcement
+	flake8 --config=python/setup.cfg python/protoc_gen_validate/validator.py
+	isort --check-only python/protoc_gen_validate/validator.py
 
 bin/shadow:
 	GOBIN=$(shell pwd)/bin go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
@@ -115,6 +118,23 @@ tests/harness/java/java-harness:
 	# generates the Java-specific test harness
 	mvn -q -f java/pom.xml clean package -DskipTests
 
+.PHONY: prepare-python-release
+prepare-python-release:
+	cp validate/validate.proto python/
+	cp LICENSE python/
+
+.PHONY: python-release
+python-release: prepare-python-release
+	rm -rf python/dist
+	python3.8 -m build --no-isolation --sdist python
+	# the below command should be identical to `python3.8 -m build --wheel`
+	# however that returns mysterious `error: could not create 'build': File exists`.
+	# setuptools copies source and data files to a temporary build directory,
+	# but why there's a collision or why setuptools stopped respecting the `build_lib` flag is unclear.
+	# As a workaround, we build a source distribution and then separately build a wheel from it.
+	python3.8 -m pip wheel --wheel-dir python/dist --no-deps python/dist/*
+	python3.8 -m twine upload --verbose --skip-existing --repository ${PYPI_REPO} --username "__token__" --password ${PGV_PYPI_TOKEN} python/dist/*
+
 .PHONY: ci
 ci: lint bazel testcases bazel-tests build_generation_tests
 
@@ -130,5 +150,6 @@ clean:
 	rm -rf \
 		tests/harness/cases/go \
 		tests/harness/cases/other_package/go
-
-
+	rm -rf \
+		python/dist
+		python/*.egg-info
diff --git a/README.md b/README.md
@@ -83,7 +83,7 @@ make build
   - `go`
   - `cc` for c++ (partially implemented)
   - `java`
-  - `python`
+- Note: Python works via runtime code generation. There's no compile-time generation. See the Python section for details.
 
 ### Examples
 
@@ -194,28 +194,25 @@ serverBuilder.addService(ServerInterceptors.intercept(svc, new ValidatingServerI
 
 #### Python
 
-Since Python is a dynamically typed language, it works with JIT code generation. So protoc does not need to be run to generate code.
-
-The file `validate/validator.py` has a `validate()` method which needs to be run with an instance of the proto you are validating. 
-
-You must install all the dependencies in the `requirements.txt` before running the validator.
+The python implementation works via JIT code generation. In other words, the `validate(msg)` function is written
+on-demand and [exec-ed](https://docs.python.org/3/library/functions.html#exec). An LRU-cache improves performance by 
+storing generated functions per descriptor. 
+ 
+The python package is available on [PyPI](https://pypi.org/project/protoc-gen-validate).
 
 To run `validate()`, do the following: 
-```
-from validator import validate, FailedValidation
-
-p = Person()
-validate(p) # This should either return None or raise a ValidationFailed exception.
+```python
+from entities_pb2 import Person
+from protoc_gen_validate.validator import validate, ValidationFailed
+
+p = Person(first_name="Foo", last_name="Bar", age=42)
+try:
+    validate(p)
+except ValidationFailed as err:
+    print(err)
 ```
 
-To see what code has been generated and run, you can do the following:
-```
-from validator import validate, print_validate, FailedValidation
-
-p = Person()
-validate(p)
-printer = print_validate(p)
-```
+You can view what code has been generated by using the `print_validate()` function.
 
 ## Constraint Rules
 

diff --git a/WORKSPACE b/WORKSPACE
@@ -8,10 +8,6 @@ load("//bazel:dependency_imports.bzl", "pgv_dependency_imports")
 
 pgv_dependency_imports()
 
-load("//bazel:pip_dependencies.bzl", "pgv_pip_dependencies")
-
-pgv_pip_dependencies()
-
 load("//:dependencies.bzl", "go_third_party")
 
 # gazelle:repository_macro dependencies.bzl%go_third_party

diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl
@@ -1,19 +1,15 @@
 load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
-load("@io_bazel_rules_python//python:pip.bzl", "pip_import", "pip_repositories")
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
-
-# Only needed for PIP support:
+load("@rules_python//python:pip.bzl", "pip_install")
 
 def _pgv_pip_dependencies():
-    pip_repositories()
-
-    # This rule translates the specified requirements.txt into
-    # @pgv_pip_deps//:requirements.bzl, which itself exposes a pip_install method.
-    pip_import(
+    # This rule translates the specified requirements.in (which must be same as install_requires from setup.cfg)
+    # into @pgv_pip_deps//:requirements.bzl.
+    pip_install(
         name = "pgv_pip_deps",
-        requirements = "//:requirements.txt",
+        requirements = "//python:requirements.in",
     )
 
 def _pgv_go_dependencies():
@@ -27,7 +23,7 @@ def pgv_dependency_imports():
     # Import @com_google_protobuf's dependencies.
     protobuf_deps()
 
-    # Import @pgv_pip_deps defined by pip's requirements.txt.
+    # Import @pgv_pip_deps defined by python/requirements.in.
     _pgv_pip_dependencies()
 
     # Import rules for the Go compiler.

diff --git a/bazel/pgv_proto_library.bzl b/bazel/pgv_proto_library.bzl
@@ -1,8 +1,7 @@
 load("@io_bazel_rules_go//proto:compiler.bzl", "go_proto_compiler")
 load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
 load("@rules_cc//cc:defs.bzl", "cc_library")
-load("@rules_python//python:defs.bzl", "py_library")
-load(":protobuf.bzl", "cc_proto_gen_validate", "java_proto_gen_validate", "python_proto_gen_validate")
+load(":protobuf.bzl", "cc_proto_gen_validate", "java_proto_gen_validate")
 
 def pgv_go_proto_library(name, proto = None, deps = [], **kwargs):
     go_proto_compiler(
@@ -63,30 +62,4 @@ def pgv_cc_proto_library(
         **kargs
     )
 
-def pgv_python_proto_library(
-        name,
-        deps = [],
-        python_deps = [],
-        **kwargs):
-    """Bazel rule to create a Python protobuf validation library from proto sources files.
-     Args:
-      name: the name of the pgv_python_proto_library
-      deps: proto_library rules that contain the necessary .proto files
-      python_deps: Python dependencies of the protos being compiled. Likely py_proto_library or pgv_python_proto_library.
-      **kwargs: other keyword arguments that are passed to the py_library.
-    """
-
-    python_proto_gen_validate(
-        name = name + "_validate",
-        deps = deps,
-    )
-    py_library(
-        name = name,
-        srcs = [name + "_validate"],
-        deps = python_deps + [
-            "@com_envoyproxy_protoc_gen_validate//validate:validate_py",
-        ],
-        **kwargs
-    )
-
 pgv_java_proto_library = java_proto_gen_validate
diff --git a/bazel/pip_dependencies.bzl b/bazel/pip_dependencies.bzl
diff --git a/bazel/protobuf.bzl b/bazel/protobuf.bzl
@@ -108,40 +108,6 @@ def _protoc_gen_validate_cc_impl(ctx):
         package_command = "true",
     )
 
-def _protoc_python_output_files(ctx, proto_file_sources):
-    python_srcs = []
-
-    for p in proto_file_sources:
-        # The returned path needs to be relative to the package directory.
-        file_path = _package_relative_path(ctx, p)
-        if file_path.endswith(".proto"):
-            file_path = file_path[:-len(".proto")]
-
-        python_srcs.append(file_path.replace("-", "_") + "_pb2.py")
-    return python_srcs
-
-def _protoc_gen_validate_python_impl(ctx):
-    """Generate Python protos using protoc-gen-validate plugin"""
-    protos = _proto_sources(ctx)
-
-    python_files = _protoc_python_output_files(ctx, protos)
-    out_files = [ctx.actions.declare_file(out) for out in python_files]
-
-    dir_out = _output_dir(ctx)
-
-    args = [
-        "--python_out=" + dir_out,
-    ]
-
-    return _protoc_gen_validate_impl(
-        ctx = ctx,
-        lang = "python",
-        protos = protos,
-        out_files = out_files,
-        protoc_args = args,
-        package_command = "true",
-    )
-
 def _protoc_gen_validate_impl(ctx, lang, protos, out_files, protoc_args, package_command):
     protoc_args.append("--plugin=protoc-gen-validate=" + ctx.executable._plugin.path)
 
@@ -317,26 +283,3 @@ java_proto_gen_validate = rule(
     },
     implementation = _java_proto_gen_validate_impl,
 )
-
-python_proto_gen_validate = rule(
-    attrs = {
-        "deps": attr.label_list(
-            mandatory = True,
-            providers = [ProtoInfo],
-        ),
-        "_protoc": attr.label(
-            cfg = "host",
-            default = Label("@com_google_protobuf//:protoc"),
-            executable = True,
-            allow_single_file = True,
-        ),
-        "_plugin": attr.label(
-            cfg = "host",
-            default = Label("@com_envoyproxy_protoc_gen_validate//:protoc-gen-validate"),
-            allow_files = True,
-            executable = True,
-        ),
-    },
-    output_to_genfiles = True,
-    implementation = _protoc_gen_validate_python_impl,
-)
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -126,12 +126,11 @@ def pgv_dependencies():
             server_urls = MAVEN_SERVER_URLS,
         )
 
-    if not native.existing_rule("io_bazel_rules_python"):
-        git_repository(
-            name = "io_bazel_rules_python",
-            remote = "https://github.com/bazelbuild/rules_python.git",
-            commit = "fdbb17a4118a1728d19e638a5291b4c4266ea5b8",
-            shallow_since = "1557865590 -0400",
+    if not native.existing_rule("rules_python"):
+        http_archive(
+            name = "rules_python",
+            url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz",
+            sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0",
         )
 
     if not native.existing_rule("rules_proto"):

diff --git a/python/BUILD b/python/BUILD
@@ -0,0 +1,11 @@
+load("@rules_python//python:defs.bzl", "py_library")
+load("@pgv_pip_deps//:requirements.bzl", "all_requirements")
+
+exports_files(["requirements.in", "setup.cfg"])
+
+py_library(
+    name = "validator_py",
+    srcs = glob(["**/*.py"]),
+    deps = all_requirements,
+    visibility = ["//visibility:public"],
+)
diff --git a/python/README.md b/python/README.md
@@ -0,0 +1,21 @@
+# Protoc-gen-validate (PGV)
+While protocol buffers effectively guarantee the types of structured data, 
+they cannot enforce semantic rules for values. This package is a python implementation
+of [protoc-gen-validate][pgv-home], which allows for runtime validation of various 
+semantic assertions expressed as annotations on the protobuf schema. The syntax for all available annotations is
+in `validate.proto`. Implemented Python annotations are listed in the [rules comparison][rules-comparison].
+
+### Example
+```python3
+from entities_pb2 import Person
+from protoc_gen_validate.validator import validate, ValidationFailed
+
+p = Person(first_name="Foo", last_name="Bar", age=42)
+try:
+    validate(p)
+except ValidationFailed as err:
+    print(err)
+```
+
+[pgv-home]: https://github.com/envoyproxy/protoc-gen-validate
+[rules-comparison]: https://github.com/envoyproxy/protoc-gen-validate/blob/main/rule_comparison.md
diff --git a/python/protoc_gen_validate/__init__.py b/python/protoc_gen_validate/__init__.py