From c744bbb5b976024f6d7568e5be01fa16a62d7cd6 Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" <48038149+Hazhzeng@users.noreply.github.com> Date: Fri, 11 Oct 2019 14:43:27 -0700 Subject: [PATCH 01/18] Pass bytes from invocation request metadata (#552) * Make http.body always accepts raw_body * Add image/png test and application/octet-stream test --- azure-pipelines-e2e.yml | 9 ++-- azure-pipelines.yml | 17 ++++---- azure_functions_worker/bindings/datumdef.py | 2 +- azure_functions_worker/constants.py | 2 + azure_functions_worker/dispatcher.py | 2 + setup.py | 4 +- tests/unittests/test_http_functions.py | 48 +++++++++++++++++++++ 7 files changed, 67 insertions(+), 17 deletions(-) diff --git a/azure-pipelines-e2e.yml b/azure-pipelines-e2e.yml index dd782ef20..8d3b7ef1c 100644 --- a/azure-pipelines-e2e.yml +++ b/azure-pipelines-e2e.yml @@ -1,7 +1,7 @@ name: 1.0.0-beta$(Date:yyyyMMdd)$(Rev:.r) variables: - DOTNET_VERSION: '2.2.300' + DOTNET_VERSION: '2.2.402' CORE_TOOLS_EXE_PATH: '$(Build.SourcesDirectory)/Azure.Functions.Core.Tools/func' jobs: @@ -17,7 +17,7 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '$(pythonVersion)' - addToPath: true + addToPath: true - powershell: .ci/e2e/setup-e2e.ps1 displayName: 'Setup custom Core Tools' @@ -31,9 +31,9 @@ jobs: displayName: 'Install Core Tools production' - task: DotNetCoreInstaller@0 inputs: - packageType: 'sdk' + packageType: 'sdk' version: $(DOTNET_VERSION) - displayName: 'Install dotnet' + displayName: 'Install dotnet' - bash: | set -e -x python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] @@ -55,4 +55,3 @@ jobs: inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' artifactName: 'test_result' - \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bfa11bea5..1936b6116 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,7 +5,7 @@ trigger: - master variables: - DOTNET_VERSION: '2.2.401' + DOTNET_VERSION: '2.2.402' jobs: - job: Tests @@ -22,7 +22,7 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '$(pythonVersion)' - addToPath: true + addToPath: true - task: ShellScript@2 inputs: disableAutoCwd: true # Execute in current directory @@ -30,12 +30,12 @@ jobs: displayName: 'Install Core Tools' - task: DotNetCoreInstaller@0 inputs: - packageType: 'sdk' + packageType: 'sdk' version: $(DOTNET_VERSION) - displayName: 'Install dotnet' + displayName: 'Install dotnet' - task: ShellScript@2 inputs: - disableAutoCwd: true + disableAutoCwd: true scriptPath: .ci/linux_devops_build.sh displayName: 'Build' - bash: | @@ -51,7 +51,7 @@ jobs: LINUXEVENTHUBCONNECTIONSTRING: $(LinuxEventHubConnectionString) LINUXSERVICEBUSCONNECTIONSTRING: $(LinuxServiceBusConnectionString) displayName: 'E2E Tests' - + - template: pack/templates/win_env_gen.yml parameters: jobName: 'WindowsEnvGen' @@ -83,10 +83,10 @@ jobs: ] pool: vmImage: 'vs2017-win2016' - steps: + steps: - task: DownloadBuildArtifacts@0 inputs: - buildType: 'current' + buildType: 'current' downloadType: 'specific' downloadPath: '$(Build.SourcesDirectory)' - task: NuGetCommand@2 @@ -99,4 +99,3 @@ jobs: inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' artifactName: 'PythonWorkerRunEnvironments' - \ No newline at end of file diff --git a/azure_functions_worker/bindings/datumdef.py b/azure_functions_worker/bindings/datumdef.py index f6f88e7cf..ce6d1dcd9 100644 --- a/azure_functions_worker/bindings/datumdef.py +++ b/azure_functions_worker/bindings/datumdef.py @@ -33,7 +33,7 @@ def from_typed_data(cls, td: protos.TypedData): k: Datum(v, 'string') for k, v in http.headers.items() }, body=( - Datum.from_typed_data(http.rawBody) + Datum.from_typed_data(http.body) or Datum(type='bytes', value=b'') ), params={ diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index 62df7a022..3edc039b0 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -1,3 +1,5 @@ # Capabilities RAW_HTTP_BODY_BYTES = "RawHttpBodyBytes" TYPED_DATA_COLLECTION = "TypedDataCollection" +RPC_HTTP_BODY_ONLY = "RpcHttpBodyOnly" +RPC_HTTP_TRIGGER_METADATA_REMOVED = "RpcHttpTriggerMetadataRemoved" diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 16ad83daf..41e76a16e 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -217,6 +217,8 @@ async def _handle__worker_init_request(self, req): capabilities = dict() capabilities[constants.RAW_HTTP_BODY_BYTES] = "true" capabilities[constants.TYPED_DATA_COLLECTION] = "true" + capabilities[constants.RPC_HTTP_BODY_ONLY] = "true" + capabilities[constants.RPC_HTTP_TRIGGER_METADATA_REMOVED] = "true" return protos.StreamingMessage( request_id=self.request_id, diff --git a/setup.py b/setup.py index 531cfc7c2..050cee628 100644 --- a/setup.py +++ b/setup.py @@ -15,9 +15,9 @@ # TODO: change this to something more stable when available. -WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/sfelyng3x6p5sus0' +WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/j7r6pk8p7mqxyuuw' '/artifacts' - '/Functions.Binaries.2.0.12642.no-runtime.zip') + '/Functions.Binaries.2.0.12701.no-runtime.zip') # Extensions necessary for non-core bindings. AZURE_EXTENSIONS = [ diff --git a/tests/unittests/test_http_functions.py b/tests/unittests/test_http_functions.py index 30658fc90..4cedd5540 100644 --- a/tests/unittests/test_http_functions.py +++ b/tests/unittests/test_http_functions.py @@ -213,3 +213,51 @@ def test_raw_body_bytes(self): finally: if (os.path.exists(received_img_file)): os.remove(received_img_file) + + def test_image_png_content_type(self): + parent_dir = pathlib.Path(__file__).parent + image_file = parent_dir / 'resources/functions.png' + with open(image_file, 'rb') as image: + img = image.read() + img_len = len(img) + r = self.webhost.request( + 'POST', 'raw_body_bytes', + headers={'Content-Type': 'image/png'}, + data=img) + + received_body_len = int(r.headers['body-len']) + self.assertEqual(received_body_len, img_len) + + body = r.content + try: + received_img_file = parent_dir / 'received_img.png' + with open(received_img_file, 'wb') as received_img: + received_img.write(body) + self.assertTrue(filecmp.cmp(received_img_file, image_file)) + finally: + if (os.path.exists(received_img_file)): + os.remove(received_img_file) + + def test_application_octet_stream_content_type(self): + parent_dir = pathlib.Path(__file__).parent + image_file = parent_dir / 'resources/functions.png' + with open(image_file, 'rb') as image: + img = image.read() + img_len = len(img) + r = self.webhost.request( + 'POST', 'raw_body_bytes', + headers={'Content-Type': 'application/octet-stream'}, + data=img) + + received_body_len = int(r.headers['body-len']) + self.assertEqual(received_body_len, img_len) + + body = r.content + try: + received_img_file = parent_dir / 'received_img.png' + with open(received_img_file, 'wb') as received_img: + received_img.write(body) + self.assertTrue(filecmp.cmp(received_img_file, image_file)) + finally: + if (os.path.exists(received_img_file)): + os.remove(received_img_file) From ee6d3fe8f492d82a0a6069630dff1c00bea75478 Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Fri, 27 Sep 2019 10:47:15 -0700 Subject: [PATCH 02/18] Generate version/os/arch bits in NuGet --- azure-pipelines.yml | 88 ++++++++++++++----- azure_functions_worker/testutils.py | 2 +- ...nctions.PythonWorkerRunEnvironments.nuspec | 14 ++- ...ctions.PythonWorkerRunEnvironments.targets | 13 ++- pack/templates/nix_env_gen.yml | 79 ++++++++--------- pack/templates/win_env_gen.yml | 76 ++++++++-------- python/prod/worker.config.json | 8 ++ python/{ => prod}/worker.py | 0 python/{ => test}/worker.config.json | 0 python/test/worker.py | 4 + 10 files changed, 168 insertions(+), 116 deletions(-) create mode 100644 python/prod/worker.config.json rename python/{ => prod}/worker.py (100%) rename python/{ => test}/worker.config.json (100%) create mode 100644 python/test/worker.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1936b6116..675634b82 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -52,34 +52,74 @@ jobs: LINUXSERVICEBUSCONNECTIONSTRING: $(LinuxServiceBusConnectionString) displayName: 'E2E Tests' -- template: pack/templates/win_env_gen.yml - parameters: - jobName: 'WindowsEnvGen' - dependency: 'Tests' +- job: Build_WINDOWS_X64 + dependsOn: 'Tests' + pool: vmImage: 'vs2017-win2016' - pythonVersion: '3.6' - artifactName: 'Windows' - -- template: pack/templates/nix_env_gen.yml - parameters: - jobName: 'LinuxEnvGen' - dependency: 'Tests' + strategy: + matrix: + Python36: + pythonVersion: '3.6' + Python37: + pythonVersion: '3.7' + steps: + - template: pack/templates/win_env_gen.yml + parameters: + pythonVersion: '$(pythonVersion)' + architecture: 'x64' + artifactName: '$(pythonVersion)_WINDOWS_X64' +- job: Build_WINDOWS_X86 + dependsOn: 'Tests' + pool: + vmImage: 'vs2017-win2016' + strategy: + matrix: + Python36: + pythonVersion: '3.6' + Python37: + pythonVersion: '3.7' + steps: + - template: pack/templates/win_env_gen.yml + parameters: + pythonVersion: '$(pythonVersion)' + architecture: 'x86' + artifactName: '$(pythonVersion)_WINDOWS_x86' +- job: Build_LINUX_X64 + dependsOn: 'Tests' + pool: vmImage: 'ubuntu-16.04' - pythonVersion: '3.6' - artifactName: 'Linux' - -- template: pack/templates/nix_env_gen.yml - parameters: - jobName: 'MacEnvGen' - dependency: 'Tests' + strategy: + matrix: + Python36: + pythonVersion: '3.6' + Python37: + pythonVersion: '3.7' + steps: + - template: pack/templates/nix_env_gen.yml + parameters: + pythonVersion: '$(pythonVersion)' + artifactName: '$(pythonVersion)_LINUX_X64' +- job: Build_OSX_X64 + dependsOn: 'Tests' + pool: vmImage: 'macOS-10.13' - pythonVersion: '3.6' - artifactName: 'Mac' + strategy: + matrix: + Python36: + pythonVersion: '3.6' + Python37: + pythonVersion: '3.7' + steps: + - template: pack/templates/nix_env_gen.yml + parameters: + pythonVersion: '$(pythonVersion)' + artifactName: '$(pythonVersion)_OSX_X64' -- job: PackageEnvironments - dependsOn: ['WindowsEnvGen', - 'LinuxEnvGen', - 'MacEnvGen' +- job: PackageWorkers + dependsOn: ['Build_WINDOWS_X64', + 'Build_WINDOWS_X86', + 'Build_LINUX_X64', + 'Build_OSX_X64' ] pool: vmImage: 'vs2017-win2016' diff --git a/azure_functions_worker/testutils.py b/azure_functions_worker/testutils.py index a33af0a6a..2a3191545 100644 --- a/azure_functions_worker/testutils.py +++ b/azure_functions_worker/testutils.py @@ -44,7 +44,7 @@ 'Microsoft.Azure.WebJobs.Script.WebHost.dll' EXTENSIONS_PATH = PROJECT_ROOT / 'build' / 'extensions' / 'bin' FUNCS_PATH = TESTS_ROOT / UNIT_TESTS_FOLDER / 'http_functions' -WORKER_PATH = PROJECT_ROOT / 'python' +WORKER_PATH = PROJECT_ROOT / 'python' / 'test' WORKER_CONFIG = PROJECT_ROOT / '.testconfig' ON_WINDOWS = platform.system() == 'Windows' diff --git a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec index a29b3385b..ad4d13d80 100644 --- a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec +++ b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec @@ -2,7 +2,7 @@ Microsoft.Azure.Functions.PythonWorkerRunEnvironments - 1.0.0-beta0 + 1.0.0 Microsoft Microsoft false @@ -10,9 +10,15 @@ © .NET Foundation. All rights reserved. - - - + + + + + + + + + \ No newline at end of file diff --git a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.targets b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.targets index 2e0514df1..25c2856fe 100644 --- a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.targets +++ b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.targets @@ -1,5 +1,5 @@ - + @@ -11,8 +11,15 @@ + + + + + - + + \ No newline at end of file diff --git a/pack/templates/nix_env_gen.yml b/pack/templates/nix_env_gen.yml index e96e758a7..cc64f69cb 100644 --- a/pack/templates/nix_env_gen.yml +++ b/pack/templates/nix_env_gen.yml @@ -1,44 +1,37 @@ parameters: - jobName: 'LinuxEnvGen' - dependency: 'Tests' - vmImage: 'ubuntu-16.04' - pythonVersion: '3.6' - artifactName: 'Linux' - -jobs: -- job: ${{ parameters.jobName }} - dependsOn: ${{ parameters.dependency }} - pool: - vmImage: ${{ parameters.vmImage }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ parameters.pythonVersion }} - addToPath: true - - task: ShellScript@2 - inputs: - disableAutoCwd: true - scriptPath: 'pack/scripts/nix_deps.sh' - - task: CopyFiles@2 - inputs: - contents: | - pack/utils/__init__.py - targetFolder: '$(Build.ArtifactStagingDirectory)/deps/azure' - flattenFolders: true - - task: CopyFiles@2 - inputs: - contents: | - python/* - targetFolder: '$(Build.ArtifactStagingDirectory)' - flattenFolders: true - - task: CopyFiles@2 - inputs: - contents: | - deps/**/* - !deps/grpc_tools/**/* - !deps/grpcio_tools*/* - targetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: ${{ parameters.artifactName }} \ No newline at end of file + pythonVersion: '' + artifactName: '' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: ${{ parameters.pythonVersion }} + addToPath: true +- task: ShellScript@2 + inputs: + disableAutoCwd: true + scriptPath: 'pack/scripts/nix_deps.sh' +- task: CopyFiles@2 + inputs: + contents: | + pack/utils/__init__.py + targetFolder: '$(Build.ArtifactStagingDirectory)/azure' + flattenFolders: true +- task: CopyFiles@2 + inputs: + contents: | + python/prod/worker.py + targetFolder: '$(Build.ArtifactStagingDirectory)' + flattenFolders: true +- task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.SourcesDirectory)/deps' + contents: | + ** + !grpc_tools/**/* + !grpcio_tools*/* + targetFolder: '$(Build.ArtifactStagingDirectory)' +- task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: ${{ parameters.artifactName }} \ No newline at end of file diff --git a/pack/templates/win_env_gen.yml b/pack/templates/win_env_gen.yml index e5b8c21ff..77afca62d 100644 --- a/pack/templates/win_env_gen.yml +++ b/pack/templates/win_env_gen.yml @@ -1,43 +1,37 @@ parameters: - jobName: 'WindowsEnvGen' - dependency: 'Tests' - vmImage: 'vs2017-win2016' - pythonVersion: '3.6' - artifactName: 'Windows' + pythonVersion: '' + artifactName: '' -jobs: -- job: ${{ parameters.jobName }} - dependsOn: ${{ parameters.dependency }} - pool: - vmImage: ${{ parameters.vmImage }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ parameters.pythonVersion }} - addToPath: true - - task: PowerShell@2 - inputs: - filePath: 'pack\scripts\win_deps.ps1' - - task: CopyFiles@2 - inputs: - contents: | - pack\utils\__init__.py - targetFolder: '$(Build.ArtifactStagingDirectory)\deps\azure' - flattenFolders: true - - task: CopyFiles@2 - inputs: - contents: | - python\* - targetFolder: '$(Build.ArtifactStagingDirectory)' - flattenFolders: true - - task: CopyFiles@2 - inputs: - contents: | - deps\**\* - !deps\grpc_tools\**\* - !deps\grpcio_tools*\* - targetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: ${{ parameters.artifactName }} \ No newline at end of file +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: ${{ parameters.pythonVersion }} + architecture: ${{ parameters.architecture }} + addToPath: true +- task: PowerShell@2 + inputs: + filePath: 'pack\scripts\win_deps.ps1' +- task: CopyFiles@2 + inputs: + contents: | + pack\utils\__init__.py + targetFolder: '$(Build.ArtifactStagingDirectory)\azure' + flattenFolders: true +- task: CopyFiles@2 + inputs: + contents: | + python\prod\worker.py + targetFolder: '$(Build.ArtifactStagingDirectory)' + flattenFolders: true +- task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.SourcesDirectory)\deps' + contents: | + ** + !grpc_tools\**\* + !grpcio_tools*\* + targetFolder: '$(Build.ArtifactStagingDirectory)' +- task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: ${{ parameters.artifactName }} \ No newline at end of file diff --git a/python/prod/worker.config.json b/python/prod/worker.config.json new file mode 100644 index 000000000..6bbac9464 --- /dev/null +++ b/python/prod/worker.config.json @@ -0,0 +1,8 @@ +{ + "description":{ + "language":"python", + "extensions":[".py"], + "defaultExecutablePath":"python", + "defaultWorkerPath":"{FUNCTIONS_WORKER_RUNTIME_VERSION}/@os/@architecture/worker.py" + } +} diff --git a/python/worker.py b/python/prod/worker.py similarity index 100% rename from python/worker.py rename to python/prod/worker.py diff --git a/python/worker.config.json b/python/test/worker.config.json similarity index 100% rename from python/worker.config.json rename to python/test/worker.config.json diff --git a/python/test/worker.py b/python/test/worker.py new file mode 100644 index 000000000..097c48a0d --- /dev/null +++ b/python/test/worker.py @@ -0,0 +1,4 @@ +from azure_functions_worker import main + +if __name__ == '__main__': + main.main() From 9ec6b77ba9c72253b65e0168088195910baed2cc Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Sun, 29 Sep 2019 12:52:21 -0700 Subject: [PATCH 03/18] Fix Windows deps generation to use Python on path --- pack/scripts/win_deps.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pack/scripts/win_deps.ps1 b/pack/scripts/win_deps.ps1 index aec192acb..51f1ac50e 100644 --- a/pack/scripts/win_deps.ps1 +++ b/pack/scripts/win_deps.ps1 @@ -1,4 +1,4 @@ -py -3.6 -m venv .env +python -m venv .env .env\scripts\activate python -m pip install . From f3d3ca6086467e0614bab031dc9ebede6437c10a Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Sun, 29 Sep 2019 14:41:25 -0700 Subject: [PATCH 04/18] Skip 3.6 32-bit as it's buggy --- azure-pipelines.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 675634b82..755910044 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -74,8 +74,6 @@ jobs: vmImage: 'vs2017-win2016' strategy: matrix: - Python36: - pythonVersion: '3.6' Python37: pythonVersion: '3.7' steps: From 1bea47bbcb6d880b8b17f1a0ee03aefc72fb078c Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Tue, 1 Oct 2019 09:44:30 -0700 Subject: [PATCH 05/18] Add default language version to description --- python/prod/worker.config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/python/prod/worker.config.json b/python/prod/worker.config.json index 6bbac9464..5d896015d 100644 --- a/python/prod/worker.config.json +++ b/python/prod/worker.config.json @@ -1,6 +1,7 @@ { "description":{ "language":"python", + "defaultLanguageVersion":"3.7", "extensions":[".py"], "defaultExecutablePath":"python", "defaultWorkerPath":"{FUNCTIONS_WORKER_RUNTIME_VERSION}/@os/@architecture/worker.py" From 91849e05592c99e1a98aaa54b933de662aa2cb04 Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Tue, 1 Oct 2019 10:43:33 -0700 Subject: [PATCH 06/18] Add %% for app setting and {} for variables --- python/prod/worker.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/prod/worker.config.json b/python/prod/worker.config.json index 5d896015d..49a889fbc 100644 --- a/python/prod/worker.config.json +++ b/python/prod/worker.config.json @@ -4,6 +4,6 @@ "defaultLanguageVersion":"3.7", "extensions":[".py"], "defaultExecutablePath":"python", - "defaultWorkerPath":"{FUNCTIONS_WORKER_RUNTIME_VERSION}/@os/@architecture/worker.py" + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/{os}/{architecture}/worker.py" } } From 07fc97179c4105a9a166fa1062ca8c8faa50f2fc Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Wed, 2 Oct 2019 15:55:04 -0700 Subject: [PATCH 07/18] Add default os/architecture and rename to defaultRuntimeVersion --- python/prod/worker.config.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/prod/worker.config.json b/python/prod/worker.config.json index 49a889fbc..43fb3990e 100644 --- a/python/prod/worker.config.json +++ b/python/prod/worker.config.json @@ -1,7 +1,9 @@ { "description":{ "language":"python", - "defaultLanguageVersion":"3.7", + "defaultRuntimeVersion":"3.7", + "defaultOSPlatform":"LINUX", + "defaultArchitecture":"X64", "extensions":[".py"], "defaultExecutablePath":"python", "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/{os}/{architecture}/worker.py" From 380d48c42b9de282db00cd219ae326c686a21b0a Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Thu, 3 Oct 2019 13:08:33 -0700 Subject: [PATCH 08/18] Revert "Add default os/architecture and rename to defaultRuntimeVersion" This reverts commit cd36aabc8596e3f4c2eaeba8d5149e22ed4d104a. --- python/prod/worker.config.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/prod/worker.config.json b/python/prod/worker.config.json index 43fb3990e..49a889fbc 100644 --- a/python/prod/worker.config.json +++ b/python/prod/worker.config.json @@ -1,9 +1,7 @@ { "description":{ "language":"python", - "defaultRuntimeVersion":"3.7", - "defaultOSPlatform":"LINUX", - "defaultArchitecture":"X64", + "defaultLanguageVersion":"3.7", "extensions":[".py"], "defaultExecutablePath":"python", "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/{os}/{architecture}/worker.py" From 576836285ab98116c9391ca1cfc522cc3d62c329 Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Thu, 3 Oct 2019 13:09:50 -0700 Subject: [PATCH 09/18] Rename to defaultRuntimeVersion --- python/prod/worker.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/prod/worker.config.json b/python/prod/worker.config.json index 49a889fbc..88bf2e595 100644 --- a/python/prod/worker.config.json +++ b/python/prod/worker.config.json @@ -1,7 +1,7 @@ { "description":{ "language":"python", - "defaultLanguageVersion":"3.7", + "defaultRuntimeVersion":"3.7", "extensions":[".py"], "defaultExecutablePath":"python", "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/{os}/{architecture}/worker.py" From 2202823ea7add3ee1714b22a0960a944f6629732 Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Thu, 3 Oct 2019 16:10:26 -0700 Subject: [PATCH 10/18] Add supported os/arch/version info --- python/prod/worker.config.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/prod/worker.config.json b/python/prod/worker.config.json index 88bf2e595..12134a7fd 100644 --- a/python/prod/worker.config.json +++ b/python/prod/worker.config.json @@ -2,6 +2,9 @@ "description":{ "language":"python", "defaultRuntimeVersion":"3.7", + "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"], + "supportedRuntimeVersions":["3.6", "3.7"], + "supportedArchitectures":["X64", "X86"], "extensions":[".py"], "defaultExecutablePath":"python", "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/{os}/{architecture}/worker.py" From d0fd84576c5bbcaf83d341d06b32e22eb023647d Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Thu, 3 Oct 2019 18:44:54 -0700 Subject: [PATCH 11/18] Exclude copying 3.6 Windows X86 in nuspec --- ...Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec index ad4d13d80..b6c24040a 100644 --- a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec +++ b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec @@ -11,7 +11,7 @@ - + From 943d3900aac414e90dd4d205942e2f8472c13ab9 Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Fri, 4 Oct 2019 10:08:43 -0700 Subject: [PATCH 12/18] Rename worker run environments to be just worker --- azure-pipelines.yml | 5 +++-- ...rosoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 755910044..7154e3f2d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -130,10 +130,11 @@ jobs: - task: NuGetCommand@2 inputs: command: pack - packagesToPack: 'pack\Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec' + packagesToPack: 'pack\Microsoft.Azure.Functions.PythonWorker.nuspec' versioningScheme: 'byEnvVar' versionEnvVar: BUILD_BUILDNUMBER # Replaces version in nuspec - task: PublishBuildArtifacts@1 inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: 'PythonWorkerRunEnvironments' + artifactName: 'PythonWorker' + diff --git a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec index b6c24040a..bd821008a 100644 --- a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec +++ b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec @@ -1,7 +1,7 @@ - Microsoft.Azure.Functions.PythonWorkerRunEnvironments + Microsoft.Azure.Functions.PythonWorker 1.0.0 Microsoft Microsoft @@ -19,6 +19,6 @@ - + \ No newline at end of file From 3ddcf6ca8adadd4fcf56167e88451ae9f57bec3e Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Fri, 4 Oct 2019 10:16:42 -0700 Subject: [PATCH 13/18] Modify lingering comment --- ...Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec index bd821008a..c6cadaa84 100644 --- a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec +++ b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec @@ -6,7 +6,7 @@ Microsoft Microsoft false - Microsoft Azure Functions Python Worker Run Environments + Microsoft Azure Functions Python Worker © .NET Foundation. All rights reserved. From f9130e3f548e512d541e87a0554761af0e97fcbc Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Fri, 4 Oct 2019 10:17:34 -0700 Subject: [PATCH 14/18] Rename worker run environment files --- ...ments.nuspec => Microsoft.Azure.Functions.PythonWorker.nuspec} | 0 ...nts.targets => Microsoft.Azure.Functions.PythonWorker.targets} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pack/{Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec => Microsoft.Azure.Functions.PythonWorker.nuspec} (100%) rename pack/{Microsoft.Azure.Functions.PythonWorkerRunEnvironments.targets => Microsoft.Azure.Functions.PythonWorker.targets} (100%) diff --git a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec b/pack/Microsoft.Azure.Functions.PythonWorker.nuspec similarity index 100% rename from pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec rename to pack/Microsoft.Azure.Functions.PythonWorker.nuspec diff --git a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.targets b/pack/Microsoft.Azure.Functions.PythonWorker.targets similarity index 100% rename from pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.targets rename to pack/Microsoft.Azure.Functions.PythonWorker.targets From d92b01ac1e755783e3cf88c764e9dfc042751f0c Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Fri, 4 Oct 2019 10:23:30 -0700 Subject: [PATCH 15/18] Follow semver when creating patch version --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7154e3f2d..ab738328c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,4 +1,4 @@ -name: 1.0.$(Date:yyyyMMdd)$(Rev:.r) +name: 1.0.$(Date:yyyyMMdd)$(Rev:r) trigger: - dev From 4bc997e80cff5d3bef38031c19d86a7e6c24e4d0 Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Sun, 6 Oct 2019 15:14:22 -0700 Subject: [PATCH 16/18] Default to Python 3.6 for now --- python/prod/worker.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/prod/worker.config.json b/python/prod/worker.config.json index 12134a7fd..81045cc12 100644 --- a/python/prod/worker.config.json +++ b/python/prod/worker.config.json @@ -1,7 +1,7 @@ { "description":{ "language":"python", - "defaultRuntimeVersion":"3.7", + "defaultRuntimeVersion":"3.6", "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"], "supportedRuntimeVersions":["3.6", "3.7"], "supportedArchitectures":["X64", "X86"], From fcacb645a9ac039293e22cef7f5ae39391d9a567 Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Sun, 13 Oct 2019 14:41:49 -0700 Subject: [PATCH 17/18] Add startup code to worker --- python/prod/worker.py | 61 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/python/prod/worker.py b/python/prod/worker.py index 097c48a0d..419462ca2 100644 --- a/python/prod/worker.py +++ b/python/prod/worker.py @@ -1,4 +1,61 @@ -from azure_functions_worker import main +import os +import sys +import platform +from pathlib import Path + +# User packages +PKGS_PATH = "site/wwwroot/.python_packages" +VENV_PKGS_PATH = "site/wwwroot/worker_venv" + +PKGS_36 = "lib/python3.6/site-packages" +PKGS = "lib/site-packages" + +# Azure environment variables +AZURE_WEBSITE_INSTANCE_ID = "WEBSITE_INSTANCE_ID" +AZURE_CONTAINER_NAME = "CONTAINER_NAME" + + +def is_azure_environment(): + return (AZURE_CONTAINER_NAME in os.environ or + AZURE_WEBSITE_INSTANCE_ID in os.environ) + + +def determine_user_pkg_paths(): + minor_version = sys.version_info[1] + + home = Path.home() + pkgs_path = os.path.join(home, PKGS_PATH) + venv_pkgs_path = os.path.join(home, VENV_PKGS_PATH) + + user_pkg_paths = [] + if minor_version == 6: + user_pkg_paths.append(os.path.join(venv_pkgs_path, PKGS_36)) + user_pkg_paths.append(os.path.join(pkgs_path, PKGS_36)) + user_pkg_paths.append(os.path.join(pkgs_path, PKGS)) + elif minor_version == 7: + user_pkg_paths.append(os.path.join(pkgs_path, PKGS)) + else: + raise RuntimeError(f'Unsupported Python version: 3.{minor_version}') + + return user_pkg_paths + if __name__ == '__main__': - main.main() + user_pkg_paths = [] + if is_azure_environment(): + user_pkg_paths = determine_user_pkg_paths() + + env = os.environ + # worker.py lives in the same directory as azure_functions_worker + func_worker_dir = str(Path(__file__).absolute().parent) + + if platform.system() == 'Windows': + joined_pkg_paths = ";".join(user_pkg_paths) + env['PYTHONPATH'] = f'{joined_pkg_paths};{func_worker_dir}' + else: + joined_pkg_paths = ":".join(user_pkg_paths) + env['PYTHONPATH'] = f'{joined_pkg_paths}:{func_worker_dir}' + + os.execve(sys.executable, + [sys.executable, '-m', 'azure_functions_worker'] + sys.argv[1:], + env) From 076716222c0e08dba154d6e45c0a5a5f01210f95 Mon Sep 17 00:00:00 2001 From: Maheer Iqbal <42051041+maiqbal11@users.noreply.github.com> Date: Tue, 15 Oct 2019 10:06:14 -0700 Subject: [PATCH 18/18] Use subprocess.run for Windows rather than execve --- python/prod/worker.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/python/prod/worker.py b/python/prod/worker.py index 419462ca2..d19a2568e 100644 --- a/python/prod/worker.py +++ b/python/prod/worker.py @@ -1,6 +1,7 @@ import os import sys import platform +import subprocess from pathlib import Path # User packages @@ -52,10 +53,14 @@ def determine_user_pkg_paths(): if platform.system() == 'Windows': joined_pkg_paths = ";".join(user_pkg_paths) env['PYTHONPATH'] = f'{joined_pkg_paths};{func_worker_dir}' + # execve doesn't work in Windows: https://bugs.python.org/issue19124 + subprocess.run([sys.executable, + '-m', 'azure_functions_worker'] + sys.argv[1:], + env=env) else: joined_pkg_paths = ":".join(user_pkg_paths) env['PYTHONPATH'] = f'{joined_pkg_paths}:{func_worker_dir}' - - os.execve(sys.executable, - [sys.executable, '-m', 'azure_functions_worker'] + sys.argv[1:], - env) + os.execve(sys.executable, + [sys.executable, '-m', 'azure_functions_worker'] + + sys.argv[1:], + env)