diff --git a/azure-pipelines-e2e.yml b/azure-pipelines-e2e.yml index dd782ef20..8d3b7ef1c 100644 --- a/azure-pipelines-e2e.yml +++ b/azure-pipelines-e2e.yml @@ -1,7 +1,7 @@ name: 1.0.0-beta$(Date:yyyyMMdd)$(Rev:.r) variables: - DOTNET_VERSION: '2.2.300' + DOTNET_VERSION: '2.2.402' CORE_TOOLS_EXE_PATH: '$(Build.SourcesDirectory)/Azure.Functions.Core.Tools/func' jobs: @@ -17,7 +17,7 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '$(pythonVersion)' - addToPath: true + addToPath: true - powershell: .ci/e2e/setup-e2e.ps1 displayName: 'Setup custom Core Tools' @@ -31,9 +31,9 @@ jobs: displayName: 'Install Core Tools production' - task: DotNetCoreInstaller@0 inputs: - packageType: 'sdk' + packageType: 'sdk' version: $(DOTNET_VERSION) - displayName: 'Install dotnet' + displayName: 'Install dotnet' - bash: | set -e -x python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev] @@ -55,4 +55,3 @@ jobs: inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' artifactName: 'test_result' - \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bfa11bea5..ab738328c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,11 +1,11 @@ -name: 1.0.$(Date:yyyyMMdd)$(Rev:.r) +name: 1.0.$(Date:yyyyMMdd)$(Rev:r) trigger: - dev - master variables: - DOTNET_VERSION: '2.2.401' + DOTNET_VERSION: '2.2.402' jobs: - job: Tests @@ -22,7 +22,7 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '$(pythonVersion)' - addToPath: true + addToPath: true - task: ShellScript@2 inputs: disableAutoCwd: true # Execute in current directory @@ -30,12 +30,12 @@ jobs: displayName: 'Install Core Tools' - task: DotNetCoreInstaller@0 inputs: - packageType: 'sdk' + packageType: 'sdk' version: $(DOTNET_VERSION) - displayName: 'Install dotnet' + displayName: 'Install dotnet' - task: ShellScript@2 inputs: - disableAutoCwd: true + disableAutoCwd: true scriptPath: .ci/linux_devops_build.sh displayName: 'Build' - bash: | @@ -51,52 +51,90 @@ jobs: LINUXEVENTHUBCONNECTIONSTRING: $(LinuxEventHubConnectionString) LINUXSERVICEBUSCONNECTIONSTRING: $(LinuxServiceBusConnectionString) displayName: 'E2E Tests' - -- template: pack/templates/win_env_gen.yml - parameters: - jobName: 'WindowsEnvGen' - dependency: 'Tests' - vmImage: 'vs2017-win2016' - pythonVersion: '3.6' - artifactName: 'Windows' -- template: pack/templates/nix_env_gen.yml - parameters: - jobName: 'LinuxEnvGen' - dependency: 'Tests' +- job: Build_WINDOWS_X64 + dependsOn: 'Tests' + pool: + vmImage: 'vs2017-win2016' + strategy: + matrix: + Python36: + pythonVersion: '3.6' + Python37: + pythonVersion: '3.7' + steps: + - template: pack/templates/win_env_gen.yml + parameters: + pythonVersion: '$(pythonVersion)' + architecture: 'x64' + artifactName: '$(pythonVersion)_WINDOWS_X64' +- job: Build_WINDOWS_X86 + dependsOn: 'Tests' + pool: + vmImage: 'vs2017-win2016' + strategy: + matrix: + Python37: + pythonVersion: '3.7' + steps: + - template: pack/templates/win_env_gen.yml + parameters: + pythonVersion: '$(pythonVersion)' + architecture: 'x86' + artifactName: '$(pythonVersion)_WINDOWS_x86' +- job: Build_LINUX_X64 + dependsOn: 'Tests' + pool: vmImage: 'ubuntu-16.04' - pythonVersion: '3.6' - artifactName: 'Linux' - -- template: pack/templates/nix_env_gen.yml - parameters: - jobName: 'MacEnvGen' - dependency: 'Tests' + strategy: + matrix: + Python36: + pythonVersion: '3.6' + Python37: + pythonVersion: '3.7' + steps: + - template: pack/templates/nix_env_gen.yml + parameters: + pythonVersion: '$(pythonVersion)' + artifactName: '$(pythonVersion)_LINUX_X64' +- job: Build_OSX_X64 + dependsOn: 'Tests' + pool: vmImage: 'macOS-10.13' - pythonVersion: '3.6' - artifactName: 'Mac' + strategy: + matrix: + Python36: + pythonVersion: '3.6' + Python37: + pythonVersion: '3.7' + steps: + - template: pack/templates/nix_env_gen.yml + parameters: + pythonVersion: '$(pythonVersion)' + artifactName: '$(pythonVersion)_OSX_X64' -- job: PackageEnvironments - dependsOn: ['WindowsEnvGen', - 'LinuxEnvGen', - 'MacEnvGen' +- job: PackageWorkers + dependsOn: ['Build_WINDOWS_X64', + 'Build_WINDOWS_X86', + 'Build_LINUX_X64', + 'Build_OSX_X64' ] pool: vmImage: 'vs2017-win2016' - steps: + steps: - task: DownloadBuildArtifacts@0 inputs: - buildType: 'current' + buildType: 'current' downloadType: 'specific' downloadPath: '$(Build.SourcesDirectory)' - task: NuGetCommand@2 inputs: command: pack - packagesToPack: 'pack\Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec' + packagesToPack: 'pack\Microsoft.Azure.Functions.PythonWorker.nuspec' versioningScheme: 'byEnvVar' versionEnvVar: BUILD_BUILDNUMBER # Replaces version in nuspec - task: PublishBuildArtifacts@1 inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: 'PythonWorkerRunEnvironments' - \ No newline at end of file + artifactName: 'PythonWorker' + diff --git a/azure_functions_worker/bindings/datumdef.py b/azure_functions_worker/bindings/datumdef.py index f6f88e7cf..ce6d1dcd9 100644 --- a/azure_functions_worker/bindings/datumdef.py +++ b/azure_functions_worker/bindings/datumdef.py @@ -33,7 +33,7 @@ def from_typed_data(cls, td: protos.TypedData): k: Datum(v, 'string') for k, v in http.headers.items() }, body=( - Datum.from_typed_data(http.rawBody) + Datum.from_typed_data(http.body) or Datum(type='bytes', value=b'') ), params={ diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index 62df7a022..3edc039b0 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -1,3 +1,5 @@ # Capabilities RAW_HTTP_BODY_BYTES = "RawHttpBodyBytes" TYPED_DATA_COLLECTION = "TypedDataCollection" +RPC_HTTP_BODY_ONLY = "RpcHttpBodyOnly" +RPC_HTTP_TRIGGER_METADATA_REMOVED = "RpcHttpTriggerMetadataRemoved" diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 16ad83daf..41e76a16e 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -217,6 +217,8 @@ async def _handle__worker_init_request(self, req): capabilities = dict() capabilities[constants.RAW_HTTP_BODY_BYTES] = "true" capabilities[constants.TYPED_DATA_COLLECTION] = "true" + capabilities[constants.RPC_HTTP_BODY_ONLY] = "true" + capabilities[constants.RPC_HTTP_TRIGGER_METADATA_REMOVED] = "true" return protos.StreamingMessage( request_id=self.request_id, diff --git a/azure_functions_worker/testutils.py b/azure_functions_worker/testutils.py index a33af0a6a..2a3191545 100644 --- a/azure_functions_worker/testutils.py +++ b/azure_functions_worker/testutils.py @@ -44,7 +44,7 @@ 'Microsoft.Azure.WebJobs.Script.WebHost.dll' EXTENSIONS_PATH = PROJECT_ROOT / 'build' / 'extensions' / 'bin' FUNCS_PATH = TESTS_ROOT / UNIT_TESTS_FOLDER / 'http_functions' -WORKER_PATH = PROJECT_ROOT / 'python' +WORKER_PATH = PROJECT_ROOT / 'python' / 'test' WORKER_CONFIG = PROJECT_ROOT / '.testconfig' ON_WINDOWS = platform.system() == 'Windows' diff --git a/pack/Microsoft.Azure.Functions.PythonWorker.nuspec b/pack/Microsoft.Azure.Functions.PythonWorker.nuspec new file mode 100644 index 000000000..c6cadaa84 --- /dev/null +++ b/pack/Microsoft.Azure.Functions.PythonWorker.nuspec @@ -0,0 +1,24 @@ + + + + Microsoft.Azure.Functions.PythonWorker + 1.0.0 + Microsoft + Microsoft + false + Microsoft Azure Functions Python Worker + © .NET Foundation. All rights reserved. + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pack/Microsoft.Azure.Functions.PythonWorker.targets b/pack/Microsoft.Azure.Functions.PythonWorker.targets new file mode 100644 index 000000000..25c2856fe --- /dev/null +++ b/pack/Microsoft.Azure.Functions.PythonWorker.targets @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec deleted file mode 100644 index a29b3385b..000000000 --- a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec +++ /dev/null @@ -1,18 +0,0 @@ - - - - Microsoft.Azure.Functions.PythonWorkerRunEnvironments - 1.0.0-beta0 - Microsoft - Microsoft - false - Microsoft Azure Functions Python Worker Run Environments - © .NET Foundation. All rights reserved. - - - - - - - - \ No newline at end of file diff --git a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.targets b/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.targets deleted file mode 100644 index 2e0514df1..000000000 --- a/pack/Microsoft.Azure.Functions.PythonWorkerRunEnvironments.targets +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/pack/scripts/win_deps.ps1 b/pack/scripts/win_deps.ps1 index aec192acb..51f1ac50e 100644 --- a/pack/scripts/win_deps.ps1 +++ b/pack/scripts/win_deps.ps1 @@ -1,4 +1,4 @@ -py -3.6 -m venv .env +python -m venv .env .env\scripts\activate python -m pip install . diff --git a/pack/templates/nix_env_gen.yml b/pack/templates/nix_env_gen.yml index e96e758a7..cc64f69cb 100644 --- a/pack/templates/nix_env_gen.yml +++ b/pack/templates/nix_env_gen.yml @@ -1,44 +1,37 @@ parameters: - jobName: 'LinuxEnvGen' - dependency: 'Tests' - vmImage: 'ubuntu-16.04' - pythonVersion: '3.6' - artifactName: 'Linux' - -jobs: -- job: ${{ parameters.jobName }} - dependsOn: ${{ parameters.dependency }} - pool: - vmImage: ${{ parameters.vmImage }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ parameters.pythonVersion }} - addToPath: true - - task: ShellScript@2 - inputs: - disableAutoCwd: true - scriptPath: 'pack/scripts/nix_deps.sh' - - task: CopyFiles@2 - inputs: - contents: | - pack/utils/__init__.py - targetFolder: '$(Build.ArtifactStagingDirectory)/deps/azure' - flattenFolders: true - - task: CopyFiles@2 - inputs: - contents: | - python/* - targetFolder: '$(Build.ArtifactStagingDirectory)' - flattenFolders: true - - task: CopyFiles@2 - inputs: - contents: | - deps/**/* - !deps/grpc_tools/**/* - !deps/grpcio_tools*/* - targetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: ${{ parameters.artifactName }} \ No newline at end of file + pythonVersion: '' + artifactName: '' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: ${{ parameters.pythonVersion }} + addToPath: true +- task: ShellScript@2 + inputs: + disableAutoCwd: true + scriptPath: 'pack/scripts/nix_deps.sh' +- task: CopyFiles@2 + inputs: + contents: | + pack/utils/__init__.py + targetFolder: '$(Build.ArtifactStagingDirectory)/azure' + flattenFolders: true +- task: CopyFiles@2 + inputs: + contents: | + python/prod/worker.py + targetFolder: '$(Build.ArtifactStagingDirectory)' + flattenFolders: true +- task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.SourcesDirectory)/deps' + contents: | + ** + !grpc_tools/**/* + !grpcio_tools*/* + targetFolder: '$(Build.ArtifactStagingDirectory)' +- task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: ${{ parameters.artifactName }} \ No newline at end of file diff --git a/pack/templates/win_env_gen.yml b/pack/templates/win_env_gen.yml index e5b8c21ff..77afca62d 100644 --- a/pack/templates/win_env_gen.yml +++ b/pack/templates/win_env_gen.yml @@ -1,43 +1,37 @@ parameters: - jobName: 'WindowsEnvGen' - dependency: 'Tests' - vmImage: 'vs2017-win2016' - pythonVersion: '3.6' - artifactName: 'Windows' + pythonVersion: '' + artifactName: '' -jobs: -- job: ${{ parameters.jobName }} - dependsOn: ${{ parameters.dependency }} - pool: - vmImage: ${{ parameters.vmImage }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ parameters.pythonVersion }} - addToPath: true - - task: PowerShell@2 - inputs: - filePath: 'pack\scripts\win_deps.ps1' - - task: CopyFiles@2 - inputs: - contents: | - pack\utils\__init__.py - targetFolder: '$(Build.ArtifactStagingDirectory)\deps\azure' - flattenFolders: true - - task: CopyFiles@2 - inputs: - contents: | - python\* - targetFolder: '$(Build.ArtifactStagingDirectory)' - flattenFolders: true - - task: CopyFiles@2 - inputs: - contents: | - deps\**\* - !deps\grpc_tools\**\* - !deps\grpcio_tools*\* - targetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: ${{ parameters.artifactName }} \ No newline at end of file +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: ${{ parameters.pythonVersion }} + architecture: ${{ parameters.architecture }} + addToPath: true +- task: PowerShell@2 + inputs: + filePath: 'pack\scripts\win_deps.ps1' +- task: CopyFiles@2 + inputs: + contents: | + pack\utils\__init__.py + targetFolder: '$(Build.ArtifactStagingDirectory)\azure' + flattenFolders: true +- task: CopyFiles@2 + inputs: + contents: | + python\prod\worker.py + targetFolder: '$(Build.ArtifactStagingDirectory)' + flattenFolders: true +- task: CopyFiles@2 + inputs: + sourceFolder: '$(Build.SourcesDirectory)\deps' + contents: | + ** + !grpc_tools\**\* + !grpcio_tools*\* + targetFolder: '$(Build.ArtifactStagingDirectory)' +- task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: ${{ parameters.artifactName }} \ No newline at end of file diff --git a/python/prod/worker.config.json b/python/prod/worker.config.json new file mode 100644 index 000000000..81045cc12 --- /dev/null +++ b/python/prod/worker.config.json @@ -0,0 +1,12 @@ +{ + "description":{ + "language":"python", + "defaultRuntimeVersion":"3.6", + "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"], + "supportedRuntimeVersions":["3.6", "3.7"], + "supportedArchitectures":["X64", "X86"], + "extensions":[".py"], + "defaultExecutablePath":"python", + "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/{os}/{architecture}/worker.py" + } +} diff --git a/python/prod/worker.py b/python/prod/worker.py new file mode 100644 index 000000000..d19a2568e --- /dev/null +++ b/python/prod/worker.py @@ -0,0 +1,66 @@ +import os +import sys +import platform +import subprocess +from pathlib import Path + +# User packages +PKGS_PATH = "site/wwwroot/.python_packages" +VENV_PKGS_PATH = "site/wwwroot/worker_venv" + +PKGS_36 = "lib/python3.6/site-packages" +PKGS = "lib/site-packages" + +# Azure environment variables +AZURE_WEBSITE_INSTANCE_ID = "WEBSITE_INSTANCE_ID" +AZURE_CONTAINER_NAME = "CONTAINER_NAME" + + +def is_azure_environment(): + return (AZURE_CONTAINER_NAME in os.environ or + AZURE_WEBSITE_INSTANCE_ID in os.environ) + + +def determine_user_pkg_paths(): + minor_version = sys.version_info[1] + + home = Path.home() + pkgs_path = os.path.join(home, PKGS_PATH) + venv_pkgs_path = os.path.join(home, VENV_PKGS_PATH) + + user_pkg_paths = [] + if minor_version == 6: + user_pkg_paths.append(os.path.join(venv_pkgs_path, PKGS_36)) + user_pkg_paths.append(os.path.join(pkgs_path, PKGS_36)) + user_pkg_paths.append(os.path.join(pkgs_path, PKGS)) + elif minor_version == 7: + user_pkg_paths.append(os.path.join(pkgs_path, PKGS)) + else: + raise RuntimeError(f'Unsupported Python version: 3.{minor_version}') + + return user_pkg_paths + + +if __name__ == '__main__': + user_pkg_paths = [] + if is_azure_environment(): + user_pkg_paths = determine_user_pkg_paths() + + env = os.environ + # worker.py lives in the same directory as azure_functions_worker + func_worker_dir = str(Path(__file__).absolute().parent) + + if platform.system() == 'Windows': + joined_pkg_paths = ";".join(user_pkg_paths) + env['PYTHONPATH'] = f'{joined_pkg_paths};{func_worker_dir}' + # execve doesn't work in Windows: https://bugs.python.org/issue19124 + subprocess.run([sys.executable, + '-m', 'azure_functions_worker'] + sys.argv[1:], + env=env) + else: + joined_pkg_paths = ":".join(user_pkg_paths) + env['PYTHONPATH'] = f'{joined_pkg_paths}:{func_worker_dir}' + os.execve(sys.executable, + [sys.executable, '-m', 'azure_functions_worker'] + + sys.argv[1:], + env) diff --git a/python/worker.config.json b/python/test/worker.config.json similarity index 100% rename from python/worker.config.json rename to python/test/worker.config.json diff --git a/python/worker.py b/python/test/worker.py similarity index 100% rename from python/worker.py rename to python/test/worker.py diff --git a/setup.py b/setup.py index 531cfc7c2..050cee628 100644 --- a/setup.py +++ b/setup.py @@ -15,9 +15,9 @@ # TODO: change this to something more stable when available. -WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/sfelyng3x6p5sus0' +WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/j7r6pk8p7mqxyuuw' '/artifacts' - '/Functions.Binaries.2.0.12642.no-runtime.zip') + '/Functions.Binaries.2.0.12701.no-runtime.zip') # Extensions necessary for non-core bindings. AZURE_EXTENSIONS = [ diff --git a/tests/unittests/test_http_functions.py b/tests/unittests/test_http_functions.py index 30658fc90..4cedd5540 100644 --- a/tests/unittests/test_http_functions.py +++ b/tests/unittests/test_http_functions.py @@ -213,3 +213,51 @@ def test_raw_body_bytes(self): finally: if (os.path.exists(received_img_file)): os.remove(received_img_file) + + def test_image_png_content_type(self): + parent_dir = pathlib.Path(__file__).parent + image_file = parent_dir / 'resources/functions.png' + with open(image_file, 'rb') as image: + img = image.read() + img_len = len(img) + r = self.webhost.request( + 'POST', 'raw_body_bytes', + headers={'Content-Type': 'image/png'}, + data=img) + + received_body_len = int(r.headers['body-len']) + self.assertEqual(received_body_len, img_len) + + body = r.content + try: + received_img_file = parent_dir / 'received_img.png' + with open(received_img_file, 'wb') as received_img: + received_img.write(body) + self.assertTrue(filecmp.cmp(received_img_file, image_file)) + finally: + if (os.path.exists(received_img_file)): + os.remove(received_img_file) + + def test_application_octet_stream_content_type(self): + parent_dir = pathlib.Path(__file__).parent + image_file = parent_dir / 'resources/functions.png' + with open(image_file, 'rb') as image: + img = image.read() + img_len = len(img) + r = self.webhost.request( + 'POST', 'raw_body_bytes', + headers={'Content-Type': 'application/octet-stream'}, + data=img) + + received_body_len = int(r.headers['body-len']) + self.assertEqual(received_body_len, img_len) + + body = r.content + try: + received_img_file = parent_dir / 'received_img.png' + with open(received_img_file, 'wb') as received_img: + received_img.write(body) + self.assertTrue(filecmp.cmp(received_img_file, image_file)) + finally: + if (os.path.exists(received_img_file)): + os.remove(received_img_file)