Gen worker builds

azure-pipelines-e2e.yml

@@ -1,7 +1,7 @@
 name: 1.0.0-beta$(Date:yyyyMMdd)$(Rev:.r)
 
 variables:
-    DOTNET_VERSION: '2.2.300'
+    DOTNET_VERSION: '2.2.402'
     CORE_TOOLS_EXE_PATH: '$(Build.SourcesDirectory)/Azure.Functions.Core.Tools/func'
 
 jobs:
@@ -17,7 +17,7 @@ jobs:
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '$(pythonVersion)'
-      addToPath: true 
+      addToPath: true
   - powershell:
       .ci/e2e/setup-e2e.ps1
     displayName: 'Setup custom Core Tools'
@@ -31,9 +31,9 @@ jobs:
     displayName: 'Install Core Tools production'
   - task: DotNetCoreInstaller@0
     inputs:
-      packageType: 'sdk' 
+      packageType: 'sdk'
       version: $(DOTNET_VERSION)
-    displayName: 'Install dotnet' 
+    displayName: 'Install dotnet'
   - bash: |
         set -e -x
         python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
@@ -55,4 +55,3 @@ jobs:
     inputs:
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'
       artifactName: 'test_result'
-

azure-pipelines.yml

@@ -1,11 +1,11 @@
-name: 1.0.$(Date:yyyyMMdd)$(Rev:.r)
+name: 1.0.$(Date:yyyyMMdd)$(Rev:r)
 
 trigger:
 - dev
 - master
 
 variables:
-    DOTNET_VERSION: '2.2.401'
+    DOTNET_VERSION: '2.2.402'
 
 jobs:
 - job: Tests
@@ -22,20 +22,20 @@ jobs:
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '$(pythonVersion)'
-      addToPath: true 
+      addToPath: true
   - task: ShellScript@2
     inputs:
       disableAutoCwd: true # Execute in current directory
       scriptPath: .ci/linux_devops_tools.sh
     displayName: 'Install Core Tools'
   - task: DotNetCoreInstaller@0
     inputs:
-      packageType: 'sdk' 
+      packageType: 'sdk'
       version: $(DOTNET_VERSION)
-    displayName: 'Install dotnet' 
+    displayName: 'Install dotnet'
   - task: ShellScript@2
     inputs:
-      disableAutoCwd: true 
+      disableAutoCwd: true
       scriptPath: .ci/linux_devops_build.sh
     displayName: 'Build'
   - bash: |
@@ -51,52 +51,90 @@ jobs:
       LINUXEVENTHUBCONNECTIONSTRING: $(LinuxEventHubConnectionString)
       LINUXSERVICEBUSCONNECTIONSTRING: $(LinuxServiceBusConnectionString)
     displayName: 'E2E Tests'
-
-- template: pack/templates/win_env_gen.yml
-  parameters:
-    jobName: 'WindowsEnvGen'
-    dependency: 'Tests'
-    vmImage: 'vs2017-win2016'
-    pythonVersion: '3.6'
-    artifactName: 'Windows'
 
-- template: pack/templates/nix_env_gen.yml
-  parameters:
-    jobName: 'LinuxEnvGen'
-    dependency: 'Tests'
+- job: Build_WINDOWS_X64
+  dependsOn: 'Tests'
+  pool: 
+    vmImage: 'vs2017-win2016'
+  strategy:
+    matrix:
+      Python36:
+        pythonVersion: '3.6'
+      Python37:
+        pythonVersion: '3.7'
+  steps:
+  - template: pack/templates/win_env_gen.yml
+    parameters:
+      pythonVersion: '$(pythonVersion)'
+      architecture: 'x64'
+      artifactName: '$(pythonVersion)_WINDOWS_X64'
+- job: Build_WINDOWS_X86
+  dependsOn: 'Tests'
+  pool: 
+    vmImage: 'vs2017-win2016'
+  strategy:
+    matrix:
+      Python37:
+        pythonVersion: '3.7'
+  steps:
+  - template: pack/templates/win_env_gen.yml
+    parameters:
+      pythonVersion: '$(pythonVersion)'
+      architecture: 'x86'
+      artifactName: '$(pythonVersion)_WINDOWS_x86'
+- job: Build_LINUX_X64
+  dependsOn: 'Tests'
+  pool: 
     vmImage: 'ubuntu-16.04'
-    pythonVersion: '3.6'
-    artifactName: 'Linux'
-
-- template: pack/templates/nix_env_gen.yml
-  parameters:
-    jobName: 'MacEnvGen'
-    dependency: 'Tests'
+  strategy:
+    matrix:
+      Python36:
+        pythonVersion: '3.6'
+      Python37:
+        pythonVersion: '3.7'
+  steps:
+  - template: pack/templates/nix_env_gen.yml
+    parameters:
+      pythonVersion: '$(pythonVersion)'
+      artifactName: '$(pythonVersion)_LINUX_X64'
+- job: Build_OSX_X64
+  dependsOn: 'Tests'
+  pool: 
     vmImage: 'macOS-10.13'
-    pythonVersion: '3.6'
-    artifactName: 'Mac'
+  strategy:
+    matrix:
+      Python36:
+        pythonVersion: '3.6'
+      Python37:
+        pythonVersion: '3.7'
+  steps:
+  - template: pack/templates/nix_env_gen.yml
+    parameters:
+      pythonVersion: '$(pythonVersion)'
+      artifactName: '$(pythonVersion)_OSX_X64'
 
-- job: PackageEnvironments
-  dependsOn: ['WindowsEnvGen',
-              'LinuxEnvGen',
-              'MacEnvGen'
+- job: PackageWorkers
+  dependsOn: ['Build_WINDOWS_X64',
+              'Build_WINDOWS_X86',
+              'Build_LINUX_X64',
+              'Build_OSX_X64'
              ]
   pool:
       vmImage: 'vs2017-win2016'
-  steps: 
+  steps:
   - task: DownloadBuildArtifacts@0
     inputs:
-      buildType: 'current' 
+      buildType: 'current'
       downloadType: 'specific'
       downloadPath: '$(Build.SourcesDirectory)'
   - task: NuGetCommand@2
     inputs:
       command: pack
-      packagesToPack: 'pack\Microsoft.Azure.Functions.PythonWorkerRunEnvironments.nuspec'
+      packagesToPack: 'pack\Microsoft.Azure.Functions.PythonWorker.nuspec'
       versioningScheme: 'byEnvVar'
       versionEnvVar: BUILD_BUILDNUMBER # Replaces version in nuspec
   - task: PublishBuildArtifacts@1
     inputs:
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-      artifactName: 'PythonWorkerRunEnvironments'
-
+      artifactName: 'PythonWorker'
+

azure_functions_worker/bindings/datumdef.py

@@ -33,7 +33,7 @@ def from_typed_data(cls, td: protos.TypedData):
                     k: Datum(v, 'string') for k, v in http.headers.items()
                 },
                 body=(
-                    Datum.from_typed_data(http.rawBody)
+                    Datum.from_typed_data(http.body)
                     or Datum(type='bytes', value=b'')
                 ),
                 params={

azure_functions_worker/constants.py

@@ -1,3 +1,5 @@
 # Capabilities
 RAW_HTTP_BODY_BYTES = "RawHttpBodyBytes"
 TYPED_DATA_COLLECTION = "TypedDataCollection"
+RPC_HTTP_BODY_ONLY = "RpcHttpBodyOnly"
+RPC_HTTP_TRIGGER_METADATA_REMOVED = "RpcHttpTriggerMetadataRemoved"

azure_functions_worker/dispatcher.py

@@ -217,6 +217,8 @@ async def _handle__worker_init_request(self, req):
         capabilities = dict()
         capabilities[constants.RAW_HTTP_BODY_BYTES] = "true"
         capabilities[constants.TYPED_DATA_COLLECTION] = "true"
+        capabilities[constants.RPC_HTTP_BODY_ONLY] = "true"
+        capabilities[constants.RPC_HTTP_TRIGGER_METADATA_REMOVED] = "true"
 
         return protos.StreamingMessage(
             request_id=self.request_id,

azure_functions_worker/testutils.py

@@ -44,7 +44,7 @@
     'Microsoft.Azure.WebJobs.Script.WebHost.dll'
 EXTENSIONS_PATH = PROJECT_ROOT / 'build' / 'extensions' / 'bin'
 FUNCS_PATH = TESTS_ROOT / UNIT_TESTS_FOLDER / 'http_functions'
-WORKER_PATH = PROJECT_ROOT / 'python'
+WORKER_PATH = PROJECT_ROOT / 'python' / 'test'
 WORKER_CONFIG = PROJECT_ROOT / '.testconfig'
 ON_WINDOWS = platform.system() == 'Windows'
 

pack/Microsoft.Azure.Functions.PythonWorker.nuspec

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd">
+  <metadata>
+    <id>Microsoft.Azure.Functions.PythonWorker</id>
+    <version>1.0.0</version>
+    <authors>Microsoft</authors>
+    <owners>Microsoft</owners>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <description>Microsoft Azure Functions Python Worker</description>
+    <copyright>© .NET Foundation. All rights reserved.</copyright>
+  </metadata>
+  <files>
+    <file src="..\3.6_WINDOWS_x64\**" target="tools\3.6\WINDOWS\X64" />
+    <!-- <file src="..\3.6_WINDOWS_X86\**" target="tools\3.6\WINDOWS\X86" /> -->
+    <file src="..\3.6_LINUX_X64\**" target="tools\3.6\LINUX\X64" />
+    <file src="..\3.6_OSX_X64\**" target="tools\3.6\OSX\X64" />
+    <file src="..\3.7_WINDOWS_X64\**" target="tools\3.7\WINDOWS\X64" />
+    <file src="..\3.7_WINDOWS_X86\**" target="tools\3.7\WINDOWS\X86" />
+    <file src="..\3.7_LINUX_X64\**" target="tools\3.7\LINUX\X64" />
+    <file src="..\3.7_OSX_X64\**" target="tools\3.7\OSX\X64" />
+    <file src="..\python\prod\worker.config.json" target="tools" />
+    <file src="Microsoft.Azure.Functions.PythonWorker.targets" target="build" />
+  </files>
+</package>

pack/Microsoft.Azure.Functions.PythonWorker.targets

@@ -0,0 +1,25 @@
+<Project>
+  <Target Name="Initialize" AfterTargets="Publish;Build">
+    <CreateItem Include="$(MSBuildThisFileDirectory)\..\">
+      <Output ItemName="MSBuildThisFileDirectoryParentDirectory" TaskParameter="Include"/>
+    </CreateItem>
+
+    <CreateProperty Value="%(MSBuildThisFileDirectoryParentDirectory.Fullpath)">
+        <Output PropertyName="NugetRoot" TaskParameter="Value"/>
+    </CreateProperty>
+
+    <ItemGroup>
+        <SourceFiles Include="$(NugetRoot)tools\**\*.*"/>
+    </ItemGroup>
+  </Target>
+
+  <Target Name="CopyOnPublish" DependsOnTargets="Initialize" AfterTargets="Publish">
+        <Copy SourceFiles="@(SourceFiles)" 
+            DestinationFiles="@(SourceFiles->'$(PublishDir)\workers\python\%(RecursiveDir)%(Filename)%(Extension)')" />
+  </Target>
+
+  <Target Name="CopyOnBuild" DependsOnTargets="Initialize" AfterTargets="Build">
+        <Copy SourceFiles="@(SourceFiles)" 
+            DestinationFiles="@(SourceFiles->'$(OutDir)\workers\python\%(RecursiveDir)%(Filename)%(Extension)')" />
+  </Target>
+</Project>

pack/scripts/win_deps.ps1

@@ -1,4 +1,4 @@
-py -3.6 -m venv .env
+python -m venv .env
 .env\scripts\activate
 python -m pip install .