From 7caa3c798785b9c61cf58b5c4cdb1cec1c2351a1 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 20 Apr 2022 09:29:54 -0700 Subject: [PATCH] Add design doc, update API name, remove scratch pad --- .../ComInterfaceGenerator/VTableStubs.md | 399 ++++++++++++++++++ .../IUnmanagedVirtualMethodTableProvider.cs | 75 +--- .../CodeSnippets.cs | 8 +- 3 files changed, 404 insertions(+), 78 deletions(-) create mode 100644 docs/design/libraries/ComInterfaceGenerator/VTableStubs.md diff --git a/docs/design/libraries/ComInterfaceGenerator/VTableStubs.md b/docs/design/libraries/ComInterfaceGenerator/VTableStubs.md new file mode 100644 index 0000000000000..ddf99389efeb0 --- /dev/null +++ b/docs/design/libraries/ComInterfaceGenerator/VTableStubs.md @@ -0,0 +1,399 @@ +# Generating Virtual Method Table Stubs + +As a building block for the COM interface source generator, we've decided to build a source generator that enables developers to mark that a given interface method should invoke a function pointer at a particular offset into an unmanaged virtual method table, or vtable. We've decided to build this generator as a building block for a few reasons: + +1. As part of the migration of dotnet/runtime to use source-generated P/Invokes, we encountered a few scenarios, particularly in the networking stacks, where non-blittable delegate interop was used because the native APIs do not have static entry points. For at least one of these scenarios, MsQuic, the native library provides a table of function pointers. From our experience, this mechanism for versioning is not uncommon and we feel that supporting native libraries that use a versioning scheme similar to this model is worthwhile for us to support. +2. There are native APIs that we are likely to interoperate with in the future that use native vtables but are not COM-oriented. In particular, the Java Native Interface API, which both dotnet/runtime and xamarin/java.interop interface with in various capacities, uses a vtable model to support exposing their APIs to C and C++. Additionally, its API does not conform to a COM-style IUnknown-based API. +3. Some COM-style APIs have some corner cases with non-COM-style interfaces. Specifically, some corners of the DirectX APIs are still vtable-based, but do not implement IUnknown. Providing this building block will allow developers to more easily consume these APIs with similar gestures as the rest of the DirectX API surface. +4. Our future COM interface source generator can build on this building block to provide sane defaults while allowing developers to use the features of this generator to override any default settings provided by the COM generator. + +## Defined types + +To support this generator, we will define the following APIs. + +The `VirtualMethodIndexAttribute` can be applied to an interface method to trigger the generator. This method will provide the index into the vtable for the method, whether or not the method implicitly takes the native `this` pointer, and which marshalling directions to support. It also has many of the same members as `LibraryImportAttribute` to consistently provide the same marshalling support across source-generated marshalling. + +```csharp +namespace System.Runtime.InteropServices; + +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +public class VirtualMethodIndexAttribute : Attribute +{ + public VirtualMethodIndexAttribute(int index) + { + Index = index; + } + + public int Index { get; } + + public bool ImplicitThisParameter { get; set; } = true; + + public CustomTypeMarshallerDirection Direction { get; set; } = CustomTypeMarshallerDirection.Ref; + + /// + /// Gets or sets how to marshal string arguments to the method. + /// + /// + /// If this field is set to a value other than , + /// must not be specified. + /// + public StringMarshalling StringMarshalling { get; set; } + + /// + /// Gets or sets the used to control how string arguments to the method are marshalled. + /// + /// + /// If this field is specified, must not be specified + /// or must be set to . + /// + public Type? StringMarshallingCustomType { get; set; } + + /// + /// Gets or sets whether the callee sets an error (SetLastError on Windows or errno + /// on other platforms) before returning from the attributed method. + /// + public bool SetLastError { get; set; } +} + +``` + +Additionally, a new interface will be provided. This new interface will be used by the source generator to fetch the native `this` pointer and the vtable that the function pointer is stored in. This interface is designed to provide an API that various native platforms, like COM, WinRT, or Swift, could use to provide support for multiple managed interface wrappers from a single native object. In particular, this interface was designed to ensure it is possible support a managed gesture to do an unmanaged "type cast" (i.e. `QueryInterface` in the COM and WinRT worlds). + +```csharp +namespace System.Runtime.InteropServices; + +public readonly ref struct VirtualMethodTableInfo +{ + public VirtualMethodTableInfo(IntPtr thisPointer, ReadOnlySpan virtualMethodTable) + { + ThisPointer = thisPointer; + VirtualMethodTable = virtualMethodTable; + } + + public IntPtr ThisPointer { get; } + public ReadOnlySpan VirtualMethodTable { get; } + + public void Deconstruct(out IntPtr thisPointer, out ReadOnlySpan virtualMethodTable) // This method allows tuple-style `var (thisPtr, vtable) = virtualMethodTableInfo;` statements from this type. + { + thisPointer = ThisPointer; + virtualMethodTable = VirtualMethodTable; + } +} + +public interface IUnmanagedVirtualMethodTableProvider where T : IEquatable +{ + VirtualMethodTableInfo GetVirtualMethodTableInfoForKey(T typeKey); +} +``` + +## Required API Shapes + +In addition to the provided APIs above, users will be required to add a `readonly static` field or `get`-able property to their user-defined interface type named `TypeKey`. The type of this member will be used as the `T` in `IUnmanagedVirtualMethodTableProvider` and the value will be passed to `GetVirtualMethodTableInfoForKey`. This mechanism is designed to enable each native API platform to provide their own casting key, for example `IID`s in COM, without interfering with each other or requiring using reflection-based types like `System.Type`. + +## Example Usage + +### Flat function table + +In this example, the native API provides a flat table of functions based on the provided version. + +```cpp +// NativeAPI.cpp + +struct NativeAPI +{ + int(*getVersion)(); + int(*add)(int x, int y); + int(*multiply)(int x, int y); +}; + +namespace +{ + int getVersion() + { + return 1; + } + int add(int x, int y) + { + return x + y; + } + int multiply(int x, int y) + { + return x * y; + } + const NativeAPI g_nativeAPI = { + &getVersion, + &add, + &multiply + }; +} + +extern "C" bool GetNativeAPI(int version, NativeAPI const** ppNativeAPI) +{ + if (version > getVersion()) + { + *ppNativeAPI = nullptr; + return false; + } + *ppNativeAPI = &g_nativeAPI; + return true; +} + +``` + +```csharp +// User-written code +// NativeAPI.cs +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly:DisableRuntimeMarshalling] + +// Define the interface of the native API +partial interface INativeAPI +{ + // There is no concept of casting for this API, but providing a type key is still required by the generator. + // Use an empty readonly record struct to provide a type that implements IEquatable but contains no data. + readonly static NoCasting TypeKey = default; + + [VirtualMethodIndex(0, ImplicitThisParameter = false, Direction = CustomTypeMarshallerDirection.In)] + int GetVersion(); + + [VirtualMethodIndex(1, ImplicitThisParameter = false, Direction = CustomTypeMarshallerDirection.In)] + int Add(int x, int y); + + [VirtualMethodIndex(2, ImplicitThisParameter = false, Direction = CustomTypeMarshallerDirection.In)] + int Multiply(int x, int y); +} + +// Define the key for native "casting" support for our scenario +readonly record struct NoCasting {} + +// Define our runtime wrapper type for the native interface. +unsafe class NativeAPI : IUnmanagedVirtualMethodTableProvider, INativeAPI.Native +{ + private CNativeAPI* _nativeAPI; + + public NativeAPI() + { + if (!CNativeAPI.GetNativeAPI(1, out _nativeAPI)) + { + throw new InvalidOperationException(); + } + } + + VirtualMethodTableInfo IUnmanagedVirtualMethodTableProvider.GetVirtualMethodTableInfoForKey(NoCasting _) + { + return new(IntPtr.Zero, MemoryMarshal.Cast(new ReadOnlySpan(_nativeAPI, 1))); + } +} + +unsafe partial struct CNativeAPI +{ + IntPtr getVersion; + IntPtr add; + IntPtr multiply; + + [LibraryImport(nameof(NativeAPI))] + public static partial bool GetNativeAPI(int version, out CNativeAPI* ppNativeAPI); +}; + +// Generated code for VirtualMethodIndex generator + +// NativeInterfaces.g.cs +partial interface INativeAPI +{ + [DynamicInterfaceCastableImplementation] + partial interface Native : INativeAPI + { + } +} + +// ManagedToNativeStubs.g.cs +partial interface INativeAPI +{ + unsafe partial interface Native + { + int INativeAPI.GetVersion() + { + var (_, vtable) = ((IUnmanagedVirtualMethodTableProvider)this).GetVirtualMethodTableInfoForKey(INativeAPI.TypeKey); + int retVal; + retVal = ((delegate* unmanaged)vtable[0])(); + return retVal; + } + } +} +partial interface INativeAPI +{ + unsafe partial interface Native + { + int INativeAPI.Add(int x, int y) + { + var (_, vtable) = ((IUnmanagedVirtualMethodTableProvider)this).GetVirtualMethodTableInfoForKey(INativeAPI.TypeKey); + int retVal; + retVal = ((delegate* unmanaged)vtable[1])(x, y); + return retVal; + } + } +} +partial interface INativeAPI +{ + unsafe partial interface Native + { + int INativeAPI.Multiply(int x, int y) + { + var (_, vtable) = ((IUnmanagedVirtualMethodTableProvider)this).GetVirtualMethodTableInfoForKey(INativeAPI.TypeKey); + int retVal; + retVal = ((delegate* unmanaged)vtable[2])(x, y); + return retVal; + } + } +} + +// LibraryImport-generated code omitted for brevity +``` + +As this generator is primarily designed to provide building blocks for future work, it has a larger requirement on user-written code. In particular, this generator does not provide any support for authoring a runtime wrapper object that stores the native pointers for the underlying object or the virtual method table. However, this lack of support also provides significant flexibility for developers. The only requirement for the runtime wrapper object type is that it implements `IUnmanagedVirtualMethodTableProvider` with a `T` matching the `TypeKey` type of the native interface. + +The emitted interface implementation can be used in two ways: + +1. The user's runtime wrapper object can directly implement the emitted `Native` interface. This method works for cases where all interfaces are statically known to exist (interfaces are not conditionally implemented on each object). +2. The user's runtime wrapper object can implement `IDynamicInterfaceCastable` and can return the handle of `INativeAPI.Native` when user code casts the wrapper to `INativeAPI`. This style is more commonly used for COM-style APIs. + +### COM interface + +```cpp +// C++ code +struct IUnknown +{ + virtual HRESULT QueryInterface(REFIID riid, void **ppvObject) = 0; + virtual ULONG AddRef() = 0; + virtual ULONG Release() = 0; +}; + +``` +```csharp +// User-defined C# code +using System; +using System.Runtime.InteropServices; + +interface IUnknown +{ + public static readonly Guid TypeKey = Guid.Parse("00000000-0000-0000-C000-000000000046"); + + [UnmanagedCallConv(CallConvs = new[] { typeof(CallConvStdcall), typeof(CallConvMemberFunction) })] + [VirtualMethodIndex(0)] + int QueryInterface(in Guid riid, out IntPtr ppvObject); + + [UnmanagedCallConv(CallConvs = new[] { typeof(CallConvStdcall), typeof(CallConvMemberFunction) })] + [VirtualMethodIndex(1)] + uint AddRef(); + + [UnmanagedCallConv(CallConvs = new[] { typeof(CallConvStdcall), typeof(CallConvMemberFunction) })] + [VirtualMethodIndex(2)] + uint Release(); +} + +class UnknownCOMObject : IUnmanagedVirtualMethodTableProvider, IDynamicInterfaceCastable +{ + private IntPtr _unknownPtr; + + public UnknownCOMObject(IntPtr unknown) + { + _unknownPtr = unknown; + } + + unsafe VirtualMethodTableInfo IUnmanagedVirtualMethodTableProvider.GetVirtualMethodTableInfoForKey(Guid iid) + { + if (iid == IUnknown.TypeKey) + { + return new VirtualMethodTableInfo(_unknownPtr, new ReadOnlySpan(**(IntPtr***)_unknownPtr), 3); + } + return default; + } + + RuntimeTypeHandle IDynamicInterfaceCastable.GetInterfaceImplementation(RuntimeTypeHandle interfaceType) + { + if (Type.GetTypeFromHandle(interfaceType) == typeof(IUnknown)) + { + return typeof(IUnknown.Native).TypeHandle; + } + return default; + } + + bool IDynamicInterfaceCastable.IsInterfaceImplemented(RuntimeTypeHandle interfaceType, bool throwIfNotImplemented) + { + return Type.GetTypeFromHandle(interfaceType) == typeof(IUnknown); + } +} + +// Generated code for VirtualMethodIndex generator + +// NativeInterfaces.g.cs +partial interface IUnknown +{ + [DynamicInterfaceCastableImplementation] + partial interface Native : IUnknown + { + } +} + +// ManagedToNativeStubs.g.cs +partial interface IUnknown +{ + partial interface Native + { + int IUnknown.QueryInterface(in Guid riid, out IntPtr ppvObject) + { + var (thisPtr, vtable) = ((IUnmanagedVirtualMethodTableProvider)this).GetVirtualMethodTableInfoForKey(IUnknown.TypeKey); + int retVal; + fixed (Guid* riid__gen_native = &riid) + fixed (IntPtr* ppvObject__gen_native = &ppvObject) + { + retVal = ((delegate* unmanaged[Stdcall, MemberFunction])vtable[0])(thisPtr, riid__gen_native, ppvObject__gen_native); + } + return retVal; + } + } +} +partial interface IUnknown +{ + partial interface Native + { + uint IUnknown.AddRef() + { + var (thisPtr, vtable) = ((IUnmanagedVirtualMethodTableProvider)this).GetVirtualMethodTableInfoForKey(IUnknown.TypeKey); + uint retVal; + retVal = ((delegate* unmanaged[Stdcall, MemberFunction])vtable[1])(thisPtr); + return retVal; + } + } +} +partial interface IUnknown +{ + partial interface Native + { + uint IUnknown.Release() + { + var (thisPtr, vtable) = ((IUnmanagedVirtualMethodTableProvider)this).GetVirtualMethodTableInfoForKey(IUnknown.TypeKey); + uint retVal; + retVal = ((delegate* unmanaged[Stdcall, MemberFunction])vtable[2])(thisPtr); + return retVal; + } + } +} + +// Native-To-Managed code omitted as the design has not been finalized yet. +``` + +This example shows how we can build COM support on top of the vtable stub generator. The generator will support specifying a custom calling convention using the already-existing `UnmanagedCallConvAttribute`, so it will automatically support forwarding any calling conventions we implement with our extensible calling convention support to the function pointer signature. + +## FAQ + +- Why emit a nested interface instead of a DIM on the existing interface? + - By emitting a nested interface, we enable flexibility in the implementation of the user-defined interface without our implementations getting in the way. With the current design, a managed implementation of a given interface would require the user to implement all members. If we emitted the member implementations directly as DIM implementations, then the compiler would happily allow a developer to only override one method and leave the rest using the native implementation, which would make the development experience of a managed implementation more difficult as there would be no IDE/compiler assistance to fully implement the contract. + +## Open Questions + +- Should we automatically apply the `[DynamicInterfaceCastableImplementation]` attribute to the generated `Native` interface? + - It is a nice convenience, but it isn't applicable in all scenarios and bloats the metadata size. Additionally, since the generated interface is `partial`, we could direct users to add it themselves to the generated interface. + diff --git a/src/libraries/System.Runtime.InteropServices/tests/Ancillary.Interop/IUnmanagedVirtualMethodTableProvider.cs b/src/libraries/System.Runtime.InteropServices/tests/Ancillary.Interop/IUnmanagedVirtualMethodTableProvider.cs index e19dcdbdd71ce..29e56c729f5c2 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/Ancillary.Interop/IUnmanagedVirtualMethodTableProvider.cs +++ b/src/libraries/System.Runtime.InteropServices/tests/Ancillary.Interop/IUnmanagedVirtualMethodTableProvider.cs @@ -29,80 +29,7 @@ public void Deconstruct(out IntPtr thisPointer, out ReadOnlySpan virtual public interface IUnmanagedVirtualMethodTableProvider where T : IEquatable { - VirtualMethodTableInfo GetFunctionPointerForIndex(T typeKey); + VirtualMethodTableInfo GetVirtualMethodTableInfoForKey(T typeKey); } - // Below here is scratch - /* - readonly record struct NoCasting { } - - //Example using IUnmanagedVirtualMethodTableProvider - public partial class MyWrapper : IUnmanagedVirtualMethodTableProvider, MyNativeAPI.Native - //public partial class MyWrapper : IUnmanagedVirtualMethodTableProvider, IDIC - { - public VirtualMethodTableInfo GetFunctionPointerForIndex(NoCasting type) => throw new NotImplementedException(); - } - - partial interface MyNativeAPI - { - public readonly static NoCasting TypeKey; - - [VirtualMethodIndex(0)] - void Foo(); - } - - // Generated: - partial interface MyNativeAPI - { - [DynamicInterfaceCastableImplementation] - internal interface Native : MyNativeAPI - { - unsafe void MyNativeAPI.Foo() - { - var (thisPtr, vtable) = ((IUnmanagedVirtualMethodTableProvider)this).GetFunctionPointerForIndex(MyNativeAPI.TypeKey); - - ((delegate* unmanaged)vtable[0])(); - } - } - } - - public abstract class GenericComWrappers : ComWrappers - where T: IComObjectWrapper - { - } - - public interface IComObjectWrapper - where T : IComObjectWrapper - { - static abstract T CreateFromIUnknown(IntPtr iUnknown); - } - - - public partial class M : GenericComWrappers - { } - - // Generated - public partial class M - { - protected override object? CreateObject(IntPtr externalComObject, CreateObjectFlags flags) => ComObject.CreateFromIUnknown(externalComObject); - } - - - public class ComObject : IUnmanagedVirtualMethodTableProvider, IDynamicInterfaceCastable, IComObjectWrapper - { - private IntPtr _iUnknown; - - private Dictionary _vtable; - - public static ComObject CreateFromIUnknown(IntPtr iUnknown) => new ComObject { _iUnknown = iUnknown }; - public virtual VirtualMethodTableInfo GetFunctionPointerForIndex(Type type) => QI; - public RuntimeTypeHandle GetInterfaceImplementation(RuntimeTypeHandle interfaceType) => throw new NotImplementedException(); - public bool IsInterfaceImplemented(RuntimeTypeHandle interfaceType, bool throwIfNotImplemented) => throw new NotImplementedException(); - } - - public class MyComObject : ComObject, IComObjectWrapper - { - public static new MyComObject CreateFromIUnknown(IntPtr iUnknown) => new MyComObject { _iUnknown = iUnknown }; - } - */ } diff --git a/src/libraries/System.Runtime.InteropServices/tests/ComInterfaceGenerator.Unit.Tests/CodeSnippets.cs b/src/libraries/System.Runtime.InteropServices/tests/ComInterfaceGenerator.Unit.Tests/CodeSnippets.cs index af21ae18bd081..0ce40ee7f5be0 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/ComInterfaceGenerator.Unit.Tests/CodeSnippets.cs +++ b/src/libraries/System.Runtime.InteropServices/tests/ComInterfaceGenerator.Unit.Tests/CodeSnippets.cs @@ -25,7 +25,7 @@ partial interface INativeAPI // Try using the generated native interface sealed class NativeAPI : IUnmanagedVirtualMethodTableProvider, INativeAPI.Native { - public VirtualMethodTableInfo GetFunctionPointerForIndex(NoCasting typeKey) => throw null; + public VirtualMethodTableInfo GetVirtualMethodTableInfoForKey(NoCasting typeKey) => throw null; } "; @@ -43,7 +43,7 @@ partial interface INativeAPI // Try using the generated native interface sealed class NativeAPI : IUnmanagedVirtualMethodTableProvider, INativeAPI.Native { - public VirtualMethodTableInfo GetFunctionPointerForIndex(NoCasting typeKey) => throw null; + public VirtualMethodTableInfo GetVirtualMethodTableInfoForKey(NoCasting typeKey) => throw null; } "; @@ -81,7 +81,7 @@ partial interface INativeAPI // Try using the generated native interface sealed class NativeAPI : IUnmanagedVirtualMethodTableProvider, INativeAPI.Native { - public VirtualMethodTableInfo GetFunctionPointerForIndex(NoCasting typeKey) => throw null; + public VirtualMethodTableInfo GetVirtualMethodTableInfoForKey(NoCasting typeKey) => throw null; } "; @@ -102,7 +102,7 @@ partial interface INativeAPI // Try using the generated native interface sealed class NativeAPI : IUnmanagedVirtualMethodTableProvider, INativeAPI.Native {{ - public VirtualMethodTableInfo GetFunctionPointerForIndex(NoCasting typeKey) => throw null; + public VirtualMethodTableInfo GetVirtualMethodTableInfoForKey(NoCasting typeKey) => throw null; }} ";