init commit

Program.cs

@@ -0,0 +1,102 @@
+
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Memory;
+using Microsoft.SemanticKernel.Connectors.Memory.Qdrant;
+using RepoUtils;
+using BlingFire;
+
+namespace SemanticQuestion10K
+{
+    internal class Program
+    {
+
+        const string memoryCollectionName = "Microsoft10K";
+        static void Main(string[] args)
+        {
+            bool parse = false;
+            bool question = false;
+            string tenkfile = "ms10k.txt";
+
+            //loop through args with an integer
+            for(int i=0; i<args.Length; i++)
+            {
+                if (args[i] == "--parse") parse = true;
+                if (args[i] == "--question") question = true;
+                if (args[i].StartsWith("--tenkfile")) tenkfile = args[i+1];
+            }
+
+            QdrantMemoryStore memoryStore = new QdrantMemoryStore(Env.Var("QDRANT_ENDPOINT"), 6333, vectorSize: 1536, ConsoleLogger.Log);
+
+            var kernel = Kernel.Builder
+            .WithLogger(ConsoleLogger.Log)
+            .Configure(c => c.AddAzureOpenAIEmbeddingGenerationService("text-embedding-ada-002", "text-embedding-ada-002", Env.Var("AZURE_OPENAI_ENDPOINT"), Env.Var("AZURE_OPENAI_KEY")))
+            //.Configure(c => c.AddOpenAIEmbeddingGenerationService("text-embedding-ada-002", "text-embedding-ada-002", Env.Var("OPENAI_API_KEY")))
+            .WithMemoryStorage(memoryStore)
+            .Build();
+
+            kernel.Config.AddAzureOpenAITextCompletionService("text-davinci-002", "text-davinci-002", Env.Var("AZURE_OPENAI_ENDPOINT"), Env.Var("AZURE_OPENAI_KEY"));
+
+            if(question) RunAsync(kernel).Wait();
+            if(parse) ParseText(kernel, tenkfile).Wait();
+        }
+
+        static async Task ParseText(IKernel kernel, string kfile)
+        {
+            string text = File.ReadAllText(kfile);
+            var allsentences = BlingFireUtils.GetSentences(text);
+
+            var i = 0;
+            foreach (var s in allsentences)
+            {
+                await kernel.Memory.SaveReferenceAsync(
+                    collection: memoryCollectionName,
+                    description: s,
+                    text: s,
+                    externalId: i.ToString(),
+                    externalSourceName: "MS10-K"
+                );
+                Console.WriteLine($"  sentence {++i} saved");
+            }
+        }
+
+        public static async Task RunAsync(IKernel kernel)
+        {
+            Console.WriteLine("Hi, welcome to Microsoft's 2022 10-K. What would you like to know?");
+            while (true)
+            {
+                Console.Write("User: ");
+                var query = Console.ReadLine();
+                if (query == null) { break; }
+                var results = kernel.Memory.SearchAsync(memoryCollectionName, query, limit: 3, minRelevanceScore: 0.77);
+
+                string FUNCTION_DEFINITION = "Act as the company Microsoft. Answer questions about your annual financial report. Only answer questions based on the info listed below. If the info below doesn't answer the question, say you don't know.\n[START INFO]\n";
+
+                await foreach (MemoryQueryResult r in results)
+                {
+                    int id = int.Parse(r.Metadata.Id);
+                    MemoryQueryResult rb2 = kernel.Memory.GetAsync(memoryCollectionName, (id - 2).ToString()).Result;
+                    MemoryQueryResult rb = kernel.Memory.GetAsync(memoryCollectionName, (id - 1).ToString()).Result;
+                    MemoryQueryResult ra = kernel.Memory.GetAsync(memoryCollectionName, (id + 1).ToString()).Result;
+                    MemoryQueryResult ra2 = kernel.Memory.GetAsync(memoryCollectionName, (id + 2).ToString()).Result;
+
+                    FUNCTION_DEFINITION += "\n " + rb2.Metadata.Id + ": " + rb2.Metadata.Description + "\n";
+                    FUNCTION_DEFINITION += "\n " + rb.Metadata.Description + "\n";
+                    FUNCTION_DEFINITION += "\n " + r.Metadata.Description + "\n";
+                    FUNCTION_DEFINITION += "\n " + ra.Metadata.Description + "\n";
+                    FUNCTION_DEFINITION += "\n " + ra2.Metadata.Id + ": " + ra2.Metadata.Description + "\n";
+                }
+
+                FUNCTION_DEFINITION += "[END INFO]\n" + query;
+
+                //  Console.WriteLine(FUNCTION_DEFINITION + "\n\n");
+                //  Console.WriteLine(FUNCTION_DEFINITION.Length);
+
+                var answer = kernel.CreateSemanticFunction(FUNCTION_DEFINITION, maxTokens: 250, temperature: 0);
+
+                var result = await answer.InvokeAsync();
+                Console.WriteLine("MS10K: " + result.Result.Trim());
+            }
+        }
+
+    }
+}

RepoUtils/ConsoleLogger.cs

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using Microsoft.Extensions.Logging;
+
+namespace RepoUtils;
+
+/// <summary>
+/// Basic logger printing to console
+/// </summary>
+internal static class ConsoleLogger
+{
+    internal static ILogger Log => LogFactory.CreateLogger<object>();
+
+    private static ILoggerFactory LogFactory => s_loggerFactory.Value;
+
+    private static readonly Lazy<ILoggerFactory> s_loggerFactory = new(LogBuilder);
+
+    private static ILoggerFactory LogBuilder()
+    {
+        return LoggerFactory.Create(builder =>
+        {
+            builder.SetMinimumLevel(LogLevel.Warning);
+
+            // builder.AddFilter("Microsoft", LogLevel.Trace);
+            // builder.AddFilter("Microsoft", LogLevel.Debug);
+            // builder.AddFilter("Microsoft", LogLevel.Information);
+            // builder.AddFilter("Microsoft", LogLevel.Warning);
+            // builder.AddFilter("Microsoft", LogLevel.Error);
+
+            builder.AddFilter("Microsoft", LogLevel.Warning);
+            builder.AddFilter("System", LogLevel.Warning);
+
+            builder.AddConsole();
+        });
+    }
+}

RepoUtils/Env.cs

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using Microsoft.Extensions.Configuration;
+
+namespace RepoUtils;
+
+internal class Env
+{
+    /// <summary>
+    /// Simple helper used to load env vars and secrets like credentials,
+    /// to avoid hard coding them in the sample code
+    /// </summary>
+    /// <param name="name">Secret name / Env var name</param>
+    /// <returns>Value found in Secret Manager or Environment Variable</returns>
+    internal static string Var(string name)
+    {
+        var configuration = new ConfigurationBuilder()
+            .AddUserSecrets<Env>()
+            .Build();
+
+        var value = configuration[name];
+        if (!string.IsNullOrEmpty(value))
+        {
+            return value;
+        }
+
+        value = Environment.GetEnvironmentVariable(name);
+        if (string.IsNullOrEmpty(value))
+        {
+            throw new YourAppException($"Secret / Env var not set: {name}");
+        }
+
+        return value;
+    }
+}

RepoUtils/RepoFiles.cs

@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.IO;
+using System.Reflection;
+
+namespace RepoUtils;
+
+internal static class RepoFiles
+{
+    /// <summary>
+    /// Scan the local folders from the repo, looking for "samples/skills" folder.
+    /// </summary>
+    /// <returns>The full path to samples/skills</returns>
+    internal static string SampleSkillsPath()
+    {
+        const string PARENT = "samples";
+        const string FOLDER = "skills";
+
+        bool SearchPath(string pathToFind, out string result, int maxAttempts = 10)
+        {
+            var currDir = Path.GetFullPath(Assembly.GetExecutingAssembly().Location);
+            bool found;
+            do
+            {
+                result = Path.Join(currDir, pathToFind);
+                found = Directory.Exists(result);
+                currDir = Path.GetFullPath(Path.Combine(currDir, ".."));
+            } while (maxAttempts-- > 0 && !found);
+
+            return found;
+        }
+
+        if (!SearchPath(PARENT + Path.DirectorySeparatorChar + FOLDER, out string path)
+            && !SearchPath(FOLDER, out path))
+        {
+            throw new YourAppException("Skills directory not found. The app needs the skills from the repo to work.");
+        }
+
+        return path;
+    }
+}

RepoUtils/YourAppException.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+
+namespace RepoUtils;
+
+public class YourAppException : Exception
+{
+    public YourAppException() : base()
+    {
+    }
+
+    public YourAppException(string message) : base(message)
+    {
+    }
+
+    public YourAppException(string message, Exception innerException) : base(message, innerException)
+    {
+    }
+}

SemanticQuestion10K.csproj

@@ -0,0 +1,33 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <UserSecretsId>42df3a95-8a02-434a-a324-79d582153226</UserSecretsId>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BlingFireNuget" Version="0.1.8" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="7.0.0" />
+    <PackageReference Include="System.Linq.Async" Version="6.0.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Reference Include="Microsoft.SemanticKernel">
+      <HintPath>..\semantic-kernel-main\dotnet\src\SemanticKernel\bin\Debug\netstandard2.1\Microsoft.SemanticKernel.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.SemanticKernel.Connectors.Memory.Qdrant">
+      <HintPath>..\semantic-kernel-main\dotnet\src\Connectors\Connectors.Memory.Qdrant\bin\Debug\netstandard2.1\Microsoft.SemanticKernel.Connectors.Memory.Qdrant.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="docs\ms10k.txt">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

SemanticQuestion10K.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.33516.290
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SemanticQuestion10K", "SemanticQuestion10K.csproj", "{13BE8342-03BC-4840-8A02-430EBDB5C588}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{13BE8342-03BC-4840-8A02-430EBDB5C588}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{13BE8342-03BC-4840-8A02-430EBDB5C588}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{13BE8342-03BC-4840-8A02-430EBDB5C588}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{13BE8342-03BC-4840-8A02-430EBDB5C588}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {95B5A78F-73F1-4317-8DCD-E581A1637843}
+	EndGlobalSection
+EndGlobal