Merge pull request #6 from Uralstech/unstable

UGemini v1.2.0

README.md

@@ -21,7 +21,7 @@ This *should* work on any reasonably modern Unity version. Built and tested in U
         - *`com.utilities`
 4. Open the Unity Package Manager window (`Window` -> `Package Manager`)
 5. Change the registry from `Unity` to `My Registries`
-6. Add the `UGemini` and *`Utilities.Encoder.Wav` packages
+6. Add the `UGemini`, *`Utilities.Async` and *`Utilities.Encoder.Wav` packages
 
 #### From GitHub Through Unity Package Manager
 
@@ -30,10 +30,20 @@ This *should* work on any reasonably modern Unity version. Built and tested in U
 3. Paste the UPM branch URL and press enter:
     - `https://github.com/Uralstech/UGemini.git#upm`
 
-*\*Adding additional dependencies:*<br/>
-Follow the steps detailed in the OpenUPM installation method and only install the *`Utilities.Encoder.Wav` package.
+*Adding additional dependencies:*<br/>
+Follow the steps detailed in the OpenUPM installation method and only install the *`Utilities.Async` and *`Utilities.Encoder.Wav` packages.
 
-*Optional, but required if you don't want to bother with encoding your AudioClips into Base64 strings manually.
+#### From GitHub Clone/Download
+
+1. Clone or download the repository from the desired branch (master, preview/unstable)
+2. Drag the package folder `UGemini/UGemini/Packages/com.uralstech.ugemini` into your Unity project's `Packages` folder
+3. In the `Packages` folder of your project, add the following line to the list in `manifest.json`:
+    `"com.uralstech.ugemini": "1.0.1",`
+
+*Adding additional dependencies:*<br/>
+Follow the steps detailed in the OpenUPM installation method and only install the *`Utilities.Async` and *`Utilities.Encoder.Wav` packages.
+
+*Optional, but `Utilities.Async` is required for streaming content and `Utilities.Encoder.Wav` is recommended if you don't want to bother with encoding your AudioClips into Base64 strings manually.
 
 ### Gemini API Support
 
@@ -52,7 +62,7 @@ Follow the steps detailed in the OpenUPM installation method and only install th
 
     - [ ] `get` method
     - [ ] `list` method
-    - [ ] `streamGenerateContent` method
+    - [x] `streamGenerateContent` method
 
 - [ ] `cachedContents` endpoint 🧪
 - [ ] `corpora` endpoint 🧪

UGemini/Packages/com.uralstech.ugemini/Runtime/Scripts/Data/ChatRequest/GeminiChatRequest.cs

@@ -1,7 +1,10 @@
 using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 using Newtonsoft.Json.Serialization;
 using System;
+using System.Collections.Generic;
 using System.ComponentModel;
+using System.Threading.Tasks;
 using Uralstech.UGemini.Tools.Declaration;
 
 namespace Uralstech.UGemini.Chat
@@ -10,8 +13,17 @@ namespace Uralstech.UGemini.Chat
     /// Request to generate a response from the model.
     /// </summary>
     [JsonObject(NamingStrategyType = typeof(CamelCaseNamingStrategy))]
-    public class GeminiChatRequest : IGeminiPostRequest
+    public class GeminiChatRequest : IGeminiStreamablePostRequest<GeminiChatResponse>
     {
+        /// <summary>
+        /// Serialization settings for deserializing partial streamed responses.
+        /// </summary>
+        private static readonly JsonSerializerSettings s_partialDataSerializerSettings = new()
+        {
+            DefaultValueHandling = DefaultValueHandling.Ignore,
+            NullValueHandling = NullValueHandling.Ignore,
+        };
+
         /// <summary>
         /// The content of the current conversation with the model.
         /// </summary>
@@ -100,8 +112,24 @@ public class GeminiChatRequest : IGeminiPostRequest
         public string ContentType => GeminiContentType.ApplicationJSON.MimeType();
 
         /// <inheritdoc/>
+        public string GetEndpointUri(GeminiRequestMetadata metadata)
+        {
+            return metadata?.IsStreaming == true
+                ? $"https://generativelanguage.googleapis.com/{ApiVersion}/models/{Model}:streamGenerateContent"
+                : $"https://generativelanguage.googleapis.com/{ApiVersion}/models/{Model}:generateContent";
+        }
+
+        /// <summary>
+        /// Callback for receiving streamed responses.
+        /// </summary>
         [JsonIgnore]
-        public string EndpointUri => $"https://generativelanguage.googleapis.com/{ApiVersion}/models/{Model}:generateContent";
+        public Func<GeminiChatResponse, Task> OnPartialResponseReceived;
+
+        /// <summary>
+        /// The streamed response.
+        /// </summary>
+        [JsonIgnore]
+        public GeminiChatResponse StreamedResponse { get; private set; }
 
         /// <summary>
         /// Creates a new <see cref="GeminiChatRequest"/>.
@@ -122,5 +150,23 @@ public string GetUtf8EncodedData()
         {
             return JsonConvert.SerializeObject(this);
         }
+
+        /// <inheritdoc/>
+        public async Task ProcessStreamedData(List<JToken> allEvents, JToken lastEvent)
+        {
+            try
+            {
+                GeminiChatResponse partialResponse = lastEvent.ToObject<GeminiChatResponse>(JsonSerializer.Create(s_partialDataSerializerSettings));
+
+                if (StreamedResponse == null)
+                    StreamedResponse = partialResponse;
+                else
+                    StreamedResponse.Append(partialResponse);
+
+                if (OnPartialResponseReceived != null)
+                    await OnPartialResponseReceived.Invoke(StreamedResponse);
+            }
+            catch { }
+        }
     }
 }

UGemini/Packages/com.uralstech.ugemini/Runtime/Scripts/Data/ChatResponse/GeminiCandidate.cs

@@ -7,7 +7,7 @@ namespace Uralstech.UGemini.Chat
     /// A response candidate generated from the model.
     /// </summary>
     [JsonObject(NamingStrategyType = typeof(CamelCaseNamingStrategy))]
-    public class GeminiCandidate
+    public class GeminiCandidate : IAppendableData<GeminiCandidate>
     {
         /// <summary>
         /// Generated content returned from the model.
@@ -54,5 +54,30 @@ public class GeminiCandidate
         /// Index of the candidate in the list of candidates.
         /// </summary>
         public int Index;
+
+        /// <inheritdoc/>
+        public void Append(GeminiCandidate data)
+        {
+            if (Content == null)
+                Content = data.Content;
+            else if (data.Content != null)
+                Content.Append(data.Content);
+
+            if (data.FinishReason != default)
+                FinishReason = data.FinishReason;
+
+            if (data.SafetyRatings != null)
+                SafetyRatings = data.SafetyRatings;
+
+            if (data.CitationMetadata != null)
+                CitationMetadata = data.CitationMetadata;
+
+            TokenCount = data.TokenCount;
+
+            if (data.GroundingAttributions != null)
+                GroundingAttributions = data.GroundingAttributions;
+
+            Index = data.Index;
+        }
     }
 }

UGemini/Packages/com.uralstech.ugemini/Runtime/Scripts/Data/ChatResponse/GeminiChatResponse.cs

@@ -1,5 +1,6 @@
 using Newtonsoft.Json;
 using Newtonsoft.Json.Serialization;
+using System;
 
 namespace Uralstech.UGemini.Chat
 {
@@ -15,7 +16,7 @@ namespace Uralstech.UGemini.Chat
     /// - feedback on each candidate is reported on <see cref="GeminiCandidate.FinishReason"/> and <see cref="GeminiCandidate.SafetyRatings"/>.
     /// </remarks>
     [JsonObject(NamingStrategyType = typeof(CamelCaseNamingStrategy))]
-    public class GeminiChatResponse
+    public class GeminiChatResponse : IAppendableData<GeminiChatResponse>
     {
         /// <summary>
         /// Candidate responses from the model.
@@ -37,5 +38,31 @@ public class GeminiChatResponse
         /// </summary>
         [JsonIgnore]
         public GeminiContentPart[] Parts => Candidates[0].Content.Parts;
+
+        /// <inheritdoc/>
+        public void Append(GeminiChatResponse data)
+        {
+            if (data.PromptFeedback != null)
+                PromptFeedback.Append(data.PromptFeedback);
+
+            if (data.UsageMetadata != null)
+                UsageMetadata.Append(data.UsageMetadata);
+
+            if (data.Candidates != null)
+            {
+                for (int i = 0; i < Candidates.Length; i++)
+                    Candidates[i].Append(data.Candidates[i]);
+
+                if (data.Candidates.Length > Candidates.Length)
+                {
+                    GeminiCandidate[] allCandidates = new GeminiCandidate[data.Candidates.Length];
+                    Candidates.CopyTo(allCandidates, 0);
+
+                    Array.Copy(data.Candidates, Candidates.Length, allCandidates, Candidates.Length, data.Candidates.Length - Candidates.Length);
+
+                    Candidates = allCandidates;
+                }
+            }
+        }
     }
 }

UGemini/Packages/com.uralstech.ugemini/Runtime/Scripts/Data/ChatResponse/GeminiPromptFeedback.cs

@@ -7,7 +7,7 @@ namespace Uralstech.UGemini.Chat
     /// A set of the feedback metadata the prompt specified in <see cref="GeminiChatResponse.Candidates"/>.
     /// </summary>
     [JsonObject(NamingStrategyType = typeof(CamelCaseNamingStrategy))]
-    public class GeminiPromptFeedback
+    public class GeminiPromptFeedback : IAppendableData<GeminiPromptFeedback>
     {
         /// <summary>
         /// If set, the prompt was blocked and no candidates are returned. Rephrase your prompt.
@@ -19,5 +19,14 @@ public class GeminiPromptFeedback
         /// Ratings for safety of the prompt. There is at most one rating per category.
         /// </summary>
         public GeminiSafetyRating[] SafetyRatings;
+
+        /// <inheritdoc/>
+        public void Append(GeminiPromptFeedback data)
+        {
+            if (data.BlockReason != default)
+                BlockReason = data.BlockReason;
+
+            SafetyRatings = data.SafetyRatings;
+        }
     }
 }

UGemini/Packages/com.uralstech.ugemini/Runtime/Scripts/Data/ChatResponse/GeminiUsageMetadata.cs

@@ -7,7 +7,7 @@ namespace Uralstech.UGemini.Chat
     /// Metadata on the generation request's token usage.
     /// </summary>
     [JsonObject(NamingStrategyType = typeof(CamelCaseNamingStrategy))]
-    public class GeminiUsageMetadata
+    public class GeminiUsageMetadata : IAppendableData<GeminiUsageMetadata>
     {
         /// <summary>
         /// Number of tokens in the prompt. When cachedContent is set, this is still the total effective prompt size. I.e. this includes the number of tokens in the cached content.
@@ -31,5 +31,14 @@ public class GeminiUsageMetadata
         /// Total token count for the generation request (prompt + candidates).
         /// </summary>
         public int TotalTokenCount;
+
+        /// <inheritdoc/>
+        public void Append(GeminiUsageMetadata data)
+        {
+            PromptTokenCount = data.PromptTokenCount;
+            CachedContentTokenCount = data.CachedContentTokenCount;
+            CandidatesTokenCount = data.CandidatesTokenCount;
+            TotalTokenCount = data.TotalTokenCount;
+        }
     }
 }

UGemini/Packages/com.uralstech.ugemini/Runtime/Scripts/Data/Common/Content/GeminiContent.cs

@@ -1,5 +1,6 @@
 using Newtonsoft.Json;
 using Newtonsoft.Json.Serialization;
+using System.Collections.Generic;
 using System.ComponentModel;
 using UnityEngine;
 using Uralstech.UGemini.FileAPI;
@@ -11,7 +12,7 @@ namespace Uralstech.UGemini
     /// The base structured datatype containing multi-part content of a message.
     /// </summary>
     [JsonObject(NamingStrategyType = typeof(CamelCaseNamingStrategy))]
-    public class GeminiContent
+    public class GeminiContent : IAppendableData<GeminiContent>
     {
         /// <summary>
         /// Ordered Parts that constitute a single message. Parts may have different MIME types.
@@ -22,7 +23,7 @@ public class GeminiContent
         /// Optional. The producer of the content.
         /// </summary>
         [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore), DefaultValue(GeminiRole.Unspecified)]
-        public GeminiRole Role = GeminiRole.Unspecified;
+        public GeminiRole Role;
 
         /// <summary>
         /// Creates a new <see cref="GeminiContent"/> from a role and message.
@@ -171,5 +172,44 @@ public static GeminiContent GetContent(GeminiFunctionResponse functionResponse)
                 }
             };
         }
+
+        /// <inheritdoc/>
+        public void Append(GeminiContent data)
+        {
+            if (data.Role != default)
+                Role = data.Role;
+
+            if (data.Parts != null)
+            {
+                List<GeminiContentPart> partsToAdd = new();
+                for (int i = 0; i < data.Parts.Length; i++)
+                {
+                    GeminiContentPart partToAppend = data.Parts[i];
+                    bool appended = false;
+
+                    for (int j = 0; j < Parts.Length; j++)
+                    {
+                        GeminiContentPart part = Parts[j];
+                        if (part.IsAppendable(partToAppend))
+                        {
+                            part.Append(partToAppend);
+                            appended = true;
+                        }
+                    }
+
+                    if (!appended)
+                        partsToAdd.Add(partToAppend);
+                }
+
+                if (partsToAdd.Count > 0)
+                {
+                    GeminiContentPart[] allParts = new GeminiContentPart[Parts.Length + partsToAdd.Count];
+                    Parts.CopyTo(allParts, 0);
+
+                    partsToAdd.CopyTo(allParts, Parts.Length);
+                    Parts = allParts;
+                }
+            }
+        }
     }
 }

UGemini/Packages/com.uralstech.ugemini/Runtime/Scripts/Data/Common/Content/GeminiContentPart.cs

@@ -9,7 +9,7 @@ namespace Uralstech.UGemini
     /// A datatype containing media that is part of a multi-part Content message. Must only contain one field at a time.
     /// </summary>
     [JsonObject(NamingStrategyType = typeof(CamelCaseNamingStrategy))]
-    public class GeminiContentPart
+    public class GeminiContentPart : IAppendableData<GeminiContentPart>
     {
         /// <summary>
         /// Inline text.
@@ -49,5 +49,53 @@ public class GeminiContentPart
         /// </remarks>
         [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore), DefaultValue(null)]
         public GeminiFileData FileData = null;
+
+        /// <inheritdoc/>
+        public void Append(GeminiContentPart data)
+        {
+            if (string.IsNullOrEmpty(Text))
+                Text = data.Text;
+            else if (!string.IsNullOrEmpty(data.Text))
+                Text += data.Text;
+
+            if (data.InlineData != null)
+                InlineData = data.InlineData;
+
+            if (data.FunctionCall != null)
+                FunctionCall = data.FunctionCall;
+
+            if (data.FunctionResponse != null)
+                FunctionResponse = data.FunctionResponse;
+
+            if (data.FileData != null)
+                FileData = data.FileData;
+        }
+
+        /// <summary>
+        /// Is the data to be appended compatible with the current <see cref="GeminiContentPart"/>?
+        /// </summary>
+        /// <param name="data">The data to be appended.</param>
+        public bool IsAppendable(GeminiContentPart data)
+        {
+            return IsEmpty || data switch
+            {
+                { Text: string text } when !string.IsNullOrEmpty(text) && !string.IsNullOrEmpty(Text) => true,
+                { InlineData: not null } when InlineData != null => data.InlineData.MimeType == InlineData.MimeType,
+                { FunctionCall: not null } when FunctionCall != null => data.FunctionCall.Name == FunctionCall.Name,
+                { FunctionResponse: not null } when FunctionResponse != null => data.FunctionResponse.Name == FunctionResponse.Name,
+                { FileData: not null } when FileData != null => data.FileData.FileUri == FileData.FileUri,
+                _ => false,
+            };
+        }
+
+        /// <summary>
+        /// Is there no content stored in this <see cref="GeminiContentPart"/>?
+        /// </summary>
+        [JsonIgnore]
+        public bool IsEmpty => string.IsNullOrEmpty(Text)
+            && InlineData == null
+            && FunctionCall == null
+            && FunctionResponse == null
+            && FileData == null;
     }
 }