diff --git a/.gitignore b/.gitignore index ab25845f8..bf1d0f700 100644 --- a/.gitignore +++ b/.gitignore @@ -347,4 +347,6 @@ site/ /LLama.Unittest/Models/*.gguf /LLama.Benchmark/Models/*.bin -/LLama.Benchmark/Models/*.gguf \ No newline at end of file +/LLama.Benchmark/Models/*.gguf + +**/appsettings.Local.json diff --git a/Assets/web-ui-dark.png b/Assets/web-ui-dark.png new file mode 100644 index 000000000..b56101326 Binary files /dev/null and b/Assets/web-ui-dark.png differ diff --git a/Assets/web-ui-light.png b/Assets/web-ui-light.png new file mode 100644 index 000000000..933a7c3a1 Binary files /dev/null and b/Assets/web-ui-light.png differ diff --git a/LLama.Web/Common/ISessionConfig.cs b/LLama.Web/Common/ISessionConfig.cs index 09bddc2d9..10fb3a56f 100644 --- a/LLama.Web/Common/ISessionConfig.cs +++ b/LLama.Web/Common/ISessionConfig.cs @@ -1,13 +1,23 @@ -namespace LLama.Web.Common +using System.ComponentModel; + +namespace LLama.Web.Common; + +public interface ISessionConfig { - public interface ISessionConfig - { - string AntiPrompt { get; set; } - List AntiPrompts { get; set; } - LLamaExecutorType ExecutorType { get; set; } - string Model { get; set; } - string OutputFilter { get; set; } - List OutputFilters { get; set; } - string Prompt { get; set; } - } + string AntiPrompt { get; set; } + + [DisplayName("Anti Prompts")] + List AntiPrompts { get; set; } + + [DisplayName("Executor Type")] + LLamaExecutorType ExecutorType { get; set; } + + string Model { get; set; } + + [DisplayName("Output Filter")] + string OutputFilter { get; set; } + + List OutputFilters { get; set; } + + string Prompt { get; set; } } \ No newline at end of file diff --git a/LLama.Web/Common/LLamaOptions.cs b/LLama.Web/Common/LLamaOptions.cs index 4a1d6e0a8..8c16b6177 100644 --- a/LLama.Web/Common/LLamaOptions.cs +++ b/LLama.Web/Common/LLamaOptions.cs @@ -1,12 +1,7 @@ -namespace LLama.Web.Common -{ - public class LLamaOptions - { - public ModelLoadType ModelLoadType { get; set; } - public List Models { get; set; } +namespace LLama.Web.Common; - public void Initialize() - { - } - } +public class LLamaOptions +{ + public ModelLoadType ModelLoadType { get; set; } + public List Models { get; set; } } diff --git a/LLama.Web/Common/SessionConfig.cs b/LLama.Web/Common/SessionConfig.cs index f0a2d22b8..f682806a1 100644 --- a/LLama.Web/Common/SessionConfig.cs +++ b/LLama.Web/Common/SessionConfig.cs @@ -1,14 +1,13 @@ -namespace LLama.Web.Common +namespace LLama.Web.Common; + +public class SessionConfig : ISessionConfig { - public class SessionConfig : ISessionConfig - { - public string Model { get; set; } - public string Prompt { get; set; } + public string Model { get; set; } + public string Prompt { get; set; } - public string AntiPrompt { get; set; } - public List AntiPrompts { get; set; } - public string OutputFilter { get; set; } - public List OutputFilters { get; set; } - public LLamaExecutorType ExecutorType { get; set; } - } + public string AntiPrompt { get; set; } + public List AntiPrompts { get; set; } + public string OutputFilter { get; set; } + public List OutputFilters { get; set; } + public LLamaExecutorType ExecutorType { get; set; } } diff --git a/LLama.Web/Extensions.cs b/LLama.Web/Extensions.cs index 130c46bd5..1fe08f6eb 100644 --- a/LLama.Web/Extensions.cs +++ b/LLama.Web/Extensions.cs @@ -1,54 +1,52 @@ -using LLama.Web.Common; +using LLama.Web.Common; -namespace LLama.Web +namespace LLama.Web; + +public static class Extensions { - public static class Extensions + /// + /// Combines the AntiPrompts list and AntiPrompt csv + /// + /// The session configuration. + /// Combined AntiPrompts with duplicates removed + public static List GetAntiPrompts(this ISessionConfig sessionConfig) { - /// - /// Combines the AntiPrompts list and AntiPrompt csv - /// - /// The session configuration. - /// Combined AntiPrompts with duplicates removed - public static List GetAntiPrompts(this ISessionConfig sessionConfig) - { - return CombineCSV(sessionConfig.AntiPrompts, sessionConfig.AntiPrompt); - } - - /// - /// Combines the OutputFilters list and OutputFilter csv - /// - /// The session configuration. - /// Combined OutputFilters with duplicates removed - public static List GetOutputFilters(this ISessionConfig sessionConfig) - { - return CombineCSV(sessionConfig.OutputFilters, sessionConfig.OutputFilter); - } + return CombineCSV(sessionConfig.AntiPrompts, sessionConfig.AntiPrompt); + } + /// + /// Combines the OutputFilters list and OutputFilter csv + /// + /// The session configuration. + /// Combined OutputFilters with duplicates removed + public static List GetOutputFilters(this ISessionConfig sessionConfig) + { + return CombineCSV(sessionConfig.OutputFilters, sessionConfig.OutputFilter); + } - /// - /// Combines a string list and a csv and removes duplicates - /// - /// The list. - /// The CSV. - /// Combined list with duplicates removed - private static List CombineCSV(List list, string csv) - { - var results = list is null || list.Count == 0 - ? CommaSeparatedToList(csv) - : CommaSeparatedToList(csv).Concat(list); - return results - .Distinct() - .ToList(); - } + /// + /// Combines a string list and a csv and removes duplicates + /// + /// The list. + /// The CSV. + /// Combined list with duplicates removed + private static List CombineCSV(List list, string csv) + { + var results = list is null || list.Count == 0 + ? CommaSeparatedToList(csv) + : CommaSeparatedToList(csv).Concat(list); + return results + .Distinct() + .ToList(); + } - private static List CommaSeparatedToList(string value) - { - if (string.IsNullOrEmpty(value)) - return new List(); + private static List CommaSeparatedToList(string value) + { + if (string.IsNullOrEmpty(value)) + return new List(); - return value.Split(",", StringSplitOptions.RemoveEmptyEntries) - .Select(x => x.Trim()) - .ToList(); - } + return value.Split(",", StringSplitOptions.RemoveEmptyEntries) + .Select(x => x.Trim()) + .ToList(); } } diff --git a/LLama.Web/Hubs/SessionConnectionHub.cs b/LLama.Web/Hubs/SessionConnectionHub.cs index 3ef46dbe4..b6a977e36 100644 --- a/LLama.Web/Hubs/SessionConnectionHub.cs +++ b/LLama.Web/Hubs/SessionConnectionHub.cs @@ -1,67 +1,63 @@ -using LLama.Web.Common; +using LLama.Web.Common; using LLama.Web.Models; using LLama.Web.Services; using Microsoft.AspNetCore.SignalR; -namespace LLama.Web.Hubs -{ - public class SessionConnectionHub : Hub - { - private readonly ILogger _logger; - private readonly IModelSessionService _modelSessionService; +namespace LLama.Web.Hubs; - public SessionConnectionHub(ILogger logger, IModelSessionService modelSessionService) - { - _logger = logger; - _modelSessionService = modelSessionService; - } +public class SessionConnectionHub : Hub +{ + private readonly ILogger _logger; + private readonly IModelSessionService _modelSessionService; - public override async Task OnConnectedAsync() - { - _logger.Log(LogLevel.Information, "[OnConnectedAsync], Id: {0}", Context.ConnectionId); + public SessionConnectionHub(ILogger logger, IModelSessionService modelSessionService) + { + _logger = logger; + _modelSessionService = modelSessionService; + } - // Notify client of successful connection - await Clients.Caller.OnStatus(Context.ConnectionId, SessionConnectionStatus.Connected); - await base.OnConnectedAsync(); - } + public override async Task OnConnectedAsync() + { + _logger.Log(LogLevel.Information, "[OnConnectedAsync], Id: {0}", Context.ConnectionId); + // Notify client of successful connection + await Clients.Caller.OnStatus(Context.ConnectionId, SessionConnectionStatus.Connected); + await base.OnConnectedAsync(); + } - public override async Task OnDisconnectedAsync(Exception exception) - { - _logger.Log(LogLevel.Information, "[OnDisconnectedAsync], Id: {0}", Context.ConnectionId); + public override async Task OnDisconnectedAsync(Exception exception) + { + _logger.Log(LogLevel.Information, "[OnDisconnectedAsync], Id: {0}", Context.ConnectionId); - // Remove connections session on disconnect - await _modelSessionService.CloseAsync(Context.ConnectionId); - await base.OnDisconnectedAsync(exception); - } + // Remove connections session on disconnect + await _modelSessionService.CloseAsync(Context.ConnectionId); + await base.OnDisconnectedAsync(exception); + } + [HubMethodName("LoadModel")] + public async Task OnLoadModel(SessionConfig sessionConfig, InferenceOptions inferenceConfig) + { + _logger.Log(LogLevel.Information, "[OnLoadModel] - Load new model, Connection: {0}", Context.ConnectionId); + await _modelSessionService.CloseAsync(Context.ConnectionId); - [HubMethodName("LoadModel")] - public async Task OnLoadModel(SessionConfig sessionConfig, InferenceOptions inferenceConfig) + // Create model session + var modelSession = await _modelSessionService.CreateAsync(Context.ConnectionId, sessionConfig, inferenceConfig); + if (modelSession is null) { - _logger.Log(LogLevel.Information, "[OnLoadModel] - Load new model, Connection: {0}", Context.ConnectionId); - await _modelSessionService.CloseAsync(Context.ConnectionId); - - // Create model session - var modelSession = await _modelSessionService.CreateAsync(Context.ConnectionId, sessionConfig, inferenceConfig); - if (modelSession is null) - { - await Clients.Caller.OnError("Failed to create model session"); - return; - } - - // Notify client - await Clients.Caller.OnStatus(Context.ConnectionId, SessionConnectionStatus.Loaded); + await Clients.Caller.OnError("Failed to create model session"); + return; } + // Notify client + await Clients.Caller.OnStatus(Context.ConnectionId, SessionConnectionStatus.Loaded); + } - [HubMethodName("SendPrompt")] - public IAsyncEnumerable OnSendPrompt(string prompt, InferenceOptions inferConfig, CancellationToken cancellationToken) - { - _logger.Log(LogLevel.Information, "[OnSendPrompt] - New prompt received, Connection: {0}", Context.ConnectionId); + [HubMethodName("SendPrompt")] + public IAsyncEnumerable OnSendPrompt(string prompt, InferenceOptions inferConfig, CancellationToken cancellationToken) + { + _logger.Log(LogLevel.Information, "[OnSendPrompt] - New prompt received, Connection: {0}", Context.ConnectionId); - var linkedCancelationToken = CancellationTokenSource.CreateLinkedTokenSource(Context.ConnectionAborted, cancellationToken); - return _modelSessionService.InferAsync(Context.ConnectionId, prompt, inferConfig, linkedCancelationToken.Token); - } + var linkedCancelationToken = CancellationTokenSource.CreateLinkedTokenSource(Context.ConnectionAborted, cancellationToken); + return _modelSessionService.InferAsync(Context.ConnectionId, prompt, inferConfig, linkedCancelationToken.Token); } } diff --git a/LLama.Web/LLama.Web.csproj b/LLama.Web/LLama.Web.csproj index 7e5bd9bdf..0ca1a6783 100644 --- a/LLama.Web/LLama.Web.csproj +++ b/LLama.Web/LLama.Web.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 disable enable @@ -15,6 +15,7 @@ + diff --git a/LLama.Web/Models/CancelModel.cs b/LLama.Web/Models/CancelModel.cs deleted file mode 100644 index de7e3f2f7..000000000 --- a/LLama.Web/Models/CancelModel.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace LLama.Web.Models -{ - public class CancelModel - { - public string ConnectionId { get; set; } - } -} diff --git a/LLama.Web/Models/LLamaModel.cs b/LLama.Web/Models/LLamaModel.cs index 5aedc5f5e..af228bd4c 100644 --- a/LLama.Web/Models/LLamaModel.cs +++ b/LLama.Web/Models/LLamaModel.cs @@ -1,108 +1,123 @@ -using LLama.Abstractions; +using LLama.Abstractions; using LLama.Web.Common; using System.Collections.Concurrent; -namespace LLama.Web.Models +namespace LLama.Web.Models; + +/// +/// Wrapper class for LLamaSharp LLamaWeights +/// +/// +public class LLamaModel : IDisposable { + private readonly ILogger _llamaLogger; + private readonly ModelOptions _config; + private readonly LLamaWeights _weights; + private readonly ConcurrentDictionary _contexts; + /// - /// Wrapper class for LLamaSharp LLamaWeights + /// Use the Create method to instantiate this class. /// - /// - public class LLamaModel : IDisposable + /// The model parameters. + /// A logger class. + /// Model weights. + private LLamaModel(ModelOptions modelParams, ILogger llamaLogger, LLamaWeights weights) { - private readonly ILogger _llamaLogger; - private readonly ModelOptions _config; - private readonly LLamaWeights _weights; - private readonly ConcurrentDictionary _contexts; - - /// - /// Initializes a new instance of the class. - /// - /// The model parameters. - public LLamaModel(ModelOptions modelParams, ILogger llamaLogger) - { - _config = modelParams; - _llamaLogger = llamaLogger; - _weights = LLamaWeights.LoadFromFile(modelParams); - _contexts = new ConcurrentDictionary(); - } - - /// - /// Gets the model configuration. - /// - public IModelParams ModelParams => _config; + _config = modelParams; + _llamaLogger = llamaLogger; + _weights = weights; + _contexts = new ConcurrentDictionary(); + } - /// - /// Gets the LLamaWeights - /// - public LLamaWeights LLamaWeights => _weights; + /// + /// Initializes a new instance of the class. + /// + /// The model parameters. + /// A logger class. + public static async Task CreateAsync(ModelOptions modelParams, ILogger llamaLogger) + { + var weights = await LLamaWeights.LoadFromFileAsync(modelParams); + return new LLamaModel(modelParams, llamaLogger, weights); + } + /// + /// Gets the model configuration. + /// + public IModelParams ModelParams => _config; - /// - /// Gets the context count. - /// - public int ContextCount => _contexts.Count; + /// + /// Gets the LLamaWeights + /// + public LLamaWeights LLamaWeights => _weights; + /// + /// Gets the context count. + /// + public int ContextCount => _contexts.Count; - /// - /// Creates a new context session on this model - /// - /// The unique context identifier - /// LLamaModelContext for this LLamaModel - /// Context exists - public Task CreateContext(string contextName) + /// + /// Creates a new context session on this model + /// + /// The unique context identifier + /// LLamaModelContext for this LLamaModel + /// Context exists + public Task CreateContext(string contextName) + { + if (_contexts.TryGetValue(contextName, out _)) { - if (_contexts.TryGetValue(contextName, out var context)) - throw new Exception($"Context with id {contextName} already exists."); - - if (_config.MaxInstances > -1 && ContextCount >= _config.MaxInstances) - throw new Exception($"Maximum model instances reached"); - - context = _weights.CreateContext(_config, _llamaLogger); - if (_contexts.TryAdd(contextName, context)) - return Task.FromResult(context); - - return Task.FromResult(null); + throw new Exception($"Context with id {contextName} already exists."); } - /// - /// Get a contexts belonging to this model - /// - /// The unique context identifier - /// LLamaModelContext for this LLamaModel with the specified contextName - public Task GetContext(string contextName) + if (_config.MaxInstances > 0 && ContextCount >= _config.MaxInstances) { - if (_contexts.TryGetValue(contextName, out var context)) - return Task.FromResult(context); - - return Task.FromResult(null); + throw new Exception($"Maximum model instances reached"); } - /// - /// Remove a context from this model - /// - /// The unique context identifier - /// true if removed, otherwise false - public Task RemoveContext(string contextName) + var context = _weights.CreateContext(_config, _llamaLogger); + if (_contexts.TryAdd(contextName, context)) { - if (!_contexts.TryRemove(contextName, out var context)) - return Task.FromResult(false); - - context?.Dispose(); - return Task.FromResult(true); + return Task.FromResult(context); } + return Task.FromResult(null); + } + + /// + /// Get a contexts belonging to this model + /// + /// The unique context identifier + /// LLamaModelContext for this LLamaModel with the specified contextName + public Task GetContext(string contextName) + { + if (_contexts.TryGetValue(contextName, out var context)) + return Task.FromResult(context); + + return Task.FromResult(null); + } - /// - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. - /// - public void Dispose() + /// + /// Remove a context from this model + /// + /// The unique context identifier + /// true if removed, otherwise false + public Task RemoveContext(string contextName) + { + if (!_contexts.TryRemove(contextName, out var context)) + return Task.FromResult(false); + + context?.Dispose(); + return Task.FromResult(true); + } + + /// + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// + public void Dispose() + { + foreach (var context in _contexts.Values) { - foreach (var context in _contexts.Values) - { - context?.Dispose(); - } - _weights.Dispose(); + context?.Dispose(); } + _weights.Dispose(); } } diff --git a/LLama.Web/Models/ModelSession.cs b/LLama.Web/Models/ModelSession.cs index c66645056..d69187093 100644 --- a/LLama.Web/Models/ModelSession.cs +++ b/LLama.Web/Models/ModelSession.cs @@ -1,142 +1,136 @@ -using LLama.Abstractions; +using LLama.Abstractions; using LLama.Web.Common; -namespace LLama.Web.Models +namespace LLama.Web.Models; + +public class ModelSession { - public class ModelSession + private readonly string _sessionId; + private readonly LLamaModel _model; + private readonly LLamaContext _context; + private readonly ILLamaExecutor _executor; + private readonly ISessionConfig _sessionConfig; + private readonly ITextStreamTransform _outputTransform; + private readonly InferenceOptions _defaultInferenceConfig; + + private CancellationTokenSource _cancellationTokenSource; + + public ModelSession(LLamaModel model, LLamaContext context, string sessionId, ISessionConfig sessionConfig, InferenceOptions inferenceOptions = null) { - private readonly string _sessionId; - private readonly LLamaModel _model; - private readonly LLamaContext _context; - private readonly ILLamaExecutor _executor; - private readonly ISessionConfig _sessionConfig; - private readonly ITextStreamTransform _outputTransform; - private readonly InferenceOptions _defaultInferenceConfig; + _model = model; + _context = context; + _sessionId = sessionId; + _sessionConfig = sessionConfig; + _defaultInferenceConfig = inferenceOptions ?? new InferenceOptions(); + _outputTransform = CreateOutputFilter(); + _executor = CreateExecutor(); + } - private CancellationTokenSource _cancellationTokenSource; + /// + /// Gets the session identifier. + /// + public string SessionId => _sessionId; + + /// + /// Gets the name of the model. + /// + public string ModelName => _sessionConfig.Model; + + /// + /// Gets the context. + /// + public LLamaContext Context => _context; + + /// + /// Gets the session configuration. + /// + public ISessionConfig SessionConfig => _sessionConfig; + + /// + /// Gets the inference parameters. + /// + public InferenceOptions InferenceParams => _defaultInferenceConfig; + + /// + /// Initializes the prompt. + /// + /// The inference configuration. + /// The cancellation token. + internal async Task InitializePrompt(InferenceOptions inferenceConfig = null, CancellationToken cancellationToken = default) + { + if (_sessionConfig.ExecutorType == LLamaExecutorType.Stateless) + return; - public ModelSession(LLamaModel model, LLamaContext context, string sessionId, ISessionConfig sessionConfig, InferenceOptions inferenceOptions = null) - { - _model = model; - _context = context; - _sessionId = sessionId; - _sessionConfig = sessionConfig; - _defaultInferenceConfig = inferenceOptions ?? new InferenceOptions(); - _outputTransform = CreateOutputFilter(); - _executor = CreateExecutor(); - } - - /// - /// Gets the session identifier. - /// - public string SessionId => _sessionId; - - /// - /// Gets the name of the model. - /// - public string ModelName => _sessionConfig.Model; - - /// - /// Gets the context. - /// - public LLamaContext Context => _context; - - /// - /// Gets the session configuration. - /// - public ISessionConfig SessionConfig => _sessionConfig; - - /// - /// Gets the inference parameters. - /// - public InferenceOptions InferenceParams => _defaultInferenceConfig; - - - - /// - /// Initializes the prompt. - /// - /// The inference configuration. - /// The cancellation token. - internal async Task InitializePrompt(InferenceOptions inferenceConfig = null, CancellationToken cancellationToken = default) - { - if (_sessionConfig.ExecutorType == LLamaExecutorType.Stateless) - return; - - if (string.IsNullOrEmpty(_sessionConfig.Prompt)) - return; - - // Run Initial prompt - var inferenceParams = ConfigureInferenceParams(inferenceConfig); - _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - await foreach (var _ in _executor.InferAsync(_sessionConfig.Prompt, inferenceParams, _cancellationTokenSource.Token)) - { - // We dont really need the response of the initial prompt, so exit on first token - break; - }; - } - - - /// - /// Runs inference on the model context - /// - /// The message. - /// The inference configuration. - /// The cancellation token. - /// - internal IAsyncEnumerable InferAsync(string message, InferenceOptions inferenceConfig = null, CancellationToken cancellationToken = default) + if (string.IsNullOrEmpty(_sessionConfig.Prompt)) + return; + + // Run Initial prompt + var inferenceParams = ConfigureInferenceParams(inferenceConfig); + _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + await foreach (var _ in _executor.InferAsync(_sessionConfig.Prompt, inferenceParams, _cancellationTokenSource.Token)) { - var inferenceParams = ConfigureInferenceParams(inferenceConfig); - _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + // We dont really need the response of the initial prompt, so exit on first token + break; + }; + } - var inferenceStream = _executor.InferAsync(message, inferenceParams, _cancellationTokenSource.Token); - if (_outputTransform is not null) - return _outputTransform.TransformAsync(inferenceStream); + /// + /// Runs inference on the model context + /// + /// The message. + /// The inference configuration. + /// The cancellation token. + /// + internal IAsyncEnumerable InferAsync(string message, InferenceOptions inferenceConfig = null, CancellationToken cancellationToken = default) + { + var inferenceParams = ConfigureInferenceParams(inferenceConfig); + _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - return inferenceStream; - } + var inferenceStream = _executor.InferAsync(message, inferenceParams, _cancellationTokenSource.Token); + if (_outputTransform is not null) + return _outputTransform.TransformAsync(inferenceStream); + return inferenceStream; + } - public void CancelInfer() - { - _cancellationTokenSource?.Cancel(); - } + public void CancelInfer() + { + _cancellationTokenSource?.Cancel(); + } - public bool IsInferCanceled() - { - return _cancellationTokenSource.IsCancellationRequested; - } - - /// - /// Configures the inference parameters. - /// - /// The inference configuration. - private IInferenceParams ConfigureInferenceParams(InferenceOptions inferenceConfig) - { - var inferenceParams = inferenceConfig ?? _defaultInferenceConfig; - inferenceParams.AntiPrompts = _sessionConfig.GetAntiPrompts(); - return inferenceParams; - } + public bool IsInferCanceled() + { + return _cancellationTokenSource.IsCancellationRequested; + } - private ITextStreamTransform CreateOutputFilter() - { - var outputFilters = _sessionConfig.GetOutputFilters(); - if (outputFilters.Count > 0) - return new LLamaTransforms.KeywordTextOutputStreamTransform(outputFilters); + /// + /// Configures the inference parameters. + /// + /// The inference configuration. + private IInferenceParams ConfigureInferenceParams(InferenceOptions inferenceConfig) + { + var inferenceParams = inferenceConfig ?? _defaultInferenceConfig; + inferenceParams.AntiPrompts = _sessionConfig.GetAntiPrompts(); + return inferenceParams; + } - return null; - } + private ITextStreamTransform CreateOutputFilter() + { + var outputFilters = _sessionConfig.GetOutputFilters(); + if (outputFilters.Count > 0) + return new LLamaTransforms.KeywordTextOutputStreamTransform(outputFilters); + return null; + } - private ILLamaExecutor CreateExecutor() + private ILLamaExecutor CreateExecutor() + { + return _sessionConfig.ExecutorType switch { - return _sessionConfig.ExecutorType switch - { - LLamaExecutorType.Interactive => new InteractiveExecutor(_context), - LLamaExecutorType.Instruct => new InstructExecutor(_context), - LLamaExecutorType.Stateless => new StatelessExecutor(_model.LLamaWeights, _context.Params), - _ => default - }; - } + LLamaExecutorType.Interactive => new InteractiveExecutor(_context), + LLamaExecutorType.Instruct => new InstructExecutor(_context), + LLamaExecutorType.Stateless => new StatelessExecutor(_model.LLamaWeights, _context.Params), + _ => default + }; } } diff --git a/LLama.Web/Models/TokenModel.cs b/LLama.Web/Models/TokenModel.cs index c95f9ec6e..54ec2aa0d 100644 --- a/LLama.Web/Models/TokenModel.cs +++ b/LLama.Web/Models/TokenModel.cs @@ -1,24 +1,23 @@ -namespace LLama.Web.Models +namespace LLama.Web.Models; + +public class TokenModel { - public class TokenModel + public TokenModel(string id, string content = null, TokenType tokenType = TokenType.Content) { - public TokenModel(string id, string content = null, TokenType tokenType = TokenType.Content) - { - Id = id; - Content = content; - TokenType = tokenType; - } - - public string Id { get; set; } - public string Content { get; set; } - public TokenType TokenType { get; set; } + Id = id; + Content = content; + TokenType = tokenType; } - public enum TokenType - { - Begin = 0, - Content = 2, - End = 4, - Cancel = 10 - } + public string Id { get; set; } + public string Content { get; set; } + public TokenType TokenType { get; set; } +} + +public enum TokenType +{ + Begin = 0, + Content = 2, + End = 4, + Cancel = 10 } diff --git a/LLama.Web/Pages/Index.cshtml b/LLama.Web/Pages/Index.cshtml index 3df4b6996..fc87988e6 100644 --- a/LLama.Web/Pages/Index.cshtml +++ b/LLama.Web/Pages/Index.cshtml @@ -1,62 +1,65 @@ -@page +@page @using LLama.Web.Common; @model IndexModel @{ - ViewData["Title"] = "Inference Demo"; + ViewData["Title"] = "Home"; } @Html.AntiForgeryToken()
-
-
-
- @ViewData["Title"] -
-
- Socket: Disconnected -
-
- +