Allow IInstructService to StopCompletion.

README.md

@@ -17,6 +17,8 @@ It's written in C# running on .Net Core that is full cross-platform framework, t
 
 **BotSharp** is in accordance with components principle strictly, decouples every part that is needed in the platform builder. So you can choose different UI/UX, or pick up a different LLM providers. They are all modulized based on unified interfaces. **BotSharp** provides an advanced Agent abstraction layer to efficiently manage complex application scenarios in enterprises, allowing enterprise developers to efficiently integrate AI into business systems.
 
+![](./docs/architecture/assets/llm_diagram.png)
+
 ### Some Features
 
 * Built-in multi-agents and conversation with state management.

src/Infrastructure/BotSharp.Core/Instructs/InstructService.cs

@@ -22,6 +22,15 @@ public partial class InstructService : IInstructService
         foreach (var hook in hooks)
         {
             await hook.BeforeCompletion(message);
+
+            // Interrupted by hook
+            if (message.StopCompletion)
+            {
+                return new InstructResult
+                {
+                    Text = message.Content
+                };
+            }
         }
 
         var completer = CompletionProvider.GetTextCompletion(_services);

src/Plugins/BotSharp.Plugin.LLamaSharp/BotSharp.Plugin.LLamaSharp.csproj

@@ -10,7 +10,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="LLamaSharp" Version="0.5.1" />
+    <PackageReference Include="LLamaSharp" Version="0.6.0" />
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.LLamaSharp/LlamaAiModel.cs

@@ -35,7 +35,7 @@ public class LlamaAiModel
 
         _params = new ModelParams(Path.Combine(_settings.ModelDir, model))
         {
-            ContextSize = _settings.MaxContextLength,
+            ContextSize = (uint)_settings.MaxContextLength,
             Seed = 1337,
             GpuLayerCount = _settings.NumberOfGpuLayer
         };

src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs

@@ -50,7 +50,7 @@ public class ChatCompletionProvider : IChatCompletion
             _logger.LogInformation(prompt);
         }
 
-        foreach (var response in executor.Infer(prompt, inferenceParams))
+        foreach (var response in executor.InferAsync(prompt, inferenceParams).ToArrayAsync().Result)
         {
             Console.Write(response);
             totalResponse += response;
@@ -108,7 +108,7 @@ public class ChatCompletionProvider : IChatCompletion
             _logger.LogInformation(prompt);
         }
 
-        foreach (var response in executor.Infer(prompt, inferenceParams))
+        await foreach (var response in executor.InferAsync(prompt, inferenceParams))
         {
             Console.Write(response);
             totalResponse += response;
@@ -151,7 +151,7 @@ public class ChatCompletionProvider : IChatCompletion
             _logger.LogInformation(agent.Instruction);
         }
 
-        foreach (var response in executor.Infer(agent.Instruction, inferenceParams))
+        await foreach (var response in executor.InferAsync(agent.Instruction, inferenceParams))
         {
             Console.Write(response);
             totalResponse += response;

src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/TextCompletionProvider.cs

@@ -36,7 +36,7 @@ public class TextCompletionProvider : ITextCompletion
 
         _tokenStatistics.StartTimer();
         string completion = "";
-        foreach (var response in executor.Infer(text, inferenceParams))
+        await foreach (var response in executor.InferAsync(text, inferenceParams))
         {
             Console.Write(response);
             completion += response;

src/WebStarter/WebStarter.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
@@ -35,7 +35,7 @@
   </ItemGroup>
     
   <ItemGroup>
-    <PackageReference Include="LLamaSharp.Backend.Cuda11" Version="0.5.1" />
+    <PackageReference Include="LLamaSharp.Backend.Cuda11" Version="0.6.0" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.16" />
     <PackageReference Include="SciSharp.TensorFlow.Redist" Version="2.11.4" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />

tests/BotSharp.Plugin.PizzaBot/Hooks/PizzaBotAgentHook.cs

@@ -4,6 +4,8 @@ namespace BotSharp.Plugin.PizzaBot.Hooks;
 
 public class PizzaBotAgentHook : AgentHookBase
 {
+    public override string SelfId => "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a";
+
     public PizzaBotAgentHook(IServiceProvider services, AgentSettings settings)
         : base(services, settings)
     {