workaround fasthttp

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

core/http/endpoints/openai/chat.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"net"
 	"time"
 
 	"github.com/gofiber/fiber/v2"
@@ -516,6 +517,29 @@ func ChatEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, evaluator
 
 			}
 
+			// NOTE: this is a bad WORKAROUND! We should find a better way to handle this.
+			// Fasthttp doesn't support context cancellation from the caller
+			// for non-streaming requests, so we need to monitor the connection directly.
+			// Monitor connection for client disconnection during non-streaming requests
+			// We access the connection directly via c.Context().Conn() to monitor it
+			// during ComputeChoices execution, not after the response is sent
+			// see: https://github.com/mudler/LocalAI/pull/7187#issuecomment-3506720906
+			var conn net.Conn = c.Context().Conn()
+			if conn != nil {
+				go func() {
+					buf := make([]byte, 1)
+					for {
+						_, err := conn.Read(buf)
+						if err != nil {
+							// Connection closed - cancel the context to stop gRPC call
+							log.Debug().Msgf("Cancelling GRPC call")
+							input.Cancel()
+							return
+						}
+					}
+				}()
+			}
+
 			result, tokenUsage, err := ComputeChoices(
 				input,
 				predInput,