feat(AI): streaming response

botcmd_translate.go

@@ -3,6 +3,7 @@ package main
 import (
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/samber/lo"
 	tele "gopkg.in/telebot.v3"
@@ -71,10 +72,6 @@ func handleTranslateBtn(c tele.Context) error {
 		return nil
 	}
 
-	// change the temporary message
-	if err := c.Edit("Sure, please wait..."); err != nil {
-		logger.Warnf("failed to alter the temporary message: %v", err)
-	}
 	// pretend to be typing
 	if err := c.Bot().Notify(msg.Chat, tele.Typing); err != nil {
 		logger.Warnf("failed to send typing action: %v", err)
@@ -98,23 +95,65 @@ func handleTranslateBtn(c tele.Context) error {
 	}
 	logger.Debugf("Openai chat request: %#+v", req)
 
-	resp, err := ai.ChatCompletion(req)
+	resp, err := ai.ChatCompletionStream(req)
 	if err != nil {
 		logger.Errorf("failed to translate: req: %#+v, err: %v", req, err)
-		_, err := c.Bot().Reply(origMsg, stickerFromID(stickerPanic), tele.Silent)
+		_, _ = c.Bot().Reply(origMsg, stickerFromID(stickerPanic), tele.Silent)
 		return err
 	}
 
-	respText := resp.Choices[0].Message.Content
+	respBuilder := strings.Builder{}
+	minWait := time.After(1 * time.Second)
+	for {
+		var (
+			nNewChunk        int
+			finished         bool
+			minWaitSatisfied bool
+		)
+	Drain:
+		for {
+			select {
+			case chunk, ok := <-resp.Stream:
+				if !ok {
+					finished = true
+					break Drain
+				}
+				nNewChunk += 1
+				respBuilder.WriteString(chunk)
+			default:
+				if minWaitSatisfied {
+					break Drain
+				}
+				<-minWait
+				minWaitSatisfied = true
+			}
+		}
+
+		if nNewChunk == 0 {
+			if chunk, ok := <-resp.Stream; !ok {
+				finished = true
+			} else {
+				respBuilder.WriteString(chunk)
+			}
+		}
+		if finished {
+			break
+		}
+
+		respoText := respBuilder.String() + "\n... (Writting)"
+		minWait = time.After(691 * time.Millisecond) // renew the timer
+		if msg, err = c.Bot().Edit(msg, respoText, tele.Silent); err != nil {
+			logger.Warnf("failed to edit the temporary message: %v", err)
+			break
+		}
+		logger.Debugf("... message edited")
+	}
+
+	respText := respBuilder.String()
 	retryBtn := translateBtnRetry
 	retryBtn.Data = targetLang
 	respMenu := &tele.ReplyMarkup{}
 	respMenu.Inline(respMenu.Row(retryBtn))
-	_, err = c.Bot().Reply(origMsg, respText, tele.Silent, respMenu)
-
-	// delete the temporary message
-	if err := c.Delete(); err != nil {
-		logger.Warnf("failed to delete the temporary message: %v", err)
-	}
+	_, err = c.Bot().Edit(msg, respText, tele.Silent, respMenu)
 	return err
 }