diff --git a/Dockerfile b/Dockerfile index 048399a..57e1e41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,4 +13,11 @@ WORKDIR /app COPY --from=builder /build/app . +# Do not forget "/v1" in the end +ENV OPENAI_API_BASE_URL="" \ + OPENAI_API_TOKEN="" \ + TELEGRAM_TOKEN="" \ + MODEL_TEXT_REQUEST="llama3.1:8b-instruct-q6_K" \ + MODEL_SUMMARIZE_REQUEST="llama3.1:8b-instruct-q6_K" + CMD ["/app/app"] diff --git a/README.md b/README.md index d5733f7..1bab54c 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,10 @@ ```shell docker run \ - -e OLLAMA_TOKEN=123 \ - -e OLLAMA_BASE_URL=http://ollama.localhost:11434/v1 \ + -e OPENAI_API_TOKEN=123 \ + -e OPENAI_API_BASE_URL=http://ollama.localhost:11434/v1 \ -e TELEGRAM_TOKEN=12345 \ + -e MODEL_TEXT_REQUEST=llama3.1:8b-instruct-q6_K + -e MODEL_TEXT_REQUEST=mistral-nemo:12b-instruct-2407-q4_K_M skobkin/telegram-llm-bot ``` diff --git a/bot/bot.go b/bot/bot.go index 6810eee..4d7d81f 100644 --- a/bot/bot.go +++ b/bot/bot.go @@ -23,16 +23,23 @@ type Bot struct { llm *llm.LlmConnector extractor *extractor.Extractor stats *stats.Stats + models ModelSelection markdownV1Replacer *strings.Replacer } -func NewBot(api *telego.Bot, llm *llm.LlmConnector, extractor *extractor.Extractor) *Bot { +func NewBot( + api *telego.Bot, + llm *llm.LlmConnector, + extractor *extractor.Extractor, + models ModelSelection, +) *Bot { return &Bot{ api: api, llm: llm, extractor: extractor, stats: stats.NewStats(), + models: models, markdownV1Replacer: strings.NewReplacer( // https://core.telegram.org/bots/api#markdown-style @@ -122,7 +129,7 @@ func (b *Bot) inlineHandler(bot *telego.Bot, update telego.Update) { slog.Error("Cannot retrieve an article using extractor", "error", err) } - llmReply, err := b.llm.Summarize(article.Text, llm.ModelLlama3Uncensored ) + llmReply, err := b.llm.Summarize(article.Text, b.models.TextRequestModel) if err != nil { slog.Error("Cannot get reply from LLM connector") @@ -148,7 +155,7 @@ func (b *Bot) inlineHandler(bot *telego.Bot, update telego.Update) { requestContext := createLlmRequestContextFromUpdate(update) - llmReply, err := b.llm.HandleSingleRequest(iq.Query, llm.ModelLlama3Uncensored, requestContext) + llmReply, err := b.llm.HandleSingleRequest(iq.Query, b.models.TextRequestModel, requestContext) if err != nil { slog.Error("Cannot get reply from LLM connector") @@ -194,7 +201,7 @@ func (b *Bot) heyHandler(bot *telego.Bot, update telego.Update) { requestContext := createLlmRequestContextFromUpdate(update) - llmReply, err := b.llm.HandleSingleRequest(userMessage, llm.ModelLlama3Uncensored, requestContext) + llmReply, err := b.llm.HandleSingleRequest(userMessage, b.models.TextRequestModel, requestContext) if err != nil { slog.Error("Cannot get reply from LLM connector") @@ -259,7 +266,7 @@ func (b *Bot) summarizeHandler(bot *telego.Bot, update telego.Update) { slog.Error("Cannot retrieve an article using extractor", "error", err) } - llmReply, err := b.llm.Summarize(article.Text, llm.ModelMistralUncensored) + llmReply, err := b.llm.Summarize(article.Text, b.models.SummarizeModel) if err != nil { slog.Error("Cannot get reply from LLM connector") diff --git a/bot/models.go b/bot/models.go new file mode 100644 index 0000000..93259d0 --- /dev/null +++ b/bot/models.go @@ -0,0 +1,6 @@ +package bot + +type ModelSelection struct { + TextRequestModel string + SummarizeModel string +} diff --git a/bot/request_context.go b/bot/request_context.go index 969abf6..93cba79 100644 --- a/bot/request_context.go +++ b/bot/request_context.go @@ -44,11 +44,13 @@ func createLlmRequestContextFromUpdate(update telego.Update) llm.RequestContext } if !rc.Inline { + // TODO: implement retrieval of chat description chat := message.Chat rc.Chat = llm.ChatContext{ - Title: chat.Title, - Description: chat.Description, - Type: chat.Type, + Title: chat.Title, + // TODO: fill when ChatFullInfo retrieved + //Description: chat.Description, + Type: chat.Type, } } diff --git a/llm/llm.go b/llm/llm.go index 73affe8..7732264 100644 --- a/llm/llm.go +++ b/llm/llm.go @@ -10,9 +10,6 @@ import ( var ( ErrLlmBackendRequestFailed = errors.New("llm back-end request failed") ErrNoChoices = errors.New("no choices in LLM response") - - ModelMistralUncensored = "dolphin-mistral:7b-v2.8-q4_K_M" - ModelLlama3Uncensored = "dolphin-llama3:8b-v2.9-q4_K_M" ) type LlmConnector struct { @@ -31,12 +28,12 @@ func NewConnector(baseUrl string, token string) *LlmConnector { } func (l *LlmConnector) HandleSingleRequest(text string, model string, requestContext RequestContext) (string, error) { - systemPrompt := "You're a bot in the Telegram chat. " + - "You're using a free model called \"" + model + "\". " + - "You see only messages addressed to you using commands due to privacy settings." + systemPrompt := "You're a bot in the Telegram chat.\n" + + "You're using a free model called \"" + model + "\".\n" + + "Currently you're not able to access chat history, so each message will be replied from a clean slate." if !requestContext.Empty { - systemPrompt += " " + requestContext.Prompt() + systemPrompt += "\n" + requestContext.Prompt() } req := openai.ChatCompletionRequest{ @@ -79,9 +76,10 @@ func (l *LlmConnector) Summarize(text string, model string) (string, error) { { Role: openai.ChatMessageRoleSystem, Content: "You're a text shortener. Give a very brief summary of the main facts " + - "point by point. Format them as a list of bullet points. " + + "point by point. Format them as a list of bullet points each starting with \"-\". " + "Avoid any commentaries and value judgement on the matter. " + - "If possible, use the same language as the original text.", + "If possible, respond in the same language as the original text." + + "Do not use any non-ASCII characters.", }, }, } @@ -108,3 +106,37 @@ func (l *LlmConnector) Summarize(text string, model string) (string, error) { return resp.Choices[0].Message.Content, nil } + +func (l *LlmConnector) GetModels() []string { + var result []string + + models, err := l.client.ListModels(context.Background()) + if err != nil { + slog.Error("llm: Model list request failed", "error", err) + + return result + } + + slog.Info("Model list retrieved", "models", models) + + for _, model := range models.Models { + result = append(result, model.ID) + } + + return result +} + +func (l *LlmConnector) HasModel(id string) bool { + model, err := l.client.GetModel(context.Background(), id) + if err != nil { + slog.Error("llm: Model request failed", "error", err) + } + + slog.Debug("llm: Returned model", "model", model) + + if model.ID != "" { + return true + } + + return false +} diff --git a/llm/request_context.go b/llm/request_context.go index fff4456..34fa144 100644 --- a/llm/request_context.go +++ b/llm/request_context.go @@ -39,15 +39,16 @@ func (c RequestContext) Prompt() string { prompt += "You're responding to inline query, so you're not in the chat right now. " } - prompt += "According to their profile, first name of the user who wrote you is \"" + c.User.FirstName + "\". " + prompt += "User profile:" + + "First name: \"" + c.User.FirstName + "\"\n" if c.User.Username != "" { - prompt += "Their username is @" + c.User.Username + ". " + prompt += "Username: @" + c.User.Username + ".\n" } if c.User.LastName != "" { - prompt += "Their last name is \"" + c.User.LastName + "\". " + prompt += "Last name: \"" + c.User.LastName + "\"\n" } if c.User.IsPremium { - prompt += "They have Telegram Premium subscription. " + prompt += "Telegram Premium subscription: active." } return prompt diff --git a/main.go b/main.go index 6611c9a..84d3f24 100644 --- a/main.go +++ b/main.go @@ -12,12 +12,31 @@ import ( ) func main() { - ollamaToken := os.Getenv("OLLAMA_TOKEN") - ollamaBaseUrl := os.Getenv("OLLAMA_BASE_URL") + apiToken := os.Getenv("OPENAI_API_TOKEN") + apiBaseUrl := os.Getenv("OPENAI_API_BASE_URL") + + models := bot.ModelSelection{ + TextRequestModel: os.Getenv("MODEL_TEXT_REQUEST"), + SummarizeModel: os.Getenv("MODEL_SUMMARIZE_REQUEST"), + } + + slog.Info("Selected", "models", models) telegramToken := os.Getenv("TELEGRAM_TOKEN") - llmc := llm.NewConnector(ollamaBaseUrl, ollamaToken) + llmc := llm.NewConnector(apiBaseUrl, apiToken) + + slog.Info("Checking models availability") + + for _, model := range []string{models.TextRequestModel, models.SummarizeModel} { + if !llmc.HasModel(model) { + slog.Error("Model not unavailable", "model", model) + os.Exit(1) + } + } + + slog.Info("All needed models are available") + ext := extractor.NewExtractor() telegramApi, err := tg.NewBot(telegramToken, tg.WithLogger(bot.NewLogger("telego: "))) @@ -26,7 +45,7 @@ func main() { os.Exit(1) } - botService := bot.NewBot(telegramApi, llmc, ext) + botService := bot.NewBot(telegramApi, llmc, ext, models) err = botService.Run() if err != nil {