Try to append to history if there is context

Fix #122

Author: wengxt

src/libime/core/historybigram.cpp

@@ -395,6 +395,32 @@ class HistoryBigramPool {
         bigram_.fillPredict(words, word, maxSize);
     }
 
+    bool maybeAppendToLatestSentence(const std::vector<WordWithCode> &context,
+                                     std::vector<WordWithCode> &newSentence) {
+        if (recent_.empty() || newSentence.empty()) {
+            return false;
+        }
+        auto &latestSentence = recent_.front();
+        if (latestSentence.size() < context.size() ||
+            !std::ranges::equal(
+                context,
+                std::views::drop(latestSentence,
+                                 latestSentence.size() - context.size()))) {
+            return false;
+        }
+
+        const int delta = 1;
+        decBigram(latestSentence.back(), {"</s>", ""}, delta);
+        for (auto &item : newSentence) {
+            unigram_.incFreq(item, delta);
+            incBigram(latestSentence.back(), item, delta);
+            latestSentence.push_back(std::move(item));
+        }
+        incBigram(latestSentence.back(), {"</s>", ""}, delta);
+
+        return true;
+    }
+
 private:
     template <typename R>
     void remove(const R &sentence) {
@@ -742,4 +768,13 @@ float HistoryBigram::scoreWithCode(
         {cur ? cur->word() : "", extractor && cur ? extractor(cur) : ""});
 }
 
+void HistoryBigram::addWithContext(const std::vector<WordWithCode> &context,
+                                   std::vector<WordWithCode> newSentence) {
+    FCITX_D();
+    if (context.empty() ||
+        !d->pools_[0].maybeAppendToLatestSentence(context, newSentence)) {
+        addWithCode(newSentence);
+    }
+}
+
 } // namespace libime

src/libime/core/historybigram.h

@@ -102,6 +102,9 @@ class LIBIMECORE_EXPORT HistoryBigram {
     int32_t rawBigramFrequency(WordWithCodeView prev,
                                WordWithCodeView cur) const;
 
+    void addWithContext(const std::vector<WordWithCode> &context,
+                        std::vector<WordWithCode> newSentence);
+
 private:
     std::unique_ptr<HistoryBigramPrivate> d_ptr;
     FCITX_DECLARE_PRIVATE(HistoryBigram);

src/libime/pinyin/pinyincontext.cpp

@@ -919,10 +919,10 @@ std::vector<std::string> PinyinContext::selectedWords() const {
     return newSentence;
 }
 
-std::vector<std::pair<std::string, std::string>>
+std::vector<HistoryBigram::WordWithCode>
 PinyinContext::selectedWordsWithPinyin() const {
     FCITX_D();
-    std::vector<std::pair<std::string, std::string>> newSentence;
+    std::vector<HistoryBigram::WordWithCode> newSentence;
     for (const auto &s : d->selected_) {
         for (const auto &item : s) {
             if (item.type_ != SelectedPinyinType::Separator) {
@@ -976,31 +976,30 @@ void PinyinContext::learn() {
         return;
     }
 
+    std::vector<HistoryBigram::WordWithCode> newSentence;
     if (auto [result, encodedWordPinyin] = d->learnWord();
         result != LearnWordResult::Ignored) {
         // Do not insert custom to history for the first time.
         if (result == LearnWordResult::Normal) {
             // Create new sentence with the whole new learned word.
-            std::vector<HistoryBigram::WordWithCode> newSentence{
-                {sentence(), encodedWordPinyin}};
-            d->ime_->model()->history().addWithCode(newSentence);
+            newSentence.push_back({sentence(), encodedWordPinyin});
+        } else {
+            return;
         }
     } else {
-        std::vector<HistoryBigram::WordWithCode> newSentence;
-        for (auto &s : d->selected_) {
-            for (auto &item : s) {
-                if (item.type_ != SelectedPinyinType::Separator) {
-                    // Non pinyin word. Skip it.
-                    if (item.encodedPinyin().empty()) {
-                        return;
-                    }
-                    newSentence.push_back(
-                        {item.word_.word(), item.encodedPinyin()});
-                }
-            }
-        }
-        d->ime_->model()->history().addWithCode(newSentence);
+        newSentence = selectedWordsWithPinyin();
+    }
+
+    if (std::ranges::any_of(newSentence, [](const auto &word) {
+            return word.second.empty();
+        })) {
+        // Don't add to history if there is any non-pinyin word.
+        return;
     }
+
+    auto context = contextWordsWithPinyin();
+    d->ime_->model()->history().addWithContext(contextWordsWithPinyin(),
+                                               std::move(newSentence));
 }
 
 void PinyinContext::setContextWords(

test/testhistorybigram.cpp

@@ -12,6 +12,7 @@
 #include <string>
 #include <unordered_set>
 #include <fcitx-utils/log.h>
+#include <fcitx-utils/stringutils.h>
 #include "libime/core/historybigram.h"
 
 namespace {
@@ -266,6 +267,24 @@ void testWithCodePredict() {
     }
 }
 
+void testAppend() {
+    using namespace libime;
+    HistoryBigram history;
+    history.addWithCode({{"你", "code1"}, {"是", "code2"}, {"一个", "code3"}});
+
+    history.addWithContext({{"是", "code2"}, {"一个", "code3"}},
+                           {{"好人", "code4"}});
+
+    history.addWithContext({{"不是", "code5"}}, {{"你的", "code6"}});
+    std::stringstream ss;
+    history.dump(ss);
+    auto lines = fcitx::stringutils::split(ss.str(), "\n");
+    FCITX_ASSERT(lines.size() == 2) << lines.size();
+    FCITX_ASSERT(lines[0] == "你的\tcode6") << lines[0];
+    FCITX_ASSERT(lines[1] == "你\tcode1 是\tcode2 一个\tcode3 好人\tcode4")
+        << lines[1];
+}
+
 } // namespace
 
 int main() {
@@ -276,5 +295,6 @@ int main() {
     testSaveAndLoadText();
     testWithCode();
     testWithCodePredict();
+    testAppend();
     return 0;
 }