Add support for store pinyin/table code in user history (#119)

Author: wengxt

src/libime/core/historybigram.cpp

@@ -7,12 +7,15 @@
 #define _FCITX_LIBIME_CORE_HISTORYBIGRAM_H_
 
 #include <cstddef>
+#include <cstdint>
+#include <functional>
 #include <istream>
 #include <memory>
 #include <ostream>
 #include <string>
 #include <string_view>
 #include <unordered_set>
+#include <utility>
 #include <vector>
 #include <fcitx-utils/macros.h>
 #include <libime/core/lattice.h>
@@ -22,8 +25,13 @@ namespace libime {
 
 class HistoryBigramPrivate;
 
+using ValidationCodeExtractor = std::function<std::string(const WordNode *)>;
+
 class LIBIMECORE_EXPORT HistoryBigram {
 public:
+    using WordWithCode = std::pair<std::string, std::string>;
+    using WordWithCodeView = std::pair<std::string_view, std::string_view>;
+
     HistoryBigram();
 
     FCITX_DECLARE_VIRTUAL_DTOR_MOVE(HistoryBigram);
@@ -43,14 +51,20 @@ class LIBIMECORE_EXPORT HistoryBigram {
     bool useOnlyUnigram() const;
 
     void forget(std::string_view word);
+    void forget(std::string_view word, std::string_view code);
 
     bool isUnknown(std::string_view v) const;
-    float score(const WordNode *prev, const WordNode *cur) const {
-        return score(prev ? prev->word() : "", cur ? cur->word() : "");
-    }
+    float score(const WordNode *prev, const WordNode *cur) const;
     float score(std::string_view prev, std::string_view cur) const;
+    float scoreWithCode(WordWithCodeView prev, WordWithCodeView cur) const;
+    float scoreWithCode(const WordNode *prev, const WordNode *cur,
+                        const ValidationCodeExtractor &extractor) const;
     void add(const SentenceResult &sentence);
     void add(const std::vector<std::string> &sentence);
+    void addWithCode(const SentenceResult &sentence,
+                     const ValidationCodeExtractor &validationCodeExtractor);
+    void
+    addWithCode(const std::vector<WordWithCode> &sentenceWithValidationCode);
 
     /// Fill the prediction based on current sentence.
     void fillPredict(std::unordered_set<std::string> &words,
@@ -59,6 +73,35 @@ class LIBIMECORE_EXPORT HistoryBigram {
 
     bool containsBigram(std::string_view prev, std::string_view cur) const;
 
+    /**
+     * Query the weighted frequency of the unigram.
+     *
+     * @since 1.1.14
+     */
+    float unigramFrequency(WordWithCodeView word) const;
+
+    /**
+     * Query the weighted frequency of the bigram.
+     *
+     * @since 1.1.14
+     */
+    float bigramFrequency(WordWithCodeView prev, WordWithCodeView cur) const;
+
+    /**
+     * Query the raw frequency of the unigram.
+     *
+     * @since 1.1.14
+     */
+    int32_t rawUnigramFrequency(WordWithCodeView word) const;
+
+    /**
+     * Query the raw frequency of the bigram.
+     *
+     * @since 1.1.14
+     */
+    int32_t rawBigramFrequency(WordWithCodeView prev,
+                               WordWithCodeView cur) const;
+
 private:
     std::unique_ptr<HistoryBigramPrivate> d_ptr;
     FCITX_DECLARE_PRIVATE(HistoryBigram);

src/libime/core/historybigram.h

@@ -32,6 +32,7 @@ class UserLanguageModelPrivate {
     bool useOnlyUnigram_ = false;
 
     HistoryBigram history_;
+    ValidationCodeExtractor extractor_;
     float weight_ = DEFAULT_USER_LANGUAGE_MODEL_USER_WEIGHT;
     // log(wa * exp(a) + wb * exp(b))
     // log(exp(log(wa) + a) + exp(b + log(wb))
@@ -128,7 +129,12 @@ float UserLanguageModel::score(const State &state, const WordNode &word,
         score = LanguageModel::score(state, word, out);
     }
     const auto *prev = d->wordFromState(state);
-    float userScore = d->history_.score(prev, &word);
+    float userScore;
+    if (d->extractor_) {
+        userScore = d->history_.scoreWithCode(prev, &word, d->extractor_);
+    } else {
+        userScore = d->history_.score(prev, &word);
+    }
     d->setWordToState(out, &word);
     return std::max(score, sum_log_prob(score + d->wa_, userScore + d->wb_));
 }
@@ -170,4 +176,9 @@ bool UserLanguageModel::containsNonUnigram(
     return LanguageModel::maxNgramLength(words) > 1;
 }
 
+void UserLanguageModel::setCodeExtractor(ValidationCodeExtractor extractor) {
+    FCITX_D();
+    d->extractor_ = std::move(extractor);
+}
+
 } // namespace libime

src/libime/core/userlanguagemodel.cpp

@@ -13,6 +13,7 @@
 #include <string_view>
 #include <vector>
 #include <fcitx-utils/macros.h>
+#include <libime/core/historybigram.h>
 #include <libime/core/languagemodel.h>
 #include <libime/core/libimecore_export.h>
 
@@ -48,6 +49,8 @@ class LIBIMECORE_EXPORT UserLanguageModel : public LanguageModel {
 
     bool containsNonUnigram(const std::vector<std::string> &words) const;
 
+    void setCodeExtractor(ValidationCodeExtractor extractor);
+
 private:
     std::unique_ptr<UserLanguageModelPrivate> d_ptr;
     FCITX_DECLARE_PRIVATE(UserLanguageModel);