Remove word from personalized dicts when it's canceled.

Bug: 15555552
Bug: 15552347
Change-Id: Ia4d8c79d079e3ce233d46d4bbad4fff1be5c5c39

java/src/com/android/inputmethod/latin/DictionaryFacilitator.java

@@ -468,15 +468,19 @@ public class DictionaryFacilitator {
                 isValid, timeStampInSeconds, mDistracterFilter);
     }
 
-    public void cancelAddingUserHistory(final PrevWordsInfo prevWordsInfo,
+    private void removeWord(final String dictName, final String word) {
-            final String committedWord) {
+        final ExpandableBinaryDictionary dictionary = mDictionaries.getSubDict(dictName);
-        final ExpandableBinaryDictionary userHistoryDictionary =
+        if (dictionary != null) {
-                mDictionaries.getSubDict(Dictionary.TYPE_USER_HISTORY);
+            dictionary.removeUnigramEntryDynamically(word);
-        if (userHistoryDictionary != null) {
-            userHistoryDictionary.removeNgramDynamically(prevWordsInfo, committedWord);
         }
     }
 
+    public void removeWordFromPersonalizedDicts(final String word) {
+        removeWord(Dictionary.TYPE_USER_HISTORY, word);
+        removeWord(Dictionary.TYPE_PERSONALIZATION, word);
+        removeWord(Dictionary.TYPE_CONTEXTUAL, word);
+    }
+
     // TODO: Revise the way to fusion suggestion results.
     public SuggestionResults getSuggestionResults(final WordComposer composer,
             final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo,

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

@@ -310,6 +310,27 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
         }
     }
 
+    /**
+     * Dynamically remove the unigram entry from the dictionary.
+     */
+    public void removeUnigramEntryDynamically(final String word) {
+        reloadDictionaryIfRequired();
+        asyncExecuteTaskWithWriteLock(new Runnable() {
+            @Override
+            public void run() {
+                if (mBinaryDictionary == null) {
+                    return;
+                }
+                runGCIfRequiredLocked(true /* mindsBlockByGC */);
+                if (!mBinaryDictionary.removeUnigramEntry(word)) {
+                    if (DEBUG) {
+                        Log.i(TAG, "Cannot remove unigram entry: " + word);
+                    }
+                }
+            }
+        });
+    }
+
     /**
      * Adds n-gram information of a word to the dictionary. May overwrite an existing entry.
      */

java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java

@@ -884,6 +884,9 @@ public final class InputLogic {
                 final String rejectedSuggestion = mWordComposer.getTypedWord();
                 mWordComposer.reset();
                 mWordComposer.setRejectedBatchModeSuggestion(rejectedSuggestion);
+                if (!TextUtils.isEmpty(rejectedSuggestion)) {
+                    mDictionaryFacilitator.removeWordFromPersonalizedDicts(rejectedSuggestion);
+                }
             } else {
                 mWordComposer.processEvent(inputTransaction.mEvent);
             }
@@ -1363,7 +1366,6 @@ public final class InputLogic {
      * @param inputTransaction The transaction in progress.
      */
     private void revertCommit(final InputTransaction inputTransaction) {
-        final PrevWordsInfo prevWordsInfo = mLastComposedWord.mPrevWordsInfo;
         final CharSequence originallyTypedWord = mLastComposedWord.mTypedWord;
         final CharSequence committedWord = mLastComposedWord.mCommittedWord;
         final String committedWordString = committedWord.toString();
@@ -1385,8 +1387,8 @@ public final class InputLogic {
             }
         }
         mConnection.deleteSurroundingText(deleteLength, 0);
-        if (!TextUtils.isEmpty(prevWordsInfo.mPrevWord) && !TextUtils.isEmpty(committedWord)) {
+        if (!TextUtils.isEmpty(committedWord)) {
-            mDictionaryFacilitator.cancelAddingUserHistory(prevWordsInfo, committedWordString);
+            mDictionaryFacilitator.removeWordFromPersonalizedDicts(committedWordString);
         }
         final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString;
         final SpannableString textToCommit = new SpannableString(stringToCommit);

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_node_reader.h

@@ -58,7 +58,7 @@ class Ver4PatriciaTrieNodeReader : public PtNodeReader {
 
     ~Ver4PatriciaTrieNodeReader() {}
 
-    virtual const PtNodeParams fetchNodeInfoInBufferFromPtNodePos(const int ptNodePos) const {
+    virtual const PtNodeParams fetchPtNodeParamsInBufferFromPtNodePos(const int ptNodePos) const {
         return fetchPtNodeInfoFromBufferAndProcessMovedPtNode(ptNodePos,
                 NOT_A_DICT_POS /* siblingNodePos */);
     }

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp

@@ -135,7 +135,7 @@ int Ver4PatriciaTriePolicy::getUnigramProbabilityOfPtNode(const int ptNodePos) c
     if (ptNodePos == NOT_A_DICT_POS) {
         return NOT_A_PROBABILITY;
     }
-    const PtNodeParams ptNodeParams(mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos));
+    const PtNodeParams ptNodeParams(mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos));
     if (ptNodeParams.isDeleted() || ptNodeParams.isBlacklisted() || ptNodeParams.isNotAWord()) {
         return NOT_A_PROBABILITY;
     }
@@ -146,7 +146,7 @@ int Ver4PatriciaTriePolicy::getShortcutPositionOfPtNode(const int ptNodePos) con
     if (ptNodePos == NOT_A_DICT_POS) {
         return NOT_A_DICT_POS;
     }
-    const PtNodeParams ptNodeParams(mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos));
+    const PtNodeParams ptNodeParams(mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos));
     if (ptNodeParams.isDeleted()) {
         return NOT_A_DICT_POS;
     }
@@ -158,7 +158,7 @@ int Ver4PatriciaTriePolicy::getBigramsPositionOfPtNode(const int ptNodePos) cons
     if (ptNodePos == NOT_A_DICT_POS) {
         return NOT_A_DICT_POS;
     }
-    const PtNodeParams ptNodeParams(mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos));
+    const PtNodeParams ptNodeParams(mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos));
     if (ptNodeParams.isDeleted()) {
         return NOT_A_DICT_POS;
     }
@@ -410,7 +410,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
         AKLOGE("getWordProperty is called for invalid word.");
         return WordProperty();
     }
-    const PtNodeParams ptNodeParams = mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos);
+    const PtNodeParams ptNodeParams = mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
     std::vector<int> codePointVector(ptNodeParams.getCodePoints(),
             ptNodeParams.getCodePoints() + ptNodeParams.getCodePointCount());
     const ProbabilityEntry probabilityEntry =

native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_writing_helper.cpp

@@ -224,7 +224,7 @@ bool Ver4PatriciaTrieWritingHelper::truncateUnigrams(
         const int ptNodePos = priorityQueue.top().getDictPos();
         priorityQueue.pop();
         const PtNodeParams ptNodeParams =
-                ptNodeReader->fetchNodeInfoInBufferFromPtNodePos(ptNodePos);
+                ptNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
         if (ptNodeParams.representsNonWordInfo()) {
             continue;
         }

native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h

@@ -126,7 +126,7 @@ class DynamicPtReadingHelper {
         if (isEnd()) {
             return PtNodeParams();
         }
-        return mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(mReadingState.mPos);
+        return mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(mReadingState.mPos);
     }
 
     AK_FORCE_INLINE bool isValidTerminalNode(const PtNodeParams &ptNodeParams) const {

native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp

@@ -87,9 +87,9 @@ bool DynamicPtUpdatingHelper::addUnigramWord(
 bool DynamicPtUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos,
         const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) {
     const PtNodeParams sourcePtNodeParams(
-            mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos));
+            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(word0Pos));
     const PtNodeParams targetPtNodeParams(
-            mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word1Pos));
+            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(word1Pos));
     return mPtNodeWriter->addNewBigramEntry(&sourcePtNodeParams, &targetPtNodeParams,
             bigramProperty, outAddedNewBigram);
 }
@@ -97,16 +97,16 @@ bool DynamicPtUpdatingHelper::addBigramWords(const int word0Pos, const int word1
 // Remove a bigram relation from word0Pos to word1Pos.
 bool DynamicPtUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) {
     const PtNodeParams sourcePtNodeParams(
-            mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos));
+            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(word0Pos));
     const PtNodeParams targetPtNodeParams(
-            mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word1Pos));
+            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(word1Pos));
     return mPtNodeWriter->removeBigramEntry(&sourcePtNodeParams, &targetPtNodeParams);
 }
 
 bool DynamicPtUpdatingHelper::addShortcutTarget(const int wordPos,
         const int *const targetCodePoints, const int targetCodePointCount,
         const int shortcutProbability) {
-    const PtNodeParams ptNodeParams(mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(wordPos));
+    const PtNodeParams ptNodeParams(mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(wordPos));
     return mPtNodeWriter->addShortcutTarget(&ptNodeParams, targetCodePoints, targetCodePointCount,
             shortcutProbability);
 }
@@ -125,7 +125,7 @@ bool DynamicPtUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int paren
 
 bool DynamicPtUpdatingHelper::setPtNodeProbability(const PtNodeParams *const originalPtNodeParams,
         const UnigramProperty *const unigramProperty, bool *const outAddedNewUnigram) {
-    if (originalPtNodeParams->isTerminal()) {
+    if (originalPtNodeParams->isTerminal() && !originalPtNodeParams->isDeleted()) {
         // Overwrites the probability.
         *outAddedNewUnigram = false;
         return mPtNodeWriter->updatePtNodeUnigramProperty(originalPtNodeParams, unigramProperty);
@@ -260,7 +260,7 @@ bool DynamicPtUpdatingHelper::reallocatePtNodeAndAddNewPtNodes(
     }
     // Load node info. Information of the 1st part will be fetched.
     const PtNodeParams ptNodeParams(
-            mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(firstPartOfReallocatedPtNodePos));
+            mPtNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(firstPartOfReallocatedPtNodePos));
     // Update children position.
     return mPtNodeWriter->updateChildrenPosition(&ptNodeParams, actualChildrenPos);
 }

native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_reader.h

@@ -27,7 +27,8 @@ namespace latinime {
 class PtNodeReader {
  public:
     virtual ~PtNodeReader() {}
-    virtual const PtNodeParams fetchNodeInfoInBufferFromPtNodePos(const int ptNodePos) const = 0;
+    virtual const PtNodeParams fetchPtNodeParamsInBufferFromPtNodePos(
+            const int ptNodePos) const = 0;
 
  protected:
     PtNodeReader() {};

native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.cpp

@@ -282,7 +282,8 @@ int PatriciaTriePolicy::getUnigramProbabilityOfPtNode(const int ptNodePos) const
     if (ptNodePos == NOT_A_DICT_POS) {
         return NOT_A_PROBABILITY;
     }
-    const PtNodeParams ptNodeParams = mPtNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos);
+    const PtNodeParams ptNodeParams =
+            mPtNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
     if (ptNodeParams.isNotAWord() || ptNodeParams.isBlacklisted()) {
         // If this is not a word, or if it's a blacklisted entry, it should behave as
         // having no probability outside of the suggestion process (where it should be used
@@ -296,14 +297,14 @@ int PatriciaTriePolicy::getShortcutPositionOfPtNode(const int ptNodePos) const {
     if (ptNodePos == NOT_A_DICT_POS) {
         return NOT_A_DICT_POS;
     }
-    return mPtNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos).getShortcutPos();
+    return mPtNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos).getShortcutPos();
 }
 
 int PatriciaTriePolicy::getBigramsPositionOfPtNode(const int ptNodePos) const {
     if (ptNodePos == NOT_A_DICT_POS) {
         return NOT_A_DICT_POS;
     }
-    return mPtNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos).getBigramsPos();
+    return mPtNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos).getBigramsPos();
 }
 
 int PatriciaTriePolicy::createAndGetLeavingChildNode(const DicNode *const dicNode,
@@ -339,7 +340,8 @@ const WordProperty PatriciaTriePolicy::getWordProperty(const int *const codePoin
         AKLOGE("getWordProperty was called for invalid word.");
         return WordProperty();
     }
-    const PtNodeParams ptNodeParams = mPtNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos);
+    const PtNodeParams ptNodeParams =
+            mPtNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
     std::vector<int> codePointVector(ptNodeParams.getCodePoints(),
             ptNodeParams.getCodePoints() + ptNodeParams.getCodePointCount());
     // Fetch bigram information.

native/jni/src/suggest/policyimpl/dictionary/structure/v2/ver2_patricia_trie_node_reader.cpp

@@ -20,7 +20,7 @@
 
 namespace latinime {
 
-const PtNodeParams Ver2ParticiaTrieNodeReader::fetchNodeInfoInBufferFromPtNodePos(
+const PtNodeParams Ver2ParticiaTrieNodeReader::fetchPtNodeParamsInBufferFromPtNodePos(
         const int ptNodePos) const {
     if (ptNodePos < 0 || ptNodePos >= mDictSize) {
         // Reading invalid position because of bug or broken dictionary.

native/jni/src/suggest/policyimpl/dictionary/structure/v2/ver2_patricia_trie_node_reader.h

@@ -36,7 +36,7 @@ class Ver2ParticiaTrieNodeReader : public PtNodeReader {
             : mDictBuffer(dictBuffer), mDictSize(dictSize), mBigramPolicy(bigramPolicy),
               mShortuctPolicy(shortcutPolicy) {}
 
-    virtual const PtNodeParams fetchNodeInfoInBufferFromPtNodePos(const int ptNodePos) const;
+    virtual const PtNodeParams fetchPtNodeParamsInBufferFromPtNodePos(const int ptNodePos) const;
 
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(Ver2ParticiaTrieNodeReader);

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h

@@ -41,7 +41,7 @@ class Ver4PatriciaTrieNodeReader : public PtNodeReader {
 
     ~Ver4PatriciaTrieNodeReader() {}
 
-    virtual const PtNodeParams fetchNodeInfoInBufferFromPtNodePos(const int ptNodePos) const {
+    virtual const PtNodeParams fetchPtNodeParamsInBufferFromPtNodePos(const int ptNodePos) const {
         return fetchPtNodeInfoFromBufferAndProcessMovedPtNode(ptNodePos,
                 NOT_A_DICT_POS /* siblingNodePos */);
     }

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp

@@ -125,7 +125,7 @@ int Ver4PatriciaTriePolicy::getUnigramProbabilityOfPtNode(const int ptNodePos) c
     if (ptNodePos == NOT_A_DICT_POS) {
         return NOT_A_PROBABILITY;
     }
-    const PtNodeParams ptNodeParams(mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos));
+    const PtNodeParams ptNodeParams(mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos));
     if (ptNodeParams.isDeleted() || ptNodeParams.isBlacklisted() || ptNodeParams.isNotAWord()) {
         return NOT_A_PROBABILITY;
     }
@@ -136,7 +136,7 @@ int Ver4PatriciaTriePolicy::getShortcutPositionOfPtNode(const int ptNodePos) con
     if (ptNodePos == NOT_A_DICT_POS) {
         return NOT_A_DICT_POS;
     }
-    const PtNodeParams ptNodeParams(mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos));
+    const PtNodeParams ptNodeParams(mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos));
     if (ptNodeParams.isDeleted()) {
         return NOT_A_DICT_POS;
     }
@@ -148,7 +148,7 @@ int Ver4PatriciaTriePolicy::getBigramsPositionOfPtNode(const int ptNodePos) cons
     if (ptNodePos == NOT_A_DICT_POS) {
         return NOT_A_DICT_POS;
     }
-    const PtNodeParams ptNodeParams(mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos));
+    const PtNodeParams ptNodeParams(mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos));
     if (ptNodeParams.isDeleted()) {
         return NOT_A_DICT_POS;
     }
@@ -222,8 +222,24 @@ bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int le
 }
 
 bool Ver4PatriciaTriePolicy::removeUnigramEntry(const int *const word, const int length) {
-    // TODO: Implement.
+    if (!mBuffers->isUpdatable()) {
+        AKLOGI("Warning: removeUnigramEntry() is called for non-updatable dictionary.");
         return false;
+    }
+    const int ptNodePos = getTerminalPtNodePositionOfWord(word, length,
+            false /* forceLowerCaseSearch */);
+    if (ptNodePos == NOT_A_DICT_POS) {
+        return false;
+    }
+    const PtNodeParams ptNodeParams = mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
+    if (!mNodeWriter.markPtNodeAsDeleted(&ptNodeParams)) {
+        AKLOGE("Cannot remove unigram. ptNodePos: %d", ptNodePos);
+        return false;
+    }
+    if (!ptNodeParams.representsNonWordInfo()) {
+        mUnigramCount--;
+    }
+    return true;
 }
 
 bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
@@ -405,7 +421,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
         AKLOGE("getWordProperty is called for invalid word.");
         return WordProperty();
     }
-    const PtNodeParams ptNodeParams = mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos);
+    const PtNodeParams ptNodeParams = mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
     std::vector<int> codePointVector(ptNodeParams.getCodePoints(),
             ptNodeParams.getCodePoints() + ptNodeParams.getCodePointCount());
     const ProbabilityEntry probabilityEntry =

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp

@@ -215,7 +215,7 @@ bool Ver4PatriciaTrieWritingHelper::truncateUnigrams(
         const int ptNodePos = priorityQueue.top().getDictPos();
         priorityQueue.pop();
         const PtNodeParams ptNodeParams =
-                ptNodeReader->fetchNodeInfoInBufferFromPtNodePos(ptNodePos);
+                ptNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
         if (ptNodeParams.representsNonWordInfo()) {
             continue;
         }