am fe395232: Remove bigram dict content.

* commit 'fe395232d69df0887863c1cbabe63def2586d29e':
  Remove bigram dict content.

native/jni/NativeFileList.mk

@@ -61,7 +61,6 @@ LATIN_IME_CORE_SRC_FILES := \
         ver2_patricia_trie_node_reader.cpp \
         ver2_pt_node_array_reader.cpp) \
     $(addprefix suggest/policyimpl/dictionary/structure/v4/, \
-        bigram/ver4_bigram_list_policy.cpp \
         ver4_dict_buffers.cpp \
         ver4_dict_constants.cpp \
         ver4_patricia_trie_node_reader.cpp \
@@ -71,7 +70,6 @@ LATIN_IME_CORE_SRC_FILES := \
         ver4_patricia_trie_writing_helper.cpp \
         ver4_pt_node_array_reader.cpp) \
     $(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \
-        bigram_dict_content.cpp \
         language_model_dict_content.cpp \
         shortcut_dict_content.cpp \
         sparse_table_dict_content.cpp \

native/jni/src/suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.cpp

@@ -1,282 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h"
-
-#include "suggest/core/dictionary/property/bigram_property.h"
-#include "suggest/policyimpl/dictionary/header/header_policy.h"
-#include "suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
-#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
-#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h"
-
-namespace latinime {
-
-void Ver4BigramListPolicy::getNextBigram(int *const outBigramPos, int *const outProbability,
-        bool *const outHasNext, int *const bigramEntryPos) const {
-    const BigramEntry bigramEntry =
-            mBigramDictContent->getBigramEntryAndAdvancePosition(bigramEntryPos);
-    if (outBigramPos) {
-        // Lookup target PtNode position.
-        *outBigramPos = mTerminalPositionLookupTable->getTerminalPtNodePosition(
-                bigramEntry.getTargetTerminalId());
-    }
-    if (outProbability) {
-        if (bigramEntry.hasHistoricalInfo()) {
-            *outProbability =
-                    ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo(),
-                            mHeaderPolicy);
-        } else {
-            *outProbability = bigramEntry.getProbability();
-        }
-    }
-    if (outHasNext) {
-        *outHasNext = bigramEntry.hasNext();
-    }
-}
-
-bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTargetTerminalId,
-        const BigramProperty *const bigramProperty, bool *const outAddedNewEntry) {
-    // 1. The word has no bigrams yet.
-    // 2. The word has bigrams, and there is the target in the list.
-    // 3. The word has bigrams, and there is an invalid entry that can be reclaimed.
-    // 4. The word has bigrams. We have to append new bigram entry to the list.
-    // 5. Same as 4, but the list is the last entry of the content file.
-    if (outAddedNewEntry) {
-        *outAddedNewEntry = false;
-    }
-    const int bigramListPos = mBigramDictContent->getBigramListHeadPos(terminalId);
-    if (bigramListPos == NOT_A_DICT_POS) {
-        // Case 1. PtNode that doesn't have a bigram list.
-        // Create new bigram list.
-        if (!mBigramDictContent->createNewBigramList(terminalId)) {
-            return false;
-        }
-        const BigramEntry newBigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
-                newTargetTerminalId);
-        const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom(&newBigramEntry,
-                bigramProperty);
-        // Write an entry.
-        int writingPos =  mBigramDictContent->getBigramListHeadPos(terminalId);
-        if (!mBigramDictContent->writeBigramEntryAndAdvancePosition(&bigramEntryToWrite,
-                &writingPos)) {
-            AKLOGE("Cannot write bigram entry. pos: %d.", writingPos);
-            return false;
-        }
-        if (!mBigramDictContent->writeTerminator(writingPos)) {
-            AKLOGE("Cannot write bigram list terminator. pos: %d.", writingPos);
-            return false;
-        }
-        if (outAddedNewEntry) {
-            *outAddedNewEntry = true;
-        }
-        return true;
-    }
-
-    int tailEntryPos = NOT_A_DICT_POS;
-    const int entryPosToUpdate = getEntryPosToUpdate(newTargetTerminalId, bigramListPos,
-            &tailEntryPos);
-    if (entryPosToUpdate == NOT_A_DICT_POS) {
-        // Case 4, 5. Add new entry to the bigram list.
-        const int contentTailPos = mBigramDictContent->getContentTailPos();
-        // If the tail entry is at the tail of content buffer, the new entry can be written without
-        // link (Case 5).
-        const bool canAppendEntry =
-                contentTailPos == tailEntryPos + mBigramDictContent->getBigramEntrySize();
-        const int newEntryPos = canAppendEntry ? tailEntryPos : contentTailPos;
-        int writingPos = newEntryPos;
-        // Write new entry at the tail position of the bigram content.
-        const BigramEntry newBigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
-                newTargetTerminalId);
-        const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom(
-                &newBigramEntry, bigramProperty);
-        if (!mBigramDictContent->writeBigramEntryAndAdvancePosition(&bigramEntryToWrite,
-                &writingPos)) {
-            AKLOGE("Cannot write bigram entry. pos: %d.", writingPos);
-            return false;
-        }
-        if (!mBigramDictContent->writeTerminator(writingPos)) {
-            AKLOGE("Cannot write bigram list terminator. pos: %d.", writingPos);
-            return false;
-        }
-        if (!canAppendEntry) {
-            // Update link of the current tail entry.
-            if (!mBigramDictContent->writeLink(newEntryPos, tailEntryPos)) {
-                AKLOGE("Cannot update bigram entry link. pos: %d, linked entry pos: %d.",
-                        tailEntryPos, newEntryPos);
-                return false;
-            }
-        }
-        if (outAddedNewEntry) {
-            *outAddedNewEntry = true;
-        }
-        return true;
-    }
-
-    // Case 2. Overwrite the existing entry. Case 3. Reclaim and reuse the existing invalid entry.
-    const BigramEntry originalBigramEntry = mBigramDictContent->getBigramEntry(entryPosToUpdate);
-    if (!originalBigramEntry.isValid()) {
-        // Case 3. Reuse the existing invalid entry. outAddedNewEntry is false when an existing
-        // entry is updated.
-        if (outAddedNewEntry) {
-            *outAddedNewEntry = true;
-        }
-    }
-    const BigramEntry updatedBigramEntry =
-            originalBigramEntry.updateTargetTerminalIdAndGetEntry(newTargetTerminalId);
-    const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom(
-            &updatedBigramEntry, bigramProperty);
-    return mBigramDictContent->writeBigramEntry(&bigramEntryToWrite, entryPosToUpdate);
-}
-
-bool Ver4BigramListPolicy::removeEntry(const int terminalId, const int targetTerminalId) {
-    const int bigramListPos = mBigramDictContent->getBigramListHeadPos(terminalId);
-    if (bigramListPos == NOT_A_DICT_POS) {
-        // Bigram list doesn't exist.
-        return false;
-    }
-    const int entryPosToUpdate = getEntryPosToUpdate(targetTerminalId, bigramListPos,
-            nullptr /* outTailEntryPos */);
-    if (entryPosToUpdate == NOT_A_DICT_POS) {
-        // Bigram entry doesn't exist.
-        return false;
-    }
-    const BigramEntry bigramEntry = mBigramDictContent->getBigramEntry(entryPosToUpdate);
-    if (targetTerminalId != bigramEntry.getTargetTerminalId()) {
-        // Bigram entry doesn't exist.
-        return false;
-    }
-    // Remove bigram entry by marking it as invalid entry and overwriting the original entry.
-    const BigramEntry updatedBigramEntry = bigramEntry.getInvalidatedEntry();
-    return mBigramDictContent->writeBigramEntry(&updatedBigramEntry, entryPosToUpdate);
-}
-
-bool Ver4BigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries(const int terminalId,
-        int *const outBigramCount) {
-    const int bigramListPos = mBigramDictContent->getBigramListHeadPos(terminalId);
-    if (bigramListPos == NOT_A_DICT_POS) {
-        // Bigram list doesn't exist.
-        return true;
-    }
-    bool hasNext = true;
-    int readingPos = bigramListPos;
-    while (hasNext) {
-        const BigramEntry bigramEntry =
-                mBigramDictContent->getBigramEntryAndAdvancePosition(&readingPos);
-        const int entryPos = readingPos - mBigramDictContent->getBigramEntrySize();
-        hasNext = bigramEntry.hasNext();
-        if (!bigramEntry.isValid()) {
-            continue;
-        }
-        const int targetPtNodePos = mTerminalPositionLookupTable->getTerminalPtNodePosition(
-                bigramEntry.getTargetTerminalId());
-        if (targetPtNodePos == NOT_A_DICT_POS) {
-            // Invalidate bigram entry.
-            const BigramEntry updatedBigramEntry = bigramEntry.getInvalidatedEntry();
-            if (!mBigramDictContent->writeBigramEntry(&updatedBigramEntry, entryPos)) {
-                return false;
-            }
-        } else if (bigramEntry.hasHistoricalInfo()) {
-            const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave(
-                    bigramEntry.getHistoricalInfo(), mHeaderPolicy);
-            if (ForgettingCurveUtils::needsToKeep(&historicalInfo, mHeaderPolicy)) {
-                const BigramEntry updatedBigramEntry =
-                        bigramEntry.updateHistoricalInfoAndGetEntry(&historicalInfo);
-                if (!mBigramDictContent->writeBigramEntry(&updatedBigramEntry, entryPos)) {
-                    return false;
-                }
-                *outBigramCount += 1;
-            } else {
-                // Remove entry.
-                const BigramEntry updatedBigramEntry = bigramEntry.getInvalidatedEntry();
-                if (!mBigramDictContent->writeBigramEntry(&updatedBigramEntry, entryPos)) {
-                    return false;
-                }
-            }
-        } else {
-            *outBigramCount += 1;
-        }
-    }
-    return true;
-}
-
-int Ver4BigramListPolicy::getBigramEntryConut(const int terminalId) {
-    const int bigramListPos = mBigramDictContent->getBigramListHeadPos(terminalId);
-    if (bigramListPos == NOT_A_DICT_POS) {
-        // Bigram list doesn't exist.
-        return 0;
-    }
-    int bigramCount = 0;
-    bool hasNext = true;
-    int readingPos = bigramListPos;
-    while (hasNext) {
-        const BigramEntry bigramEntry =
-                mBigramDictContent->getBigramEntryAndAdvancePosition(&readingPos);
-        hasNext = bigramEntry.hasNext();
-        if (bigramEntry.isValid()) {
-            bigramCount++;
-        }
-    }
-    return bigramCount;
-}
-
-int Ver4BigramListPolicy::getEntryPosToUpdate(const int targetTerminalIdToFind,
-        const int bigramListPos, int *const outTailEntryPos) const {
-    if (outTailEntryPos) {
-        *outTailEntryPos = NOT_A_DICT_POS;
-    }
-    int invalidEntryPos = NOT_A_DICT_POS;
-    int readingPos = bigramListPos;
-    while (true) {
-        const BigramEntry bigramEntry =
-                mBigramDictContent->getBigramEntryAndAdvancePosition(&readingPos);
-        const int entryPos = readingPos - mBigramDictContent->getBigramEntrySize();
-        if (!bigramEntry.hasNext()) {
-            if (outTailEntryPos) {
-                *outTailEntryPos = entryPos;
-            }
-            break;
-        }
-        if (bigramEntry.getTargetTerminalId() == targetTerminalIdToFind) {
-            // Entry with same target is found.
-            return entryPos;
-        } else if (!bigramEntry.isValid()) {
-            // Invalid entry that can be reused is found.
-            invalidEntryPos = entryPos;
-        }
-    }
-    return invalidEntryPos;
-}
-
-const BigramEntry Ver4BigramListPolicy::createUpdatedBigramEntryFrom(
-        const BigramEntry *const originalBigramEntry,
-        const BigramProperty *const bigramProperty) const {
-    // TODO: Consolidate historical info and probability.
-    if (mHeaderPolicy->hasHistoricalInfoOfWords()) {
-        const HistoricalInfo historicalInfoForUpdate(bigramProperty->getTimestamp(),
-                bigramProperty->getLevel(), bigramProperty->getCount());
-        const HistoricalInfo updatedHistoricalInfo =
-                ForgettingCurveUtils::createUpdatedHistoricalInfo(
-                        originalBigramEntry->getHistoricalInfo(), bigramProperty->getProbability(),
-                        &historicalInfoForUpdate, mHeaderPolicy);
-        return originalBigramEntry->updateHistoricalInfoAndGetEntry(&updatedHistoricalInfo);
-    } else {
-        return originalBigramEntry->updateProbabilityAndGetEntry(bigramProperty->getProbability());
-    }
-}
-
-} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h

@@ -1,72 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_VER4_BIGRAM_LIST_POLICY_H
-#define LATINIME_VER4_BIGRAM_LIST_POLICY_H
-
-#include "defines.h"
-#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_entry.h"
-
-namespace latinime {
-
-class BigramDictContent;
-class BigramProperty;
-class HeaderPolicy;
-class TerminalPositionLookupTable;
-
-class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy {
- public:
-    Ver4BigramListPolicy(BigramDictContent *const bigramDictContent,
-            const TerminalPositionLookupTable *const terminalPositionLookupTable,
-            const HeaderPolicy *const headerPolicy)
-            : mBigramDictContent(bigramDictContent),
-              mTerminalPositionLookupTable(terminalPositionLookupTable),
-              mHeaderPolicy(headerPolicy) {}
-
-    void getNextBigram(int *const outBigramPos, int *const outProbability,
-            bool *const outHasNext, int *const bigramEntryPos) const;
-
-    bool skipAllBigrams(int *const pos) const {
-        // Do nothing because we don't need to skip bigram lists in ver4 dictionaries.
-        return true;
-    }
-
-    bool addNewEntry(const int terminalId, const int newTargetTerminalId,
-            const BigramProperty *const bigramProperty, bool *const outAddedNewEntry);
-
-    bool removeEntry(const int terminalId, const int targetTerminalId);
-
-    bool updateAllBigramEntriesAndDeleteUselessEntries(const int terminalId,
-            int *const outBigramCount);
-
-    int getBigramEntryConut(const int terminalId);
-
- private:
-    DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4BigramListPolicy);
-
-    int getEntryPosToUpdate(const int targetTerminalIdToFind, const int bigramListPos,
-            int *const outTailEntryPos) const;
-
-    const BigramEntry createUpdatedBigramEntryFrom(const BigramEntry *const originalBigramEntry,
-            const BigramProperty *const bigramProperty) const;
-
-    BigramDictContent *const mBigramDictContent;
-    const TerminalPositionLookupTable *const mTerminalPositionLookupTable;
-    const HeaderPolicy *const mHeaderPolicy;
-};
-} // namespace latinime
-#endif /* LATINIME_VER4_BIGRAM_LIST_POLICY_H */

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp

@@ -1,219 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h"
-
-#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
-
-namespace latinime {
-
-const int BigramDictContent::INVALID_LINKED_ENTRY_POS = Ver4DictConstants::NOT_A_TERMINAL_ID;
-
-const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
-        int *const bigramEntryPos) const {
-    const BufferWithExtendableBuffer *const bigramListBuffer = getContentBuffer();
-    const int bigramEntryTailPos = (*bigramEntryPos) + getBigramEntrySize();
-    if (*bigramEntryPos < 0 || bigramEntryTailPos > bigramListBuffer->getTailPosition()) {
-        AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bigramEntryTailPos: %d, "
-                "bufSize: %d", *bigramEntryPos, bigramEntryTailPos,
-                        bigramListBuffer->getTailPosition());
-        ASSERT(false);
-        return BigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
-                Ver4DictConstants::NOT_A_TERMINAL_ID);
-    }
-    const int bigramFlags = bigramListBuffer->readUintAndAdvancePosition(
-            Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, bigramEntryPos);
-    const bool isLink = (bigramFlags & Ver4DictConstants::BIGRAM_IS_LINK_MASK) != 0;
-    int probability = NOT_A_PROBABILITY;
-    int timestamp = NOT_A_TIMESTAMP;
-    int level = 0;
-    int count = 0;
-    if (mHasHistoricalInfo) {
-        timestamp = bigramListBuffer->readUintAndAdvancePosition(
-                Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos);
-        level = bigramListBuffer->readUintAndAdvancePosition(
-                Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, bigramEntryPos);
-        count = bigramListBuffer->readUintAndAdvancePosition(
-                Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos);
-    } else {
-        probability = bigramListBuffer->readUintAndAdvancePosition(
-                Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
-    }
-    const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition(
-            Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos);
-    const int targetTerminalId =
-            (encodedTargetTerminalId == Ver4DictConstants::INVALID_BIGRAM_TARGET_TERMINAL_ID) ?
-                    Ver4DictConstants::NOT_A_TERMINAL_ID : encodedTargetTerminalId;
-    if (isLink) {
-        const int linkedEntryPos = targetTerminalId;
-        if (linkedEntryPos == INVALID_LINKED_ENTRY_POS) {
-            // Bigram list terminator is found.
-            return BigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
-                    Ver4DictConstants::NOT_A_TERMINAL_ID);
-        }
-        *bigramEntryPos = linkedEntryPos;
-        return getBigramEntryAndAdvancePosition(bigramEntryPos);
-    }
-    // hasNext is always true because we should continue to read the next entry until the terminator
-    // is found.
-    if (mHasHistoricalInfo) {
-        const HistoricalInfo historicalInfo(timestamp, level, count);
-        return BigramEntry(true /* hasNext */, probability, &historicalInfo, targetTerminalId);
-    } else {
-        return BigramEntry(true /* hasNext */, probability, targetTerminalId);
-    }
-}
-
-bool BigramDictContent::writeBigramEntryAndAdvancePosition(
-        const BigramEntry *const bigramEntryToWrite, int *const entryWritingPos) {
-    return writeBigramEntryAttributesAndAdvancePosition(false /* isLink */,
-            bigramEntryToWrite->getProbability(), bigramEntryToWrite->getTargetTerminalId(),
-            bigramEntryToWrite->getHistoricalInfo()->getTimeStamp(),
-            bigramEntryToWrite->getHistoricalInfo()->getLevel(),
-            bigramEntryToWrite->getHistoricalInfo()->getCount(),
-            entryWritingPos);
-}
-
-bool BigramDictContent::writeBigramEntryAttributesAndAdvancePosition(
-        const bool isLink, const int probability, const int targetTerminalId,
-        const int timestamp, const int level, const int count, int *const entryWritingPos) {
-    BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer();
-    const int bigramFlags = isLink ? Ver4DictConstants::BIGRAM_IS_LINK_MASK : 0;
-    if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags,
-            Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) {
-        AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags);
-        return false;
-    }
-    if (mHasHistoricalInfo) {
-        if (!bigramListBuffer->writeUintAndAdvancePosition(timestamp,
-                Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) {
-            AKLOGE("Cannot write bigram timestamps. pos: %d, timestamp: %d", *entryWritingPos,
-                    timestamp);
-            return false;
-        }
-        if (!bigramListBuffer->writeUintAndAdvancePosition(level,
-                Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, entryWritingPos)) {
-            AKLOGE("Cannot write bigram level. pos: %d, level: %d", *entryWritingPos,
-                    level);
-            return false;
-        }
-        if (!bigramListBuffer->writeUintAndAdvancePosition(count,
-                Ver4DictConstants::WORD_COUNT_FIELD_SIZE, entryWritingPos)) {
-            AKLOGE("Cannot write bigram count. pos: %d, count: %d", *entryWritingPos,
-                    count);
-            return false;
-        }
-    } else {
-        if (!bigramListBuffer->writeUintAndAdvancePosition(probability,
-                Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
-            AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
-                    probability);
-            return false;
-        }
-    }
-    const int targetTerminalIdToWrite = (targetTerminalId == Ver4DictConstants::NOT_A_TERMINAL_ID) ?
-            Ver4DictConstants::INVALID_BIGRAM_TARGET_TERMINAL_ID : targetTerminalId;
-    if (!bigramListBuffer->writeUintAndAdvancePosition(targetTerminalIdToWrite,
-            Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, entryWritingPos)) {
-        AKLOGE("Cannot write bigram target terminal id. pos: %d, target terminal id: %d",
-                *entryWritingPos, targetTerminalId);
-        return false;
-    }
-    return true;
-}
-
-bool BigramDictContent::writeLink(const int linkedEntryPos, const int writingPos) {
-    const int targetTerminalId = linkedEntryPos;
-    int pos = writingPos;
-    return writeBigramEntryAttributesAndAdvancePosition(true /* isLink */,
-            NOT_A_PROBABILITY /* probability */, targetTerminalId, NOT_A_TIMESTAMP, 0 /* level */,
-            0 /* count */, &pos);
-}
-
-bool BigramDictContent::runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
-        const BigramDictContent *const originalBigramDictContent,
-        int *const outBigramEntryCount) {
-    for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
-            it != terminalIdMap->end(); ++it) {
-        const int originalBigramListPos =
-                originalBigramDictContent->getBigramListHeadPos(it->first);
-        if (originalBigramListPos == NOT_A_DICT_POS) {
-            // This terminal does not have a bigram list.
-            continue;
-        }
-        const int bigramListPos = getContentBuffer()->getTailPosition();
-        int bigramEntryCount = 0;
-        // Copy bigram list with GC from original content.
-        if (!runGCBigramList(originalBigramListPos, originalBigramDictContent, bigramListPos,
-                terminalIdMap, &bigramEntryCount)) {
-            AKLOGE("Cannot complete GC for the bigram list. original pos: %d, pos: %d",
-                    originalBigramListPos, bigramListPos);
-            return false;
-        }
-        if (bigramEntryCount == 0) {
-            // All bigram entries are useless. This terminal does not have a bigram list.
-            continue;
-        }
-        *outBigramEntryCount += bigramEntryCount;
-        // Set bigram list position to the lookup table.
-        if (!getUpdatableAddressLookupTable()->set(it->second, bigramListPos)) {
-            AKLOGE("Cannot set bigram list position. terminal id: %d, pos: %d",
-                    it->second, bigramListPos);
-            return false;
-        }
-    }
-    return true;
-}
-
-// Returns whether GC for the bigram list was succeeded or not.
-bool BigramDictContent::runGCBigramList(const int bigramListPos,
-        const BigramDictContent *const sourceBigramDictContent, const int toPos,
-        const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
-        int *const outEntryCount) {
-    bool hasNext = true;
-    int readingPos = bigramListPos;
-    int writingPos = toPos;
-    while (hasNext) {
-        const BigramEntry originalBigramEntry =
-                sourceBigramDictContent->getBigramEntryAndAdvancePosition(&readingPos);
-        hasNext = originalBigramEntry.hasNext();
-        if (!originalBigramEntry.isValid()) {
-            continue;
-        }
-        TerminalPositionLookupTable::TerminalIdMap::const_iterator it =
-                terminalIdMap->find(originalBigramEntry.getTargetTerminalId());
-        if (it == terminalIdMap->end()) {
-            // Target word has been removed.
-            continue;
-        }
-        const BigramEntry updatedBigramEntry =
-                originalBigramEntry.updateTargetTerminalIdAndGetEntry(it->second);
-        if (!writeBigramEntryAndAdvancePosition(&updatedBigramEntry, &writingPos)) {
-            AKLOGE("Cannot write bigram entry to run GC. pos: %d", writingPos);
-            return false;
-        }
-        *outEntryCount += 1;
-    }
-    if (*outEntryCount > 0) {
-        if (!writeTerminator(writingPos)) {
-            AKLOGE("Cannot write terminator to run GC. pos: %d", writingPos);
-            return false;
-        }
-    }
-    return true;
-}
-
-} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h

@@ -1,128 +0,0 @@
-/*
- * Copyright (C) 2013, The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_BIGRAM_DICT_CONTENT_H
-#define LATINIME_BIGRAM_DICT_CONTENT_H
-
-#include <cstdio>
-
-#include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_entry.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
-#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
-
-namespace latinime {
-
-class ReadWriteByteArrayView;
-
-class BigramDictContent : public SparseTableDictContent {
- public:
-    BigramDictContent(const ReadWriteByteArrayView *const buffers, const bool hasHistoricalInfo)
-            : SparseTableDictContent(buffers, Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE,
-                      Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE),
-              mHasHistoricalInfo(hasHistoricalInfo) {}
-
-    BigramDictContent(const bool hasHistoricalInfo)
-            : SparseTableDictContent(Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE,
-                      Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE),
-              mHasHistoricalInfo(hasHistoricalInfo) {}
-
-    int getContentTailPos() const {
-        return getContentBuffer()->getTailPosition();
-    }
-
-    const BigramEntry getBigramEntry(const int bigramEntryPos) const {
-        int readingPos = bigramEntryPos;
-        return getBigramEntryAndAdvancePosition(&readingPos);
-    }
-
-    const BigramEntry getBigramEntryAndAdvancePosition(int *const bigramEntryPos) const;
-
-    // Returns head position of bigram list for a PtNode specified by terminalId.
-    int getBigramListHeadPos(const int terminalId) const {
-        const SparseTable *const addressLookupTable = getAddressLookupTable();
-        if (!addressLookupTable->contains(terminalId)) {
-            return NOT_A_DICT_POS;
-        }
-        return addressLookupTable->get(terminalId);
-    }
-
-    bool writeBigramEntryAtTail(const BigramEntry *const bigramEntryToWrite) {
-        int writingPos = getContentBuffer()->getTailPosition();
-        return writeBigramEntryAndAdvancePosition(bigramEntryToWrite, &writingPos);
-    }
-
-    bool writeBigramEntry(const BigramEntry *const bigramEntryToWrite, const int entryWritingPos) {
-        int writingPos = entryWritingPos;
-        return writeBigramEntryAndAdvancePosition(bigramEntryToWrite, &writingPos);
-    }
-
-    bool writeBigramEntryAndAdvancePosition(const BigramEntry *const bigramEntryToWrite,
-            int *const entryWritingPos);
-
-    bool writeTerminator(const int writingPos) {
-        // Terminator is a link to the invalid position.
-        return writeLink(INVALID_LINKED_ENTRY_POS, writingPos);
-    }
-
-    bool writeLink(const int linkedPos, const int writingPos);
-
-    bool createNewBigramList(const int terminalId) {
-        const int bigramListPos = getContentBuffer()->getTailPosition();
-        return getUpdatableAddressLookupTable()->set(terminalId, bigramListPos);
-    }
-
-    bool flushToFile(FILE *const file) const {
-        return flush(file);
-    }
-
-    bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
-            const BigramDictContent *const originalBigramDictContent,
-            int *const outBigramEntryCount);
-
-    int getBigramEntrySize() const {
-        if (mHasHistoricalInfo) {
-            return Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE
-                    + Ver4DictConstants::TIME_STAMP_FIELD_SIZE
-                    + Ver4DictConstants::WORD_LEVEL_FIELD_SIZE
-                    + Ver4DictConstants::WORD_COUNT_FIELD_SIZE
-                    + Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE;
-        } else {
-            return Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE
-                    + Ver4DictConstants::PROBABILITY_SIZE
-                    + Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE;
-        }
-    }
-
- private:
-    DISALLOW_COPY_AND_ASSIGN(BigramDictContent);
-
-    static const int INVALID_LINKED_ENTRY_POS;
-
-    bool writeBigramEntryAttributesAndAdvancePosition(
-            const bool isLink, const int probability, const int targetTerminalId,
-            const int timestamp, const int level, const int count, int *const entryWritingPos);
-
-    bool runGCBigramList(const int bigramListPos,
-            const BigramDictContent *const sourceBigramDictContent, const int toPos,
-            const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
-            int *const outEntryCount);
-
-    bool mHasHistoricalInfo;
-};
-} // namespace latinime
-#endif /* LATINIME_BIGRAM_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_entry.h

@@ -1,99 +0,0 @@
-/*
- * Copyright (C) 2013, The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef LATINIME_BIGRAM_ENTRY_H
-#define LATINIME_BIGRAM_ENTRY_H
-
-#include "defines.h"
-#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
-#include "suggest/policyimpl/dictionary/utils/historical_info.h"
-
-namespace latinime {
-
-class BigramEntry {
- public:
-    BigramEntry(const BigramEntry& bigramEntry)
-            : mHasNext(bigramEntry.mHasNext), mProbability(bigramEntry.mProbability),
-              mHistoricalInfo(), mTargetTerminalId(bigramEntry.mTargetTerminalId) {}
-
-    // Entry with historical information.
-    BigramEntry(const bool hasNext, const int probability, const int targetTerminalId)
-            : mHasNext(hasNext), mProbability(probability), mHistoricalInfo(),
-              mTargetTerminalId(targetTerminalId) {}
-
-    // Entry with historical information.
-    BigramEntry(const bool hasNext, const int probability,
-            const HistoricalInfo *const historicalInfo, const int targetTerminalId)
-            : mHasNext(hasNext), mProbability(probability), mHistoricalInfo(*historicalInfo),
-              mTargetTerminalId(targetTerminalId) {}
-
-    const BigramEntry getInvalidatedEntry() const {
-        return updateTargetTerminalIdAndGetEntry(Ver4DictConstants::NOT_A_TERMINAL_ID);
-    }
-
-    const BigramEntry updateHasNextAndGetEntry(const bool hasNext) const {
-        return BigramEntry(hasNext, mProbability, &mHistoricalInfo, mTargetTerminalId);
-    }
-
-    const BigramEntry updateTargetTerminalIdAndGetEntry(const int newTargetTerminalId) const {
-        return BigramEntry(mHasNext, mProbability, &mHistoricalInfo, newTargetTerminalId);
-    }
-
-    const BigramEntry updateProbabilityAndGetEntry(const int probability) const {
-        return BigramEntry(mHasNext, probability, &mHistoricalInfo, mTargetTerminalId);
-    }
-
-    const BigramEntry updateHistoricalInfoAndGetEntry(
-            const HistoricalInfo *const historicalInfo) const {
-        return BigramEntry(mHasNext, mProbability, historicalInfo, mTargetTerminalId);
-    }
-
-    bool isValid() const {
-        return mTargetTerminalId != Ver4DictConstants::NOT_A_TERMINAL_ID;
-    }
-
-    bool hasNext() const {
-        return mHasNext;
-    }
-
-    int getProbability() const {
-        return mProbability;
-    }
-
-    bool hasHistoricalInfo() const {
-        return mHistoricalInfo.isValid();
-    }
-
-    const HistoricalInfo *getHistoricalInfo() const {
-        return &mHistoricalInfo;
-    }
-
-    int getTargetTerminalId() const {
-        return mTargetTerminalId;
-    }
-
- private:
-    // Copy constructor is public to use this class as a type of return value.
-    DISALLOW_DEFAULT_CONSTRUCTOR(BigramEntry);
-    DISALLOW_ASSIGNMENT_OPERATOR(BigramEntry);
-
-    const bool mHasNext;
-    const int mProbability;
-    const HistoricalInfo mHistoricalInfo;
-    const int mTargetTerminalId;
-};
-} // namespace latinime
-#endif /* LATINIME_BIGRAM_ENTRY_H */

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp

@@ -159,11 +159,6 @@ bool Ver4DictBuffers::flushDictBuffers(FILE *const file) const {
         AKLOGE("Language model dict content cannot be written.");
         return false;
     }
-    // Write bigram dict content.
-    if (!mBigramDictContent.flushToFile(file)) {
-        AKLOGE("Bigram dict content cannot be written.");
-        return false;
-    }
     // Write shortcut dict content.
     if (!mShortcutDictContent.flushToFile(file)) {
         AKLOGE("Shortcut dict content cannot be written.");
@@ -186,8 +181,6 @@ Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer,
                   contentBuffers[Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX]),
           mLanguageModelDictContent(contentBuffers[Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX],
                   mHeaderPolicy.hasHistoricalInfoOfWords()),
-          mBigramDictContent(&contentBuffers[Ver4DictConstants::BIGRAM_BUFFERS_INDEX],
-                  mHeaderPolicy.hasHistoricalInfoOfWords()),
           mShortcutDictContent(&contentBuffers[Ver4DictConstants::SHORTCUT_BUFFERS_INDEX]),
           mIsUpdatable(mDictBuffer->isUpdatable()) {}
 
@@ -196,7 +189,6 @@ Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const i
           mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE),
           mExpandableTrieBuffer(maxTrieSize), mTerminalPositionLookupTable(),
           mLanguageModelDictContent(headerPolicy->hasHistoricalInfoOfWords()),
-          mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(),
+          mShortcutDictContent(),  mIsUpdatable(true) {}
-          mIsUpdatable(true) {}
 
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h

@@ -22,7 +22,6 @@
 
 #include "defines.h"
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
-#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h"
 #include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h"
 #include "suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h"
 #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h"
@@ -53,7 +52,6 @@ class Ver4DictBuffers {
         return mExpandableTrieBuffer.isNearSizeLimit()
                 || mTerminalPositionLookupTable.isNearSizeLimit()
                 || mLanguageModelDictContent.isNearSizeLimit()
-                || mBigramDictContent.isNearSizeLimit()
                 || mShortcutDictContent.isNearSizeLimit();
     }
 
@@ -89,14 +87,6 @@ class Ver4DictBuffers {
         return &mLanguageModelDictContent;
     }
 
-    AK_FORCE_INLINE BigramDictContent *getMutableBigramDictContent() {
-        return &mBigramDictContent;
-    }
-
-    AK_FORCE_INLINE const BigramDictContent *getBigramDictContent() const {
-        return &mBigramDictContent;
-    }
-
     AK_FORCE_INLINE ShortcutDictContent *getMutableShortcutDictContent() {
         return &mShortcutDictContent;
     }
@@ -135,7 +125,6 @@ class Ver4DictBuffers {
     BufferWithExtendableBuffer mExpandableTrieBuffer;
     TerminalPositionLookupTable mTerminalPositionLookupTable;
     LanguageModelDictContent mLanguageModelDictContent;
-    BigramDictContent mBigramDictContent;
     ShortcutDictContent mShortcutDictContent;
     const int mIsUpdatable;
 };

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp

@@ -29,20 +29,18 @@ const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;
 
 // NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT for Trie and TerminalAddressLookupTable.
 // NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT for language model.
-// NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for bigram and shortcut.
+// NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for shortcut.
 const size_t Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE =
         NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 2
                 + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT
-                + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT * 2;
+                + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT;
 const int Ver4DictConstants::TRIE_BUFFER_INDEX = 0;
 const int Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX =
         TRIE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
 const int Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX =
         TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
-const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX =
-        LANGUAGE_MODEL_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT;
 const int Ver4DictConstants::SHORTCUT_BUFFERS_INDEX =
-        BIGRAM_BUFFERS_INDEX + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT;
+        LANGUAGE_MODEL_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT;
 
 const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1;
 const int Ver4DictConstants::PROBABILITY_SIZE = 1;
@@ -56,21 +54,9 @@ const int Ver4DictConstants::WORD_COUNT_FIELD_SIZE = 1;
 
 const uint8_t Ver4DictConstants::FLAG_REPRESENTS_BEGINNING_OF_SENTENCE = 0x1;
 
-const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 16;
-const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE = 4;
 const int Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE = 64;
 const int Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE = 4;
 
-const int Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE = 3;
-// Unsigned int max value of BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE-byte is used for representing
-// invalid terminal ID in bigram lists.
-const int Ver4DictConstants::INVALID_BIGRAM_TARGET_TERMINAL_ID =
-        (1 << (BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE * 8)) - 1;
-const int Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE = 1;
-const int Ver4DictConstants::BIGRAM_PROBABILITY_MASK = 0x0F;
-const int Ver4DictConstants::BIGRAM_IS_LINK_MASK = 0x80;
-const int Ver4DictConstants::BIGRAM_LARGE_PROBABILITY_FIELD_SIZE = 1;
-
 const int Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE = 1;
 const int Ver4DictConstants::SHORTCUT_PROBABILITY_MASK = 0x0F;
 const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80;

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h

@@ -52,19 +52,9 @@ class Ver4DictConstants {
     // Flags in probability entry.
     static const uint8_t FLAG_REPRESENTS_BEGINNING_OF_SENTENCE;
 
-    static const int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE;
-    static const int BIGRAM_ADDRESS_TABLE_DATA_SIZE;
     static const int SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE;
     static const int SHORTCUT_ADDRESS_TABLE_DATA_SIZE;
 
-    static const int BIGRAM_FLAGS_FIELD_SIZE;
-    static const int BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE;
-    static const int INVALID_BIGRAM_TARGET_TERMINAL_ID;
-    static const int BIGRAM_IS_LINK_MASK;
-    static const int BIGRAM_PROBABILITY_MASK;
-    // Used when bigram list has time stamp.
-    static const int BIGRAM_LARGE_PROBABILITY_FIELD_SIZE;
-
     static const int SHORTCUT_FLAGS_FIELD_SIZE;
     static const int SHORTCUT_PROBABILITY_MASK;
     static const int SHORTCUT_HAS_NEXT_MASK;

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp

@@ -21,7 +21,6 @@
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h"
-#include "suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h"
 #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h"
 #include "suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h"
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
@@ -230,12 +229,6 @@ bool Ver4PatriciaTrieNodeWriter::addNgramEntry(const WordIdArrayView prevWordIds
     if (!probabilityEntry.isValid() && outAddedNewBigram) {
         *outAddedNewBigram = true;
     }
-    // TODO: Remove.
-    if (!mBigramPolicy->addNewEntry(prevWordIds[0], wordId, bigramProperty, outAddedNewBigram)) {
-        AKLOGE("Cannot add new bigram entry. prevWordId: %d, wordId: %d",
-                prevWordIds[0], wordId);
-        return false;
-    }
     return true;
 }
 
@@ -244,19 +237,15 @@ bool Ver4PatriciaTrieNodeWriter::removeNgramEntry(const WordIdArrayView prevWord
     // TODO: Support n-gram.
     LanguageModelDictContent *const languageModelDictContent =
             mBuffers->getMutableLanguageModelDictContent();
-    if (!languageModelDictContent->removeNgramProbabilityEntry(prevWordIds.limit(1 /* maxSize */),
+    return languageModelDictContent->removeNgramProbabilityEntry(prevWordIds.limit(1 /* maxSize */),
-            wordId)) {
+            wordId);
-        // TODO: Uncomment.
-        // return false;
-    }
-    // TODO: Remove.
-    return mBigramPolicy->removeEntry(prevWordIds[0], wordId);
 }
 
+// TODO: Remove when we stop supporting v402 format.
 bool Ver4PatriciaTrieNodeWriter::updateAllBigramEntriesAndDeleteUselessEntries(
             const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) {
-    return mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(
+    // Do nothing.
-            sourcePtNodeParams->getTerminalId(), outBigramEntryCount);
+    return true;
 }
 
 bool Ver4PatriciaTrieNodeWriter::updateAllPositionFields(
@@ -291,12 +280,6 @@ bool Ver4PatriciaTrieNodeWriter::updateAllPositionFields(
     if (!updateChildrenPosition(toBeUpdatedPtNodeParams, childrenPos)) {
         return false;
     }
-
-    // Counts bigram entries.
-    if (outBigramEntryCount) {
-        *outBigramEntryCount = mBigramPolicy->getBigramEntryConut(
-                toBeUpdatedPtNodeParams->getTerminalId());
-    }
     return true;
 }
 

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h

@@ -27,7 +27,6 @@ namespace latinime {
 
 class BufferWithExtendableBuffer;
 class HeaderPolicy;
-class Ver4BigramListPolicy;
 class Ver4DictBuffers;
 class Ver4PatriciaTrieNodeReader;
 class Ver4PtNodeArrayReader;
@@ -42,10 +41,9 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
             Ver4DictBuffers *const buffers, const HeaderPolicy *const headerPolicy,
             const PtNodeReader *const ptNodeReader,
             const PtNodeArrayReader *const ptNodeArrayReader,
-            Ver4BigramListPolicy *const bigramPolicy, Ver4ShortcutListPolicy *const shortcutPolicy)
+            Ver4ShortcutListPolicy *const shortcutPolicy)
             : mTrieBuffer(trieBuffer), mBuffers(buffers), mHeaderPolicy(headerPolicy),
-              mReadingHelper(ptNodeReader, ptNodeArrayReader), mBigramPolicy(bigramPolicy),
+              mReadingHelper(ptNodeReader, ptNodeArrayReader), mShortcutPolicy(shortcutPolicy) {}
-              mShortcutPolicy(shortcutPolicy) {}
 
     virtual ~Ver4PatriciaTrieNodeWriter() {}
 
@@ -114,7 +112,6 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter {
     Ver4DictBuffers *const mBuffers;
     const HeaderPolicy *const mHeaderPolicy;
     DynamicPtReadingHelper mReadingHelper;
-    Ver4BigramListPolicy *const mBigramPolicy;
     Ver4ShortcutListPolicy *const mShortcutPolicy;
 };
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h

@@ -23,7 +23,6 @@
 #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
 #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h"
-#include "suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h"
 #include "suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h"
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h"
@@ -42,14 +41,12 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
     Ver4PatriciaTriePolicy(Ver4DictBuffers::Ver4DictBuffersPtr buffers)
             : mBuffers(std::move(buffers)), mHeaderPolicy(mBuffers->getHeaderPolicy()),
               mDictBuffer(mBuffers->getWritableTrieBuffer()),
-              mBigramPolicy(mBuffers->getMutableBigramDictContent(),
-                      mBuffers->getTerminalPositionLookupTable(), mHeaderPolicy),
               mShortcutPolicy(mBuffers->getMutableShortcutDictContent(),
                       mBuffers->getTerminalPositionLookupTable()),
               mNodeReader(mDictBuffer, mBuffers->getLanguageModelDictContent(), mHeaderPolicy),
               mPtNodeArrayReader(mDictBuffer),
               mNodeWriter(mDictBuffer, mBuffers.get(), mHeaderPolicy, &mNodeReader,
-                      &mPtNodeArrayReader, &mBigramPolicy, &mShortcutPolicy),
+                      &mPtNodeArrayReader, &mShortcutPolicy),
               mUpdatingHelper(mDictBuffer, &mNodeReader, &mNodeWriter),
               mWritingHelper(mBuffers.get()),
               mUnigramCount(mHeaderPolicy->getUnigramCount()),
@@ -132,7 +129,6 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
     const Ver4DictBuffers::Ver4DictBuffersPtr mBuffers;
     const HeaderPolicy *const mHeaderPolicy;
     BufferWithExtendableBuffer *const mDictBuffer;
-    Ver4BigramListPolicy mBigramPolicy;
     Ver4ShortcutListPolicy mShortcutPolicy;
     Ver4PatriciaTrieNodeReader mNodeReader;
     Ver4PtNodeArrayReader mPtNodeArrayReader;

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp

@@ -20,7 +20,6 @@
 #include <queue>
 
 #include "suggest/policyimpl/dictionary/header/header_policy.h"
-#include "suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h"
 #include "suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h"
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
 #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
@@ -77,13 +76,10 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
     Ver4PatriciaTrieNodeReader ptNodeReader(mBuffers->getTrieBuffer(),
             mBuffers->getLanguageModelDictContent(), headerPolicy);
     Ver4PtNodeArrayReader ptNodeArrayReader(mBuffers->getTrieBuffer());
-    Ver4BigramListPolicy bigramPolicy(mBuffers->getMutableBigramDictContent(),
-            mBuffers->getTerminalPositionLookupTable(), headerPolicy);
     Ver4ShortcutListPolicy shortcutPolicy(mBuffers->getMutableShortcutDictContent(),
             mBuffers->getTerminalPositionLookupTable());
     Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(),
-            mBuffers, headerPolicy, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy,
+            mBuffers, headerPolicy, &ptNodeReader, &ptNodeArrayReader, &shortcutPolicy);
-            &shortcutPolicy);
 
     int entryCountTable[MAX_PREV_WORD_COUNT_FOR_N_GRAM + 1];
     if (!mBuffers->getMutableLanguageModelDictContent()->updateAllProbabilityEntries(headerPolicy,
@@ -118,16 +114,6 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
             &traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted)) {
         return false;
     }
-    const int unigramCount = traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted
-            .getValidUnigramCount();
-    const int maxUnigramCount = headerPolicy->getMaxUnigramCount();
-    if (headerPolicy->isDecayingDict() && unigramCount > maxUnigramCount) {
-        if (!truncateUnigrams(&ptNodeReader, &ptNodeWriter, maxUnigramCount)) {
-            AKLOGE("Cannot remove unigrams. current: %d, max: %d", unigramCount,
-                    maxUnigramCount);
-            return false;
-        }
-    }
 
     readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
     DynamicPtGcEventListeners::TraversePolicyToUpdateBigramProbability
@@ -136,21 +122,12 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
             &traversePolicyToUpdateBigramProbability)) {
         return false;
     }
-    const int bigramCount = traversePolicyToUpdateBigramProbability.getValidBigramEntryCount();
-    const int maxBigramCount = headerPolicy->getMaxBigramCount();
-    if (headerPolicy->isDecayingDict() && bigramCount > maxBigramCount) {
-        if (!truncateBigrams(maxBigramCount)) {
-            AKLOGE("Cannot remove bigrams. current: %d, max: %d", bigramCount, maxBigramCount);
-            return false;
-        }
-    }
 
     // Mapping from positions in mBuffer to positions in bufferToWrite.
     PtNodeWriter::DictPositionRelocationMap dictPositionRelocationMap;
     readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos);
     Ver4PatriciaTrieNodeWriter ptNodeWriterForNewBuffers(buffersToWrite->getWritableTrieBuffer(),
-            buffersToWrite, headerPolicy, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy,
+            buffersToWrite, headerPolicy, &ptNodeReader, &ptNodeArrayReader, &shortcutPolicy);
-            &shortcutPolicy);
     DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer
             traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&ptNodeWriterForNewBuffers,
                     buffersToWrite->getWritableTrieBuffer(), &dictPositionRelocationMap);
@@ -163,12 +140,10 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
     Ver4PatriciaTrieNodeReader newPtNodeReader(buffersToWrite->getTrieBuffer(),
             buffersToWrite->getLanguageModelDictContent(), headerPolicy);
     Ver4PtNodeArrayReader newPtNodeArrayreader(buffersToWrite->getTrieBuffer());
-    Ver4BigramListPolicy newBigramPolicy(buffersToWrite->getMutableBigramDictContent(),
-            buffersToWrite->getTerminalPositionLookupTable(), headerPolicy);
     Ver4ShortcutListPolicy newShortcutPolicy(buffersToWrite->getMutableShortcutDictContent(),
             buffersToWrite->getTerminalPositionLookupTable());
     Ver4PatriciaTrieNodeWriter newPtNodeWriter(buffersToWrite->getWritableTrieBuffer(),
-            buffersToWrite, headerPolicy, &newPtNodeReader, &newPtNodeArrayreader, &newBigramPolicy,
+            buffersToWrite, headerPolicy, &newPtNodeReader, &newPtNodeArrayreader,
             &newShortcutPolicy);
     // Re-assign terminal IDs for valid terminal PtNodes.
     TerminalPositionLookupTable::TerminalIdMap terminalIdMap;
@@ -181,11 +156,6 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
             mBuffers->getLanguageModelDictContent(), nullptr /* outNgramCount */)) {
         return false;
     }
-    // Run GC for bigram dict content.
-    if(!buffersToWrite->getMutableBigramDictContent()->runGC(&terminalIdMap,
-            mBuffers->getBigramDictContent(), outBigramCount)) {
-        return false;
-    }
     // Run GC for shortcut dict content.
     if(!buffersToWrite->getMutableShortcutDictContent()->runGC(&terminalIdMap,
             mBuffers->getShortcutDictContent())) {
@@ -213,93 +183,6 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
     return true;
 }
 
-// TODO: Remove.
-bool Ver4PatriciaTrieWritingHelper::truncateUnigrams(
-        const Ver4PatriciaTrieNodeReader *const ptNodeReader,
-        Ver4PatriciaTrieNodeWriter *const ptNodeWriter, const int maxUnigramCount) {
-    const TerminalPositionLookupTable *const terminalPosLookupTable =
-            mBuffers->getTerminalPositionLookupTable();
-    const int nextTerminalId = terminalPosLookupTable->getNextTerminalId();
-    std::priority_queue<DictProbability, std::vector<DictProbability>, DictProbabilityComparator>
-            priorityQueue;
-    for (int i = 0; i < nextTerminalId; ++i) {
-        const int terminalPos = terminalPosLookupTable->getTerminalPtNodePosition(i);
-        if (terminalPos == NOT_A_DICT_POS) {
-            continue;
-        }
-        const ProbabilityEntry probabilityEntry =
-                mBuffers->getLanguageModelDictContent()->getProbabilityEntry(i);
-        const int probability = probabilityEntry.hasHistoricalInfo() ?
-                ForgettingCurveUtils::decodeProbability(
-                        probabilityEntry.getHistoricalInfo(), mBuffers->getHeaderPolicy()) :
-                probabilityEntry.getProbability();
-        priorityQueue.push(DictProbability(terminalPos, probability,
-                probabilityEntry.getHistoricalInfo()->getTimeStamp()));
-    }
-
-    // Delete unigrams.
-    while (static_cast<int>(priorityQueue.size()) > maxUnigramCount) {
-        const int ptNodePos = priorityQueue.top().getDictPos();
-        priorityQueue.pop();
-        const PtNodeParams ptNodeParams =
-                ptNodeReader->fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
-        if (ptNodeParams.representsNonWordInfo()) {
-            continue;
-        }
-        if (!ptNodeWriter->markPtNodeAsWillBecomeNonTerminal(&ptNodeParams)) {
-            AKLOGE("Cannot mark PtNode as willBecomeNonterminal. PtNode pos: %d", ptNodePos);
-            return false;
-        }
-    }
-    return true;
-}
-
-// TODO: Remove.
-bool Ver4PatriciaTrieWritingHelper::truncateBigrams(const int maxBigramCount) {
-    const TerminalPositionLookupTable *const terminalPosLookupTable =
-            mBuffers->getTerminalPositionLookupTable();
-    const int nextTerminalId = terminalPosLookupTable->getNextTerminalId();
-    std::priority_queue<DictProbability, std::vector<DictProbability>, DictProbabilityComparator>
-            priorityQueue;
-    BigramDictContent *const bigramDictContent = mBuffers->getMutableBigramDictContent();
-    for (int i = 0; i < nextTerminalId; ++i) {
-        const int bigramListPos = bigramDictContent->getBigramListHeadPos(i);
-        if (bigramListPos == NOT_A_DICT_POS) {
-            continue;
-        }
-        bool hasNext = true;
-        int readingPos = bigramListPos;
-        while (hasNext) {
-            const BigramEntry bigramEntry =
-                    bigramDictContent->getBigramEntryAndAdvancePosition(&readingPos);
-            const int entryPos = readingPos - bigramDictContent->getBigramEntrySize();
-            hasNext = bigramEntry.hasNext();
-            if (!bigramEntry.isValid()) {
-                continue;
-            }
-            const int probability = bigramEntry.hasHistoricalInfo() ?
-                    ForgettingCurveUtils::decodeProbability(
-                            bigramEntry.getHistoricalInfo(), mBuffers->getHeaderPolicy()) :
-                    bigramEntry.getProbability();
-            priorityQueue.push(DictProbability(entryPos, probability,
-                    bigramEntry.getHistoricalInfo()->getTimeStamp()));
-        }
-    }
-
-    // Delete bigrams.
-    while (static_cast<int>(priorityQueue.size()) > maxBigramCount) {
-        const int entryPos = priorityQueue.top().getDictPos();
-        const BigramEntry bigramEntry = bigramDictContent->getBigramEntry(entryPos);
-        const BigramEntry invalidatedBigramEntry = bigramEntry.getInvalidatedEntry();
-        if (!bigramDictContent->writeBigramEntry(&invalidatedBigramEntry, entryPos)) {
-            AKLOGE("Cannot write bigram entry to remove. pos: %d", entryPos);
-            return false;
-        }
-        priorityQueue.pop();
-    }
-    return true;
-}
-
 bool Ver4PatriciaTrieWritingHelper::TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds
         ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) {
     if (!ptNodeParams->isTerminal()) {

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h

@@ -66,49 +66,6 @@ class Ver4PatriciaTrieWritingHelper {
         const TerminalPositionLookupTable::TerminalIdMap *const mTerminalIdMap;
     };
 
-    // For truncateUnigrams() and truncateBigrams().
-    class DictProbability {
-     public:
-        DictProbability(const int dictPos, const int probability, const int timestamp)
-                : mDictPos(dictPos), mProbability(probability), mTimestamp(timestamp) {}
-
-        int getDictPos() const {
-            return mDictPos;
-        }
-
-        int getProbability() const {
-            return mProbability;
-        }
-
-        int getTimestamp() const {
-            return mTimestamp;
-        }
-
-     private:
-        DISALLOW_DEFAULT_CONSTRUCTOR(DictProbability);
-
-        int mDictPos;
-        int mProbability;
-        int mTimestamp;
-    };
-
-    // For truncateUnigrams() and truncateBigrams().
-    class DictProbabilityComparator {
-     public:
-        bool operator()(const DictProbability &left, const DictProbability &right) {
-            if (left.getProbability() != right.getProbability()) {
-                return left.getProbability() > right.getProbability();
-            }
-            if (left.getTimestamp() != right.getTimestamp()) {
-                return left.getTimestamp() < right.getTimestamp();
-            }
-            return left.getDictPos() > right.getDictPos();
-        }
-
-     private:
-        DISALLOW_ASSIGNMENT_OPERATOR(DictProbabilityComparator);
-    };
-
     bool runGC(const int rootPtNodeArrayPos, const HeaderPolicy *const headerPolicy,
             Ver4DictBuffers *const buffersToWrite, int *const outUnigramCount,
             int *const outBigramCount);