Merge "Use DictionaryStructureWithBufferPolicy for traversing."

native/jni/src/suggest/core/dicnode/dic_node_utils.cpp

@@ -21,7 +21,6 @@
 #include "suggest/core/dicnode/dic_node.h"
 #include "suggest/core/dicnode/dic_node_proximity_filter.h"
 #include "suggest/core/dicnode/dic_node_vector.h"
-#include "suggest/core/dictionary/binary_dictionary_info.h"
 #include "suggest/core/dictionary/multi_bigram_map.h"
 #include "suggest/core/dictionary/probability_utils.h"
 #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
@@ -33,17 +32,17 @@ namespace latinime {
 // Node initialization utils //
 ///////////////////////////////
 
-/* static */ void DicNodeUtils::initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo,
+/* static */ void DicNodeUtils::initAsRoot(
+        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
         const int prevWordNodePos, DicNode *const newRootNode) {
-    newRootNode->initAsRoot(binaryDictionaryInfo->getStructurePolicy()->getRootPosition(),
+    newRootNode->initAsRoot(dictionaryStructurePolicy->getRootPosition(), prevWordNodePos);
-            prevWordNodePos);
 }
 
 /*static */ void DicNodeUtils::initAsRootWithPreviousWord(
-        const BinaryDictionaryInfo *const binaryDictionaryInfo,
+        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
         DicNode *const prevWordLastNode, DicNode *const newRootNode) {
     newRootNode->initAsRootWithPreviousWord(
-            prevWordLastNode, binaryDictionaryInfo->getStructurePolicy()->getRootPosition());
+            prevWordLastNode, dictionaryStructurePolicy->getRootPosition());
 }
 
 /* static */ void DicNodeUtils::initByCopy(DicNode *srcNode, DicNode *destNode) {
@@ -67,12 +66,13 @@ namespace latinime {
 }
 
 /* static */ void DicNodeUtils::getAllChildDicNodes(DicNode *dicNode,
-        const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes) {
+        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
-    getProximityChildDicNodes(dicNode, binaryDictionaryInfo, 0, 0, false, childDicNodes);
+        DicNodeVector *childDicNodes) {
+    getProximityChildDicNodes(dicNode, dictionaryStructurePolicy, 0, 0, false, childDicNodes);
 }
 
 /* static */ void DicNodeUtils::getProximityChildDicNodes(DicNode *dicNode,
-        const BinaryDictionaryInfo *const binaryDictionaryInfo,
+        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
         const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly,
         DicNodeVector *childDicNodes) {
     if (dicNode->isTotalInputSizeExceedingLimit()) {
@@ -82,7 +82,7 @@ namespace latinime {
     if (!dicNode->isLeavingNode()) {
         DicNodeUtils::createAndGetPassingChildNode(dicNode, &childrenFilter, childDicNodes);
     } else {
-        binaryDictionaryInfo->getStructurePolicy()->createAndGetAllChildNodes(dicNode,
+        dictionaryStructurePolicy->createAndGetAllChildNodes(dicNode,
                 &childrenFilter, childDicNodes);
     }
 }
@@ -94,12 +94,13 @@ namespace latinime {
  * Computes the combined bigram / unigram cost for the given dicNode.
  */
 /* static */ float DicNodeUtils::getBigramNodeImprobability(
-        const BinaryDictionaryInfo *const binaryDictionaryInfo,
+        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
         const DicNode *const node, MultiBigramMap *multiBigramMap) {
     if (node->hasMultipleWords() && !node->isValidMultipleWordSuggestion()) {
         return static_cast<float>(MAX_VALUE_FOR_WEIGHTING);
     }
-    const int probability = getBigramNodeProbability(binaryDictionaryInfo, node, multiBigramMap);
+    const int probability = getBigramNodeProbability(dictionaryStructurePolicy, node,
+            multiBigramMap);
     // TODO: This equation to calculate the improbability looks unreasonable.  Investigate this.
     const float cost = static_cast<float>(MAX_PROBABILITY - probability)
             / static_cast<float>(MAX_PROBABILITY);
@@ -107,7 +108,7 @@ namespace latinime {
 }
 
 /* static */ int DicNodeUtils::getBigramNodeProbability(
-        const BinaryDictionaryInfo *const binaryDictionaryInfo,
+        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
         const DicNode *const node, MultiBigramMap *multiBigramMap) {
     const int unigramProbability = node->getProbability();
     const int wordPos = node->getPos();
@@ -118,8 +119,8 @@ namespace latinime {
         return ProbabilityUtils::backoff(unigramProbability);
     }
     if (multiBigramMap) {
-        return multiBigramMap->getBigramProbability(binaryDictionaryInfo->getStructurePolicy(),
+        return multiBigramMap->getBigramProbability(dictionaryStructurePolicy, prevWordPos,
-                prevWordPos, wordPos, unigramProbability);
+                wordPos, unigramProbability);
     }
     return ProbabilityUtils::backoff(unigramProbability);
 }

native/jni/src/suggest/core/dicnode/dic_node_utils.h

@@ -23,10 +23,10 @@
 
 namespace latinime {
 
-class BinaryDictionaryInfo;
 class DicNode;
 class DicNodeProximityFilter;
 class DicNodeVector;
+class DictionaryStructureWithBufferPolicy;
 class ProximityInfoState;
 class MultiBigramMap;
 
@@ -34,18 +34,22 @@ class DicNodeUtils {
  public:
     static int appendTwoWords(const int *src0, const int16_t length0, const int *src1,
             const int16_t length1, int *dest);
-    static void initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo,
+    static void initAsRoot(
+            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
             const int prevWordNodePos, DicNode *newRootNode);
-    static void initAsRootWithPreviousWord(const BinaryDictionaryInfo *const binaryDictionaryInfo,
+    static void initAsRootWithPreviousWord(
+            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
             DicNode *prevWordLastNode, DicNode *newRootNode);
     static void initByCopy(DicNode *srcNode, DicNode *destNode);
     static void getAllChildDicNodes(DicNode *dicNode,
-            const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes);
+            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
-    static float getBigramNodeImprobability(const BinaryDictionaryInfo *const binaryDictionaryInfo,
+            DicNodeVector *childDicNodes);
+    static float getBigramNodeImprobability(
+            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
             const DicNode *const node, MultiBigramMap *const multiBigramMap);
     // TODO: Move to private
     static void getProximityChildDicNodes(DicNode *dicNode,
-            const BinaryDictionaryInfo *const binaryDictionaryInfo,
+            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
             const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly,
             DicNodeVector *childDicNodes);
 
@@ -54,7 +58,8 @@ class DicNodeUtils {
     // Max number of bigrams to look up
     static const int MAX_BIGRAMS_CONSIDERED_PER_CONTEXT = 500;
 
-    static int getBigramNodeProbability(const BinaryDictionaryInfo *const binaryDictionaryInfo,
+    static int getBigramNodeProbability(
+            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
             const DicNode *const node, MultiBigramMap *multiBigramMap);
     static void createAndGetPassingChildNode(DicNode *dicNode,
             const DicNodeProximityFilter *const childrenFilter, DicNodeVector *childDicNodes);

native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp

@@ -22,15 +22,16 @@
 
 #include "defines.h"
 #include "suggest/core/dictionary/binary_dictionary_bigrams_iterator.h"
-#include "suggest/core/dictionary/binary_dictionary_info.h"
 #include "suggest/core/dictionary/dictionary.h"
 #include "suggest/core/dictionary/probability_utils.h"
+#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
 #include "utils/char_utils.h"
 
 namespace latinime {
 
-BigramDictionary::BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo)
+BigramDictionary::BigramDictionary(
-        : mBinaryDictionaryInfo(binaryDictionaryInfo) {
+        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy)
+        : mDictionaryStructurePolicy(dictionaryStructurePolicy) {
     if (DEBUG_DICT) {
         AKLOGI("BigramDictionary - constructor");
     }
@@ -112,12 +113,11 @@ int BigramDictionary::getPredictions(const int *prevWord, const int prevWordLeng
     int bigramCount = 0;
     int unigramProbability = 0;
     int bigramBuffer[MAX_WORD_LENGTH];
-    const DictionaryStructureWithBufferPolicy *const structurePolicy =
+    BinaryDictionaryBigramsIterator bigramsIt(
-            mBinaryDictionaryInfo->getStructurePolicy();
+            mDictionaryStructurePolicy->getBigramsStructurePolicy(), pos);
-    BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(), pos);
     while (bigramsIt.hasNext()) {
         bigramsIt.next();
-        const int length = structurePolicy->
+        const int length = mDictionaryStructurePolicy->
                 getCodePointsAndProbabilityAndReturnCodePointCount(bigramsIt.getBigramPos(),
                         MAX_WORD_LENGTH, bigramBuffer, &unigramProbability);
         // Due to space constraints, the probability for bigrams is approximate - the lower the
@@ -139,12 +139,10 @@ int BigramDictionary::getPredictions(const int *prevWord, const int prevWordLeng
 int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const int prevWordLength,
         const bool forceLowerCaseSearch) const {
     if (0 >= prevWordLength) return NOT_A_DICT_POS;
-    const DictionaryStructureWithBufferPolicy *const structurePolicy =
+    int pos = mDictionaryStructurePolicy->getTerminalNodePositionOfWord(prevWord, prevWordLength,
-            mBinaryDictionaryInfo->getStructurePolicy();
-    int pos = structurePolicy->getTerminalNodePositionOfWord(prevWord, prevWordLength,
             forceLowerCaseSearch);
     if (NOT_A_VALID_WORD_POS == pos) return NOT_A_DICT_POS;
-    return structurePolicy->getBigramsPositionOfNode(pos);
+    return mDictionaryStructurePolicy->getBigramsPositionOfNode(pos);
 }
 
 bool BigramDictionary::isValidBigram(const int *word0, int length0, const int *word1,
@@ -152,13 +150,12 @@ bool BigramDictionary::isValidBigram(const int *word0, int length0, const int *w
     int pos = getBigramListPositionForWord(word0, length0, false /* forceLowerCaseSearch */);
     // getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams
     if (NOT_A_DICT_POS == pos) return false;
-    const DictionaryStructureWithBufferPolicy *const structurePolicy =
+    int nextWordPos = mDictionaryStructurePolicy->getTerminalNodePositionOfWord(word1, length1,
-            mBinaryDictionaryInfo->getStructurePolicy();
-    int nextWordPos = structurePolicy->getTerminalNodePositionOfWord(word1, length1,
             false /* forceLowerCaseSearch */);
     if (NOT_A_VALID_WORD_POS == nextWordPos) return false;
 
-    BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(), pos);
+    BinaryDictionaryBigramsIterator bigramsIt(
+            mDictionaryStructurePolicy->getBigramsStructurePolicy(), pos);
     while (bigramsIt.hasNext()) {
         bigramsIt.next();
         if (bigramsIt.getBigramPos() == nextWordPos) {

native/jni/src/suggest/core/dictionary/bigram_dictionary.h

@@ -21,11 +21,11 @@
 
 namespace latinime {
 
-class BinaryDictionaryInfo;
+class DictionaryStructureWithBufferPolicy;
 
 class BigramDictionary {
  public:
-    BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo);
+    BigramDictionary(const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy);
 
     int getPredictions(const int *word, int length, int *outBigramCodePoints,
             int *outBigramProbability, int *outputTypes) const;
@@ -40,7 +40,7 @@ class BigramDictionary {
     int getBigramListPositionForWord(const int *prevWord, const int prevWordLength,
             const bool forceLowerCaseSearch) const;
 
-    const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
+    const DictionaryStructureWithBufferPolicy *const mDictionaryStructurePolicy;
 };
 } // namespace latinime
 #endif // LATINIME_BIGRAM_DICTIONARY_H

native/jni/src/suggest/core/dictionary/binary_dictionary_header.h

@@ -24,8 +24,6 @@
 
 namespace latinime {
 
-class BinaryDictionaryInfo;
-
 /**
  * This class abstracts dictionary header structures and provide interface to access dictionary
  * header information.

native/jni/src/suggest/core/dictionary/binary_dictionary_header_reading_utils.cpp

@@ -45,9 +45,8 @@ const BinaryDictionaryHeaderReadingUtils::DictionaryFlags
 /* static */ int BinaryDictionaryHeaderReadingUtils::getHeaderSize(const uint8_t *const dictBuf) {
     // See the format of the header in the comment in
     // BinaryDictionaryFormatUtils::detectFormatVersion()
-    return ByteArrayUtils::readUint32(dictBuf,
+    return ByteArrayUtils::readUint32(dictBuf, VERSION_2_HEADER_MAGIC_NUMBER_SIZE
-            VERSION_2_HEADER_MAGIC_NUMBER_SIZE + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE
+            + VERSION_2_HEADER_DICTIONARY_VERSION_SIZE + VERSION_2_HEADER_FLAG_SIZE);
-                    + VERSION_2_HEADER_FLAG_SIZE);
 }
 
 /* static */ BinaryDictionaryHeaderReadingUtils::DictionaryFlags

native/jni/src/suggest/core/dictionary/binary_dictionary_info.h

@@ -21,9 +21,7 @@
 
 #include "defines.h"
 #include "jni.h"
-#include "suggest/core/dictionary/binary_dictionary_format_utils.h"
 #include "suggest/core/dictionary/binary_dictionary_header.h"
-#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
 #include "utils/log_utils.h"
 
 namespace latinime {
@@ -34,18 +32,11 @@ class BinaryDictionaryInfo {
             const int dictSize, const int mmapFd, const int dictBufOffset, const bool isUpdatable)
             : mDictBuf(dictBuf), mDictSize(dictSize), mMmapFd(mmapFd),
               mDictBufOffset(dictBufOffset), mIsUpdatable(isUpdatable),
-              mDictionaryFormat(BinaryDictionaryFormatUtils::detectFormatVersion(
+              mDictionaryHeader(dictBuf) {
-                      mDictBuf, mDictSize)),
-              mDictionaryHeader(dictBuf), mDictRoot(mDictBuf + mDictionaryHeader.getSize()),
-              // TODO: Remove.
-              mStructurePolicy(DictionaryStructureWithBufferPolicyFactory
-                      ::newDictionaryStructurePolicy(this)) {
         logDictionaryInfo(env);
     }
 
-    ~BinaryDictionaryInfo() {
+    ~BinaryDictionaryInfo() {}
-        delete mStructurePolicy;
-    }
 
     AK_FORCE_INLINE const uint8_t *getDictBuf() const {
         return mDictBuf;
@@ -63,30 +54,12 @@ class BinaryDictionaryInfo {
         return mDictBufOffset;
     }
 
-    AK_FORCE_INLINE const uint8_t *getDictRoot() const {
-        return mDictRoot;
-    }
-
-    AK_FORCE_INLINE BinaryDictionaryFormatUtils::FORMAT_VERSION getFormat() const {
-        return mDictionaryFormat;
-    }
-
-    // TODO: Move to DictionaryStructurePolicy.
-    AK_FORCE_INLINE const BinaryDictionaryHeader *getHeader() const {
-        return &mDictionaryHeader;
-    }
-
     AK_FORCE_INLINE bool isDynamicallyUpdatable() const {
         // TODO: Support dynamic dictionary formats.
         const bool isUpdatableDictionaryFormat = false;
         return mIsUpdatable && isUpdatableDictionaryFormat;
     }
 
-    // TODO: remove
-    AK_FORCE_INLINE const DictionaryStructureWithBufferPolicy *getStructurePolicy() const {
-        return mStructurePolicy;
-    }
-
  private:
     DISALLOW_COPY_AND_ASSIGN(BinaryDictionaryInfo);
 
@@ -95,13 +68,9 @@ class BinaryDictionaryInfo {
     const int mMmapFd;
     const int mDictBufOffset;
     const bool mIsUpdatable;
-    const BinaryDictionaryFormatUtils::FORMAT_VERSION mDictionaryFormat;
     // TODO: Move BinaryDictionaryHeader to policyimpl and introduce dedicated API to the
-    // DictionaryStructurePolicy.
+    // DictionaryStructureWithBufferPolicy.
     const BinaryDictionaryHeader mDictionaryHeader;
-    const uint8_t *const mDictRoot;
-    // TODO: remove
-    const DictionaryStructureWithBufferPolicy *const mStructurePolicy;
 
     AK_FORCE_INLINE void logDictionaryInfo(JNIEnv *const env) const {
         const int BUFFER_SIZE = 16;

native/jni/src/suggest/core/dictionary/dictionary.cpp

@@ -18,7 +18,6 @@
 
 #include "suggest/core/dictionary/dictionary.h"
 
-#include <map> // TODO: remove
 #include <stdint.h>
 
 #include "defines.h"
@@ -27,6 +26,7 @@
 #include "suggest/core/session/dic_traverse_session.h"
 #include "suggest/core/suggest.h"
 #include "suggest/core/suggest_options.h"
+#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
 #include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h"
 #include "suggest/policyimpl/typing/typing_suggest_policy_factory.h"
 
@@ -36,7 +36,10 @@ Dictionary::Dictionary(JNIEnv *env, void *dict, int dictSize, int mmapFd,
         int dictBufOffset, bool isUpdatable)
         : mBinaryDictionaryInfo(env, static_cast<const uint8_t *>(dict), dictSize, mmapFd,
                 dictBufOffset, isUpdatable),
-          mBigramDictionary(new BigramDictionary(&mBinaryDictionaryInfo)),
+          mDictionaryStructureWithBufferPolicy(DictionaryStructureWithBufferPolicyFactory
+                  ::newDictionaryStructureWithBufferPolicy(
+                          static_cast<const uint8_t *>(dict), dictSize)),
+          mBigramDictionary(new BigramDictionary(mDictionaryStructureWithBufferPolicy)),
           mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())),
           mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) {
 }
@@ -45,6 +48,7 @@ Dictionary::~Dictionary() {
     delete mBigramDictionary;
     delete mGestureSuggest;
     delete mTypingSuggest;
+    delete mDictionaryStructureWithBufferPolicy;
 }
 
 int Dictionary::getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession,
@@ -83,14 +87,12 @@ int Dictionary::getBigrams(const int *word, int length, int *outWords, int *freq
 }
 
 int Dictionary::getProbability(const int *word, int length) const {
-    const DictionaryStructureWithBufferPolicy *const structurePolicy =
+    int pos = getDictionaryStructurePolicy()->getTerminalNodePositionOfWord(word, length,
-            mBinaryDictionaryInfo.getStructurePolicy();
-    int pos = structurePolicy->getTerminalNodePositionOfWord(word, length,
             false /* forceLowerCaseSearch */);
     if (NOT_A_VALID_WORD_POS == pos) {
         return NOT_A_PROBABILITY;
     }
-    return structurePolicy->getUnigramProbability(pos);
+    return getDictionaryStructurePolicy()->getUnigramProbability(pos);
 }
 
 bool Dictionary::isValidBigram(const int *word0, int length0, const int *word1, int length1) const {

native/jni/src/suggest/core/dictionary/dictionary.h

@@ -26,6 +26,7 @@
 namespace latinime {
 
 class BigramDictionary;
+class DictionaryStructureWithBufferPolicy;
 class DicTraverseSession;
 class ProximityInfo;
 class SuggestInterface;
@@ -77,16 +78,22 @@ class Dictionary {
     void removeBigramWords(const int *const word0, const int length0, const int *const word1,
             const int length1);
 
+    // TODO: Remove.
     const BinaryDictionaryInfo *getBinaryDictionaryInfo() const {
         return &mBinaryDictionaryInfo;
     }
 
+    const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {
+        return mDictionaryStructureWithBufferPolicy;
+    }
+
     virtual ~Dictionary();
 
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);
 
     const BinaryDictionaryInfo mBinaryDictionaryInfo;
+    DictionaryStructureWithBufferPolicy *const mDictionaryStructureWithBufferPolicy;
     const BigramDictionary *mBigramDictionary;
     SuggestInterface *mGestureSuggest;
     SuggestInterface *mTypingSuggest;

native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h

@@ -21,6 +21,7 @@
 
 namespace latinime {
 
+class BinaryDictionaryHeader;
 class DicNode;
 class DicNodeVector;
 class DictionaryBigramsStructurePolicy;
@@ -65,6 +66,9 @@ class DictionaryStructureWithBufferPolicy {
 
     virtual int getBigramsPositionOfNode(const int nodePos) const = 0;
 
+    // TODO: Use policy to access header information.
+    virtual const BinaryDictionaryHeader *getHeader() const = 0;
+
     virtual const DictionaryBigramsStructurePolicy *getBigramsStructurePolicy() const = 0;
 
     virtual const DictionaryShortcutsStructurePolicy *getShortcutsStructurePolicy() const = 0;

native/jni/src/suggest/core/policy/weighting.cpp

@@ -148,7 +148,7 @@ static inline void profile(const CorrectionType correctionType, DicNode *const n
     case CT_TERMINAL: {
         const float languageImprobability =
                 DicNodeUtils::getBigramNodeImprobability(
-                        traverseSession->getBinaryDictionaryInfo(), dicNode, multiBigramMap);
+                        traverseSession->getDictionaryStructurePolicy(), dicNode, multiBigramMap);
         return weighting->getTerminalLanguageCost(traverseSession, dicNode, languageImprobability);
     }
     case CT_TERMINAL_INSERTION:

native/jni/src/suggest/core/session/dic_traverse_session.cpp

@@ -19,29 +19,28 @@
 #include "defines.h"
 #include "jni.h"
 #include "suggest/core/dictionary/binary_dictionary_header.h"
-#include "suggest/core/dictionary/binary_dictionary_info.h"
 #include "suggest/core/dictionary/dictionary.h"
+#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
 
 namespace latinime {
 
 void DicTraverseSession::init(const Dictionary *const dictionary, const int *prevWord,
         int prevWordLength, const SuggestOptions *const suggestOptions) {
     mDictionary = dictionary;
-    const BinaryDictionaryInfo *const binaryDictionaryInfo =
+    mMultiWordCostMultiplier = getDictionaryStructurePolicy()->getHeader()
-            mDictionary->getBinaryDictionaryInfo();
+            ->getMultiWordCostMultiplier();
-    mMultiWordCostMultiplier = binaryDictionaryInfo->getHeader()->getMultiWordCostMultiplier();
     mSuggestOptions = suggestOptions;
     if (!prevWord) {
         mPrevWordPos = NOT_A_VALID_WORD_POS;
         return;
     }
     // TODO: merge following similar calls to getTerminalPosition into one case-insensitive call.
-    mPrevWordPos = binaryDictionaryInfo->getStructurePolicy()->getTerminalNodePositionOfWord(
+    mPrevWordPos = getDictionaryStructurePolicy()->getTerminalNodePositionOfWord(
             prevWord, prevWordLength, false /* forceLowerCaseSearch */);
     if (mPrevWordPos == NOT_A_VALID_WORD_POS) {
         // Check bigrams for lower-cased previous word if original was not found. Useful for
         // auto-capitalized words like "The [current_word]".
-        mPrevWordPos = binaryDictionaryInfo->getStructurePolicy()->getTerminalNodePositionOfWord(
+        mPrevWordPos = getDictionaryStructurePolicy()->getTerminalNodePositionOfWord(
                 prevWord, prevWordLength, true /* forceLowerCaseSearch */);
     }
 }
@@ -56,8 +55,9 @@ void DicTraverseSession::setupForGetSuggestions(const ProximityInfo *pInfo,
             maxSpatialDistance, maxPointerCount);
 }
 
-const BinaryDictionaryInfo *DicTraverseSession::getBinaryDictionaryInfo() const {
+const DictionaryStructureWithBufferPolicy *DicTraverseSession::getDictionaryStructurePolicy()
-    return mDictionary->getBinaryDictionaryInfo();
+        const {
+    return mDictionary->getDictionaryStructurePolicy();
 }
 
 void DicTraverseSession::resetCache(const int nextActiveCacheSize, const int maxWords) {

native/jni/src/suggest/core/session/dic_traverse_session.h

@@ -28,8 +28,8 @@
 
 namespace latinime {
 
-class BinaryDictionaryInfo;
 class Dictionary;
+class DictionaryStructureWithBufferPolicy;
 class ProximityInfo;
 class SuggestOptions;
 
@@ -75,8 +75,7 @@ class DicTraverseSession {
             const int maxPointerCount);
     void resetCache(const int nextActiveCacheSize, const int maxWords);
 
-    // TODO: Use DictionaryStructurePolicy instead of BinaryDictionaryInfo.
+    const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const;
-    const BinaryDictionaryInfo *getBinaryDictionaryInfo() const;
 
     //--------------------
     // getters and setters

native/jni/src/suggest/core/suggest.cpp

@@ -19,13 +19,12 @@
 #include "suggest/core/dicnode/dic_node.h"
 #include "suggest/core/dicnode/dic_node_priority_queue.h"
 #include "suggest/core/dicnode/dic_node_vector.h"
-// TODO: Use DictionaryStructurePolicy instead of BinaryDictionaryInfo.
-#include "suggest/core/dictionary/binary_dictionary_info.h"
 #include "suggest/core/dictionary/binary_dictionary_shortcut_iterator.h"
 #include "suggest/core/dictionary/dictionary.h"
 #include "suggest/core/dictionary/digraph_utils.h"
 #include "suggest/core/dictionary/shortcut_utils.h"
 #include "suggest/core/layout/proximity_info.h"
+#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
 #include "suggest/core/policy/scoring.h"
 #include "suggest/core/policy/traversal.h"
 #include "suggest/core/policy/weighting.h"
@@ -108,7 +107,7 @@ void Suggest::initializeSearch(DicTraverseSession *traverseSession, int commitPo
                 MAX_RESULTS);
         // Create a new dic node here
         DicNode rootNode;
-        DicNodeUtils::initAsRoot(traverseSession->getBinaryDictionaryInfo(),
+        DicNodeUtils::initAsRoot(traverseSession->getDictionaryStructurePolicy(),
                 traverseSession->getPrevWordPos(), &rootNode);
         traverseSession->getDicTraverseCache()->copyPushActive(&rootNode);
     }
@@ -212,11 +211,10 @@ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequen
         }
 
         if (!terminalDicNode->hasMultipleWords()) {
-            const DictionaryStructureWithBufferPolicy *const structurePolicy =
-                    traverseSession->getBinaryDictionaryInfo()->getStructurePolicy();
             BinaryDictionaryShortcutIterator shortcutIt(
-                    structurePolicy->getShortcutsStructurePolicy(),
+                    traverseSession->getDictionaryStructurePolicy()->getShortcutsStructurePolicy(),
-                    structurePolicy->getShortcutPositionOfNode(terminalDicNode->getPos()));
+                    traverseSession->getDictionaryStructurePolicy()
+                            ->getShortcutPositionOfNode(terminalDicNode->getPos()));
             // Shortcut is not supported for multiple words suggestions.
             // TODO: Check shortcuts during traversal for multiple words suggestions.
             const bool sameAsTyped = TRAVERSAL->sameAsTyped(traverseSession, terminalDicNode);
@@ -299,7 +297,7 @@ void Suggest::expandCurrentDicNodes(DicTraverseSession *traverseSession) const {
             }
 
             DicNodeUtils::getAllChildDicNodes(
-                    &dicNode, traverseSession->getBinaryDictionaryInfo(), &childDicNodes);
+                    &dicNode, traverseSession->getDictionaryStructurePolicy(), &childDicNodes);
 
             const int childDicNodesSize = childDicNodes.getSizeAndLock();
             for (int i = 0; i < childDicNodesSize; ++i) {
@@ -310,7 +308,7 @@ void Suggest::expandCurrentDicNodes(DicTraverseSession *traverseSession) const {
                     continue;
                 }
                 if (DigraphUtils::hasDigraphForCodePoint(
-                        traverseSession->getBinaryDictionaryInfo()->getHeader(),
+                        traverseSession->getDictionaryStructurePolicy()->getHeader(),
                         childDicNode->getNodeCodePoint())) {
                     correctionDicNode.initByCopy(childDicNode);
                     correctionDicNode.advanceDigraphIndex();
@@ -448,7 +446,7 @@ void Suggest::processDicNodeAsOmission(
         DicTraverseSession *traverseSession, DicNode *dicNode) const {
     DicNodeVector childDicNodes;
     DicNodeUtils::getAllChildDicNodes(
-            dicNode, traverseSession->getBinaryDictionaryInfo(), &childDicNodes);
+            dicNode, traverseSession->getDictionaryStructurePolicy(), &childDicNodes);
 
     const int size = childDicNodes.getSizeAndLock();
     for (int i = 0; i < size; i++) {
@@ -473,7 +471,8 @@ void Suggest::processDicNodeAsInsertion(DicTraverseSession *traverseSession,
         DicNode *dicNode) const {
     const int16_t pointIndex = dicNode->getInputIndex(0);
     DicNodeVector childDicNodes;
-    DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getBinaryDictionaryInfo(),
+    DicNodeUtils::getProximityChildDicNodes(dicNode,
+            traverseSession->getDictionaryStructurePolicy(),
             traverseSession->getProximityInfoState(0), pointIndex + 1, true, &childDicNodes);
     const int size = childDicNodes.getSizeAndLock();
     for (int i = 0; i < size; i++) {
@@ -491,14 +490,15 @@ void Suggest::processDicNodeAsTransposition(DicTraverseSession *traverseSession,
         DicNode *dicNode) const {
     const int16_t pointIndex = dicNode->getInputIndex(0);
     DicNodeVector childDicNodes1;
-    DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getBinaryDictionaryInfo(),
+    DicNodeUtils::getProximityChildDicNodes(dicNode,
+            traverseSession->getDictionaryStructurePolicy(),
             traverseSession->getProximityInfoState(0), pointIndex + 1, false, &childDicNodes1);
     const int childSize1 = childDicNodes1.getSizeAndLock();
     for (int i = 0; i < childSize1; i++) {
         if (childDicNodes1[i]->hasChildren()) {
             DicNodeVector childDicNodes2;
             DicNodeUtils::getProximityChildDicNodes(
-                    childDicNodes1[i], traverseSession->getBinaryDictionaryInfo(),
+                    childDicNodes1[i], traverseSession->getDictionaryStructurePolicy(),
                     traverseSession->getProximityInfoState(0), pointIndex, false, &childDicNodes2);
             const int childSize2 = childDicNodes2.getSizeAndLock();
             for (int j = 0; j < childSize2; j++) {
@@ -539,7 +539,7 @@ void Suggest::createNextWordDicNode(DicTraverseSession *traverseSession, DicNode
     // Create a non-cached node here.
     DicNode newDicNode;
     DicNodeUtils::initAsRootWithPreviousWord(
-            traverseSession->getBinaryDictionaryInfo(), dicNode, &newDicNode);
+            traverseSession->getDictionaryStructurePolicy(), dicNode, &newDicNode);
     const CorrectionType correctionType = spaceSubstitution ?
             CT_NEW_WORD_SPACE_SUBSTITUTION : CT_NEW_WORD_SPACE_OMITTION;
     Weighting::addCostAndForwardInputIndex(WEIGHTING, correctionType, traverseSession, dicNode,

native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp

@@ -16,21 +16,23 @@
 
 #include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
 
+#include <stdint.h>
+
 #include "defines.h"
-#include "suggest/core/dictionary/binary_dictionary_info.h"
+#include "suggest/core/dictionary/binary_dictionary_format_utils.h"
 #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h"
 #include "suggest/policyimpl/dictionary/patricia_trie_policy.h"
 
 namespace latinime {
 
 /* static */ DictionaryStructureWithBufferPolicy *DictionaryStructureWithBufferPolicyFactory
-        ::newDictionaryStructurePolicy(
+        ::newDictionaryStructureWithBufferPolicy(const uint8_t *const dictBuf,
-        const BinaryDictionaryInfo *const binaryDictionaryInfo) {
+                const int dictSize) {
-    switch (binaryDictionaryInfo->getFormat()) {
+    switch (BinaryDictionaryFormatUtils::detectFormatVersion(dictBuf, dictSize)) {
         case BinaryDictionaryFormatUtils::VERSION_2:
-            return new PatriciaTriePolicy(binaryDictionaryInfo->getDictRoot());
+            return new PatriciaTriePolicy(dictBuf);
         case BinaryDictionaryFormatUtils::VERSION_3:
-            return new DynamicPatriciaTriePolicy(binaryDictionaryInfo->getDictRoot());
+            return new DynamicPatriciaTriePolicy(dictBuf);
         default:
             ASSERT(false);
             return 0;

native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h

@@ -17,18 +17,17 @@
 #ifndef LATINIME_DICTIONARY_STRUCTURE_WITH_BUFFER_POLICY_FACTORY_H
 #define LATINIME_DICTIONARY_STRUCTURE_WITH_BUFFER_POLICY_FACTORY_H
 
-#include "defines.h"
+#include <stdint.h>
 
+#include "defines.h"
 #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
 
 namespace latinime {
 
-class BinaryDictionaryInfo;
-
 class DictionaryStructureWithBufferPolicyFactory {
  public:
-    static DictionaryStructureWithBufferPolicy *newDictionaryStructurePolicy(
+    static DictionaryStructureWithBufferPolicy *newDictionaryStructureWithBufferPolicy(
-            const BinaryDictionaryInfo *const binaryDictionaryInfo);
+            const uint8_t *const dictBuf, const int dictSize);
 
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory);

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h

@@ -20,6 +20,7 @@
 #include <stdint.h>
 
 #include "defines.h"
+#include "suggest/core/dictionary/binary_dictionary_header.h"
 #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
 #include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
 #include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h"
@@ -31,8 +32,9 @@ class DicNodeVector;
 
 class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
  public:
-    DynamicPatriciaTriePolicy(const uint8_t *const dictRoot)
+    DynamicPatriciaTriePolicy(const uint8_t *const dictBuf)
-            : mDictRoot(dictRoot), mBigramListPolicy(dictRoot), mShortcutListPolicy(dictRoot) {}
+            : mHeader(dictBuf), mDictRoot(dictBuf + mHeader.getSize()),
+              mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {}
 
     ~DynamicPatriciaTriePolicy() {}
 
@@ -56,6 +58,11 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
 
     int getBigramsPositionOfNode(const int nodePos) const;
 
+    // TODO: Remove and use policy to access header information.
+    const BinaryDictionaryHeader *getHeader() const {
+        return &mHeader;
+    }
+
     const DictionaryBigramsStructurePolicy *getBigramsStructurePolicy() const {
         return &mBigramListPolicy;
     }
@@ -68,6 +75,7 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
     DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy);
     static const int MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP;
 
+    const BinaryDictionaryHeader mHeader;
     // TODO: Consolidate mDictRoot.
     const uint8_t *const mDictRoot;
     const BigramListPolicy mBigramListPolicy;

native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h

@@ -20,6 +20,7 @@
 #include <stdint.h>
 
 #include "defines.h"
+#include "suggest/core/dictionary/binary_dictionary_header.h"
 #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
 #include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h"
 #include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h"
@@ -31,8 +32,9 @@ class DicNodeVector;
 
 class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
  public:
-    PatriciaTriePolicy(const uint8_t *const dictRoot)
+    PatriciaTriePolicy(const uint8_t *const dictBuf)
-            : mDictRoot(dictRoot), mBigramListPolicy(dictRoot), mShortcutListPolicy(dictRoot) {}
+            : mHeader(dictBuf), mDictRoot(dictBuf + mHeader.getSize()),
+              mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {}
 
     ~PatriciaTriePolicy() {}
 
@@ -56,6 +58,11 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
 
     int getBigramsPositionOfNode(const int nodePos) const;
 
+    // TODO: Remove and use policy to access header information.
+    const BinaryDictionaryHeader *getHeader() const {
+        return &mHeader;
+    }
+
     const DictionaryBigramsStructurePolicy *getBigramsStructurePolicy() const {
         return &mBigramListPolicy;
     }
@@ -67,6 +74,7 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy);
 
+    const BinaryDictionaryHeader mHeader;
     const uint8_t *const mDictRoot;
     const BigramListPolicy mBigramListPolicy;
     const ShortcutListPolicy mShortcutListPolicy;

native/jni/src/suggest/policyimpl/typing/typing_weighting.h

@@ -155,7 +155,8 @@ class TypingWeighting : public Weighting {
     float getNewWordBigramLanguageCost(const DicTraverseSession *const traverseSession,
             const DicNode *const dicNode,
             MultiBigramMap *const multiBigramMap) const {
-        return DicNodeUtils::getBigramNodeImprobability(traverseSession->getBinaryDictionaryInfo(),
+        return DicNodeUtils::getBigramNodeImprobability(
+                traverseSession->getDictionaryStructurePolicy(),
                 dicNode, multiBigramMap) * ScoringParams::DISTANCE_WEIGHT_LANGUAGE;
     }