am 202e416b: Add bigram writing methods.

* commit '202e416b51ef4cf3553afeb305ca4b14dd6105e5':
  Add bigram writing methods.

native/jni/Android.mk

@@ -68,6 +68,7 @@ LATIN_IME_CORE_SRC_FILES := \
     suggest/core/session/dic_traverse_session.cpp \
     $(addprefix suggest/policyimpl/dictionary/, \
         bigram/bigram_list_reading_utils.cpp \
+        bigram/dynamic_bigram_list_policy.cpp \
         header/header_policy.cpp \
         header/header_reading_utils.cpp \
         shortcut/shortcut_list_reading_utils.cpp \

native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp

@@ -117,9 +117,15 @@ int BigramDictionary::getPredictions(const int *prevWord, const int prevWordLeng
             mDictionaryStructurePolicy->getBigramsStructurePolicy(), pos);
     while (bigramsIt.hasNext()) {
         bigramsIt.next();
-        const int length = mDictionaryStructurePolicy->
+        if (bigramsIt.getBigramPos() == NOT_A_VALID_WORD_POS) {
+            continue;
+        }
+        const int codePointCount = mDictionaryStructurePolicy->
                 getCodePointsAndProbabilityAndReturnCodePointCount(bigramsIt.getBigramPos(),
                         MAX_WORD_LENGTH, bigramBuffer, &unigramProbability);
+        if (codePointCount <= 0) {
+            continue;
+        }
         // Due to space constraints, the probability for bigrams is approximate - the lower the
         // unigram probability, the worse the precision. The theoritical maximum error in
         // resulting probability is 8 - although in the practice it's never bigger than 3 or 4
@@ -127,8 +133,8 @@ int BigramDictionary::getPredictions(const int *prevWord, const int prevWordLeng
         // here, but it can't get too bad.
         const int probability = ProbabilityUtils::computeProbabilityForBigram(
                 unigramProbability, bigramsIt.getProbability());
-        addWordBigram(bigramBuffer, length, probability, outBigramProbability, outBigramCodePoints,
+        addWordBigram(bigramBuffer, codePointCount, probability, outBigramProbability,
-                outputTypes);
+                outBigramCodePoints, outputTypes);
         ++bigramCount;
     }
     return min(bigramCount, MAX_RESULTS);

native/jni/src/suggest/core/dictionary/multi_bigram_map.h

@@ -73,6 +73,9 @@ class MultiBigramMap {
                     bigramsListPos);
             while (bigramsIt.hasNext()) {
                 bigramsIt.next();
+                if (bigramsIt.getBigramPos() == NOT_A_VALID_WORD_POS) {
+                    continue;
+                }
                 mBigramMap[bigramsIt.getBigramPos()] = bigramsIt.getProbability();
                 mBloomFilter.setInFilter(bigramsIt.getBigramPos());
             }

native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_policy.h

@@ -33,16 +33,16 @@ class BigramListPolicy : public DictionaryBigramsStructurePolicy {
 
     void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext,
             int *const pos) const {
-        const BigramListReadingUtils::BigramFlags flags =
+        const BigramListReadWriteUtils::BigramFlags flags =
-                BigramListReadingUtils::getFlagsAndForwardPointer(mBigramsBuf, pos);
+                BigramListReadWriteUtils::getFlagsAndForwardPointer(mBigramsBuf, pos);
-        *outBigramPos = BigramListReadingUtils::getBigramAddressAndForwardPointer(
+        *outBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer(
                         mBigramsBuf, flags, pos);
-        *outProbability = BigramListReadingUtils::getProbabilityFromFlags(flags);
+        *outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(flags);
-        *outHasNext = BigramListReadingUtils::hasNext(flags);
+        *outHasNext = BigramListReadWriteUtils::hasNext(flags);
     }
 
     void skipAllBigrams(int *const pos) const {
-        BigramListReadingUtils::skipExistingBigrams(mBigramsBuf, pos);
+        BigramListReadWriteUtils::skipExistingBigrams(mBigramsBuf, pos);
     }
 
  private:

native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.cpp

@@ -20,24 +20,25 @@
 
 namespace latinime {
 
-const BigramListReadingUtils::BigramFlags BigramListReadingUtils::MASK_ATTRIBUTE_ADDRESS_TYPE =
+const BigramListReadWriteUtils::BigramFlags BigramListReadWriteUtils::MASK_ATTRIBUTE_ADDRESS_TYPE =
         0x30;
-const BigramListReadingUtils::BigramFlags
+const BigramListReadWriteUtils::BigramFlags
-        BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10;
+        BigramListReadWriteUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10;
-const BigramListReadingUtils::BigramFlags
+const BigramListReadWriteUtils::BigramFlags
-        BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20;
+        BigramListReadWriteUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20;
-const BigramListReadingUtils::BigramFlags
+const BigramListReadWriteUtils::BigramFlags
-        BigramListReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30;
+        BigramListReadWriteUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30;
-const BigramListReadingUtils::BigramFlags
+const BigramListReadWriteUtils::BigramFlags
-        BigramListReadingUtils::FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40;
+        BigramListReadWriteUtils::FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40;
 // Flag for presence of more attributes
-const BigramListReadingUtils::BigramFlags BigramListReadingUtils::FLAG_ATTRIBUTE_HAS_NEXT = 0x80;
+const BigramListReadWriteUtils::BigramFlags BigramListReadWriteUtils::FLAG_ATTRIBUTE_HAS_NEXT =
+        0x80;
 // Mask for attribute probability, stored on 4 bits inside the flags byte.
-const BigramListReadingUtils::BigramFlags
+const BigramListReadWriteUtils::BigramFlags
-        BigramListReadingUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F;
+        BigramListReadWriteUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F;
-const int BigramListReadingUtils::ATTRIBUTE_ADDRESS_SHIFT = 4;
+const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4;
 
-/* static */ int BigramListReadingUtils::getBigramAddressAndForwardPointer(
+/* static */ int BigramListReadWriteUtils::getBigramAddressAndForwardPointer(
         const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos) {
     int offset = 0;
     const int origin = *pos;
@@ -52,6 +53,9 @@ const int BigramListReadingUtils::ATTRIBUTE_ADDRESS_SHIFT = 4;
             offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos);
             break;
     }
+    if (offset == 0) {
+        return NOT_A_VALID_WORD_POS;
+    }
     if (isOffsetNegative(flags)) {
         return origin - offset;
     } else {

native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h

@@ -14,9 +14,10 @@
  * limitations under the License.
  */
 
-#ifndef LATINIME_BIGRAM_LIST_READING_UTILS_H
+#ifndef LATINIME_BIGRAM_LIST_READ_WRITE_UTILS_H
-#define LATINIME_BIGRAM_LIST_READING_UTILS_H
+#define LATINIME_BIGRAM_LIST_READ_WRITE_UTILS_H
 
+#include <cstdlib>
 #include <stdint.h>
 
 #include "defines.h"
@@ -24,7 +25,8 @@
 
 namespace latinime {
 
-class BigramListReadingUtils {
+// TODO: Change name of this file according to the class name.
+class BigramListReadWriteUtils {
 public:
    typedef uint8_t BigramFlags;
 
@@ -55,8 +57,62 @@ public:
    static int getBigramAddressAndForwardPointer(const uint8_t *const bigramsBuf,
            const BigramFlags flags, int *const pos);
 
+   // Returns the size of the bigram position field that is stored in bigram flags.
+   static AK_FORCE_INLINE int attributeAddressSize(const BigramFlags flags) {
+       return (flags & MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT;
+       /* Note: this is a value-dependant optimization of what may probably be
+          more readably written this way:
+          switch (flags * BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) {
+          case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: return 1;
+          case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: return 2;
+          case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTE: return 3;
+          default: return 0;
+          }
+       */
+   }
+
+   static AK_FORCE_INLINE BigramFlags setHasNextFlag(const BigramFlags flags) {
+       return flags | FLAG_ATTRIBUTE_HAS_NEXT;
+   }
+
+   // Returns true if the bigram entry is valid and put entry values into out*.
+   static AK_FORCE_INLINE bool createBigramEntryAndGetFlagsAndOffsetAndOffsetFieldSize(
+           const int entryPos, const int targetPos, const int probability, const bool hasNext,
+           BigramFlags *const outBigramFlags, uint32_t *const outOffset,
+           int *const outOffsetFieldSize) {
+       if (targetPos == NOT_A_VALID_WORD_POS) {
+           return false;
+       }
+       BigramFlags flags = probability & MASK_ATTRIBUTE_PROBABILITY;
+       if (hasNext) {
+           flags |= FLAG_ATTRIBUTE_HAS_NEXT;
+       }
+       const int targetFieldPos = entryPos + 1;
+       const int offset = targetPos - targetFieldPos;
+       if (offset < 0) {
+           flags |= FLAG_ATTRIBUTE_OFFSET_NEGATIVE;
+       }
+       const uint32_t absOffest = abs(offset);
+       if ((absOffest >> 24) != 0) {
+           // Offset is too large.
+           return false;
+       } else if ((absOffest >> 16) != 0) {
+           flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES;
+           *outOffsetFieldSize = 3;
+       } else if ((absOffest >> 8) != 0) {
+           flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES;
+           *outOffsetFieldSize = 2;
+       } else {
+           flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
+           *outOffsetFieldSize = 1;
+       }
+       *outBigramFlags = flags;
+       *outOffset = absOffest;
+       return true;
+   }
+
 private:
-   DISALLOW_IMPLICIT_CONSTRUCTORS(BigramListReadingUtils);
+   DISALLOW_IMPLICIT_CONSTRUCTORS(BigramListReadWriteUtils);
 
    static const BigramFlags MASK_ATTRIBUTE_ADDRESS_TYPE;
    static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
@@ -70,19 +126,6 @@ private:
    static AK_FORCE_INLINE bool isOffsetNegative(const BigramFlags flags) {
        return (flags & FLAG_ATTRIBUTE_OFFSET_NEGATIVE) != 0;
    }
-
-   static AK_FORCE_INLINE int attributeAddressSize(const BigramFlags flags) {
-       return (flags & MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT;
-       /* Note: this is a value-dependant optimization of what may probably be
-          more readably written this way:
-          switch (flags * BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) {
-          case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: return 1;
-          case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: return 2;
-          case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTE: return 3;
-          default: return 0;
-          }
-       */
-   }
 };
 } // namespace latinime
-#endif // LATINIME_BIGRAM_LIST_READING_UTILS_H
+#endif // LATINIME_BIGRAM_LIST_READ_WRITE_UTILS_H

native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp

@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h"
+
+namespace latinime {
+
+bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos) {
+    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos);
+    const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer);
+    if (usesAdditionalBuffer) {
+        *fromPos -= mBuffer->getOriginalBufferSize();
+    }
+    BigramListReadWriteUtils::BigramFlags flags;
+    do {
+        flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, fromPos);
+        int bigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer(
+                buffer, flags, fromPos);
+        if (bigramPos == NOT_A_VALID_WORD_POS) {
+            // skip invalid bigram entry.
+            continue;
+        }
+        if (usesAdditionalBuffer) {
+            bigramPos += mBuffer->getOriginalBufferSize();
+        }
+        BigramListReadWriteUtils::BigramFlags newBigramFlags;
+        uint32_t newBigramOffset;
+        int newBigramOffsetFieldSize;
+        if(!BigramListReadWriteUtils::createBigramEntryAndGetFlagsAndOffsetAndOffsetFieldSize(
+                *toPos, bigramPos, BigramListReadWriteUtils::getProbabilityFromFlags(flags),
+                BigramListReadWriteUtils::hasNext(flags), &newBigramFlags, &newBigramOffset,
+                &newBigramOffsetFieldSize)) {
+            continue;
+        }
+        // Write bigram entry. Target buffer is always the additional buffer.
+        if (!mBuffer->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */,toPos)) {
+            return false;
+        }
+        if (!mBuffer->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize,
+                toPos)) {
+            return false;
+        }
+    } while(BigramListReadWriteUtils::hasNext(flags));
+    if (usesAdditionalBuffer) {
+        *fromPos += mBuffer->getOriginalBufferSize();
+    }
+    return true;
+}
+
+bool DynamicBigramListPolicy::addBigramEntry(const int bigramPos, const int probability,
+        int *const pos) {
+    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos);
+    const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer);
+    if (usesAdditionalBuffer) {
+        *pos -= mBuffer->getOriginalBufferSize();
+    }
+    BigramListReadWriteUtils::BigramFlags flags;
+    do {
+        int entryPos = *pos;
+        if (usesAdditionalBuffer) {
+            entryPos += mBuffer->getOriginalBufferSize();
+        }
+        flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, pos);
+        BigramListReadWriteUtils::getBigramAddressAndForwardPointer(buffer, flags, pos);
+        if (BigramListReadWriteUtils::hasNext(flags)) {
+            continue;
+        }
+        // The current last entry is found.
+        // First, update the flags of the last entry.
+        const BigramListReadWriteUtils::BigramFlags updatedFlags =
+                BigramListReadWriteUtils::setHasNextFlag(flags);
+        if (!mBuffer->writeUintAndAdvancePosition(updatedFlags, 1 /* size */, &entryPos)) {
+            return false;
+        }
+        // Then, add a new entry after the last entry.
+        BigramListReadWriteUtils::BigramFlags newBigramFlags;
+        uint32_t newBigramOffset;
+        int newBigramOffsetFieldSize;
+        if(!BigramListReadWriteUtils::createBigramEntryAndGetFlagsAndOffsetAndOffsetFieldSize(
+                *pos, bigramPos, BigramListReadWriteUtils::getProbabilityFromFlags(flags),
+                BigramListReadWriteUtils::hasNext(flags), &newBigramFlags, &newBigramOffset,
+                &newBigramOffsetFieldSize)) {
+            continue;
+        }
+        int newEntryPos = *pos;
+        if (usesAdditionalBuffer) {
+            newEntryPos += mBuffer->getOriginalBufferSize();
+        }
+        // Write bigram flags.
+        if (!mBuffer->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */,
+                &newEntryPos)) {
+            return false;
+        }
+        // Write bigram positon offset.
+        if (!mBuffer->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize,
+                &newEntryPos)) {
+            return false;
+        }
+    } while(BigramListReadWriteUtils::hasNext(flags));
+    if (usesAdditionalBuffer) {
+        *pos += mBuffer->getOriginalBufferSize();
+    }
+    return true;
+}
+
+bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int targetBigramPos) {
+    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(bigramListPos);
+    const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer);
+    int pos = bigramListPos;
+    if (usesAdditionalBuffer) {
+        pos -= mBuffer->getOriginalBufferSize();
+    }
+    BigramListReadWriteUtils::BigramFlags flags;
+    do {
+        flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, &pos);
+        int bigramOffsetFieldPos = pos;
+        if (usesAdditionalBuffer) {
+            bigramOffsetFieldPos += mBuffer->getOriginalBufferSize();
+        }
+        int bigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer(
+                buffer, flags, &pos);
+        if (usesAdditionalBuffer && bigramPos != NOT_A_VALID_WORD_POS) {
+            bigramPos += mBuffer->getOriginalBufferSize();
+        }
+        if (bigramPos != targetBigramPos) {
+            continue;
+        }
+        // Target entry is found. Write 0 into the bigram pos field to mark the bigram invalid.
+        const int bigramOffsetFieldSize =
+                BigramListReadWriteUtils::attributeAddressSize(flags);
+        if (!mBuffer->writeUintAndAdvancePosition(0 /* data */, bigramOffsetFieldSize,
+                &bigramOffsetFieldPos)) {
+            return false;
+        }
+        return true;
+    } while(BigramListReadWriteUtils::hasNext(flags));
+    return false;
+}
+
+} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h

@@ -31,7 +31,7 @@ namespace latinime {
  */
 class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy {
  public:
-    DynamicBigramListPolicy(const BufferWithExtendableBuffer *const buffer)
+    DynamicBigramListPolicy(BufferWithExtendableBuffer *const buffer)
             : mBuffer(buffer) {}
 
     ~DynamicBigramListPolicy() {}
@@ -43,15 +43,15 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy {
         if (usesAdditionalBuffer) {
             *pos -= mBuffer->getOriginalBufferSize();
         }
-        const BigramListReadingUtils::BigramFlags flags =
+        const BigramListReadWriteUtils::BigramFlags flags =
-                BigramListReadingUtils::getFlagsAndForwardPointer(buffer, pos);
+                BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, pos);
-        *outBigramPos = BigramListReadingUtils::getBigramAddressAndForwardPointer(
+        *outBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer(
                 buffer, flags, pos);
-        if (usesAdditionalBuffer) {
+        if (usesAdditionalBuffer && *outBigramPos != NOT_A_VALID_WORD_POS) {
             *outBigramPos += mBuffer->getOriginalBufferSize();
         }
-        *outProbability = BigramListReadingUtils::getProbabilityFromFlags(flags);
+        *outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(flags);
-        *outHasNext = BigramListReadingUtils::hasNext(flags);
+        *outHasNext = BigramListReadWriteUtils::hasNext(flags);
         if (usesAdditionalBuffer) {
             *pos += mBuffer->getOriginalBufferSize();
         }
@@ -63,16 +63,25 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy {
         if (usesAdditionalBuffer) {
             *pos -= mBuffer->getOriginalBufferSize();
         }
-        BigramListReadingUtils::skipExistingBigrams(buffer, pos);
+        BigramListReadWriteUtils::skipExistingBigrams(buffer, pos);
         if (usesAdditionalBuffer) {
             *pos += mBuffer->getOriginalBufferSize();
         }
     }
 
+    // Copy bigrams from the bigram list that starts at fromPos to toPos and advance these
+    // positions after bigram lists. This method skips invalid bigram entries.
+    bool copyAllBigrams(int *const fromPos, int *const toPos);
+
+    bool addBigramEntry(const int bigramPos, const int probability, int *const pos);
+
+    // Return if targetBigramPos is found or not.
+    bool removeBigram(const int bigramListPos, const int targetBigramPos);
+
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicBigramListPolicy);
 
-    const BufferWithExtendableBuffer *const mBuffer;
+    BufferWithExtendableBuffer *const mBuffer;
 };
 } // namespace latinime
 #endif // LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H