Merge "Implement SparseTable in native code."

native/jni/Android.mk

@@ -96,7 +96,8 @@ LATIN_IME_CORE_SRC_FILES := \
         dict_file_writing_utils.cpp \
         forgetting_curve_utils.cpp \
         format_utils.cpp \
-        mmapped_buffer.cpp) \
+        mmapped_buffer.cpp \
+        sparse_table.cpp) \
     suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \
     $(addprefix suggest/policyimpl/typing/, \
         scoring_params.cpp \

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h

@@ -21,6 +21,7 @@
 #include "suggest/policyimpl/dictionary/structure/v4/content/dict_content.h"
 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
 #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
+#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
 
 namespace latinime {
 
@@ -29,7 +30,8 @@ class SparseTableDictContent : public DictContent {
  public:
     AK_FORCE_INLINE SparseTableDictContent(const char *const dictDirPath,
             const char *const lookupTableFileName, const char *const addressTableFileName,
-            const char *const contentFileName, const bool isUpdatable)
+            const char *const contentFileName, const bool isUpdatable,
+            const int sparseTableBlockSize, const int sparseTableDataSize)
             : mLookupTableBuffer(
                       MmappedBuffer::openBuffer(dictDirPath, lookupTableFileName, isUpdatable)),
               mAddressTableBuffer(
@@ -45,7 +47,9 @@ class SparseTableDictContent : public DictContent {
                       BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
               mExpandableContentBuffer(mContentBuffer.get() ? mContentBuffer.get()->getBuffer() : 0,
                       mContentBuffer.get() ? mContentBuffer.get()->getBufferSize() : 0,
-                      BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE) {}
+                      BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE),
+              mAddressLookupTable(&mExpandableLookupTableBuffer, &mExpandableAddressTableBuffer,
+                      sparseTableBlockSize, sparseTableDataSize) {}
 
     virtual ~SparseTableDictContent() {}
 
@@ -64,6 +68,7 @@ class SparseTableDictContent : public DictContent {
     BufferWithExtendableBuffer mExpandableLookupTableBuffer;
     BufferWithExtendableBuffer mExpandableAddressTableBuffer;
     BufferWithExtendableBuffer mExpandableContentBuffer;
+    SparseTable mAddressLookupTable;
 };
 } // namespace latinime
 #endif /* LATINIME_SPARSE_TABLE_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h

@@ -67,11 +67,15 @@ class Ver4DictBuffers {
               mBigramDictContent(dictDirPath,
                       Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
                       Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
-                      Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable),
+                      Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable,
+                      Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE,
+                      Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE),
               mShortcutDictContent(dictDirPath,
                       Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
                       Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
-                      Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable) {}
+                      Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable,
+                      Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE,
+                      Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE) {}
 
     const MmappedBuffer::MmappedBufferPtr mDictBuffer;
     SingleDictContent mTerminalAddressTable;

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp

@@ -34,4 +34,9 @@ const int Ver4DictConstants::NOT_A_TERMINAL_ID = -1;
 const int Ver4DictConstants::PROBABILITY_SIZE = 1;
 const int Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE = 1;
 
+const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4;
+const int Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE = 4;
+const int Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE = 16;
+const int Ver4DictConstants::SHORTCUT_ADDRESS_TABLE_DATA_SIZE = 4;
+
 } // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h

@@ -38,6 +38,10 @@ class Ver4DictConstants {
     static const int PROBABILITY_SIZE;
     static const int FLAGS_IN_PROBABILITY_FILE_SIZE;
 
+    static const int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE;
+    static const int BIGRAM_ADDRESS_TABLE_DATA_SIZE;
+    static const int SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE;
+    static const int SHORTCUT_ADDRESS_TABLE_DATA_SIZE;
  private:
     DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants);
 };

native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp

@@ -23,17 +23,16 @@ const int BufferWithExtendableBuffer::NEAR_BUFFER_LIMIT_THRESHOLD_PERCENTILE = 9
 // TODO: Needs to allocate larger memory corresponding to the current vector size.
 const size_t BufferWithExtendableBuffer::EXTEND_ADDITIONAL_BUFFER_SIZE_STEP = 128 * 1024;
 
+uint32_t BufferWithExtendableBuffer::readUint(const int size, const int pos) const {
+    const bool readingPosIsInAdditionalBuffer = isInAdditionalBuffer(pos);
+    const int posInBuffer = readingPosIsInAdditionalBuffer ? pos - mOriginalBufferSize : pos;
+    return ByteArrayUtils::readUint(getBuffer(readingPosIsInAdditionalBuffer), size, posInBuffer);
+}
+
 uint32_t BufferWithExtendableBuffer::readUintAndAdvancePosition(const int size,
         int *const pos) const {
-    const bool readingPosIsInAdditionalBuffer = isInAdditionalBuffer(*pos);
+    const int value = readUint(size, *pos);
-    if (readingPosIsInAdditionalBuffer) {
+    *pos += size;
-        *pos -= mOriginalBufferSize;
-    }
-    const int value = ByteArrayUtils::readUintAndAdvancePosition(
-            getBuffer(readingPosIsInAdditionalBuffer), size, pos);
-    if (readingPosIsInAdditionalBuffer) {
-        *pos += mOriginalBufferSize;
-    }
     return value;
 }
 

native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h

@@ -71,6 +71,8 @@ class BufferWithExtendableBuffer {
         }
     }
 
+    uint32_t readUint(const int size, const int pos) const;
+
     uint32_t readUintAndAdvancePosition(const int size, int *const pos) const;
 
     AK_FORCE_INLINE int getOriginalBufferSize() const {

native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h

@@ -114,19 +114,19 @@ class ByteArrayUtils {
         return buffer[(*pos)++];
     }
 
-    static AK_FORCE_INLINE int readUintAndAdvancePosition(const uint8_t *const buffer,
+    static AK_FORCE_INLINE int readUint(const uint8_t *const buffer,
-            const int size, int *const pos) {
+            const int size, const int pos) {
         // size must be in 1 to 4.
         ASSERT(size >= 1 && size <= 4);
         switch (size) {
             case 1:
-                return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos);
+                return ByteArrayUtils::readUint8(buffer, pos);
             case 2:
-                return ByteArrayUtils::readUint16AndAdvancePosition(buffer, pos);
+                return ByteArrayUtils::readUint16(buffer, pos);
             case 3:
-                return ByteArrayUtils::readUint24AndAdvancePosition(buffer, pos);
+                return ByteArrayUtils::readUint24(buffer, pos);
             case 4:
-                return ByteArrayUtils::readUint32AndAdvancePosition(buffer, pos);
+                return ByteArrayUtils::readUint32(buffer, pos);
             default:
                 return 0;
         }

native/jni/src/suggest/policyimpl/dictionary/utils/sparse_table.cpp

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "suggest/policyimpl/dictionary/utils/sparse_table.h"
+
+namespace latinime {
+
+const int SparseTable::NOT_EXIST = -1;
+
+bool SparseTable::contains(const int id) const {
+    if (id < 0 || mIndexTableBuffer->getTailPosition() <= id * mDataSize) {
+        return false;
+    }
+    const int indexTableIndex = id / mBlockSize;
+    const int readingPos = indexTableIndex * mDataSize;
+    const int index = mIndexTableBuffer->readUint(mDataSize, readingPos);
+    return index != NOT_EXIST;
+}
+
+uint32_t SparseTable::get(const int id) const {
+    const int indexTableIndex = id / mBlockSize;
+    int readingPos = indexTableIndex * mDataSize;
+    const int index = mIndexTableBuffer->readUint(mDataSize, readingPos);
+    const int offset = id % mBlockSize;
+    readingPos = (index * mDataSize + offset) * mBlockSize;
+    return mContentTableBuffer->readUint(mDataSize, readingPos);
+}
+
+} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/utils/sparse_table.h

@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LATINIME_SPARSE_TABLE_H
+#define LATINIME_SPARSE_TABLE_H
+
+#include <stdint.h>
+
+#include "defines.h"
+#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
+
+namespace latinime {
+
+// Note that there is a corresponding implementation in SparseTable.java.
+// TODO: Support multiple content buffers.
+class SparseTable {
+ public:
+    SparseTable(BufferWithExtendableBuffer *const indexTableBuffer,
+            BufferWithExtendableBuffer *const contentTableBuffer, const int blockSize,
+            const int dataSize)
+            : mIndexTableBuffer(indexTableBuffer), mContentTableBuffer(contentTableBuffer),
+              mBlockSize(blockSize), mDataSize(dataSize) {}
+
+    bool contains(const int id) const;
+
+    uint32_t get(const int id) const;
+
+ private:
+    DISALLOW_IMPLICIT_CONSTRUCTORS(SparseTable);
+
+    static const int NOT_EXIST;
+
+    BufferWithExtendableBuffer *const mIndexTableBuffer;
+    BufferWithExtendableBuffer *const mContentTableBuffer;
+    const int mBlockSize;
+    const int mDataSize;
+};
+} // namespace latinime
+#endif /* LATINIME_SPARSE_TABLE_H */