nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Introduce Ver4DictBuffers to handle multiple buffers.

Browse source 
Bug: 11073222
Change-Id: Icacca8133c0030b19ca826fbf9525f41a1ac6cc8
This commit is contained in:
Keisuke Kuroyanagi 2013-10-24 23:45:01 -07:00
parent 1e34cc1698
commit 75d7f0fbf5
6 changed files with 185 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
native/jni/Android.mk
View file

@ -85,7 +85,9 @@ LATIN_IME_CORE_SRC_FILES := \
dynamic_patricia_trie_reading_utils.cpp \
dynamic_patricia_trie_writing_helper.cpp \
dynamic_patricia_trie_writing_utils.cpp) \
suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp \
$(addprefix suggest/policyimpl/dictionary/structure/v4/, \
ver4_dict_constants.cpp \
ver4_patricia_trie_policy.cpp) \
$(addprefix suggest/policyimpl/dictionary/utils/, \
buffer_with_extendable_buffer.cpp \
byte_array_utils.cpp \

26
native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp
View file

@ -17,10 +17,13 @@
#include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h"
#include <stdint.h>
#include <string>
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
@ -45,9 +48,26 @@ namespace latinime {
case FormatUtils::VERSION_3:
return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr(
new DynamicPatriciaTriePolicy(mmappedBuffer));
case FormatUtils::VERSION_4:
// TODO: Support version 4 dictionary format.
// Fall through.
case FormatUtils::VERSION_4: {
std::string dictDirPath(path);
const std::string::size_type pos =
dictDirPath.rfind(Ver4DictConstants::TRIE_FILE_EXTENSION);
if (pos == std::string::npos) {
// Dictionary file name is not valid as a version 4 dictionary.
return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr(0);
}
// Removing extension to get the base path.
dictDirPath.erase(pos);
const Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(
Ver4DictBuffers::openVer4DictBuffers(dictDirPath.c_str(), mmappedBuffer));
if (!dictBuffers.get()->isValid()) {
AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements.");
ASSERT(false);
return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr(0);
}
return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr(
new Ver4PatriciaTriePolicy(dictBuffers));
}
default:
AKLOGE("DICT: dictionary format is unknown, bad magic number");
ASSERT(false);

75
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h Normal file
View file

@ -0,0 +1,75 @@
/*
* Copyright (C) 2013, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_VER4_DICT_BUFFER_H
#define LATINIME_VER4_DICT_BUFFER_H
#include "defines.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h"
namespace latinime {
class Ver4DictBuffers {
public:
typedef ExclusiveOwnershipPointer<Ver4DictBuffers> Ver4DictBuffersPtr;
static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr &dictBuffer) {
const bool isUpdatable = dictBuffer.get() ? dictBuffer.get()->isUpdatable() : false;
return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable));
}
AK_FORCE_INLINE bool isValid() const {
return mDictBuffer.get() != 0 && mProbabilityDictContent.isValid()
&& mTerminalAddressTable.isValid() && mBigramDictContent.isValid()
&& mShortcutDictContent.isValid();
}
AK_FORCE_INLINE const uint8_t *getRawDictBuffer() const {
return mDictBuffer.get()->getBuffer();
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictBuffers);
AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath,
const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable)
: mDictBuffer(dictBuffer),
mTerminalAddressTable(dictDirPath,
Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable),
mProbabilityDictContent(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION,
isUpdatable),
mBigramDictContent(dictDirPath,
Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable),
mShortcutDictContent(dictDirPath,
Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION,
Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable) {}
const MmappedBuffer::MmappedBufferPtr mDictBuffer;
SingleDictContent mTerminalAddressTable;
SingleDictContent mProbabilityDictContent;
SparseTableDictContent mBigramDictContent;
SparseTableDictContent mShortcutDictContent;
};
} // namespace latinime
#endif /* LATINIME_VER4_DICT_BUFFER_H */

33
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp Normal file
View file

@ -0,0 +1,33 @@
/*
* Copyright (C) 2013, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h"
namespace latinime {
const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie";
const char *const Ver4DictConstants::FREQ_FILE_EXTENSION = ".freq";
// tat = Terminal Address Table
const char *const Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat";
const char *const Ver4DictConstants::BIGRAM_FILE_EXTENSION = ".bigram_freq";
const char *const Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION = ".bigram_lookup";
const char *const Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION = ".bigram_index_freq";
const char *const Ver4DictConstants::SHORTCUT_FILE_EXTENSION = ".shortcut_shortcut";
const char *const Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION = ".shortcut_lookup";
const char *const Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION =
".shortcut_index_shortcut";
} // namespace latinime

41
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h Normal file
View file

@ -0,0 +1,41 @@
/*
* Copyright (C) 2013, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LATINIME_VER4_DICT_CONSTANTS_H
#define LATINIME_VER4_DICT_CONSTANTS_H
#include "defines.h"
namespace latinime {
// Note that there are corresponding definitions in FormatSpec.java.
class Ver4DictConstants {
public:
static const char *const TRIE_FILE_EXTENSION;
static const char *const FREQ_FILE_EXTENSION;
static const char *const TERMINAL_ADDRESS_TABLE_FILE_EXTENSION;
static const char *const BIGRAM_FILE_EXTENSION;
static const char *const BIGRAM_LOOKUP_TABLE_FILE_EXTENSION;
static const char *const BIGRAM_CONTENT_TABLE_FILE_EXTENSION;
static const char *const SHORTCUT_FILE_EXTENSION;
static const char *const SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION;
static const char *const SHORTCUT_CONTENT_TABLE_FILE_EXTENSION;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4DictConstants);
};
} // namespace latinime
#endif /* LATINIME_VER4_DICT_CONSTANTS_H */

12
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h
View file

@ -19,6 +19,9 @@
#include "defines.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "suggest/policyimpl/dictionary/header/header_policy.h"
#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h"
#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
namespace latinime {
@ -28,7 +31,9 @@ class DicNodeVector;
// TODO: Implement.
class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
public:
~Ver4PatriciaTriePolicy() {}
Ver4PatriciaTriePolicy(const Ver4DictBuffers::Ver4DictBuffersPtr &buffers)
: mBuffers(buffers),
mHeaderPolicy(mBuffers.get()->getRawDictBuffer(), FormatUtils::VERSION_4) {};
AK_FORCE_INLINE int getRootPosition() const {
return 0;
@ -53,7 +58,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
int getBigramsPositionOfPtNode(const int ptNodePos) const;
const DictionaryHeaderStructurePolicy *getHeaderStructurePolicy() const {
return 0;
return &mHeaderPolicy;
}
const DictionaryBigramsStructurePolicy *getBigramsStructurePolicy() const {
@ -83,6 +88,9 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTriePolicy);
const Ver4DictBuffers::Ver4DictBuffersPtr mBuffers;
const HeaderPolicy mHeaderPolicy;
};
} // namespace latinime
#endif // LATINIME_VER4_PATRICIA_TRIE_POLICY_H