Save language model in the body buffer.

Bug: 14425059
Change-Id: Iaec277f7bed03d6c6780c6ce90fbe5fe799e175e
main
Keisuke Kuroyanagi 2014-08-01 20:19:16 +09:00
parent c0c674cdc0
commit c4696b2eb6
6 changed files with 56 additions and 3 deletions

View File

@ -72,6 +72,7 @@ LATIN_IME_CORE_SRC_FILES := \
ver4_pt_node_array_reader.cpp) \ ver4_pt_node_array_reader.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \ $(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \
bigram_dict_content.cpp \ bigram_dict_content.cpp \
language_model_dict_content.cpp \
probability_dict_content.cpp \ probability_dict_content.cpp \
shortcut_dict_content.cpp \ shortcut_dict_content.cpp \
sparse_table_dict_content.cpp \ sparse_table_dict_content.cpp \

View File

@ -0,0 +1,25 @@
/*
* Copyright (C) 2014, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h"
namespace latinime {
bool LanguageModelDictContent::save(FILE *const file) const {
return mTrieMap.save(file);
}
} // namespace latinime

View File

@ -17,16 +17,28 @@
#ifndef LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H #ifndef LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H
#define LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H #define LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H
#include <cstdio>
#include "defines.h" #include "defines.h"
#include "suggest/policyimpl/dictionary/utils/trie_map.h"
#include "utils/byte_array_view.h"
namespace latinime { namespace latinime {
class LanguageModelDictContent { class LanguageModelDictContent {
public: public:
explicit LanguageModelDictContent(const bool hasHistoricalInfo) {} LanguageModelDictContent(const ReadWriteByteArrayView trieMapBuffer,
const bool hasHistoricalInfo)
: mTrieMap(trieMapBuffer) {}
explicit LanguageModelDictContent(const bool hasHistoricalInfo) : mTrieMap() {}
bool save(FILE *const file) const;
private: private:
DISALLOW_COPY_AND_ASSIGN(LanguageModelDictContent); DISALLOW_COPY_AND_ASSIGN(LanguageModelDictContent);
TrieMap mTrieMap;
}; };
} // namespace latinime } // namespace latinime
#endif /* LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H */ #endif /* LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H */

View File

@ -162,6 +162,11 @@ bool Ver4DictBuffers::flushDictBuffers(FILE *const file) const {
AKLOGE("Probability dict content cannot be written."); AKLOGE("Probability dict content cannot be written.");
return false; return false;
} }
// Write language model content.
if (!mLanguageModelDictContent.save(file)) {
AKLOGE("Language model dict content cannot be written.");
return false;
}
// Write bigram dict content. // Write bigram dict content.
if (!mBigramDictContent.flushToFile(file)) { if (!mBigramDictContent.flushToFile(file)) {
AKLOGE("Bigram dict content cannot be written."); AKLOGE("Bigram dict content cannot be written.");
@ -195,7 +200,11 @@ Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer,
contentBuffers[Ver4DictConstants::PROBABILITY_BUFFER_INDEX], contentBuffers[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::PROBABILITY_BUFFER_INDEX], contentBufferSizes[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],
mHeaderPolicy.hasHistoricalInfoOfWords()), mHeaderPolicy.hasHistoricalInfoOfWords()),
mLanguageModelDictContent(mHeaderPolicy.hasHistoricalInfoOfWords()), mLanguageModelDictContent(
ReadWriteByteArrayView(
contentBuffers[Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX]),
mHeaderPolicy.hasHistoricalInfoOfWords()),
mBigramDictContent(&contentBuffers[Ver4DictConstants::BIGRAM_BUFFERS_INDEX], mBigramDictContent(&contentBuffers[Ver4DictConstants::BIGRAM_BUFFERS_INDEX],
&contentBufferSizes[Ver4DictConstants::BIGRAM_BUFFERS_INDEX], &contentBufferSizes[Ver4DictConstants::BIGRAM_BUFFERS_INDEX],
mHeaderPolicy.hasHistoricalInfoOfWords()), mHeaderPolicy.hasHistoricalInfoOfWords()),

View File

@ -31,14 +31,17 @@ const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;
// NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for bigram and shortcut. // NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for bigram and shortcut.
const size_t Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE = const size_t Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE =
NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 3 NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 3
+ NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT
+ NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT * 2; + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT * 2;
const int Ver4DictConstants::TRIE_BUFFER_INDEX = 0; const int Ver4DictConstants::TRIE_BUFFER_INDEX = 0;
const int Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX = const int Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX =
TRIE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; TRIE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::PROBABILITY_BUFFER_INDEX = const int Ver4DictConstants::PROBABILITY_BUFFER_INDEX =
TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX = const int Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX =
PROBABILITY_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; PROBABILITY_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX =
LANGUAGE_MODEL_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT;
const int Ver4DictConstants::SHORTCUT_BUFFERS_INDEX = const int Ver4DictConstants::SHORTCUT_BUFFERS_INDEX =
BIGRAM_BUFFERS_INDEX + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT; BIGRAM_BUFFERS_INDEX + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT;
@ -73,5 +76,6 @@ const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80;
const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT = 1; const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT = 1;
const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT = 3; const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT = 3;
const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT = 1;
} // namespace latinime } // namespace latinime

View File

@ -36,6 +36,7 @@ class Ver4DictConstants {
static const int TRIE_BUFFER_INDEX; static const int TRIE_BUFFER_INDEX;
static const int TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX; static const int TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX;
static const int PROBABILITY_BUFFER_INDEX; static const int PROBABILITY_BUFFER_INDEX;
static const int LANGUAGE_MODEL_BUFFER_INDEX;
static const int BIGRAM_BUFFERS_INDEX; static const int BIGRAM_BUFFERS_INDEX;
static const int SHORTCUT_BUFFERS_INDEX; static const int SHORTCUT_BUFFERS_INDEX;
@ -71,6 +72,7 @@ class Ver4DictConstants {
static const size_t NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT; static const size_t NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
static const size_t NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT; static const size_t NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT;
static const size_t NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT;
}; };
} // namespace latinime } // namespace latinime
#endif /* LATINIME_VER4_DICT_CONSTANTS_H */ #endif /* LATINIME_VER4_DICT_CONSTANTS_H */