nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Implement getHeaderInfoNative."

Browse source
This commit is contained in:
Keisuke Kuroyanagi 2014-02-06 11:26:48 +00:00 committed by Android (Google) Code Review
commit fc6d0f8738
12 changed files with 103 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View file

@ -217,9 +217,8 @@ public final class BinaryDictionary extends Dictionary {
outAttributeValues.get(i));
attributes.put(attributeKey, attributeValue);
}
final boolean hasHistoricalInfo =
attributes.get(DictionaryHeader.HAS_HISTORICAL_INFO_KEY).equals(
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
final boolean hasHistoricalInfo = DictionaryHeader.ATTRIBUTE_VALUE_TRUE.equals(
attributes.get(DictionaryHeader.HAS_HISTORICAL_INFO_KEY));
return new DictionaryHeader(outHeaderSize[0], new DictionaryOptions(attributes),
new FormatSpec.FormatOptions(outFormatVersion[0], hasHistoricalInfo));
}

11
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View file

@ -23,10 +23,12 @@ import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.makedict.DictionaryHeader;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.WordProperty;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.utils.AsyncResultHolder;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.CombinedFormatUtils;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LanguageModelParam;
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
@ -785,7 +787,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
getExecutor(mDictName).execute(new Runnable() {
@Override
public void run() {
Log.d(TAG, "dictionary=" + mDictName);
Log.d(TAG, "Dump dictionary: " + mDictName);
try {
final DictionaryHeader header = mBinaryDictionary.getHeader();
Log.d(TAG, CombinedFormatUtils.formatAttributeMap(
header.mDictionaryOptions.mAttributes));
} catch (final UnsupportedFormatException e) {
Log.d(TAG, "Cannot fetch header information.", e);
}
int token = 0;
do {
final BinaryDictionary.GetNextWordPropertyResult result =

6
java/src/com/android/inputmethod/latin/utils/CombinedFormatUtils.java
View file

@ -37,11 +37,11 @@ public class CombinedFormatUtils {
public static String formatAttributeMap(final HashMap<String, String> attributeMap) {
final StringBuilder builder = new StringBuilder();
builder.append(DICTIONARY_TAG + "=");
if (attributeMap.containsKey(DictionaryHeader.DICTIONARY_DESCRIPTION_KEY)) {
builder.append(attributeMap.get(DictionaryHeader.DICTIONARY_DESCRIPTION_KEY));
if (attributeMap.containsKey(DictionaryHeader.DICTIONARY_ID_KEY)) {
builder.append(attributeMap.get(DictionaryHeader.DICTIONARY_ID_KEY));
}
for (final String key : attributeMap.keySet()) {
if (key == DictionaryHeader.DICTIONARY_DESCRIPTION_KEY) {
if (key.equals(DictionaryHeader.DICTIONARY_ID_KEY)) {
continue;
}
final String value = attributeMap.get(key);

38
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View file

@ -53,7 +53,7 @@ static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclas
return false;
}
HeaderReadWriteUtils::AttributeMap attributeMap;
DictionaryHeaderStructurePolicy::AttributeMap attributeMap;
for (int i = 0; i < keyCount; i++) {
jstring keyString = static_cast<jstring>(
env->GetObjectArrayElement(attributeKeyStringArray, i));
@ -61,7 +61,7 @@ static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclas
char keyChars[keyUtf8Length + 1];
env->GetStringUTFRegion(keyString, 0, env->GetStringLength(keyString), keyChars);
keyChars[keyUtf8Length] = '\0';
HeaderReadWriteUtils::AttributeMap::key_type key;
DictionaryHeaderStructurePolicy::AttributeMap::key_type key;
HeaderReadWriteUtils::insertCharactersIntoVector(keyChars, &key);
jstring valueString = static_cast<jstring>(
@ -70,7 +70,7 @@ static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclas
char valueChars[valueUtf8Length + 1];
env->GetStringUTFRegion(valueString, 0, env->GetStringLength(valueString), valueChars);
valueChars[valueUtf8Length] = '\0';
HeaderReadWriteUtils::AttributeMap::mapped_type value;
DictionaryHeaderStructurePolicy::AttributeMap::mapped_type value;
HeaderReadWriteUtils::insertCharactersIntoVector(valueChars, &value);
attributeMap[key] = value;
}
@ -145,16 +145,42 @@ static void latinime_BinaryDictionary_getHeaderInfo(JNIEnv *env, jclass clazz, j
jobject outAttributeValues) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return;
const int formatVersion = dictionary->getFormatVersionNumber();
const DictionaryHeaderStructurePolicy *const headerPolicy =
dictionary->getDictionaryStructurePolicy()->getHeaderStructurePolicy();
const int headerSize = headerPolicy->getSize();
env->SetIntArrayRegion(outHeaderSize, 0 /* start */, 1 /* len */, &headerSize);
const int formatVersion = headerPolicy->getFormatVersionNumber();
env->SetIntArrayRegion(outFormatVersion, 0 /* start */, 1 /* len */, &formatVersion);
// TODO: Implement
// Output attribute map
jclass arrayListClass = env->FindClass("java/util/ArrayList");
jmethodID addMethodId = env->GetMethodID(arrayListClass, "add", "(Ljava/lang/Object;)Z");
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap =
headerPolicy->getAttributeMap();
for (DictionaryHeaderStructurePolicy::AttributeMap::const_iterator it = attributeMap->begin();
it != attributeMap->end(); ++it) {
// Output key
jintArray keyCodePointArray = env->NewIntArray(it->first.size());
env->SetIntArrayRegion(
keyCodePointArray, 0 /* start */, it->first.size(), &it->first.at(0));
env->CallVoidMethod(outAttributeKeys, addMethodId, keyCodePointArray);
env->DeleteLocalRef(keyCodePointArray);
// Output value
jintArray valueCodePointArray = env->NewIntArray(it->second.size());
env->SetIntArrayRegion(
valueCodePointArray, 0 /* start */, it->second.size(), &it->second.at(0));
env->CallVoidMethod(outAttributeValues, addMethodId, valueCodePointArray);
env->DeleteLocalRef(valueCodePointArray);
}
env->DeleteLocalRef(arrayListClass);
return;
}
static int latinime_BinaryDictionary_getFormatVersion(JNIEnv *env, jclass clazz, jlong dict) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return 0;
return dictionary->getFormatVersionNumber();
const DictionaryHeaderStructurePolicy *const headerPolicy =
dictionary->getDictionaryStructurePolicy()->getHeaderStructurePolicy();
return headerPolicy->getFormatVersionNumber();
}
static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, jlong dict,

5
native/jni/src/suggest/core/dictionary/dictionary.h
View file

@ -105,11 +105,6 @@ class Dictionary {
return mDictionaryStructureWithBufferPolicy.get();
}
int getFormatVersionNumber() const {
return mDictionaryStructureWithBufferPolicy.get()->getHeaderStructurePolicy()
->getFormatVersionNumber();
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);

9
native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h
View file

@ -17,6 +17,9 @@
#ifndef LATINIME_DICTIONARY_HEADER_STRUCTURE_POLICY_H
#define LATINIME_DICTIONARY_HEADER_STRUCTURE_POLICY_H
#include <map>
#include <vector>
#include "defines.h"
namespace latinime {
@ -27,10 +30,16 @@ namespace latinime {
*/
class DictionaryHeaderStructurePolicy {
public:
typedef std::map<std::vector<int>, std::vector<int> > AttributeMap;
virtual ~DictionaryHeaderStructurePolicy() {}
virtual int getFormatVersionNumber() const = 0;
virtual int getSize() const = 0;
virtual const AttributeMap *getAttributeMap() const = 0;
virtual bool requiresGermanUmlautProcessing() const = 0;
virtual float getMultiWordCostMultiplier() const = 0;

11
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
View file

@ -46,7 +46,8 @@ void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *out
}
std::vector<int> keyCodePointVector;
HeaderReadWriteUtils::insertCharactersIntoVector(key, &keyCodePointVector);
HeaderReadWriteUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyCodePointVector);
DictionaryHeaderStructurePolicy::AttributeMap::const_iterator it =
mAttributeMap.find(keyCodePointVector);
if (it == mAttributeMap.end()) {
// The key was not found.
outValue[0] = '?';
@ -82,7 +83,7 @@ bool HeaderPolicy::fillInAndWriteHeaderToBuffer(const bool updatesLastDecayedTim
const int unigramCount, const int bigramCount,
const int extendedRegionSize, BufferWithExtendableBuffer *const outBuffer) const {
int writingPos = 0;
HeaderReadWriteUtils::AttributeMap attributeMapToWrite(mAttributeMap);
DictionaryHeaderStructurePolicy::AttributeMap attributeMapToWrite(mAttributeMap);
fillInHeader(updatesLastDecayedTime, unigramCount, bigramCount,
extendedRegionSize, &attributeMapToWrite);
if (!HeaderReadWriteUtils::writeDictionaryVersion(outBuffer, mDictFormatVersion,
@ -113,7 +114,7 @@ bool HeaderPolicy::fillInAndWriteHeaderToBuffer(const bool updatesLastDecayedTim
void HeaderPolicy::fillInHeader(const bool updatesLastDecayedTime, const int unigramCount,
const int bigramCount, const int extendedRegionSize,
HeaderReadWriteUtils::AttributeMap *outAttributeMap) const {
DictionaryHeaderStructurePolicy::AttributeMap *outAttributeMap) const {
HeaderReadWriteUtils::setIntAttribute(outAttributeMap, UNIGRAM_COUNT_KEY, unigramCount);
HeaderReadWriteUtils::setIntAttribute(outAttributeMap, BIGRAM_COUNT_KEY, bigramCount);
HeaderReadWriteUtils::setIntAttribute(outAttributeMap, EXTENDED_REGION_SIZE_KEY,
@ -129,9 +130,9 @@ void HeaderPolicy::fillInHeader(const bool updatesLastDecayedTime, const int uni
}
}
/* static */ HeaderReadWriteUtils::AttributeMap
/* static */ DictionaryHeaderStructurePolicy::AttributeMap
HeaderPolicy::createAttributeMapAndReadAllAttributes(const uint8_t *const dictBuf) {
HeaderReadWriteUtils::AttributeMap attributeMap;
DictionaryHeaderStructurePolicy::AttributeMap attributeMap;
HeaderReadWriteUtils::fetchAllHeaderAttributes(dictBuf, &attributeMap);
return attributeMap;
}

12
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View file

@ -57,7 +57,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
// Constructs header information using an attribute map.
HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion,
const std::vector<int> locale,
const HeaderReadWriteUtils::AttributeMap *const attributeMap)
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap)
: mDictFormatVersion(dictFormatVersion),
mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
attributeMap)), mSize(0), mAttributeMap(*attributeMap), mLocale(locale),
@ -155,6 +155,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
return !isDecayingDict();
}
const DictionaryHeaderStructurePolicy::AttributeMap *getAttributeMap() const {
return &mAttributeMap;
}
void readHeaderValueOrQuestionMark(const char *const key,
int *outValue, int outValueSize) const;
@ -164,7 +168,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
void fillInHeader(const bool updatesLastDecayedTime,
const int unigramCount, const int bigramCount, const int extendedRegionSize,
HeaderReadWriteUtils::AttributeMap *outAttributeMap) const;
DictionaryHeaderStructurePolicy::AttributeMap *outAttributeMap) const;
private:
DISALLOW_COPY_AND_ASSIGN(HeaderPolicy);
@ -185,7 +189,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const FormatUtils::FORMAT_VERSION mDictFormatVersion;
const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags;
const int mSize;
HeaderReadWriteUtils::AttributeMap mAttributeMap;
DictionaryHeaderStructurePolicy::AttributeMap mAttributeMap;
const std::vector<int> mLocale;
const float mMultiWordCostMultiplier;
const bool mRequiresGermanUmlautProcessing;
@ -201,7 +205,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
float readMultipleWordCostMultiplier() const;
bool readRequiresGermanUmlautProcessing() const;
static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes(
static DictionaryHeaderStructurePolicy::AttributeMap createAttributeMapAndReadAllAttributes(
const uint8_t *const dictBuf);
};
} // namespace latinime

4
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
View file

@ -36,6 +36,8 @@ const int HeaderReadWriteUtils::HEADER_SIZE_FIELD_SIZE = 4;
const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
typedef DictionaryHeaderStructurePolicy::AttributeMap AttributeMap;
/* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) {
// See the format of the header in the comment in
// BinaryDictionaryFormatUtils::detectFormatVersion()
@ -51,7 +53,7 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
/* static */ HeaderReadWriteUtils::DictionaryFlags
HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
const AttributeMap *const attributeMap) {
return NO_FLAGS;
}

43
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h
View file

@ -17,11 +17,10 @@
#ifndef LATINIME_HEADER_READ_WRITE_UTILS_H
#define LATINIME_HEADER_READ_WRITE_UTILS_H
#include <map>
#include <stdint.h>
#include <vector>
#include "defines.h"
#include "suggest/core/policy/dictionary_header_structure_policy.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
namespace latinime {
@ -31,7 +30,6 @@ class BufferWithExtendableBuffer;
class HeaderReadWriteUtils {
public:
typedef uint16_t DictionaryFlags;
typedef std::map<std::vector<int>, std::vector<int> > AttributeMap;
static int getHeaderSize(const uint8_t *const dictBuf);
@ -43,10 +41,10 @@ class HeaderReadWriteUtils {
}
static DictionaryFlags createAndGetDictionaryFlagsUsingAttributeMap(
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap);
static void fetchAllHeaderAttributes(const uint8_t *const dictBuf,
AttributeMap *const headerAttributes);
DictionaryHeaderStructurePolicy::AttributeMap *const headerAttributes);
static bool writeDictionaryVersion(BufferWithExtendableBuffer *const buffer,
const FormatUtils::FORMAT_VERSION version, int *const writingPos);
@ -58,31 +56,38 @@ class HeaderReadWriteUtils {
const int size, int *const writingPos);
static bool writeHeaderAttributes(BufferWithExtendableBuffer *const buffer,
const AttributeMap *const headerAttributes, int *const writingPos);
const DictionaryHeaderStructurePolicy::AttributeMap *const headerAttributes,
int *const writingPos);
/**
* Methods for header attributes.
*/
static void setCodePointVectorAttribute(AttributeMap *const headerAttributes,
static void setCodePointVectorAttribute(
DictionaryHeaderStructurePolicy::AttributeMap *const headerAttributes,
const char *const key, const std::vector<int> value);
static void setBoolAttribute(AttributeMap *const headerAttributes,
static void setBoolAttribute(
DictionaryHeaderStructurePolicy::AttributeMap *const headerAttributes,
const char *const key, const bool value);
static void setIntAttribute(AttributeMap *const headerAttributes,
static void setIntAttribute(
DictionaryHeaderStructurePolicy::AttributeMap *const headerAttributes,
const char *const key, const int value);
static const std::vector<int> readCodePointVectorAttributeValue(
const AttributeMap *const headerAttributes, const char *const key);
const DictionaryHeaderStructurePolicy::AttributeMap *const headerAttributes,
const char *const key);
static bool readBoolAttributeValue(const AttributeMap *const headerAttributes,
static bool readBoolAttributeValue(
const DictionaryHeaderStructurePolicy::AttributeMap *const headerAttributes,
const char *const key, const bool defaultValue);
static int readIntAttributeValue(const AttributeMap *const headerAttributes,
static int readIntAttributeValue(
const DictionaryHeaderStructurePolicy::AttributeMap *const headerAttributes,
const char *const key, const int defaultValue);
static void insertCharactersIntoVector(const char *const characters,
AttributeMap::key_type *const key);
DictionaryHeaderStructurePolicy::AttributeMap::key_type *const key);
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderReadWriteUtils);
@ -98,11 +103,15 @@ class HeaderReadWriteUtils {
// Value for the "flags" field. It's unused at the moment.
static const DictionaryFlags NO_FLAGS;
static void setIntAttributeInner(AttributeMap *const headerAttributes,
const AttributeMap::key_type *const key, const int value);
static void setIntAttributeInner(
DictionaryHeaderStructurePolicy::AttributeMap *const headerAttributes,
const DictionaryHeaderStructurePolicy::AttributeMap::key_type *const key,
const int value);
static int readIntAttributeValueInner(const AttributeMap *const headerAttributes,
const AttributeMap::key_type *const key, const int defaultValue);
static int readIntAttributeValueInner(
const DictionaryHeaderStructurePolicy::AttributeMap *const headerAttributes,
const DictionaryHeaderStructurePolicy::AttributeMap::key_type *const key,
const int defaultValue);
};
}
#endif /* LATINIME_HEADER_READ_WRITE_UTILS_H */

4
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View file

@ -32,7 +32,7 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
/* static */ bool DictFileWritingUtils::createEmptyDictFile(const char *const filePath,
const int dictVersion, const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
TimeKeeper::setCurrentTime();
switch (dictVersion) {
case FormatUtils::VERSION_4:
@ -46,7 +46,7 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, localeAsCodePointVector, attributeMap);
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
Ver4DictBuffers::createVer4DictBuffers(&headerPolicy);

4
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
View file

@ -32,7 +32,7 @@ class DictFileWritingUtils {
static bool createEmptyDictFile(const char *const filePath, const int dictVersion,
const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap);
static bool flushAllHeaderAndBodyToFile(const char *const filePath,
BufferWithExtendableBuffer *const dictHeader,
@ -46,7 +46,7 @@ class DictFileWritingUtils {
static bool createEmptyV4DictFile(const char *const filePath,
const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap);
static bool flushBufferToFile(const char *const filePath,
const BufferWithExtendableBuffer *const buffer);