nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "[HD04] Make the locale mandatory."

main
Jean Chalard 2014-02-05 09:16:16 +00:00 committed by Android (Google) Code Review
parent bbb6e8f10f 43cf9076b2
commit b162911fa4
14 changed files with 82 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View File

@ -139,7 +139,7 @@ public final class BinaryDictionary extends Dictionary {
}
private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,
String[] attributeKeyStringArray, String[] attributeValueStringArray);
String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray);
private static native long openNative(String sourceDir, long dictOffset, long dictSize,
boolean isUpdatable);
private static native void getHeaderInfoNative(long dict, int[] outHeaderSize,
@ -179,7 +179,7 @@ public final class BinaryDictionary extends Dictionary {
private static native String getPropertyNative(long dict, String query);
public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
final Map<String, String> attributeMap) {
final Locale locale, final Map<String, String> attributeMap) {
final String[] keyArray = new String[attributeMap.size()];
final String[] valueArray = new String[attributeMap.size()];
int index = 0;
@ -188,7 +188,8 @@ public final class BinaryDictionary extends Dictionary {
valueArray[index] = attributeMap.get(key);
index++;
}
return createEmptyDictFileNative(filePath, dictVersion, keyArray, valueArray);
return createEmptyDictFileNative(filePath, dictVersion, locale.toString(), keyArray,
valueArray);
}
// TODO: Move native dict into session

6
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -289,10 +289,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
Log.e(TAG, "Can't remove a file: " + file.getName());
}
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
mBinaryDictionary = new BinaryDictionary(
file.getAbsolutePath(), 0 /* offset */, file.length(),
true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
true /* useFullEditDistance */, mLocale, mDictType, mIsUpdatable);
} else {
mDictionaryWriter.clear();
}
@ -594,7 +594,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
Log.e(TAG, "Can't remove a file: " + file.getName());
}
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
} else {
if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
mBinaryDictionary.flushWithGC();

4
java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
View File

@ -55,7 +55,9 @@ public class Ver4DictEncoder implements DictEncoder {
throw new UnsupportedFormatException("Given path is not a directory.");
}
if (!BinaryDictionary.createEmptyDictFile(mDictPlacedDir.getAbsolutePath(),
FormatSpec.VERSION4, dict.mOptions.mAttributes)) {
FormatSpec.VERSION4, LocaleUtils.constructLocaleFromString(
dict.mOptions.mAttributes.get(DictionaryHeader.DICTIONARY_LOCALE_KEY)),
dict.mOptions.mAttributes)) {
throw new IOException("Cannot create dictionary file : "
+ mDictPlacedDir.getAbsolutePath());
}

12
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View File

@ -29,6 +29,7 @@
#include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "utils/autocorrection_threshold_utils.h"
#include "utils/char_utils.h"
#include "utils/time_keeper.h"
namespace latinime {
@ -37,13 +38,15 @@ class ProximityInfo;
// TODO: Move to makedict.
static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclass clazz,
jstring filePath, jlong dictVersion, jobjectArray attributeKeyStringArray,
jstring filePath, jlong dictVersion, jstring locale, jobjectArray attributeKeyStringArray,
jobjectArray attributeValueStringArray) {
const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
char filePathChars[filePathUtf8Length + 1];
env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
filePathChars[filePathUtf8Length] = '\0';
jsize localeLength = env->GetStringLength(locale);
jchar localeCodePoints[localeLength];
env->GetStringRegion(locale, 0, localeLength, localeCodePoints);
const int keyCount = env->GetArrayLength(attributeKeyStringArray);
const int valueCount = env->GetArrayLength(attributeValueStringArray);
if (keyCount != valueCount) {
@ -73,7 +76,7 @@ static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclas
}
return DictFileWritingUtils::createEmptyDictFile(filePathChars, static_cast<int>(dictVersion),
&attributeMap);
CharUtils::convertShortArrayToIntVector(localeCodePoints, localeLength), &attributeMap);
}
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
@ -503,7 +506,8 @@ static int latinime_BinaryDictionary_setCurrentTimeForTest(JNIEnv *env, jclass c
static const JNINativeMethod sMethods[] = {
{
const_cast<char *>("createEmptyDictFileNative"),
const_cast<char *>("(Ljava/lang/String;J[Ljava/lang/String;[Ljava/lang/String;)Z"),
const_cast<char *>(
"(Ljava/lang/String;JLjava/lang/String;[Ljava/lang/String;[Ljava/lang/String;)Z"),
reinterpret_cast<void *>(latinime_BinaryDictionary_createEmptyDictFile)
},
{

6
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
View File

@ -32,6 +32,7 @@ const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE
// Historical info is information that is needed to support decaying such as timestamp, level and
// count.
const char *const HeaderPolicy::HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO";
const char *const HeaderPolicy::LOCALE_KEY = "locale"; // match Java declaration
const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100;
const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f;
@ -59,6 +60,10 @@ void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *out
outValue[terminalIndex] = '\0';
}
const std::vector<int> HeaderPolicy::readLocale() const {
return HeaderReadWriteUtils::readCodePointVectorAttributeValue(&mAttributeMap, LOCALE_KEY);
}
float HeaderPolicy::readMultipleWordCostMultiplier() const {
const int demotionRate = HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
MULTIPLE_WORDS_DEMOTION_RATE_KEY, DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE);
@ -116,6 +121,7 @@ void HeaderPolicy::fillInHeader(const bool updatesLastDecayedTime, const int uni
// Set the current time as the generation time.
HeaderReadWriteUtils::setIntAttribute(outAttributeMap, DATE_KEY,
TimeKeeper::peekCurrentTime());
HeaderReadWriteUtils::setCodePointVectorAttribute(outAttributeMap, LOCALE_KEY, mLocale);
if (updatesLastDecayedTime) {
// Set current time as the last updated time.
HeaderReadWriteUtils::setIntAttribute(outAttributeMap, LAST_DECAYED_TIME_KEY,

13
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View File

@ -23,6 +23,7 @@
#include "suggest/core/policy/dictionary_header_structure_policy.h"
#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
#include "utils/char_utils.h"
#include "utils/time_keeper.h"
namespace latinime {
@ -35,6 +36,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
mDictionaryFlags(HeaderReadWriteUtils::getFlags(dictBuf)),
mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)),
mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)),
mLocale(readLocale()),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
@ -54,10 +56,11 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
// Constructs header information using an attribute map.
HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion,
const std::vector<int> locale,
const HeaderReadWriteUtils::AttributeMap *const attributeMap)
: mDictFormatVersion(dictFormatVersion),
mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
attributeMap)), mSize(0), mAttributeMap(*attributeMap),
attributeMap)), mSize(0), mAttributeMap(*attributeMap), mLocale(locale),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
@ -68,12 +71,13 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
DATE_KEY, TimeKeeper::peekCurrentTime() /* defaultValue */)),
mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0),
mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue(
&mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {}
&mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {
}
// Temporary dummy header.
HeaderPolicy()
: mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0),
mAttributeMap(), mMultiWordCostMultiplier(0.0f),
mAttributeMap(), mLocale(CharUtils::EMPTY_STRING), mMultiWordCostMultiplier(0.0f),
mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false),
mDate(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0),
mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {}
@ -174,6 +178,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
static const char *const BIGRAM_COUNT_KEY;
static const char *const EXTENDED_REGION_SIZE_KEY;
static const char *const HAS_HISTORICAL_INFO_KEY;
static const char *const LOCALE_KEY;
static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE;
static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE;
@ -181,6 +186,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags;
const int mSize;
HeaderReadWriteUtils::AttributeMap mAttributeMap;
const std::vector<int> mLocale;
const float mMultiWordCostMultiplier;
const bool mRequiresGermanUmlautProcessing;
const bool mIsDecayingDict;
@ -191,6 +197,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const int mExtendedRegionSize;
const bool mHasHistoricalInfoOfWords;
const std::vector<int> readLocale() const;
float readMultipleWordCostMultiplier() const;
bool readRequiresGermanUmlautProcessing() const;

19
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
View File

@ -130,6 +130,13 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
return true;
}
/* static */ void HeaderReadWriteUtils::setCodePointVectorAttribute(
AttributeMap *const headerAttributes, const char *const key, const std::vector<int> value) {
AttributeMap::key_type keyVector;
insertCharactersIntoVector(key, &keyVector);
(*headerAttributes)[keyVector] = value;
}
/* static */ void HeaderReadWriteUtils::setBoolAttribute(AttributeMap *const headerAttributes,
const char *const key, const bool value) {
setIntAttribute(headerAttributes, key, value ? 1 : 0);
@ -151,6 +158,18 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
(*headerAttributes)[*key] = valueVector;
}
/* static */ const std::vector<int> HeaderReadWriteUtils::readCodePointVectorAttributeValue(
const AttributeMap *const headerAttributes, const char *const key) {
AttributeMap::key_type keyVector;
insertCharactersIntoVector(key, &keyVector);
AttributeMap::const_iterator it = headerAttributes->find(keyVector);
if (it == headerAttributes->end()) {
return std::vector<int>();
} else {
return it->second;
}
}
/* static */ bool HeaderReadWriteUtils::readBoolAttributeValue(
const AttributeMap *const headerAttributes, const char *const key,
const bool defaultValue) {

6
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h
View File

@ -63,12 +63,18 @@ class HeaderReadWriteUtils {
/**
* Methods for header attributes.
*/
static void setCodePointVectorAttribute(AttributeMap *const headerAttributes,
const char *const key, const std::vector<int> value);
static void setBoolAttribute(AttributeMap *const headerAttributes,
const char *const key, const bool value);
static void setIntAttribute(AttributeMap *const headerAttributes,
const char *const key, const int value);
static const std::vector<int> readCodePointVectorAttributeValue(
const AttributeMap *const headerAttributes, const char *const key);
static bool readBoolAttributeValue(const AttributeMap *const headerAttributes,
const char *const key, const bool defaultValue);

8
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View File

@ -31,11 +31,12 @@ namespace latinime {
const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = ".tmp";
/* static */ bool DictFileWritingUtils::createEmptyDictFile(const char *const filePath,
const int dictVersion, const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
const int dictVersion, const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
TimeKeeper::setCurrentTime();
switch (dictVersion) {
case FormatUtils::VERSION_4:
return createEmptyV4DictFile(filePath, attributeMap);
return createEmptyV4DictFile(filePath, localeAsCodePointVector, attributeMap);
default:
AKLOGE("Cannot create dictionary %s because format version %d is not supported.",
filePath, dictVersion);
@ -44,8 +45,9 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
}
/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap);
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, localeAsCodePointVector, attributeMap);
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
Ver4DictBuffers::createVer4DictBuffers(&headerPolicy);
headerPolicy.fillInAndWriteHeaderToBuffer(true /* updatesLastDecayedTime */,

2
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
View File

@ -31,6 +31,7 @@ class DictFileWritingUtils {
static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE;
static bool createEmptyDictFile(const char *const filePath, const int dictVersion,
const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
static bool flushAllHeaderAndBodyToFile(const char *const filePath,
@ -44,6 +45,7 @@ class DictFileWritingUtils {
DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils);
static bool createEmptyV4DictFile(const char *const filePath,
const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
static bool flushBufferToFile(const char *const filePath,

2
native/jni/src/utils/char_utils.cpp
View File

@ -1273,4 +1273,6 @@ static int compare_pair_capital(const void *a, const void *b) {
/* U+04F0 */ 0x0423, 0x0443, 0x0423, 0x0443, 0x0427, 0x0447, 0x04F6, 0x04F7,
/* U+04F8 */ 0x042B, 0x044B, 0x04FA, 0x04FB, 0x04FC, 0x04FD, 0x04FE, 0x04FF,
};
/* static */ const std::vector<int> CharUtils::EMPTY_STRING(1 /* size */, '\0' /* value */);
} // namespace latinime

9
native/jni/src/utils/char_utils.h
View File

@ -18,6 +18,7 @@
#define LATINIME_CHAR_UTILS_H
#include <cctype>
#include <vector>
#include "defines.h"
@ -85,7 +86,15 @@ class CharUtils {
return spaceCount;
}
static AK_FORCE_INLINE std::vector<int> convertShortArrayToIntVector(
const unsigned short *const source, const int length) {
std::vector<int> destination;
destination.insert(destination.end(), source, source + length);
return destination; // Copies the vector
}
static unsigned short latin_tolower(const unsigned short c);
static const std::vector<int> EMPTY_STRING;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(CharUtils);

6
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View File

@ -28,6 +28,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
import java.io.File;
import java.io.IOException;
@ -104,15 +105,14 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
FileUtils.deleteRecursively(file);
Map<String, String> attributeMap = new HashMap<String, String>();
attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, dictId);
attributeMap.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, dictId);
attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY,
String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY,
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
FormatSpec.VERSION4, attributeMap)) {
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
LocaleUtils.constructLocaleFromString(TEST_LOCALE), attributeMap)) {
return file;
} else {
throw new IOException("Empty dictionary " + file.getAbsolutePath()

4
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View File

@ -69,8 +69,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
file.delete();
file.mkdir();
Map<String, String> attributeMap = new HashMap<String, String>();
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
FormatSpec.VERSION4, attributeMap)) {
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
Locale.ENGLISH, attributeMap)) {
return file;
} else {
throw new IOException("Empty dictionary " + file.getAbsolutePath()