nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[HD04] Make the locale mandatory. 

Bug: 11281748
Change-Id: I69281b0053bec404c3e3c713ade3f65a140f51b1
main
Jean Chalard 2014-02-04 23:51:05 +09:00
parent aac53dd7f5
commit 43cf9076b2
14 changed files with 82 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
java/src/com/android/inputmethod/latin/BinaryDictionary.java
View File

@ -139,7 +139,7 @@ public final class BinaryDictionary extends Dictionary {
}
private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,
String[] attributeKeyStringArray, String[] attributeValueStringArray);
String locale, String[] attributeKeyStringArray, String[] attributeValueStringArray);
private static native long openNative(String sourceDir, long dictOffset, long dictSize,
boolean isUpdatable);
private static native void getHeaderInfoNative(long dict, int[] outHeaderSize,
@ -179,7 +179,7 @@ public final class BinaryDictionary extends Dictionary {
private static native String getPropertyNative(long dict, String query);
public static boolean createEmptyDictFile(final String filePath, final long dictVersion,
final Map<String, String> attributeMap) {
final Locale locale, final Map<String, String> attributeMap) {
final String[] keyArray = new String[attributeMap.size()];
final String[] valueArray = new String[attributeMap.size()];
int index = 0;
@ -188,7 +188,8 @@ public final class BinaryDictionary extends Dictionary {
valueArray[index] = attributeMap.get(key);
index++;
}
return createEmptyDictFileNative(filePath, dictVersion, keyArray, valueArray);
return createEmptyDictFileNative(filePath, dictVersion, locale.toString(), keyArray,
valueArray);
}
// TODO: Move native dict into session

6
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -289,10 +289,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
Log.e(TAG, "Can't remove a file: " + file.getName());
}
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
mBinaryDictionary = new BinaryDictionary(
file.getAbsolutePath(), 0 /* offset */, file.length(),
true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
true /* useFullEditDistance */, mLocale, mDictType, mIsUpdatable);
} else {
mDictionaryWriter.clear();
}
@ -594,7 +594,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
Log.e(TAG, "Can't remove a file: " + file.getName());
}
BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());
DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap());
} else {
if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
mBinaryDictionary.flushWithGC();

4
java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
View File

@ -55,7 +55,9 @@ public class Ver4DictEncoder implements DictEncoder {
throw new UnsupportedFormatException("Given path is not a directory.");
}
if (!BinaryDictionary.createEmptyDictFile(mDictPlacedDir.getAbsolutePath(),
FormatSpec.VERSION4, dict.mOptions.mAttributes)) {
FormatSpec.VERSION4, LocaleUtils.constructLocaleFromString(
dict.mOptions.mAttributes.get(DictionaryHeader.DICTIONARY_LOCALE_KEY)),
dict.mOptions.mAttributes)) {
throw new IOException("Cannot create dictionary file : "
+ mDictPlacedDir.getAbsolutePath());
}

12
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
View File

@ -29,6 +29,7 @@
#include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h"
#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
#include "utils/autocorrection_threshold_utils.h"
#include "utils/char_utils.h"
#include "utils/time_keeper.h"
namespace latinime {
@ -37,13 +38,15 @@ class ProximityInfo;
// TODO: Move to makedict.
static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclass clazz,
jstring filePath, jlong dictVersion, jobjectArray attributeKeyStringArray,
jstring filePath, jlong dictVersion, jstring locale, jobjectArray attributeKeyStringArray,
jobjectArray attributeValueStringArray) {
const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
char filePathChars[filePathUtf8Length + 1];
env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
filePathChars[filePathUtf8Length] = '\0';
jsize localeLength = env->GetStringLength(locale);
jchar localeCodePoints[localeLength];
env->GetStringRegion(locale, 0, localeLength, localeCodePoints);
const int keyCount = env->GetArrayLength(attributeKeyStringArray);
const int valueCount = env->GetArrayLength(attributeValueStringArray);
if (keyCount != valueCount) {
@ -73,7 +76,7 @@ static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclas
}
return DictFileWritingUtils::createEmptyDictFile(filePathChars, static_cast<int>(dictVersion),
&attributeMap);
CharUtils::convertShortArrayToIntVector(localeCodePoints, localeLength), &attributeMap);
}
static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
@ -503,7 +506,8 @@ static int latinime_BinaryDictionary_setCurrentTimeForTest(JNIEnv *env, jclass c
static const JNINativeMethod sMethods[] = {
{
const_cast<char *>("createEmptyDictFileNative"),
const_cast<char *>("(Ljava/lang/String;J[Ljava/lang/String;[Ljava/lang/String;)Z"),
const_cast<char *>(
"(Ljava/lang/String;JLjava/lang/String;[Ljava/lang/String;[Ljava/lang/String;)Z"),
reinterpret_cast<void *>(latinime_BinaryDictionary_createEmptyDictFile)
},
{

6
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp
View File

@ -32,6 +32,7 @@ const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE
// Historical info is information that is needed to support decaying such as timestamp, level and
// count.
const char *const HeaderPolicy::HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO";
const char *const HeaderPolicy::LOCALE_KEY = "locale"; // match Java declaration
const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100;
const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f;
@ -59,6 +60,10 @@ void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *out
outValue[terminalIndex] = '\0';
}
const std::vector<int> HeaderPolicy::readLocale() const {
return HeaderReadWriteUtils::readCodePointVectorAttributeValue(&mAttributeMap, LOCALE_KEY);
}
float HeaderPolicy::readMultipleWordCostMultiplier() const {
const int demotionRate = HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap,
MULTIPLE_WORDS_DEMOTION_RATE_KEY, DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE);
@ -116,6 +121,7 @@ void HeaderPolicy::fillInHeader(const bool updatesLastDecayedTime, const int uni
// Set the current time as the generation time.
HeaderReadWriteUtils::setIntAttribute(outAttributeMap, DATE_KEY,
TimeKeeper::peekCurrentTime());
HeaderReadWriteUtils::setCodePointVectorAttribute(outAttributeMap, LOCALE_KEY, mLocale);
if (updatesLastDecayedTime) {
// Set current time as the last updated time.
HeaderReadWriteUtils::setIntAttribute(outAttributeMap, LAST_DECAYED_TIME_KEY,

13
native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
View File

@ -23,6 +23,7 @@
#include "suggest/core/policy/dictionary_header_structure_policy.h"
#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h"
#include "suggest/policyimpl/dictionary/utils/format_utils.h"
#include "utils/char_utils.h"
#include "utils/time_keeper.h"
namespace latinime {
@ -35,6 +36,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
mDictionaryFlags(HeaderReadWriteUtils::getFlags(dictBuf)),
mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)),
mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)),
mLocale(readLocale()),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
@ -54,10 +56,11 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
// Constructs header information using an attribute map.
HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion,
const std::vector<int> locale,
const HeaderReadWriteUtils::AttributeMap *const attributeMap)
: mDictFormatVersion(dictFormatVersion),
mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
attributeMap)), mSize(0), mAttributeMap(*attributeMap),
attributeMap)), mSize(0), mAttributeMap(*attributeMap), mLocale(locale),
mMultiWordCostMultiplier(readMultipleWordCostMultiplier()),
mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()),
mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap,
@ -68,12 +71,13 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
DATE_KEY, TimeKeeper::peekCurrentTime() /* defaultValue */)),
mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0),
mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue(
&mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {}
&mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {
}
// Temporary dummy header.
HeaderPolicy()
: mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0),
mAttributeMap(), mMultiWordCostMultiplier(0.0f),
mAttributeMap(), mLocale(CharUtils::EMPTY_STRING), mMultiWordCostMultiplier(0.0f),
mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false),
mDate(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0),
mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {}
@ -174,6 +178,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
static const char *const BIGRAM_COUNT_KEY;
static const char *const EXTENDED_REGION_SIZE_KEY;
static const char *const HAS_HISTORICAL_INFO_KEY;
static const char *const LOCALE_KEY;
static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE;
static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE;
@ -181,6 +186,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags;
const int mSize;
HeaderReadWriteUtils::AttributeMap mAttributeMap;
const std::vector<int> mLocale;
const float mMultiWordCostMultiplier;
const bool mRequiresGermanUmlautProcessing;
const bool mIsDecayingDict;
@ -191,6 +197,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const int mExtendedRegionSize;
const bool mHasHistoricalInfoOfWords;
const std::vector<int> readLocale() const;
float readMultipleWordCostMultiplier() const;
bool readRequiresGermanUmlautProcessing() const;

19
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp
View File

@ -130,6 +130,13 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
return true;
}
/* static */ void HeaderReadWriteUtils::setCodePointVectorAttribute(
AttributeMap *const headerAttributes, const char *const key, const std::vector<int> value) {
AttributeMap::key_type keyVector;
insertCharactersIntoVector(key, &keyVector);
(*headerAttributes)[keyVector] = value;
}
/* static */ void HeaderReadWriteUtils::setBoolAttribute(AttributeMap *const headerAttributes,
const char *const key, const bool value) {
setIntAttribute(headerAttributes, key, value ? 1 : 0);
@ -151,6 +158,18 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
(*headerAttributes)[*key] = valueVector;
}
/* static */ const std::vector<int> HeaderReadWriteUtils::readCodePointVectorAttributeValue(
const AttributeMap *const headerAttributes, const char *const key) {
AttributeMap::key_type keyVector;
insertCharactersIntoVector(key, &keyVector);
AttributeMap::const_iterator it = headerAttributes->find(keyVector);
if (it == headerAttributes->end()) {
return std::vector<int>();
} else {
return it->second;
}
}
/* static */ bool HeaderReadWriteUtils::readBoolAttributeValue(
const AttributeMap *const headerAttributes, const char *const key,
const bool defaultValue) {

6
native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h
View File

@ -63,12 +63,18 @@ class HeaderReadWriteUtils {
/**
* Methods for header attributes.
*/
static void setCodePointVectorAttribute(AttributeMap *const headerAttributes,
const char *const key, const std::vector<int> value);
static void setBoolAttribute(AttributeMap *const headerAttributes,
const char *const key, const bool value);
static void setIntAttribute(AttributeMap *const headerAttributes,
const char *const key, const int value);
static const std::vector<int> readCodePointVectorAttributeValue(
const AttributeMap *const headerAttributes, const char *const key);
static bool readBoolAttributeValue(const AttributeMap *const headerAttributes,
const char *const key, const bool defaultValue);

8
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp
View File

@ -31,11 +31,12 @@ namespace latinime {
const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = ".tmp";
/* static */ bool DictFileWritingUtils::createEmptyDictFile(const char *const filePath,
const int dictVersion, const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
const int dictVersion, const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
TimeKeeper::setCurrentTime();
switch (dictVersion) {
case FormatUtils::VERSION_4:
return createEmptyV4DictFile(filePath, attributeMap);
return createEmptyV4DictFile(filePath, localeAsCodePointVector, attributeMap);
default:
AKLOGE("Cannot create dictionary %s because format version %d is not supported.",
filePath, dictVersion);
@ -44,8 +45,9 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
}
/* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath,
const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap);
HeaderPolicy headerPolicy(FormatUtils::VERSION_4, localeAsCodePointVector, attributeMap);
Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers =
Ver4DictBuffers::createVer4DictBuffers(&headerPolicy);
headerPolicy.fillInAndWriteHeaderToBuffer(true /* updatesLastDecayedTime */,

2
native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h
View File

@ -31,6 +31,7 @@ class DictFileWritingUtils {
static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE;
static bool createEmptyDictFile(const char *const filePath, const int dictVersion,
const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
static bool flushAllHeaderAndBodyToFile(const char *const filePath,
@ -44,6 +45,7 @@ class DictFileWritingUtils {
DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils);
static bool createEmptyV4DictFile(const char *const filePath,
const std::vector<int> localeAsCodePointVector,
const HeaderReadWriteUtils::AttributeMap *const attributeMap);
static bool flushBufferToFile(const char *const filePath,

2
native/jni/src/utils/char_utils.cpp
View File

@ -1273,4 +1273,6 @@ static int compare_pair_capital(const void *a, const void *b) {
/* U+04F0 */ 0x0423, 0x0443, 0x0423, 0x0443, 0x0427, 0x0447, 0x04F6, 0x04F7,
/* U+04F8 */ 0x042B, 0x044B, 0x04FA, 0x04FB, 0x04FC, 0x04FD, 0x04FE, 0x04FF,
};
/* static */ const std::vector<int> CharUtils::EMPTY_STRING(1 /* size */, '\0' /* value */);
} // namespace latinime

9
native/jni/src/utils/char_utils.h
View File

@ -18,6 +18,7 @@
#define LATINIME_CHAR_UTILS_H
#include <cctype>
#include <vector>
#include "defines.h"
@ -85,7 +86,15 @@ class CharUtils {
return spaceCount;
}
static AK_FORCE_INLINE std::vector<int> convertShortArrayToIntVector(
const unsigned short *const source, const int length) {
std::vector<int> destination;
destination.insert(destination.end(), source, source + length);
return destination; // Copies the vector
}
static unsigned short latin_tolower(const unsigned short c);
static const std::vector<int> EMPTY_STRING;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(CharUtils);

6
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
View File

@ -28,6 +28,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
import java.io.File;
import java.io.IOException;
@ -104,15 +105,14 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
FileUtils.deleteRecursively(file);
Map<String, String> attributeMap = new HashMap<String, String>();
attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, dictId);
attributeMap.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, dictId);
attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY,
String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY,
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY,
DictionaryHeader.ATTRIBUTE_VALUE_TRUE);
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
FormatSpec.VERSION4, attributeMap)) {
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
LocaleUtils.constructLocaleFromString(TEST_LOCALE), attributeMap)) {
return file;
} else {
throw new IOException("Empty dictionary " + file.getAbsolutePath()

4
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View File

@ -69,8 +69,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
file.delete();
file.mkdir();
Map<String, String> attributeMap = new HashMap<String, String>();
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),
FormatSpec.VERSION4, attributeMap)) {
if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4,
Locale.ENGLISH, attributeMap)) {
return file;
} else {
throw new IOException("Empty dictionary " + file.getAbsolutePath()