Merge "Return whether the dynamic dict operation was success."
This commit is contained in:
commit
3faf4fc5e7
13 changed files with 151 additions and 91 deletions
|
@ -183,9 +183,9 @@ public final class BinaryDictionary extends Dictionary {
|
||||||
private static native void getHeaderInfoNative(long dict, int[] outHeaderSize,
|
private static native void getHeaderInfoNative(long dict, int[] outHeaderSize,
|
||||||
int[] outFormatVersion, ArrayList<int[]> outAttributeKeys,
|
int[] outFormatVersion, ArrayList<int[]> outAttributeKeys,
|
||||||
ArrayList<int[]> outAttributeValues);
|
ArrayList<int[]> outAttributeValues);
|
||||||
private static native void flushNative(long dict, String filePath);
|
private static native boolean flushNative(long dict, String filePath);
|
||||||
private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC);
|
private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC);
|
||||||
private static native void flushWithGCNative(long dict, String filePath);
|
private static native boolean flushWithGCNative(long dict, String filePath);
|
||||||
private static native void closeNative(long dict);
|
private static native void closeNative(long dict);
|
||||||
private static native int getFormatVersionNative(long dict);
|
private static native int getFormatVersionNative(long dict);
|
||||||
private static native int getProbabilityNative(long dict, int[] word);
|
private static native int getProbabilityNative(long dict, int[] word);
|
||||||
|
@ -203,12 +203,12 @@ public final class BinaryDictionary extends Dictionary {
|
||||||
int[] outputSuggestionCount, int[] outputCodePoints, int[] outputScores,
|
int[] outputSuggestionCount, int[] outputCodePoints, int[] outputScores,
|
||||||
int[] outputIndices, int[] outputTypes, int[] outputAutoCommitFirstWordConfidence,
|
int[] outputIndices, int[] outputTypes, int[] outputAutoCommitFirstWordConfidence,
|
||||||
float[] inOutLanguageWeight);
|
float[] inOutLanguageWeight);
|
||||||
private static native void addUnigramWordNative(long dict, int[] word, int probability,
|
private static native boolean addUnigramWordNative(long dict, int[] word, int probability,
|
||||||
int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence,
|
int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence,
|
||||||
boolean isNotAWord, boolean isBlacklisted, int timestamp);
|
boolean isNotAWord, boolean isBlacklisted, int timestamp);
|
||||||
private static native void addBigramWordsNative(long dict, int[] word0,
|
private static native boolean addBigramWordsNative(long dict, int[] word0,
|
||||||
boolean isBeginningOfSentence, int[] word1, int probability, int timestamp);
|
boolean isBeginningOfSentence, int[] word1, int probability, int timestamp);
|
||||||
private static native void removeBigramWordsNative(long dict, int[] word0,
|
private static native boolean removeBigramWordsNative(long dict, int[] word0,
|
||||||
boolean isBeginningOfSentence, int[] word1);
|
boolean isBeginningOfSentence, int[] word1);
|
||||||
private static native int addMultipleDictionaryEntriesNative(long dict,
|
private static native int addMultipleDictionaryEntriesNative(long dict,
|
||||||
LanguageModelParam[] languageModelParams, int startIndex);
|
LanguageModelParam[] languageModelParams, int startIndex);
|
||||||
|
@ -417,45 +417,53 @@ public final class BinaryDictionary extends Dictionary {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add a unigram entry to binary dictionary with unigram attributes in native code.
|
// Add a unigram entry to binary dictionary with unigram attributes in native code.
|
||||||
public void addUnigramEntry(final String word, final int probability,
|
public boolean addUnigramEntry(final String word, final int probability,
|
||||||
final String shortcutTarget, final int shortcutProbability,
|
final String shortcutTarget, final int shortcutProbability,
|
||||||
final boolean isBeginningOfSentence, final boolean isNotAWord,
|
final boolean isBeginningOfSentence, final boolean isNotAWord,
|
||||||
final boolean isBlacklisted, final int timestamp) {
|
final boolean isBlacklisted, final int timestamp) {
|
||||||
if (word == null || (word.isEmpty() && !isBeginningOfSentence)) {
|
if (word == null || (word.isEmpty() && !isBeginningOfSentence)) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
final int[] codePoints = StringUtils.toCodePointArray(word);
|
final int[] codePoints = StringUtils.toCodePointArray(word);
|
||||||
final int[] shortcutTargetCodePoints = (shortcutTarget != null) ?
|
final int[] shortcutTargetCodePoints = (shortcutTarget != null) ?
|
||||||
StringUtils.toCodePointArray(shortcutTarget) : null;
|
StringUtils.toCodePointArray(shortcutTarget) : null;
|
||||||
addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints,
|
if (!addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints,
|
||||||
shortcutProbability, isBeginningOfSentence, isNotAWord, isBlacklisted, timestamp);
|
shortcutProbability, isBeginningOfSentence, isNotAWord, isBlacklisted, timestamp)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
mHasUpdated = true;
|
mHasUpdated = true;
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add an n-gram entry to the binary dictionary with timestamp in native code.
|
// Add an n-gram entry to the binary dictionary with timestamp in native code.
|
||||||
public void addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word,
|
public boolean addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word,
|
||||||
final int probability,
|
final int probability, final int timestamp) {
|
||||||
final int timestamp) {
|
|
||||||
if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) {
|
if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord);
|
final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord);
|
||||||
final int[] codePoints1 = StringUtils.toCodePointArray(word);
|
final int[] codePoints1 = StringUtils.toCodePointArray(word);
|
||||||
addBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence,
|
if (!addBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence,
|
||||||
codePoints1, probability, timestamp);
|
codePoints1, probability, timestamp)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
mHasUpdated = true;
|
mHasUpdated = true;
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove an n-gram entry from the binary dictionary in native code.
|
// Remove an n-gram entry from the binary dictionary in native code.
|
||||||
public void removeNgramEntry(final PrevWordsInfo prevWordsInfo, final String word) {
|
public boolean removeNgramEntry(final PrevWordsInfo prevWordsInfo, final String word) {
|
||||||
if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) {
|
if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord);
|
final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord);
|
||||||
final int[] codePoints1 = StringUtils.toCodePointArray(word);
|
final int[] codePoints1 = StringUtils.toCodePointArray(word);
|
||||||
removeBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence,
|
if (!removeBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence,
|
||||||
codePoints1);
|
codePoints1)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
mHasUpdated = true;
|
mHasUpdated = true;
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) {
|
public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) {
|
||||||
|
@ -485,26 +493,33 @@ public final class BinaryDictionary extends Dictionary {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Flush to dict file if the dictionary has been updated.
|
// Flush to dict file if the dictionary has been updated.
|
||||||
public void flush() {
|
public boolean flush() {
|
||||||
if (!isValidDictionary()) return;
|
if (!isValidDictionary()) return false;
|
||||||
if (mHasUpdated) {
|
if (mHasUpdated) {
|
||||||
flushNative(mNativeDict, mDictFilePath);
|
if (!flushNative(mNativeDict, mDictFilePath)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
reopen();
|
reopen();
|
||||||
}
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run GC and flush to dict file if the dictionary has been updated.
|
// Run GC and flush to dict file if the dictionary has been updated.
|
||||||
public void flushWithGCIfHasUpdated() {
|
public boolean flushWithGCIfHasUpdated() {
|
||||||
if (mHasUpdated) {
|
if (mHasUpdated) {
|
||||||
flushWithGC();
|
return flushWithGC();
|
||||||
}
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run GC and flush to dict file.
|
// Run GC and flush to dict file.
|
||||||
public void flushWithGC() {
|
public boolean flushWithGC() {
|
||||||
if (!isValidDictionary()) return;
|
if (!isValidDictionary()) return false;
|
||||||
flushWithGCNative(mNativeDict, mDictFilePath);
|
if (!flushWithGCNative(mNativeDict, mDictFilePath)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
reopen();
|
reopen();
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -298,8 +298,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
protected void addUnigramLocked(final String word, final int frequency,
|
protected void addUnigramLocked(final String word, final int frequency,
|
||||||
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
|
final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord,
|
||||||
final boolean isBlacklisted, final int timestamp) {
|
final boolean isBlacklisted, final int timestamp) {
|
||||||
mBinaryDictionary.addUnigramEntry(word, frequency, shortcutTarget, shortcutFreq,
|
if (!mBinaryDictionary.addUnigramEntry(word, frequency, shortcutTarget, shortcutFreq,
|
||||||
false /* isBeginningOfSentence */, isNotAWord, isBlacklisted, timestamp);
|
false /* isBeginningOfSentence */, isNotAWord, isBlacklisted, timestamp)) {
|
||||||
|
Log.e(TAG, "Cannot add unigram entry. word: " + word);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -322,7 +324,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
|
||||||
|
|
||||||
protected void addNgramEntryLocked(final PrevWordsInfo prevWordsInfo, final String word,
|
protected void addNgramEntryLocked(final PrevWordsInfo prevWordsInfo, final String word,
|
||||||
final int frequency, final int timestamp) {
|
final int frequency, final int timestamp) {
|
||||||
mBinaryDictionary.addNgramEntry(prevWordsInfo, word, frequency, timestamp);
|
if (!mBinaryDictionary.addNgramEntry(prevWordsInfo, word, frequency, timestamp)) {
|
||||||
|
Log.e(TAG, "Cannot add n-gram entry.");
|
||||||
|
Log.e(TAG, " PrevWordsInfo: " + prevWordsInfo);
|
||||||
|
Log.e(TAG, " word: " + word);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -53,4 +53,10 @@ public class PrevWordsInfo {
|
||||||
public boolean isValid() {
|
public boolean isValid() {
|
||||||
return mPrevWord != null;
|
return mPrevWord != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "PrevWord: " + mPrevWord + ", isBeginningOfSentence: "
|
||||||
|
+ mIsBeginningOfSentence + ".";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -95,15 +95,15 @@ static jlong latinime_BinaryDictionary_createOnMemory(JNIEnv *env, jclass clazz,
|
||||||
return reinterpret_cast<jlong>(dictionary);
|
return reinterpret_cast<jlong>(dictionary);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void latinime_BinaryDictionary_flush(JNIEnv *env, jclass clazz, jlong dict,
|
static bool latinime_BinaryDictionary_flush(JNIEnv *env, jclass clazz, jlong dict,
|
||||||
jstring filePath) {
|
jstring filePath) {
|
||||||
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
||||||
if (!dictionary) return;
|
if (!dictionary) return false;
|
||||||
const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
|
const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
|
||||||
char filePathChars[filePathUtf8Length + 1];
|
char filePathChars[filePathUtf8Length + 1];
|
||||||
env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
|
env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
|
||||||
filePathChars[filePathUtf8Length] = '\0';
|
filePathChars[filePathUtf8Length] = '\0';
|
||||||
dictionary->flush(filePathChars);
|
return dictionary->flush(filePathChars);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool latinime_BinaryDictionary_needsToRunGC(JNIEnv *env, jclass clazz,
|
static bool latinime_BinaryDictionary_needsToRunGC(JNIEnv *env, jclass clazz,
|
||||||
|
@ -113,15 +113,15 @@ static bool latinime_BinaryDictionary_needsToRunGC(JNIEnv *env, jclass clazz,
|
||||||
return dictionary->needsToRunGC(mindsBlockByGC == JNI_TRUE);
|
return dictionary->needsToRunGC(mindsBlockByGC == JNI_TRUE);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void latinime_BinaryDictionary_flushWithGC(JNIEnv *env, jclass clazz, jlong dict,
|
static bool latinime_BinaryDictionary_flushWithGC(JNIEnv *env, jclass clazz, jlong dict,
|
||||||
jstring filePath) {
|
jstring filePath) {
|
||||||
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
||||||
if (!dictionary) return;
|
if (!dictionary) return false;
|
||||||
const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
|
const jsize filePathUtf8Length = env->GetStringUTFLength(filePath);
|
||||||
char filePathChars[filePathUtf8Length + 1];
|
char filePathChars[filePathUtf8Length + 1];
|
||||||
env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
|
env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars);
|
||||||
filePathChars[filePathUtf8Length] = '\0';
|
filePathChars[filePathUtf8Length] = '\0';
|
||||||
dictionary->flushWithGC(filePathChars);
|
return dictionary->flushWithGC(filePathChars);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) {
|
static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) {
|
||||||
|
@ -324,13 +324,13 @@ static void latinime_BinaryDictionary_getWordProperty(JNIEnv *env, jclass clazz,
|
||||||
outShortcutProbabilities);
|
outShortcutProbabilities);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz, jlong dict,
|
static bool latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz, jlong dict,
|
||||||
jintArray word, jint probability, jintArray shortcutTarget, jint shortcutProbability,
|
jintArray word, jint probability, jintArray shortcutTarget, jint shortcutProbability,
|
||||||
jboolean isBeginningOfSentence, jboolean isNotAWord, jboolean isBlacklisted,
|
jboolean isBeginningOfSentence, jboolean isNotAWord, jboolean isBlacklisted,
|
||||||
jint timestamp) {
|
jint timestamp) {
|
||||||
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
||||||
if (!dictionary) {
|
if (!dictionary) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
jsize codePointCount = env->GetArrayLength(word);
|
jsize codePointCount = env->GetArrayLength(word);
|
||||||
int codePoints[codePointCount];
|
int codePoints[codePointCount];
|
||||||
|
@ -344,15 +344,15 @@ static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz,
|
||||||
// Use 1 for count to indicate the word has inputted.
|
// Use 1 for count to indicate the word has inputted.
|
||||||
const UnigramProperty unigramProperty(isBeginningOfSentence, isNotAWord,
|
const UnigramProperty unigramProperty(isBeginningOfSentence, isNotAWord,
|
||||||
isBlacklisted, probability, timestamp, 0 /* level */, 1 /* count */, &shortcuts);
|
isBlacklisted, probability, timestamp, 0 /* level */, 1 /* count */, &shortcuts);
|
||||||
dictionary->addUnigramEntry(codePoints, codePointCount, &unigramProperty);
|
return dictionary->addUnigramEntry(codePoints, codePointCount, &unigramProperty);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz, jlong dict,
|
static bool latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz, jlong dict,
|
||||||
jintArray word0, jboolean isBeginningOfSentence, jintArray word1, jint probability,
|
jintArray word0, jboolean isBeginningOfSentence, jintArray word1, jint probability,
|
||||||
jint timestamp) {
|
jint timestamp) {
|
||||||
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
||||||
if (!dictionary) {
|
if (!dictionary) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
jsize word0Length = env->GetArrayLength(word0);
|
jsize word0Length = env->GetArrayLength(word0);
|
||||||
int word0CodePoints[word0Length];
|
int word0CodePoints[word0Length];
|
||||||
|
@ -366,14 +366,14 @@ static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz,
|
||||||
const BigramProperty bigramProperty(&bigramTargetCodePoints, probability,
|
const BigramProperty bigramProperty(&bigramTargetCodePoints, probability,
|
||||||
timestamp, 0 /* level */, 1 /* count */);
|
timestamp, 0 /* level */, 1 /* count */);
|
||||||
const PrevWordsInfo prevWordsInfo(word0CodePoints, word0Length, isBeginningOfSentence);
|
const PrevWordsInfo prevWordsInfo(word0CodePoints, word0Length, isBeginningOfSentence);
|
||||||
dictionary->addNgramEntry(&prevWordsInfo, &bigramProperty);
|
return dictionary->addNgramEntry(&prevWordsInfo, &bigramProperty);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void latinime_BinaryDictionary_removeBigramWords(JNIEnv *env, jclass clazz, jlong dict,
|
static bool latinime_BinaryDictionary_removeBigramWords(JNIEnv *env, jclass clazz, jlong dict,
|
||||||
jintArray word0, jboolean isBeginningOfSentence, jintArray word1) {
|
jintArray word0, jboolean isBeginningOfSentence, jintArray word1) {
|
||||||
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
|
||||||
if (!dictionary) {
|
if (!dictionary) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
jsize word0Length = env->GetArrayLength(word0);
|
jsize word0Length = env->GetArrayLength(word0);
|
||||||
int word0CodePoints[word0Length];
|
int word0CodePoints[word0Length];
|
||||||
|
@ -382,7 +382,7 @@ static void latinime_BinaryDictionary_removeBigramWords(JNIEnv *env, jclass claz
|
||||||
int word1CodePoints[word1Length];
|
int word1CodePoints[word1Length];
|
||||||
env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints);
|
env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints);
|
||||||
const PrevWordsInfo prevWordsInfo(word0CodePoints, word0Length, isBeginningOfSentence);
|
const PrevWordsInfo prevWordsInfo(word0CodePoints, word0Length, isBeginningOfSentence);
|
||||||
dictionary->removeNgramEntry(&prevWordsInfo, word1CodePoints, word1Length);
|
return dictionary->removeNgramEntry(&prevWordsInfo, word1CodePoints, word1Length);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns how many language model params are processed.
|
// Returns how many language model params are processed.
|
||||||
|
@ -610,7 +610,7 @@ static const JNINativeMethod sMethods[] = {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
const_cast<char *>("flushNative"),
|
const_cast<char *>("flushNative"),
|
||||||
const_cast<char *>("(JLjava/lang/String;)V"),
|
const_cast<char *>("(JLjava/lang/String;)Z"),
|
||||||
reinterpret_cast<void *>(latinime_BinaryDictionary_flush)
|
reinterpret_cast<void *>(latinime_BinaryDictionary_flush)
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
@ -620,7 +620,7 @@ static const JNINativeMethod sMethods[] = {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
const_cast<char *>("flushWithGCNative"),
|
const_cast<char *>("flushWithGCNative"),
|
||||||
const_cast<char *>("(JLjava/lang/String;)V"),
|
const_cast<char *>("(JLjava/lang/String;)Z"),
|
||||||
reinterpret_cast<void *>(latinime_BinaryDictionary_flushWithGC)
|
reinterpret_cast<void *>(latinime_BinaryDictionary_flushWithGC)
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
@ -651,17 +651,17 @@ static const JNINativeMethod sMethods[] = {
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
const_cast<char *>("addUnigramWordNative"),
|
const_cast<char *>("addUnigramWordNative"),
|
||||||
const_cast<char *>("(J[II[IIZZZI)V"),
|
const_cast<char *>("(J[II[IIZZZI)Z"),
|
||||||
reinterpret_cast<void *>(latinime_BinaryDictionary_addUnigramWord)
|
reinterpret_cast<void *>(latinime_BinaryDictionary_addUnigramWord)
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
const_cast<char *>("addBigramWordsNative"),
|
const_cast<char *>("addBigramWordsNative"),
|
||||||
const_cast<char *>("(J[IZ[III)V"),
|
const_cast<char *>("(J[IZ[III)Z"),
|
||||||
reinterpret_cast<void *>(latinime_BinaryDictionary_addBigramWords)
|
reinterpret_cast<void *>(latinime_BinaryDictionary_addBigramWords)
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
const_cast<char *>("removeBigramWordsNative"),
|
const_cast<char *>("removeBigramWordsNative"),
|
||||||
const_cast<char *>("(J[IZ[I)V"),
|
const_cast<char *>("(J[IZ[I)Z"),
|
||||||
reinterpret_cast<void *>(latinime_BinaryDictionary_removeBigramWords)
|
reinterpret_cast<void *>(latinime_BinaryDictionary_removeBigramWords)
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
|
@ -80,38 +80,38 @@ int Dictionary::getBigramProbability(const PrevWordsInfo *const prevWordsInfo, c
|
||||||
return mBigramDictionary.getBigramProbability(prevWordsInfo, word, length);
|
return mBigramDictionary.getBigramProbability(prevWordsInfo, word, length);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::addUnigramEntry(const int *const word, const int length,
|
bool Dictionary::addUnigramEntry(const int *const word, const int length,
|
||||||
const UnigramProperty *const unigramProperty) {
|
const UnigramProperty *const unigramProperty) {
|
||||||
if (unigramProperty->representsBeginningOfSentence()
|
if (unigramProperty->representsBeginningOfSentence()
|
||||||
&& !mDictionaryStructureWithBufferPolicy->getHeaderStructurePolicy()
|
&& !mDictionaryStructureWithBufferPolicy->getHeaderStructurePolicy()
|
||||||
->supportsBeginningOfSentence()) {
|
->supportsBeginningOfSentence()) {
|
||||||
AKLOGE("The dictionary doesn't support Beginning-of-Sentence.");
|
AKLOGE("The dictionary doesn't support Beginning-of-Sentence.");
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
mDictionaryStructureWithBufferPolicy->addUnigramEntry(word, length, unigramProperty);
|
return mDictionaryStructureWithBufferPolicy->addUnigramEntry(word, length, unigramProperty);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
|
bool Dictionary::addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
|
||||||
const BigramProperty *const bigramProperty) {
|
const BigramProperty *const bigramProperty) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
mDictionaryStructureWithBufferPolicy->addNgramEntry(prevWordsInfo, bigramProperty);
|
return mDictionaryStructureWithBufferPolicy->addNgramEntry(prevWordsInfo, bigramProperty);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
|
bool Dictionary::removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
|
||||||
const int *const word, const int length) {
|
const int *const word, const int length) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
mDictionaryStructureWithBufferPolicy->removeNgramEntry(prevWordsInfo, word, length);
|
return mDictionaryStructureWithBufferPolicy->removeNgramEntry(prevWordsInfo, word, length);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::flush(const char *const filePath) {
|
bool Dictionary::flush(const char *const filePath) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
mDictionaryStructureWithBufferPolicy->flush(filePath);
|
return mDictionaryStructureWithBufferPolicy->flush(filePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
void Dictionary::flushWithGC(const char *const filePath) {
|
bool Dictionary::flushWithGC(const char *const filePath) {
|
||||||
TimeKeeper::setCurrentTime();
|
TimeKeeper::setCurrentTime();
|
||||||
mDictionaryStructureWithBufferPolicy->flushWithGC(filePath);
|
return mDictionaryStructureWithBufferPolicy->flushWithGC(filePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Dictionary::needsToRunGC(const bool mindsBlockByGC) {
|
bool Dictionary::needsToRunGC(const bool mindsBlockByGC) {
|
||||||
|
|
|
@ -76,18 +76,18 @@ class Dictionary {
|
||||||
int getBigramProbability(const PrevWordsInfo *const prevWordsInfo,
|
int getBigramProbability(const PrevWordsInfo *const prevWordsInfo,
|
||||||
const int *word, int length) const;
|
const int *word, int length) const;
|
||||||
|
|
||||||
void addUnigramEntry(const int *const codePoints, const int codePointCount,
|
bool addUnigramEntry(const int *const codePoints, const int codePointCount,
|
||||||
const UnigramProperty *const unigramProperty);
|
const UnigramProperty *const unigramProperty);
|
||||||
|
|
||||||
void addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
|
bool addNgramEntry(const PrevWordsInfo *const prevWordsInfo,
|
||||||
const BigramProperty *const bigramProperty);
|
const BigramProperty *const bigramProperty);
|
||||||
|
|
||||||
void removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word,
|
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word,
|
||||||
const int length);
|
const int length);
|
||||||
|
|
||||||
void flush(const char *const filePath);
|
bool flush(const char *const filePath);
|
||||||
|
|
||||||
void flushWithGC(const char *const filePath);
|
bool flushWithGC(const char *const filePath);
|
||||||
|
|
||||||
bool needsToRunGC(const bool mindsBlockByGC);
|
bool needsToRunGC(const bool mindsBlockByGC);
|
||||||
|
|
||||||
|
|
|
@ -81,9 +81,11 @@ class DictionaryStructureWithBufferPolicy {
|
||||||
virtual bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
|
virtual bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo,
|
||||||
const int *const word, const int length) = 0;
|
const int *const word, const int length) = 0;
|
||||||
|
|
||||||
virtual void flush(const char *const filePath) = 0;
|
// Returns whether the flush was success or not.
|
||||||
|
virtual bool flush(const char *const filePath) = 0;
|
||||||
|
|
||||||
virtual void flushWithGC(const char *const filePath) = 0;
|
// Returns whether the GC and flush were success or not.
|
||||||
|
virtual bool flushWithGC(const char *const filePath) = 0;
|
||||||
|
|
||||||
virtual bool needsToRunGC(const bool mindsBlockByGC) const = 0;
|
virtual bool needsToRunGC(const bool mindsBlockByGC) const = 0;
|
||||||
|
|
||||||
|
|
|
@ -296,26 +296,30 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Ver4PatriciaTriePolicy::flush(const char *const filePath) {
|
bool Ver4PatriciaTriePolicy::flush(const char *const filePath) {
|
||||||
if (!mBuffers->isUpdatable()) {
|
if (!mBuffers->isUpdatable()) {
|
||||||
AKLOGI("Warning: flush() is called for non-updatable dictionary. filePath: %s", filePath);
|
AKLOGI("Warning: flush() is called for non-updatable dictionary. filePath: %s", filePath);
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
if (!mWritingHelper.writeToDictFile(filePath, mUnigramCount, mBigramCount)) {
|
if (!mWritingHelper.writeToDictFile(filePath, mUnigramCount, mBigramCount)) {
|
||||||
AKLOGE("Cannot flush the dictionary to file.");
|
AKLOGE("Cannot flush the dictionary to file.");
|
||||||
mIsCorrupted = true;
|
mIsCorrupted = true;
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
|
bool Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
|
||||||
if (!mBuffers->isUpdatable()) {
|
if (!mBuffers->isUpdatable()) {
|
||||||
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
|
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
if (!mWritingHelper.writeToDictFileWithGC(getRootPosition(), filePath)) {
|
if (!mWritingHelper.writeToDictFileWithGC(getRootPosition(), filePath)) {
|
||||||
AKLOGE("Cannot flush the dictionary to file with GC.");
|
AKLOGE("Cannot flush the dictionary to file with GC.");
|
||||||
mIsCorrupted = true;
|
mIsCorrupted = true;
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
|
bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
|
||||||
|
|
|
@ -117,9 +117,9 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word,
|
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word,
|
||||||
const int length);
|
const int length);
|
||||||
|
|
||||||
void flush(const char *const filePath);
|
bool flush(const char *const filePath);
|
||||||
|
|
||||||
void flushWithGC(const char *const filePath);
|
bool flushWithGC(const char *const filePath);
|
||||||
|
|
||||||
bool needsToRunGC(const bool mindsBlockByGC) const;
|
bool needsToRunGC(const bool mindsBlockByGC) const;
|
||||||
|
|
||||||
|
|
|
@ -102,14 +102,16 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void flush(const char *const filePath) {
|
bool flush(const char *const filePath) {
|
||||||
// This method should not be called for non-updatable dictionary.
|
// This method should not be called for non-updatable dictionary.
|
||||||
AKLOGI("Warning: flush() is called for non-updatable dictionary.");
|
AKLOGI("Warning: flush() is called for non-updatable dictionary.");
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void flushWithGC(const char *const filePath) {
|
bool flushWithGC(const char *const filePath) {
|
||||||
// This method should not be called for non-updatable dictionary.
|
// This method should not be called for non-updatable dictionary.
|
||||||
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
|
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool needsToRunGC(const bool mindsBlockByGC) const {
|
bool needsToRunGC(const bool mindsBlockByGC) const {
|
||||||
|
|
|
@ -304,26 +304,30 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Ver4PatriciaTriePolicy::flush(const char *const filePath) {
|
bool Ver4PatriciaTriePolicy::flush(const char *const filePath) {
|
||||||
if (!mBuffers->isUpdatable()) {
|
if (!mBuffers->isUpdatable()) {
|
||||||
AKLOGI("Warning: flush() is called for non-updatable dictionary. filePath: %s", filePath);
|
AKLOGI("Warning: flush() is called for non-updatable dictionary. filePath: %s", filePath);
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
if (!mWritingHelper.writeToDictFile(filePath, mUnigramCount, mBigramCount)) {
|
if (!mWritingHelper.writeToDictFile(filePath, mUnigramCount, mBigramCount)) {
|
||||||
AKLOGE("Cannot flush the dictionary to file.");
|
AKLOGE("Cannot flush the dictionary to file.");
|
||||||
mIsCorrupted = true;
|
mIsCorrupted = true;
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
|
bool Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) {
|
||||||
if (!mBuffers->isUpdatable()) {
|
if (!mBuffers->isUpdatable()) {
|
||||||
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
|
AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary.");
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
if (!mWritingHelper.writeToDictFileWithGC(getRootPosition(), filePath)) {
|
if (!mWritingHelper.writeToDictFileWithGC(getRootPosition(), filePath)) {
|
||||||
AKLOGE("Cannot flush the dictionary to file with GC.");
|
AKLOGE("Cannot flush the dictionary to file with GC.");
|
||||||
mIsCorrupted = true;
|
mIsCorrupted = true;
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
|
bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const {
|
||||||
|
|
|
@ -99,9 +99,9 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
|
||||||
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word1,
|
bool removeNgramEntry(const PrevWordsInfo *const prevWordsInfo, const int *const word1,
|
||||||
const int length1);
|
const int length1);
|
||||||
|
|
||||||
void flush(const char *const filePath);
|
bool flush(const char *const filePath);
|
||||||
|
|
||||||
void flushWithGC(const char *const filePath);
|
bool flushWithGC(const char *const filePath);
|
||||||
|
|
||||||
bool needsToRunGC(const bool mindsBlockByGC) const;
|
bool needsToRunGC(const bool mindsBlockByGC) const;
|
||||||
|
|
||||||
|
|
|
@ -75,33 +75,54 @@ public class Ver4DictEncoder implements DictEncoder {
|
||||||
for (final WordProperty wordProperty : dict) {
|
for (final WordProperty wordProperty : dict) {
|
||||||
// TODO: switch to addMultipleDictionaryEntries when they support shortcuts
|
// TODO: switch to addMultipleDictionaryEntries when they support shortcuts
|
||||||
if (null == wordProperty.mShortcutTargets || wordProperty.mShortcutTargets.isEmpty()) {
|
if (null == wordProperty.mShortcutTargets || wordProperty.mShortcutTargets.isEmpty()) {
|
||||||
binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(),
|
if (!binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(),
|
||||||
null /* shortcutTarget */, 0 /* shortcutProbability */,
|
null /* shortcutTarget */, 0 /* shortcutProbability */,
|
||||||
wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord,
|
wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord,
|
||||||
wordProperty.mIsBlacklistEntry, 0 /* timestamp */);
|
wordProperty.mIsBlacklistEntry, 0 /* timestamp */)) {
|
||||||
|
MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
|
for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
|
||||||
binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(),
|
if (!binaryDict.addUnigramEntry(wordProperty.mWord,
|
||||||
|
wordProperty.getProbability(),
|
||||||
shortcutTarget.mWord, shortcutTarget.getProbability(),
|
shortcutTarget.mWord, shortcutTarget.getProbability(),
|
||||||
wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord,
|
wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord,
|
||||||
wordProperty.mIsBlacklistEntry, 0 /* timestamp */);
|
wordProperty.mIsBlacklistEntry, 0 /* timestamp */)) {
|
||||||
|
MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord
|
||||||
|
+ ", shortcutTarget: " + shortcutTarget.mWord);
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
|
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
|
||||||
binaryDict.flushWithGC();
|
if (!binaryDict.flushWithGC()) {
|
||||||
|
MakedictLog.e("Cannot flush dict with GC.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (final WordProperty word0Property : dict) {
|
for (final WordProperty word0Property : dict) {
|
||||||
if (null == word0Property.mBigrams) continue;
|
if (null == word0Property.mBigrams) continue;
|
||||||
for (final WeightedString word1 : word0Property.mBigrams) {
|
for (final WeightedString word1 : word0Property.mBigrams) {
|
||||||
binaryDict.addNgramEntry(new PrevWordsInfo(word0Property.mWord), word1.mWord,
|
final PrevWordsInfo prevWordsInfo = new PrevWordsInfo(word0Property.mWord);
|
||||||
word1.getProbability(), 0 /* timestamp */);
|
if (!binaryDict.addNgramEntry(prevWordsInfo, word1.mWord,
|
||||||
|
word1.getProbability(), 0 /* timestamp */)) {
|
||||||
|
MakedictLog.e("Cannot add n-gram entry for "
|
||||||
|
+ prevWordsInfo + " -> " + word1.mWord);
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
|
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
|
||||||
binaryDict.flushWithGC();
|
if (!binaryDict.flushWithGC()) {
|
||||||
|
MakedictLog.e("Cannot flush dict with GC.");
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
binaryDict.flushWithGC();
|
}
|
||||||
|
if (!binaryDict.flushWithGC()) {
|
||||||
|
MakedictLog.e("Cannot flush dict with GC.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
binaryDict.close();
|
binaryDict.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue