nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "DictionaryWriter to abstract binary dictionary writing."

main
Keisuke Kuroynagi 2013-07-04 14:15:24 +00:00 committed by Android (Google) Code Review
parent cac66f29eb edd1992ed3
commit db296305d6
5 changed files with 254 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin;
import android.content.Context;
import android.util.Log;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
// TODO: Quit extending Dictionary after implementing dynamic binary dictionary.
abstract public class AbstractDictionaryWriter extends Dictionary {
/** Used for Log actions from this class */
private static final String TAG = AbstractDictionaryWriter.class.getSimpleName();
private final Context mContext;
public AbstractDictionaryWriter(final Context context, final String dictType) {
super(dictType);
mContext = context;
}
abstract public void clear();
abstract public void addUnigramWord(final String word, final String shortcutTarget,
final int frequency, final boolean isNotAWord);
abstract public void addBigramWords(final String word0, final String word1,
final int frequency, final boolean isValid);
abstract public void removeBigramWords(final String word0, final String word1);
abstract protected void writeBinaryDictionary(final FileOutputStream out)
throws IOException, UnsupportedFormatException;
public void write(final String fileName) {
final String tempFileName = fileName + ".temp";
final File file = new File(mContext.getFilesDir(), fileName);
final File tempFile = new File(mContext.getFilesDir(), tempFileName);
FileOutputStream out = null;
try {
out = new FileOutputStream(tempFile);
writeBinaryDictionary(out);
out.flush();
out.close();
tempFile.renameTo(file);
} catch (IOException e) {
Log.e(TAG, "IO exception while writing file", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "Unsupported format", e);
} finally {
if (out != null) {
try {
out.close();
} catch (IOException e) {
// ignore
}
}
}
}
}

6
java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
View File

@ -109,7 +109,6 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
@Override
public void loadDictionaryAsync() {
clearFusionDictionary();
loadDeviceAccountsEmailAddresses();
loadDictionaryAsyncForUri(ContactsContract.Profile.CONTENT_URI);
// TODO: Switch this URL to the newer ContactsContract too
@ -235,6 +234,11 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
return end;
}
@Override
protected boolean needsToReloadBeforeWriting() {
return true;
}
@Override
protected boolean hasContentChanged() {
final long startTime = SystemClock.uptimeMillis();

107
java/src/com/android/inputmethod/latin/DictionaryWriter.java Normal file
View File

@ -0,0 +1,107 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin;
import android.content.Context;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
/**
* An in memory dictionary for memorizing entries and writing a binary dictionary.
*/
public class DictionaryWriter extends AbstractDictionaryWriter {
// TODO: Regenerate version 3 binary dictionary.
private static final int BINARY_DICT_VERSION = 2;
private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
new FormatSpec.FormatOptions(BINARY_DICT_VERSION);
private FusionDictionary mFusionDictionary;
public DictionaryWriter(final Context context, final String dictType) {
super(context, dictType);
clear();
}
@Override
public void clear() {
final HashMap<String, String> attributes = CollectionUtils.newHashMap();
mFusionDictionary = new FusionDictionary(new Node(),
new FusionDictionary.DictionaryOptions(attributes, false, false));
}
/**
* Adds a word unigram to the fusion dictionary.
*/
// TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries,
// considering performance regression.
@Override
public void addUnigramWord(final String word, final String shortcutTarget, final int frequency,
final boolean isNotAWord) {
if (shortcutTarget == null) {
mFusionDictionary.add(word, frequency, null, isNotAWord);
} else {
// TODO: Do this in the subclass, with this class taking an arraylist.
final ArrayList<WeightedString> shortcutTargets = CollectionUtils.newArrayList();
shortcutTargets.add(new WeightedString(shortcutTarget, frequency));
mFusionDictionary.add(word, frequency, shortcutTargets, isNotAWord);
}
}
@Override
public void addBigramWords(final String word0, final String word1, final int frequency,
final boolean isValid) {
mFusionDictionary.setBigram(word0, word1, frequency);
}
@Override
public void removeBigramWords(final String word0, final String word1) {
// This class don't support removing bigram words.
}
@Override
protected void writeBinaryDictionary(final FileOutputStream out)
throws IOException, UnsupportedFormatException {
BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS);
}
@Override
public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
final String prevWord, final ProximityInfo proximityInfo,
boolean blockOffensiveWords) {
// This class doesn't support suggestion.
return null;
}
@Override
public boolean isValidWord(String word) {
// This class doesn't support dictionary retrieval.
return false;
}
}

151
java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
View File

@ -22,17 +22,9 @@ import android.util.Log;
import com.android.inputmethod.keyboard.ProximityInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@ -76,8 +68,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
*/
private BinaryDictionary mBinaryDictionary;
/** The expandable fusion dictionary used to generate the binary dictionary. */
private FusionDictionary mFusionDictionary;
/** The in-memory dictionary used to generate the binary dictionary. */
private AbstractDictionaryWriter mDictionaryWriter;
/**
* The name of this dictionary, used as the filename for storing the binary dictionary. Multiple
@ -92,11 +84,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
/** Controls access to the local binary dictionary for this instance. */
private final DictionaryController mLocalDictionaryController = new DictionaryController();
// TODO: Regenerate version 3 binary dictionary.
private static final int BINARY_DICT_VERSION = 2;
private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
new FormatSpec.FormatOptions(BINARY_DICT_VERSION);
/**
* Abstract method for loading the unigrams and bigrams of a given dictionary in a background
* thread.
@ -138,7 +125,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
mContext = context;
mBinaryDictionary = null;
mSharedDictionaryController = getSharedDictionaryController(filename);
clearFusionDictionary();
mDictionaryWriter = new DictionaryWriter(context, dictType);
}
protected static String getFilenameWithLocale(final String name, final String localeStr) {
@ -157,47 +144,51 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
mBinaryDictionary.close();
mBinaryDictionary = null;
}
mDictionaryWriter.close();
} finally {
mLocalDictionaryController.writeLock().unlock();
}
}
/**
* Clears the fusion dictionary on the Java side. Note: Does not modify the binary dictionary on
* the native side.
* Adds a word unigram to the dictionary. Used for loading a dictionary.
*/
public void clearFusionDictionary() {
final HashMap<String, String> attributes = CollectionUtils.newHashMap();
mFusionDictionary = new FusionDictionary(new Node(),
new FusionDictionary.DictionaryOptions(attributes, false, false));
protected void addWord(final String word, final String shortcutTarget,
final int frequency, final boolean isNotAWord) {
mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
}
/**
* Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes
* are done to update the binary dictionary.
* Sets a word bigram in the dictionary. Used for loading a dictionary.
*/
// TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries,
// considering performance regression.
protected void addWord(final String word, final String shortcutTarget, final int frequency,
final boolean isNotAWord) {
if (shortcutTarget == null) {
mFusionDictionary.add(word, frequency, null, isNotAWord);
} else {
// TODO: Do this in the subclass, with this class taking an arraylist.
final ArrayList<WeightedString> shortcutTargets = CollectionUtils.newArrayList();
shortcutTargets.add(new WeightedString(shortcutTarget, frequency));
mFusionDictionary.add(word, frequency, shortcutTargets, isNotAWord);
}
}
/**
* Sets a word bigram in the fusion dictionary. Call updateBinaryDictionary when all changes are
* done to update the binary dictionary.
*/
// TODO: Create "cache dictionary" to cache fresh bigrams for frequently updated dictionaries,
// considering performance regression.
protected void setBigram(final String prevWord, final String word, final int frequency) {
mFusionDictionary.setBigram(prevWord, word, frequency);
mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */);
}
/**
* Dynamically adds a word unigram to the dictionary.
*/
protected void addWordDynamically(final String word, final String shortcutTarget,
final int frequency, final boolean isNotAWord) {
mLocalDictionaryController.writeLock().lock();
try {
mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
} finally {
mLocalDictionaryController.writeLock().unlock();
}
}
/**
* Dynamically sets a word bigram in the dictionary.
*/
protected void setBigramDynamically(final String prevWord, final String word,
final int frequency) {
mLocalDictionaryController.writeLock().lock();
try {
mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */);
} finally {
mLocalDictionaryController.writeLock().unlock();
}
}
@Override
@ -207,9 +198,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
asyncReloadDictionaryIfRequired();
if (mLocalDictionaryController.readLock().tryLock()) {
try {
if (mBinaryDictionary != null) {
return mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo,
final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
mDictionaryWriter.getSuggestions(composer, prevWord, proximityInfo,
blockOffensiveWords);
if (mBinaryDictionary != null) {
final ArrayList<SuggestedWordInfo> binarySuggestion =
mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo,
blockOffensiveWords);
if (inMemDictSuggestion == null) {
return binarySuggestion;
} else if (binarySuggestion == null) {
return inMemDictSuggestion;
} else {
binarySuggestion.addAll(binarySuggestion);
return binarySuggestion;
}
} else {
return inMemDictSuggestion;
}
} finally {
mLocalDictionaryController.readLock().unlock();
@ -240,22 +245,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
return mBinaryDictionary.isValidWord(word);
}
protected boolean isValidBigram(final String word1, final String word2) {
if (mBinaryDictionary == null) return false;
return mBinaryDictionary.isValidBigram(word1, word2);
}
protected boolean isValidBigramInner(final String word1, final String word2) {
if (mLocalDictionaryController.readLock().tryLock()) {
try {
return isValidBigramLocked(word1, word2);
} finally {
mLocalDictionaryController.readLock().unlock();
}
}
return false;
}
protected boolean isValidBigramLocked(final String word1, final String word2) {
if (mBinaryDictionary == null) return false;
return mBinaryDictionary.isValidBigram(word1, word2);
@ -274,7 +263,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
* Loads the current binary dictionary from internal storage. Assumes the dictionary file
* exists.
*/
protected void loadBinaryDictionary() {
private void loadBinaryDictionary() {
if (DEBUG) {
Log.d(TAG, "Loading binary dictionary: " + mFilename + " request="
+ mSharedDictionaryController.mLastUpdateRequestTime + " update="
@ -305,6 +294,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
}
}
/**
* Abstract method for checking if it is required to reload the dictionary before writing
* a binary dictionary.
*/
abstract protected boolean needsToReloadBeforeWriting();
/**
* Generates and writes a new binary dictionary based on the contents of the fusion dictionary.
*/
@ -314,33 +309,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
+ mSharedDictionaryController.mLastUpdateRequestTime + " update="
+ mSharedDictionaryController.mLastUpdateTime);
}
if (needsToReloadBeforeWriting()) {
mDictionaryWriter.clear();
loadDictionaryAsync();
final String tempFileName = mFilename + ".temp";
final File file = new File(mContext.getFilesDir(), mFilename);
final File tempFile = new File(mContext.getFilesDir(), tempFileName);
FileOutputStream out = null;
try {
out = new FileOutputStream(tempFile);
BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS);
out.flush();
out.close();
tempFile.renameTo(file);
clearFusionDictionary();
} catch (IOException e) {
Log.e(TAG, "IO exception while writing file", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "Unsupported format", e);
} finally {
if (out != null) {
try {
out.close();
} catch (IOException e) {
// ignore
}
}
}
mDictionaryWriter.write(mFilename);
}
/**

6
java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
View File

@ -240,7 +240,6 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary {
private void addWords(final Cursor cursor) {
final boolean hasShortcutColumn = Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN;
clearFusionDictionary();
if (cursor == null) return;
if (cursor.moveToFirst()) {
final int indexWord = cursor.getColumnIndex(Words.WORD);
@ -267,4 +266,9 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary {
protected boolean hasContentChanged() {
return true;
}
@Override
protected boolean needsToReloadBeforeWriting() {
return true;
}
}