nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge "Add getDictDecoder."

Browse source
This commit is contained in:
Ken Wakasa 2013-09-12 11:17:03 +00:00 committed by Android (Google) Code Review
commit a6dc3056ff
13 changed files with 80 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
View file

@ -21,9 +21,10 @@ import android.content.SharedPreferences;
import android.content.res.AssetFileDescriptor;
import android.util.Log;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.DictionaryInfoUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
@ -228,7 +229,7 @@ final public class BinaryDictionaryGetter {
private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) {
try {
// Read the version of the file
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f);
final FileHeader header = dictDecoder.readHeader();
final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY);

6
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View file

@ -148,7 +148,7 @@ public final class BinaryDictIOUtils {
* @throws IOException if the file can't be read.
* @throws UnsupportedFormatException if the format of the file is not recognized.
*/
/* package */ static void readUnigramsAndBigramsBinary(final Ver3DictDecoder dictDecoder,
/* package */ static void readUnigramsAndBigramsBinary(final DictDecoder dictDecoder,
final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
UnsupportedFormatException {
@ -169,7 +169,7 @@ public final class BinaryDictIOUtils {
* @throws UnsupportedFormatException if the format of the file is not recognized.
*/
@UsedForTesting
/* package */ static int getTerminalPosition(final Ver3DictDecoder dictDecoder,
/* package */ static int getTerminalPosition(final DictDecoder dictDecoder,
final String word) throws IOException, UnsupportedFormatException {
if (word == null) return FormatSpec.NOT_VALID_WORD;
dictDecoder.setPosition(0);
@ -519,7 +519,7 @@ public final class BinaryDictIOUtils {
final File file, final long offset, final long length)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE];
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file,
new DictDecoder.DictionaryBufferFactory() {
@Override
public DictBuffer getDictionaryBuffer(File file)

8
java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
View file

@ -118,6 +118,14 @@ public interface DictDecoder {
public boolean readForwardLinkAndAdvancePosition();
public boolean hasNextPtNodeArray();
/**
* Opens the dictionary file and makes DictBuffer.
*/
@UsedForTesting
public void openDictBuffer() throws FileNotFoundException, IOException;
@UsedForTesting
public boolean isOpenedDictBuffer();
// Flags for DictionaryBufferFactory.
public static final int USE_READONLY_BYTEBUFFER = 0x01000000;
public static final int USE_BYTEARRAY = 0x02000000;

25
java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
View file

@ -18,8 +18,11 @@ package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import java.io.File;
/**
* Dictionary File Format Specification.
*/
@ -341,6 +344,28 @@ public final class FormatSpec {
}
}
/**
* Returns new dictionary decoder.
*
* @param dictFile the dictionary file.
* @param bufferType the flag indicating buffer type which is used by the dictionary decoder.
* @return new dictionary decoder if the dictionary file exists, otherwise null.
*/
public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) {
if (!dictFile.isFile()) return null;
return new Ver3DictDecoder(dictFile, bufferType);
}
public static DictDecoder getDictDecoder(final File dictFile,
final DictionaryBufferFactory factory) {
if (!dictFile.isFile()) return null;
return new Ver3DictDecoder(dictFile, factory);
}
public static DictDecoder getDictDecoder(final File dictFile) {
return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER);
}
private FormatSpec() {
// This utility class is not publicly instantiable.
}

14
java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java
View file

@ -173,11 +173,7 @@ public class Ver3DictDecoder implements DictDecoder {
private final DictionaryBufferFactory mBufferFactory;
private DictBuffer mDictBuffer;
public Ver3DictDecoder(final File file) {
this(file, USE_READONLY_BYTEBUFFER);
}
public Ver3DictDecoder(final File file, final int factoryFlag) {
/* package */ Ver3DictDecoder(final File file, final int factoryFlag) {
mDictionaryBinaryFile = file;
mDictBuffer = null;
@ -192,15 +188,21 @@ public class Ver3DictDecoder implements DictDecoder {
}
}
public Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) {
/* package */ Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) {
mDictionaryBinaryFile = file;
mBufferFactory = factory;
}
@Override
public void openDictBuffer() throws FileNotFoundException, IOException {
mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile);
}
@Override
public boolean isOpenedDictBuffer() {
return mDictBuffer != null;
}
/* package */ DictBuffer getDictBuffer() {
return mDictBuffer;
}

16
java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
View file

@ -25,14 +25,13 @@ import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
@ -173,16 +172,19 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
// Load the dictionary from binary file
final File dictFile = new File(mContext.getFilesDir(), mFileName);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile,
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile,
DictDecoder.USE_BYTEARRAY);
if (dictDecoder == null) {
// This is an expected condition: we don't have a user history dictionary for this
// language yet. It will be created sometime later.
return;
}
try {
dictDecoder.openDictBuffer();
UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
} catch (FileNotFoundException e) {
// This is an expected condition: we don't have a user history dictionary for this
// language yet. It will be created sometime later.
} catch (IOException e) {
Log.e(TAG, "IOException on opening a bytebuffer", e);
Log.d(TAG, "IOException on opening a bytebuffer", e);
} finally {
if (PROFILE_SAVE_RESTORE) {
final long diff = System.currentTimeMillis() - now;

4
java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
View file

@ -20,13 +20,13 @@ import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.PendingAttribute;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
import java.io.IOException;
@ -125,7 +125,7 @@ public final class UserHistoryDictIOUtils {
/**
* Reads dictionary from file.
*/
public static void readDictionaryBinary(final Ver3DictDecoder dictDecoder,
public static void readDictionaryBinary(final DictDecoder dictDecoder,
final OnAddWordListener dict) {
final TreeMap<Integer, String> unigrams = CollectionUtils.newTreeMap();
final TreeMap<Integer, Integer> frequencies = CollectionUtils.newTreeMap();

33
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View file

@ -131,18 +131,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
// Utilities for test
/**
* Makes new DictDecoder according to BUFFER_TYPE.
*/
private Ver3DictDecoder getDictDecoder(final File file, final int bufferType) {
if (bufferType == USE_BYTE_BUFFER) {
return new Ver3DictDecoder(file, DictDecoder.USE_READONLY_BYTEBUFFER);
} else if (bufferType == USE_BYTE_ARRAY) {
return new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
}
return null;
}
/**
* Generates a random word.
*/
@ -285,9 +273,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
FusionDictionary dict = null;
try {
final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType);
dictDecoder.openDictBuffer();
assertNotNull(dictDecoder.getDictBuffer());
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
now = System.currentTimeMillis();
dict = dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */);
diff = System.currentTimeMillis() - now;
@ -443,9 +429,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
long now = -1, diff = -1;
try {
final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType);
dictDecoder.openDictBuffer();
assertNotNull("Can't get buffer.", dictDecoder.getDictBuffer());
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
now = System.currentTimeMillis();
dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams);
diff = System.currentTimeMillis() - now;
@ -531,9 +515,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
}
// Tests for getTerminalPosition
private String getWordFromBinary(final Ver3DictDecoder dictDecoder, final int address) {
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
if (dictBuffer.position() != 0) dictBuffer.position(0);
private String getWordFromBinary(final DictDecoder dictDecoder, final int address) {
if (dictDecoder.getPosition() != 0) dictDecoder.setPosition(0);
FileHeader fileHeader = null;
try {
@ -548,7 +531,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
address, fileHeader.mFormatOptions).mWord;
}
private long runGetTerminalPosition(final Ver3DictDecoder dictDecoder, final String word,
private long runGetTerminalPosition(final DictDecoder dictDecoder, final String word,
int index, boolean contained) {
final int expectedFrequency = (UNIGRAM_FREQ + index) % 255;
long diff = -1;
@ -584,14 +567,14 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer();
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
}
assertNotNull("Can't get the buffer", dictDecoder.getDictBuffer());
assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer());
try {
// too long word
@ -648,7 +631,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
}
assertNotNull("Can't get the buffer", dictDecoder.getDictBuffer());
assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer());
try {
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,

16
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
View file

@ -140,7 +140,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
int position = FormatSpec.NOT_VALID_WORD;
try {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
DictDecoder.USE_READONLY_BYTEBUFFER);
position = dictDecoder.getTerminalPosition(word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
@ -149,7 +150,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
/**
* Find a word using the Ver3DictDecoder.
* Find a word using the DictDecoder.
*
* @param dictDecoder the dict decoder
* @param word the word searched
@ -157,21 +158,20 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
* @throws IOException
* @throws UnsupportedFormatException
*/
private static PtNodeInfo findWordByBinaryDictReader(final Ver3DictDecoder dictDecoder,
private static PtNodeInfo findWordByBinaryDictReader(final DictDecoder dictDecoder,
final String word) throws IOException, UnsupportedFormatException {
int position = dictDecoder.getTerminalPosition(word);
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
if (position != FormatSpec.NOT_VALID_WORD) {
dictBuffer.position(0);
dictDecoder.setPosition(0);
final FileHeader header = dictDecoder.readHeader();
dictBuffer.position(position);
dictDecoder.setPosition(position);
return dictDecoder.readPtNode(position, header.mFormatOptions);
}
return null;
}
private PtNodeInfo findWordFromFile(final File file, final String word) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
PtNodeInfo info = null;
try {
dictDecoder.openDictBuffer();
@ -234,7 +234,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
private void checkReverseLookup(final File file, final String word, final int position) {
try {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
final FileHeader fileHeader = dictDecoder.readHeader();
assertEquals(word,
BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize,

2
tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
View file

@ -143,7 +143,7 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
}
private void readDictFromFile(final File file, final OnAddWordListener listener) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer();
} catch (FileNotFoundException e) {

4
tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
View file

@ -18,9 +18,9 @@ package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import org.xml.sax.SAXException;
@ -185,7 +185,7 @@ public final class BinaryDictOffdeviceUtils {
crash(filename, new RuntimeException(
filename + " does not seem to be a dictionary file"));
} else {
final DictDecoder dictDecoder = new Ver3DictDecoder(decodedSpec.mFile,
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file,
DictDecoder.USE_BYTEARRAY);
if (report) {
System.out.println("Format : Binary dictionary format");

2
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View file

@ -267,7 +267,7 @@ public class DictionaryMaker {
private static FusionDictionary readBinaryFile(final String binaryFilename)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final File file = new File(binaryFilename);
final DictDecoder dictDecoder = new Ver3DictDecoder(file);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */);
}

4
tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
View file

@ -18,12 +18,12 @@ package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import junit.framework.TestCase;
@ -69,7 +69,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
}
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final DictDecoder dictDecoder = new Ver3DictDecoder(decodeSpec.mFile);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile);
final FusionDictionary resultDict = dictDecoder.readDictionaryBinary(
null /* dict : an optional dictionary to add words to, or null */,
false /* deleteDictIfBroken */);