nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

am e9a10ff0: Add DictDecoder.readDictionaryBinary. 

* commit 'e9a10ff0f026b5ec458f116afc7a75806574cbcd':
  Add DictDecoder.readDictionaryBinary.
main
Yuichiro Hanada 2013-08-23 04:32:36 -07:00 committed by Android Git Automerger
parent b6a70ead78 e9a10ff0f0
commit 6bdc19173f
13 changed files with 125 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
View File

@ -229,8 +229,6 @@ final public class BinaryDictionaryGetter {
try {
// Read the version of the file
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f);
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
final FileHeader header = dictDecoder.readHeader();
final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY);

10
java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
View File

@ -559,17 +559,9 @@ public final class BinaryDictDecoderUtils {
* @return the created (or merged) dictionary.
*/
@UsedForTesting
public static FusionDictionary readDictionaryBinary(final Ver3DictDecoder dictDecoder,
/* package */ static FusionDictionary readDictionaryBinary(final Ver3DictDecoder dictDecoder,
final FusionDictionary dict) throws FileNotFoundException, IOException,
UnsupportedFormatException {
// if the buffer has not been opened, open the buffer with bytebuffer.
if (dictDecoder.getDictBuffer() == null) dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
if (dictDecoder.getDictBuffer() == null) {
MakedictLog.e("Cannot open the buffer");
}
// Read header
final FileHeader fileHeader = dictDecoder.readHeader();

7
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View File

@ -521,8 +521,8 @@ public final class BinaryDictIOUtils {
final File file, final long offset, final long length)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE];
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
dictDecoder.openDictBuffer(new DictDecoder.DictionaryBufferFactory() {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
new DictDecoder.DictionaryBufferFactory() {
@Override
public DictBuffer getDictionaryBuffer(File file)
throws FileNotFoundException, IOException {
@ -534,7 +534,8 @@ public final class BinaryDictIOUtils {
inStream.close();
}
}
});
}
);
return dictDecoder.readHeader();
}

20
java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
View File

@ -44,6 +44,26 @@ public interface DictDecoder {
*/
public CharGroupInfo readPtNode(final int ptNodePos, final FormatOptions formatOptions);
/**
* Reads a buffer and returns the memory representation of the dictionary.
*
* This high-level method takes a buffer and reads its contents, populating a
* FusionDictionary structure. The optional dict argument is an existing dictionary to
* which words from the buffer should be added. If it is null, a new dictionary is created.
*
* @param dict an optional dictionary to add words to, or null.
* @return the created (or merged) dictionary.
*/
@UsedForTesting
public FusionDictionary readDictionaryBinary(final FusionDictionary dict)
throws FileNotFoundException, IOException, UnsupportedFormatException;
// Flags for DictionaryBufferFactory.
public static final int USE_READONLY_BYTEBUFFER = 0x01000000;
public static final int USE_BYTEARRAY = 0x02000000;
public static final int USE_WRITABLE_BYTEBUFFER = 0x04000000;
public static final int MASK_DICTBUFFER = 0x0F000000;
public interface DictionaryBufferFactory {
public DictBuffer getDictionaryBuffer(final File file)
throws FileNotFoundException, IOException;

45
java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java
View File

@ -165,31 +165,53 @@ public class Ver3DictDecoder implements DictDecoder {
}
private final File mDictionaryBinaryFile;
private final DictionaryBufferFactory mBufferFactory;
private DictBuffer mDictBuffer;
public Ver3DictDecoder(final File file) {
this(file, USE_READONLY_BYTEBUFFER);
}
public Ver3DictDecoder(final File file, final int factoryFlag) {
mDictionaryBinaryFile = file;
mDictBuffer = null;
if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) {
mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory();
} else if ((factoryFlag & MASK_DICTBUFFER) == USE_BYTEARRAY) {
mBufferFactory = new DictionaryBufferFromByteArrayFactory();
} else if ((factoryFlag & MASK_DICTBUFFER) == USE_WRITABLE_BYTEBUFFER) {
mBufferFactory = new DictionaryBufferFromWritableByteBufferFactory();
} else {
mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory();
}
}
public void openDictBuffer(final DictDecoder.DictionaryBufferFactory factory)
throws FileNotFoundException, IOException {
mDictBuffer = factory.getDictionaryBuffer(mDictionaryBinaryFile);
public Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) {
mDictionaryBinaryFile = file;
mBufferFactory = factory;
}
public DictBuffer getDictBuffer() {
public void openDictBuffer() throws FileNotFoundException, IOException {
mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile);
}
/* package */ DictBuffer getDictBuffer() {
return mDictBuffer;
}
@UsedForTesting
public DictBuffer openAndGetDictBuffer(final DictDecoder.DictionaryBufferFactory factory)
throws FileNotFoundException, IOException {
openDictBuffer(factory);
/* package */ DictBuffer openAndGetDictBuffer() throws FileNotFoundException, IOException {
openDictBuffer();
return getDictBuffer();
}
@Override
public FileHeader readHeader() throws IOException, UnsupportedFormatException {
if (mDictBuffer == null) {
openDictBuffer();
}
final int version = HeaderReader.readVersion(mDictBuffer);
final int optionsFlags = HeaderReader.readOptionFlags(mDictBuffer);
@ -278,4 +300,13 @@ public class Ver3DictDecoder implements DictDecoder {
return new CharGroupInfo(ptNodePos, addressPointer, flags, characters, frequency,
parentAddress, childrenAddress, shortcutTargets, bigrams);
}
@Override
public FusionDictionary readDictionaryBinary(final FusionDictionary dict)
throws FileNotFoundException, IOException, UnsupportedFormatException {
if (mDictBuffer == null) {
openDictBuffer();
}
return BinaryDictDecoderUtils.readDictionaryBinary(this, dict);
}
}

10
java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
View File

@ -28,6 +28,7 @@ import com.android.inputmethod.latin.ExpandableDictionary;
import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.WordComposer;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
@ -242,11 +243,12 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
};
// Load the dictionary from binary file
final Ver3DictDecoder reader = new Ver3DictDecoder(
new File(getContext().getFilesDir(), fileName));
final File dictFile = new File(getContext().getFilesDir(), fileName);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile,
DictDecoder.USE_BYTEARRAY);
try {
reader.openDictBuffer(new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
UserHistoryDictIOUtils.readDictionaryBinary(reader, listener);
dictDecoder.openDictBuffer();
UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
} catch (FileNotFoundException e) {
// This is an expected condition: we don't have a user history dictionary for this
// language yet. It will be created sometime later.

33
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View File

@ -33,7 +33,6 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@ -133,17 +132,15 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
// Utilities for test
/**
* Makes new DictBuffer according to BUFFER_TYPE.
* Makes new DictDecoder according to BUFFER_TYPE.
*/
private void getDictBuffer(final Ver3DictDecoder dictDecoder, final int bufferType)
throws FileNotFoundException, IOException {
private Ver3DictDecoder getDictDecoder(final File file, final int bufferType) {
if (bufferType == USE_BYTE_BUFFER) {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
return new Ver3DictDecoder(file, DictDecoder.USE_READONLY_BYTEBUFFER);
} else if (bufferType == USE_BYTE_ARRAY) {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
return new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
}
return null;
}
/**
@ -284,14 +281,14 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcutMap,
final int bufferType) {
long now, diff = -1;
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
FusionDictionary dict = null;
try {
getDictBuffer(dictDecoder, bufferType);
final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType);
dictDecoder.openDictBuffer();
assertNotNull(dictDecoder.getDictBuffer());
now = System.currentTimeMillis();
dict = BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder, null);
dict = dictDecoder.readDictionaryBinary(null);
diff = System.currentTimeMillis() - now;
} catch (IOException e) {
Log.e(TAG, "IOException while reading dictionary", e);
@ -444,9 +441,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
final Map<Integer, Integer> resultFreqs = CollectionUtils.newTreeMap();
long now = -1, diff = -1;
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
try {
getDictBuffer(dictDecoder, bufferType);
final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType);
dictDecoder.openDictBuffer();
assertNotNull("Can't get buffer.", dictDecoder.getDictBuffer());
now = System.currentTimeMillis();
BinaryDictIOUtils.readUnigramsAndBigramsBinary(dictDecoder, resultWords, resultFreqs,
@ -587,10 +584,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
dictDecoder.openDictBuffer();
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
@ -648,10 +644,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
dictDecoder.openDictBuffer();
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);

37
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
View File

@ -31,7 +31,6 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
@ -140,23 +139,13 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
private int getWordPosition(final File file, final String word) {
int position = FormatSpec.NOT_VALID_WORD;
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
dictDecoder.openDictBuffer();
position = BinaryDictIOUtils.getTerminalPosition(dictDecoder, word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
// do nothing
}
}
}
return position;
}
@ -184,11 +173,10 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
private CharGroupInfo findWordFromFile(final File file, final String word) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
CharGroupInfo info = null;
try {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
dictDecoder.openDictBuffer();
info = findWordByBinaryDictReader(dictDecoder, word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
@ -200,11 +188,12 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
private long insertAndCheckWord(final File file, final String word, final int frequency,
final boolean exist, final ArrayList<WeightedString> bigrams,
final ArrayList<WeightedString> shortcuts) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
BufferedOutputStream outStream = null;
long amountOfTime = -1;
try {
dictDecoder.openDictBuffer(new DictionaryBufferFromWritableByteBufferFactory());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
DictDecoder.USE_WRITABLE_BYTEBUFFER);
dictDecoder.openDictBuffer();
outStream = new BufferedOutputStream(new FileOutputStream(file, true));
if (!exist) {
@ -234,9 +223,10 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
private void deleteWord(final File file, final String word) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
try {
dictDecoder.openDictBuffer(new DictionaryBufferFromWritableByteBufferFactory());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
DictDecoder.USE_WRITABLE_BYTEBUFFER);
dictDecoder.openDictBuffer();
DynamicBinaryDictIOUtils.deleteWord(dictDecoder, word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
@ -244,10 +234,9 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
private void checkReverseLookup(final File file, final String word, final int position) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
try {
final DictBuffer dictBuffer = dictDecoder.openAndGetDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final FileHeader fileHeader = dictDecoder.readHeader();
assertEquals(word,
BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize,

10
tests/src/com/android/inputmethod/latin/makedict/Ver3DictDecoderTests.java
View File

@ -68,9 +68,9 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
}
assertNotNull(testFile);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory);
try {
dictDecoder.openDictBuffer(factory);
dictDecoder.openDictBuffer();
} catch (Exception e) {
Log.e(TAG, "Failed to open the buffer", e);
}
@ -78,7 +78,7 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
writeDataToFile(testFile);
try {
dictDecoder.openDictBuffer(factory);
dictDecoder.openDictBuffer();
} catch (Exception e) {
Log.e(TAG, "Raised the exception while opening buffer", e);
}
@ -110,7 +110,7 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
Log.e(TAG, "IOException while the creating temporary file", e);
}
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory);
// the default return value of getBuffer() must be null.
assertNull("the default return value of getBuffer() is not null",
@ -122,7 +122,7 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
DictBuffer dictBuffer = null;
try {
dictBuffer = dictDecoder.openAndGetDictBuffer(factory);
dictBuffer = dictDecoder.openAndGetDictBuffer();
} catch (IOException e) {
Log.e(TAG, "Failed to open and get the buffer", e);
}

6
tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
View File

@ -21,6 +21,7 @@ import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
@ -142,10 +143,9 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
}
private void readDictFromFile(final File file, final OnAddWordListener listener) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
dictDecoder.openDictBuffer();
} catch (FileNotFoundException e) {
Log.e(TAG, "file not found", e);
} catch (IOException e) {

8
tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
@ -184,15 +185,14 @@ public final class BinaryDictOffdeviceUtils {
crash(filename, new RuntimeException(
filename + " does not seem to be a dictionary file"));
} else {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(decodedSpec.mFile);
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
final DictDecoder dictDecoder = new Ver3DictDecoder(decodedSpec.mFile,
DictDecoder.USE_BYTEARRAY);
if (report) {
System.out.println("Format : Binary dictionary format");
System.out.println("Packaging : " + decodedSpec.describeChain());
System.out.println("Uncompressed size : " + decodedSpec.mFile.length());
}
return BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder, null);
return dictDecoder.readDictionaryBinary(null);
}
}
} catch (IOException e) {

7
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
@ -266,10 +267,8 @@ public class DictionaryMaker {
private static FusionDictionary readBinaryFile(final String binaryFilename)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final File file = new File(binaryFilename);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
return BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder, null);
final DictDecoder dictDecoder = new Ver3DictDecoder(file);
return dictDecoder.readDictionaryBinary(null);
}
/**

8
tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
View File

@ -16,7 +16,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
@ -69,10 +69,8 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
}
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(decodeSpec.mFile);
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
final FusionDictionary resultDict = BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder,
final DictDecoder dictDecoder = new Ver3DictDecoder(decodeSpec.mFile);
final FusionDictionary resultDict = dictDecoder.readDictionaryBinary(
null /* dict : an optional dictionary to add words to, or null */);
assertEquals("Dictionary can't be read back correctly",
FusionDictionary.findWordInTree(resultDict.mRootNodeArray, "foo").getFrequency(),