Add DictDecoder.readDictionaryBinary.

Bug: 10434720
Change-Id: I14690a6e0f922ed1bab3a4b6c9a457ae84d4c1a4
main
Yuichiro Hanada 2013-08-22 22:43:20 +09:00
parent b64157bf47
commit e9a10ff0f0
13 changed files with 125 additions and 100 deletions

View File

@ -229,8 +229,6 @@ final public class BinaryDictionaryGetter {
try {
// Read the version of the file
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f);
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
final FileHeader header = dictDecoder.readHeader();
final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY);

View File

@ -559,17 +559,9 @@ public final class BinaryDictDecoderUtils {
* @return the created (or merged) dictionary.
*/
@UsedForTesting
public static FusionDictionary readDictionaryBinary(final Ver3DictDecoder dictDecoder,
/* package */ static FusionDictionary readDictionaryBinary(final Ver3DictDecoder dictDecoder,
final FusionDictionary dict) throws FileNotFoundException, IOException,
UnsupportedFormatException {
// if the buffer has not been opened, open the buffer with bytebuffer.
if (dictDecoder.getDictBuffer() == null) dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
if (dictDecoder.getDictBuffer() == null) {
MakedictLog.e("Cannot open the buffer");
}
// Read header
final FileHeader fileHeader = dictDecoder.readHeader();

View File

@ -521,20 +521,21 @@ public final class BinaryDictIOUtils {
final File file, final long offset, final long length)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE];
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
dictDecoder.openDictBuffer(new DictDecoder.DictionaryBufferFactory() {
@Override
public DictBuffer getDictionaryBuffer(File file)
throws FileNotFoundException, IOException {
final FileInputStream inStream = new FileInputStream(file);
try {
inStream.read(buffer);
return new ByteArrayDictBuffer(buffer);
} finally {
inStream.close();
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
new DictDecoder.DictionaryBufferFactory() {
@Override
public DictBuffer getDictionaryBuffer(File file)
throws FileNotFoundException, IOException {
final FileInputStream inStream = new FileInputStream(file);
try {
inStream.read(buffer);
return new ByteArrayDictBuffer(buffer);
} finally {
inStream.close();
}
}
}
}
});
);
return dictDecoder.readHeader();
}

View File

@ -44,6 +44,26 @@ public interface DictDecoder {
*/
public CharGroupInfo readPtNode(final int ptNodePos, final FormatOptions formatOptions);
/**
* Reads a buffer and returns the memory representation of the dictionary.
*
* This high-level method takes a buffer and reads its contents, populating a
* FusionDictionary structure. The optional dict argument is an existing dictionary to
* which words from the buffer should be added. If it is null, a new dictionary is created.
*
* @param dict an optional dictionary to add words to, or null.
* @return the created (or merged) dictionary.
*/
@UsedForTesting
public FusionDictionary readDictionaryBinary(final FusionDictionary dict)
throws FileNotFoundException, IOException, UnsupportedFormatException;
// Flags for DictionaryBufferFactory.
public static final int USE_READONLY_BYTEBUFFER = 0x01000000;
public static final int USE_BYTEARRAY = 0x02000000;
public static final int USE_WRITABLE_BYTEBUFFER = 0x04000000;
public static final int MASK_DICTBUFFER = 0x0F000000;
public interface DictionaryBufferFactory {
public DictBuffer getDictionaryBuffer(final File file)
throws FileNotFoundException, IOException;

View File

@ -165,31 +165,53 @@ public class Ver3DictDecoder implements DictDecoder {
}
private final File mDictionaryBinaryFile;
private final DictionaryBufferFactory mBufferFactory;
private DictBuffer mDictBuffer;
public Ver3DictDecoder(final File file) {
this(file, USE_READONLY_BYTEBUFFER);
}
public Ver3DictDecoder(final File file, final int factoryFlag) {
mDictionaryBinaryFile = file;
mDictBuffer = null;
if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) {
mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory();
} else if ((factoryFlag & MASK_DICTBUFFER) == USE_BYTEARRAY) {
mBufferFactory = new DictionaryBufferFromByteArrayFactory();
} else if ((factoryFlag & MASK_DICTBUFFER) == USE_WRITABLE_BYTEBUFFER) {
mBufferFactory = new DictionaryBufferFromWritableByteBufferFactory();
} else {
mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory();
}
}
public void openDictBuffer(final DictDecoder.DictionaryBufferFactory factory)
throws FileNotFoundException, IOException {
mDictBuffer = factory.getDictionaryBuffer(mDictionaryBinaryFile);
public Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) {
mDictionaryBinaryFile = file;
mBufferFactory = factory;
}
public DictBuffer getDictBuffer() {
public void openDictBuffer() throws FileNotFoundException, IOException {
mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile);
}
/* package */ DictBuffer getDictBuffer() {
return mDictBuffer;
}
@UsedForTesting
public DictBuffer openAndGetDictBuffer(final DictDecoder.DictionaryBufferFactory factory)
throws FileNotFoundException, IOException {
openDictBuffer(factory);
/* package */ DictBuffer openAndGetDictBuffer() throws FileNotFoundException, IOException {
openDictBuffer();
return getDictBuffer();
}
@Override
public FileHeader readHeader() throws IOException, UnsupportedFormatException {
if (mDictBuffer == null) {
openDictBuffer();
}
final int version = HeaderReader.readVersion(mDictBuffer);
final int optionsFlags = HeaderReader.readOptionFlags(mDictBuffer);
@ -278,4 +300,13 @@ public class Ver3DictDecoder implements DictDecoder {
return new CharGroupInfo(ptNodePos, addressPointer, flags, characters, frequency,
parentAddress, childrenAddress, shortcutTargets, bigrams);
}
@Override
public FusionDictionary readDictionaryBinary(final FusionDictionary dict)
throws FileNotFoundException, IOException, UnsupportedFormatException {
if (mDictBuffer == null) {
openDictBuffer();
}
return BinaryDictDecoderUtils.readDictionaryBinary(this, dict);
}
}

View File

@ -28,6 +28,7 @@ import com.android.inputmethod.latin.ExpandableDictionary;
import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.WordComposer;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
@ -242,11 +243,12 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
};
// Load the dictionary from binary file
final Ver3DictDecoder reader = new Ver3DictDecoder(
new File(getContext().getFilesDir(), fileName));
final File dictFile = new File(getContext().getFilesDir(), fileName);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile,
DictDecoder.USE_BYTEARRAY);
try {
reader.openDictBuffer(new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
UserHistoryDictIOUtils.readDictionaryBinary(reader, listener);
dictDecoder.openDictBuffer();
UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
} catch (FileNotFoundException e) {
// This is an expected condition: we don't have a user history dictionary for this
// language yet. It will be created sometime later.

View File

@ -33,7 +33,6 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@ -133,17 +132,15 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
// Utilities for test
/**
* Makes new DictBuffer according to BUFFER_TYPE.
* Makes new DictDecoder according to BUFFER_TYPE.
*/
private void getDictBuffer(final Ver3DictDecoder dictDecoder, final int bufferType)
throws FileNotFoundException, IOException {
private Ver3DictDecoder getDictDecoder(final File file, final int bufferType) {
if (bufferType == USE_BYTE_BUFFER) {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
} else if (bufferType == USE_BYTE_ARRAY) {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
return new Ver3DictDecoder(file, DictDecoder.USE_READONLY_BYTEBUFFER);
} else if (bufferType == USE_BYTE_ARRAY) {
return new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
}
return null;
}
/**
@ -284,14 +281,14 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcutMap,
final int bufferType) {
long now, diff = -1;
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
FusionDictionary dict = null;
try {
getDictBuffer(dictDecoder, bufferType);
final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType);
dictDecoder.openDictBuffer();
assertNotNull(dictDecoder.getDictBuffer());
now = System.currentTimeMillis();
dict = BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder, null);
dict = dictDecoder.readDictionaryBinary(null);
diff = System.currentTimeMillis() - now;
} catch (IOException e) {
Log.e(TAG, "IOException while reading dictionary", e);
@ -444,9 +441,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
final Map<Integer, Integer> resultFreqs = CollectionUtils.newTreeMap();
long now = -1, diff = -1;
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
try {
getDictBuffer(dictDecoder, bufferType);
final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType);
dictDecoder.openDictBuffer();
assertNotNull("Can't get buffer.", dictDecoder.getDictBuffer());
now = System.currentTimeMillis();
BinaryDictIOUtils.readUnigramsAndBigramsBinary(dictDecoder, resultWords, resultFreqs,
@ -587,10 +584,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
dictDecoder.openDictBuffer();
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
@ -648,10 +644,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
dictDecoder.openDictBuffer();
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);

View File

@ -31,7 +31,6 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
@ -140,23 +139,13 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
private int getWordPosition(final File file, final String word) {
int position = FormatSpec.NOT_VALID_WORD;
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
dictDecoder.openDictBuffer();
position = BinaryDictIOUtils.getTerminalPosition(dictDecoder, word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
// do nothing
}
}
}
return position;
}
@ -184,11 +173,10 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
private CharGroupInfo findWordFromFile(final File file, final String word) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
CharGroupInfo info = null;
try {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
dictDecoder.openDictBuffer();
info = findWordByBinaryDictReader(dictDecoder, word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
@ -200,11 +188,12 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
private long insertAndCheckWord(final File file, final String word, final int frequency,
final boolean exist, final ArrayList<WeightedString> bigrams,
final ArrayList<WeightedString> shortcuts) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
BufferedOutputStream outStream = null;
long amountOfTime = -1;
try {
dictDecoder.openDictBuffer(new DictionaryBufferFromWritableByteBufferFactory());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
DictDecoder.USE_WRITABLE_BYTEBUFFER);
dictDecoder.openDictBuffer();
outStream = new BufferedOutputStream(new FileOutputStream(file, true));
if (!exist) {
@ -234,9 +223,10 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
private void deleteWord(final File file, final String word) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
try {
dictDecoder.openDictBuffer(new DictionaryBufferFromWritableByteBufferFactory());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
DictDecoder.USE_WRITABLE_BYTEBUFFER);
dictDecoder.openDictBuffer();
DynamicBinaryDictIOUtils.deleteWord(dictDecoder, word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
@ -244,10 +234,9 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
private void checkReverseLookup(final File file, final String word, final int position) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
try {
final DictBuffer dictBuffer = dictDecoder.openAndGetDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final FileHeader fileHeader = dictDecoder.readHeader();
assertEquals(word,
BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize,

View File

@ -68,9 +68,9 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
}
assertNotNull(testFile);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory);
try {
dictDecoder.openDictBuffer(factory);
dictDecoder.openDictBuffer();
} catch (Exception e) {
Log.e(TAG, "Failed to open the buffer", e);
}
@ -78,7 +78,7 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
writeDataToFile(testFile);
try {
dictDecoder.openDictBuffer(factory);
dictDecoder.openDictBuffer();
} catch (Exception e) {
Log.e(TAG, "Raised the exception while opening buffer", e);
}
@ -110,7 +110,7 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
Log.e(TAG, "IOException while the creating temporary file", e);
}
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile, factory);
// the default return value of getBuffer() must be null.
assertNull("the default return value of getBuffer() is not null",
@ -122,7 +122,7 @@ public class Ver3DictDecoderTests extends AndroidTestCase {
DictBuffer dictBuffer = null;
try {
dictBuffer = dictDecoder.openAndGetDictBuffer(factory);
dictBuffer = dictDecoder.openAndGetDictBuffer();
} catch (IOException e) {
Log.e(TAG, "Failed to open and get the buffer", e);
}

View File

@ -21,6 +21,7 @@ import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
@ -142,10 +143,9 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
}
private void readDictFromFile(final File file, final OnAddWordListener listener) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
dictDecoder.openDictBuffer();
} catch (FileNotFoundException e) {
Log.e(TAG, "file not found", e);
} catch (IOException e) {

View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
@ -184,15 +185,14 @@ public final class BinaryDictOffdeviceUtils {
crash(filename, new RuntimeException(
filename + " does not seem to be a dictionary file"));
} else {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(decodedSpec.mFile);
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory());
final DictDecoder dictDecoder = new Ver3DictDecoder(decodedSpec.mFile,
DictDecoder.USE_BYTEARRAY);
if (report) {
System.out.println("Format : Binary dictionary format");
System.out.println("Packaging : " + decodedSpec.describeChain());
System.out.println("Uncompressed size : " + decodedSpec.mFile.length());
}
return BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder, null);
return dictDecoder.readDictionaryBinary(null);
}
}
} catch (IOException e) {

View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
@ -266,10 +267,8 @@ public class DictionaryMaker {
private static FusionDictionary readBinaryFile(final String binaryFilename)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final File file = new File(binaryFilename);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
return BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder, null);
final DictDecoder dictDecoder = new Ver3DictDecoder(file);
return dictDecoder.readDictionaryBinary(null);
}
/**

View File

@ -16,7 +16,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
@ -69,10 +69,8 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
}
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(decodeSpec.mFile);
dictDecoder.openDictBuffer(
new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory());
final FusionDictionary resultDict = BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder,
final DictDecoder dictDecoder = new Ver3DictDecoder(decodeSpec.mFile);
final FusionDictionary resultDict = dictDecoder.readDictionaryBinary(
null /* dict : an optional dictionary to add words to, or null */);
assertEquals("Dictionary can't be read back correctly",
FusionDictionary.findWordInTree(resultDict.mRootNodeArray, "foo").getFrequency(),