nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add BinaryDictReader. 

Bug: 9618601

Change-Id: Ief07fa0c3c4f7f5999a3fafcef4e47b6b6fd8143
main
Yuichiro Hanada 2013-08-12 14:47:20 +09:00
parent 7242bf1a15
commit 3feacba1eb
11 changed files with 327 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View File

@ -145,21 +145,21 @@ public final class BinaryDictIOUtils {
* Reads unigrams and bigrams from the binary file.
* Doesn't make the memory representation of the dictionary.
*
* @param buffer the buffer to read.
* @param reader the reader.
* @param words the map to store the address as a key and the word as a value.
* @param frequencies the map to store the address as a key and the frequency as a value.
* @param bigrams the map to store the address as a key and the list of address as a value.
* @throws IOException
* @throws UnsupportedFormatException
*/
public static void readUnigramsAndBigramsBinary(final FusionDictionaryBufferInterface buffer,
public static void readUnigramsAndBigramsBinary(final BinaryDictReader reader,
final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
UnsupportedFormatException {
// Read header
final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
readUnigramsAndBigramsBinaryInner(buffer, header.mHeaderSize, words, frequencies, bigrams,
header.mFormatOptions);
final FileHeader header = BinaryDictInputOutput.readHeader(reader.getBuffer());
readUnigramsAndBigramsBinaryInner(reader.getBuffer(), header.mHeaderSize, words,
frequencies, bigrams, header.mFormatOptions);
}
/**

20
java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java
View File

@ -66,6 +66,7 @@ public final class BinaryDictInputOutput {
public void position(int newPosition);
public void put(final byte b);
public int limit();
@UsedForTesting
public int capacity();
}
@ -1722,23 +1723,30 @@ public final class BinaryDictInputOutput {
* FusionDictionary structure. The optional dict argument is an existing dictionary to
* which words from the buffer should be added. If it is null, a new dictionary is created.
*
* @param buffer the buffer to read.
* @param reader the reader.
* @param dict an optional dictionary to add words to, or null.
* @return the created (or merged) dictionary.
*/
@UsedForTesting
public static FusionDictionary readDictionaryBinary(
final FusionDictionaryBufferInterface buffer, final FusionDictionary dict)
throws IOException, UnsupportedFormatException {
public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
final FusionDictionary dict) throws FileNotFoundException, IOException,
UnsupportedFormatException {
// clear cache
wordCache.clear();
// if the buffer has not been opened, open the buffer with bytebuffer.
if (reader.getBuffer() == null) reader.openBuffer(
new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
if (reader.getBuffer() == null) {
MakedictLog.e("Cannot open the buffer");
}
// Read header
final FileHeader header = readHeader(buffer);
final FileHeader header = readHeader(reader.getBuffer());
Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>();
Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
final Node root = readNode(buffer, header.mHeaderSize, reverseNodeMapping,
final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping,
reverseGroupMapping, header.mFormatOptions);
FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions);

109
java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java Normal file
View File

@ -0,0 +1,109 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
public class BinaryDictReader {
public interface FusionDictionaryBufferFactory {
public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
throws FileNotFoundException, IOException;
}
/**
* Creates FusionDictionaryBuffer from a ByteBuffer
*/
public static final class FusionDictionaryBufferFromByteBufferFactory
implements FusionDictionaryBufferFactory {
@Override
public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
throws FileNotFoundException, IOException {
FileInputStream inStream = null;
ByteBuffer buffer = null;
try {
inStream = new FileInputStream(file);
buffer = inStream.getChannel().map(FileChannel.MapMode.READ_ONLY,
0, file.length());
} finally {
if (inStream != null) {
inStream.close();
}
}
if (buffer != null) {
return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
}
return null;
}
}
/**
* Creates FusionDictionaryBuffer from a byte array
*/
public static final class FusionDictionaryBufferFromByteArrayFactory
implements FusionDictionaryBufferFactory {
@Override
public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
throws FileNotFoundException, IOException {
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
final byte[] array = new byte[(int) file.length()];
inStream.read(array);
return new ByteArrayWrapper(array);
} finally {
if (inStream != null) {
inStream.close();
}
}
}
}
private final File mDictionaryBinaryFile;
private FusionDictionaryBufferInterface mFusionDictionaryBuffer;
public BinaryDictReader(final File file) {
mDictionaryBinaryFile = file;
mFusionDictionaryBuffer = null;
}
public void openBuffer(final FusionDictionaryBufferFactory factory)
throws FileNotFoundException, IOException {
mFusionDictionaryBuffer = factory.getFusionDictionaryBuffer(mDictionaryBinaryFile);
}
public FusionDictionaryBufferInterface getBuffer() {
return mFusionDictionaryBuffer;
}
@UsedForTesting
public FusionDictionaryBufferInterface openAndGetBuffer(
final FusionDictionaryBufferFactory factory)
throws FileNotFoundException, IOException {
openBuffer(factory);
return getBuffer();
}
}

21
java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
View File

@ -28,9 +28,9 @@ import com.android.inputmethod.latin.ExpandableDictionary;
import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.WordComposer;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
@ -39,7 +39,6 @@ import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils;
import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
@ -234,27 +233,17 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
};
// Load the dictionary from binary file
FileInputStream inStream = null;
final BinaryDictReader reader = new BinaryDictReader(
new File(getContext().getFilesDir(), fileName));
try {
final File file = new File(getContext().getFilesDir(), fileName);
final byte[] buffer = new byte[(int)file.length()];
inStream = new FileInputStream(file);
inStream.read(buffer);
UserHistoryDictIOUtils.readDictionaryBinary(
new ByteArrayWrapper(buffer), listener);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
UserHistoryDictIOUtils.readDictionaryBinary(reader, listener);
} catch (FileNotFoundException e) {
// This is an expected condition: we don't have a user history dictionary for this
// language yet. It will be created sometime later.
} catch (IOException e) {
Log.e(TAG, "IOException on opening a bytebuffer", e);
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
// do nothing
}
}
if (PROFILE_SAVE_RESTORE) {
final long diff = System.currentTimeMillis() - now;
Log.d(TAG, "PROF: Load UserHistoryDictionary: "

6
java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
View File

@ -21,7 +21,7 @@ import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
@ -118,13 +118,13 @@ public final class UserHistoryDictIOUtils {
/**
* Reads dictionary from file.
*/
public static void readDictionaryBinary(final FusionDictionaryBufferInterface buffer,
public static void readDictionaryBinary(final BinaryDictReader reader,
final OnAddWordListener dict) {
final Map<Integer, String> unigrams = CollectionUtils.newTreeMap();
final Map<Integer, Integer> frequencies = CollectionUtils.newTreeMap();
final Map<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap();
try {
BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, unigrams, frequencies,
BinaryDictIOUtils.readUnigramsAndBigramsBinary(reader, unigrams, frequencies,
bigrams);
} catch (IOException e) {
Log.e(TAG, "IO exception while reading file", e);

70
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java
View File

@ -27,15 +27,13 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@ -120,31 +118,13 @@ public class BinaryDictIOTests extends AndroidTestCase {
/**
* Makes new buffer according to BUFFER_TYPE.
*/
private FusionDictionaryBufferInterface getBuffer(final File file, final int bufferType) {
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
if (bufferType == USE_BYTE_ARRAY) {
final byte[] array = new byte[(int)file.length()];
inStream.read(array);
return new ByteArrayWrapper(array);
} else if (bufferType == USE_BYTE_BUFFER){
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, file.length());
return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
}
} catch (IOException e) {
Log.e(TAG, "IOException while making buffer", e);
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
Log.e(TAG, "IOException while closing stream", e);
}
}
private void getBuffer(final BinaryDictReader reader, final int bufferType)
throws FileNotFoundException, IOException {
if (bufferType == USE_BYTE_BUFFER) {
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
} else if (bufferType == USE_BYTE_ARRAY) {
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
}
return null;
}
/**
@ -285,13 +265,14 @@ public class BinaryDictIOTests extends AndroidTestCase {
final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcutMap,
final int bufferType) {
long now, diff = -1;
final FusionDictionaryBufferInterface buffer = getBuffer(file, bufferType);
assertNotNull(buffer);
final BinaryDictReader reader = new BinaryDictReader(file);
FusionDictionary dict = null;
try {
getBuffer(reader, bufferType);
assertNotNull(reader.getBuffer());
now = System.currentTimeMillis();
dict = BinaryDictInputOutput.readDictionaryBinary(buffer, null);
dict = BinaryDictInputOutput.readDictionaryBinary(reader, null);
diff = System.currentTimeMillis() - now;
} catch (IOException e) {
Log.e(TAG, "IOException while reading dictionary", e);
@ -421,11 +402,12 @@ public class BinaryDictIOTests extends AndroidTestCase {
final Map<Integer, Integer> resultFreqs = CollectionUtils.newTreeMap();
long now = -1, diff = -1;
final FusionDictionaryBufferInterface buffer = getBuffer(file, bufferType);
assertNotNull("Can't get buffer.", buffer);
final BinaryDictReader reader = new BinaryDictReader(file);
try {
getBuffer(reader, bufferType);
assertNotNull("Can't get buffer.", reader.getBuffer());
now = System.currentTimeMillis();
BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, resultWords, resultFreqs,
BinaryDictIOUtils.readUnigramsAndBigramsBinary(reader, resultWords, resultFreqs,
resultBigrams);
diff = System.currentTimeMillis() - now;
} catch (IOException e) {
@ -562,7 +544,16 @@ public class BinaryDictIOTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final FusionDictionaryBufferInterface buffer = getBuffer(file, USE_BYTE_ARRAY);
final BinaryDictReader reader = new BinaryDictReader(file);
FusionDictionaryBufferInterface buffer = null;
try {
buffer = reader.openAndGetBuffer(
new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
}
assertNotNull("Can't get the buffer", buffer);
try {
// too long word
@ -614,7 +605,16 @@ public class BinaryDictIOTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final FusionDictionaryBufferInterface buffer = getBuffer(file, USE_BYTE_ARRAY);
final BinaryDictReader reader = new BinaryDictReader(file);
FusionDictionaryBufferInterface buffer = null;
try {
buffer = reader.openAndGetBuffer(
new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
}
assertNotNull("Can't get the buffer", buffer);
try {
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,

138
tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java Normal file
View File

@ -0,0 +1,138 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFactory;
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory;
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory;
import android.test.AndroidTestCase;
import android.util.Log;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
/**
* Unit tests for BinaryDictReader
*/
public class BinaryDictReaderTests extends AndroidTestCase {
private static final String TAG = BinaryDictReaderTests.class.getSimpleName();
private final byte[] data = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
// Utilities for testing
public void writeDataToFile(final File file) {
FileOutputStream outStream = null;
try {
outStream = new FileOutputStream(file);
outStream.write(data);
} catch (IOException e) {
fail ("Can't write data to the test file");
} finally {
if (outStream != null) {
try {
outStream.close();
} catch (IOException e) {
Log.e(TAG, "Failed to close the output stream", e);
}
}
}
}
@SuppressWarnings("null")
public void runTestOpenBuffer(final String testName,
final FusionDictionaryBufferFactory factory) {
File testFile = null;
try {
testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir());
} catch (IOException e) {
Log.e(TAG, "IOException while the creating temporary file", e);
}
assertNotNull(testFile);
final BinaryDictReader reader = new BinaryDictReader(testFile);
try {
reader.openBuffer(factory);
} catch (Exception e) {
Log.e(TAG, "Failed to open the buffer", e);
}
writeDataToFile(testFile);
try {
reader.openBuffer(factory);
} catch (Exception e) {
Log.e(TAG, "Raised the exception while opening buffer", e);
}
assertEquals(testFile.length(), reader.getBuffer().capacity());
}
public void testOpenBufferWithByteBuffer() {
runTestOpenBuffer("testOpenBufferWithByteBuffer",
new FusionDictionaryBufferFromByteBufferFactory());
}
public void testOpenBufferWithByteArray() {
runTestOpenBuffer("testOpenBufferWithByteArray",
new FusionDictionaryBufferFromByteArrayFactory());
}
@SuppressWarnings("null")
public void runTestGetBuffer(final String testName,
final FusionDictionaryBufferFactory factory) {
File testFile = null;
try {
testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir());
} catch (IOException e) {
Log.e(TAG, "IOException while the creating temporary file", e);
}
final BinaryDictReader reader = new BinaryDictReader(testFile);
// the default return value of getBuffer() must be null.
assertNull("the default return value of getBuffer() is not null", reader.getBuffer());
writeDataToFile(testFile);
assertTrue(testFile.exists());
Log.d(TAG, "file length = " + testFile.length());
FusionDictionaryBufferInterface buffer = null;
try {
buffer = reader.openAndGetBuffer(factory);
} catch (IOException e) {
Log.e(TAG, "Failed to open and get the buffer", e);
}
assertNotNull("the buffer must not be null", buffer);
for (int i = 0; i < data.length; ++i) {
assertEquals(data[i], buffer.readUnsignedByte());
}
}
public void testGetBufferWithByteBuffer() {
runTestGetBuffer("testGetBufferWithByteBuffer",
new FusionDictionaryBufferFromByteBufferFactory());
}
public void testGetBufferWithByteArray() {
runTestGetBuffer("testGetBufferWithByteArray",
new FusionDictionaryBufferFromByteArrayFactory());
}
}

21
tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
View File

@ -21,16 +21,15 @@ import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
@ -147,27 +146,15 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
}
private void readDictFromFile(final File file, final OnAddWordListener listener) {
FileInputStream inStream = null;
final BinaryDictReader reader = new BinaryDictReader(file);
try {
inStream = new FileInputStream(file);
final byte[] buffer = new byte[(int)file.length()];
inStream.read(buffer);
UserHistoryDictIOUtils.readDictionaryBinary(new ByteArrayWrapper(buffer), listener);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
} catch (FileNotFoundException e) {
Log.e(TAG, "file not found", e);
} catch (IOException e) {
Log.e(TAG, "IOException", e);
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
// do nothing
}
}
}
UserHistoryDictIOUtils.readDictionaryBinary(reader, listener);
}
public void testGenerateFusionDictionary() {

10
tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
@ -185,16 +186,15 @@ public final class BinaryDictOffdeviceUtils {
crash(filename, new RuntimeException(
filename + " does not seem to be a dictionary file"));
} else {
final FileInputStream inStream = new FileInputStream(decodedSpec.mFile);
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, decodedSpec.mFile.length());
final BinaryDictReader reader = new BinaryDictReader(decodedSpec.mFile);
reader.openBuffer(
new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
if (report) {
System.out.println("Format : Binary dictionary format");
System.out.println("Packaging : " + decodedSpec.describeChain());
System.out.println("Uncompressed size : " + decodedSpec.mFile.length());
}
return BinaryDictInputOutput.readDictionaryBinary(
new BinaryDictInputOutput.ByteBufferWrapper(buffer), null);
return BinaryDictInputOutput.readDictionaryBinary(reader, null);
}
}
} catch (IOException e) {

23
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.MakedictLog;
@ -265,24 +266,10 @@ public class DictionaryMaker {
*/
private static FusionDictionary readBinaryFile(final String binaryFilename)
throws FileNotFoundException, IOException, UnsupportedFormatException {
FileInputStream inStream = null;
try {
final File file = new File(binaryFilename);
inStream = new FileInputStream(file);
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, file.length());
return BinaryDictInputOutput.readDictionaryBinary(
new BinaryDictInputOutput.ByteBufferWrapper(buffer), null);
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
// do nothing
}
}
}
final File file = new File(binaryFilename);
final BinaryDictReader reader = new BinaryDictReader(file);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
return BinaryDictInputOutput.readDictionaryBinary(reader, null);
}
/**

9
tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
@ -69,11 +70,9 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
}
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final FileInputStream inStream = new FileInputStream(decodeSpec.mFile);
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, decodeSpec.mFile.length());
final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(
new BinaryDictInputOutput.ByteBufferWrapper(buffer),
final BinaryDictReader reader = new BinaryDictReader(decodeSpec.mFile);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(reader,
null /* dict : an optional dictionary to add words to, or null */);
assertEquals("Dictionary can't be read back correctly",
resultDict.findWordInTree(resultDict.mRoot, "foo").getFrequency(), TEST_FREQ);