am f27297d2: Merge "Add BinaryDictReader."
* commit 'f27297d2ece8a247a8b9e18d4e65dc0d0ba68b77': Add BinaryDictReader.main
commit
a3fe14b4cc
|
@ -145,21 +145,21 @@ public final class BinaryDictIOUtils {
|
|||
* Reads unigrams and bigrams from the binary file.
|
||||
* Doesn't make the memory representation of the dictionary.
|
||||
*
|
||||
* @param buffer the buffer to read.
|
||||
* @param reader the reader.
|
||||
* @param words the map to store the address as a key and the word as a value.
|
||||
* @param frequencies the map to store the address as a key and the frequency as a value.
|
||||
* @param bigrams the map to store the address as a key and the list of address as a value.
|
||||
* @throws IOException
|
||||
* @throws UnsupportedFormatException
|
||||
*/
|
||||
public static void readUnigramsAndBigramsBinary(final FusionDictionaryBufferInterface buffer,
|
||||
public static void readUnigramsAndBigramsBinary(final BinaryDictReader reader,
|
||||
final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
|
||||
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
|
||||
UnsupportedFormatException {
|
||||
// Read header
|
||||
final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
|
||||
readUnigramsAndBigramsBinaryInner(buffer, header.mHeaderSize, words, frequencies, bigrams,
|
||||
header.mFormatOptions);
|
||||
final FileHeader header = BinaryDictInputOutput.readHeader(reader.getBuffer());
|
||||
readUnigramsAndBigramsBinaryInner(reader.getBuffer(), header.mHeaderSize, words,
|
||||
frequencies, bigrams, header.mFormatOptions);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -66,6 +66,7 @@ public final class BinaryDictInputOutput {
|
|||
public void position(int newPosition);
|
||||
public void put(final byte b);
|
||||
public int limit();
|
||||
@UsedForTesting
|
||||
public int capacity();
|
||||
}
|
||||
|
||||
|
@ -1722,23 +1723,30 @@ public final class BinaryDictInputOutput {
|
|||
* FusionDictionary structure. The optional dict argument is an existing dictionary to
|
||||
* which words from the buffer should be added. If it is null, a new dictionary is created.
|
||||
*
|
||||
* @param buffer the buffer to read.
|
||||
* @param reader the reader.
|
||||
* @param dict an optional dictionary to add words to, or null.
|
||||
* @return the created (or merged) dictionary.
|
||||
*/
|
||||
@UsedForTesting
|
||||
public static FusionDictionary readDictionaryBinary(
|
||||
final FusionDictionaryBufferInterface buffer, final FusionDictionary dict)
|
||||
throws IOException, UnsupportedFormatException {
|
||||
public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
|
||||
final FusionDictionary dict) throws FileNotFoundException, IOException,
|
||||
UnsupportedFormatException {
|
||||
// clear cache
|
||||
wordCache.clear();
|
||||
|
||||
// if the buffer has not been opened, open the buffer with bytebuffer.
|
||||
if (reader.getBuffer() == null) reader.openBuffer(
|
||||
new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
|
||||
if (reader.getBuffer() == null) {
|
||||
MakedictLog.e("Cannot open the buffer");
|
||||
}
|
||||
|
||||
// Read header
|
||||
final FileHeader header = readHeader(buffer);
|
||||
final FileHeader header = readHeader(reader.getBuffer());
|
||||
|
||||
Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>();
|
||||
Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
|
||||
final Node root = readNode(buffer, header.mHeaderSize, reverseNodeMapping,
|
||||
final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping,
|
||||
reverseGroupMapping, header.mFormatOptions);
|
||||
|
||||
FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions);
|
||||
|
|
|
@ -0,0 +1,109 @@
|
|||
/*
|
||||
* Copyright (C) 2013 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.android.inputmethod.latin.makedict;
|
||||
|
||||
import com.android.inputmethod.annotations.UsedForTesting;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
|
||||
public class BinaryDictReader {
|
||||
|
||||
public interface FusionDictionaryBufferFactory {
|
||||
public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
|
||||
throws FileNotFoundException, IOException;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates FusionDictionaryBuffer from a ByteBuffer
|
||||
*/
|
||||
public static final class FusionDictionaryBufferFromByteBufferFactory
|
||||
implements FusionDictionaryBufferFactory {
|
||||
@Override
|
||||
public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
|
||||
throws FileNotFoundException, IOException {
|
||||
FileInputStream inStream = null;
|
||||
ByteBuffer buffer = null;
|
||||
try {
|
||||
inStream = new FileInputStream(file);
|
||||
buffer = inStream.getChannel().map(FileChannel.MapMode.READ_ONLY,
|
||||
0, file.length());
|
||||
} finally {
|
||||
if (inStream != null) {
|
||||
inStream.close();
|
||||
}
|
||||
}
|
||||
if (buffer != null) {
|
||||
return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates FusionDictionaryBuffer from a byte array
|
||||
*/
|
||||
public static final class FusionDictionaryBufferFromByteArrayFactory
|
||||
implements FusionDictionaryBufferFactory {
|
||||
@Override
|
||||
public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
|
||||
throws FileNotFoundException, IOException {
|
||||
FileInputStream inStream = null;
|
||||
try {
|
||||
inStream = new FileInputStream(file);
|
||||
final byte[] array = new byte[(int) file.length()];
|
||||
inStream.read(array);
|
||||
return new ByteArrayWrapper(array);
|
||||
} finally {
|
||||
if (inStream != null) {
|
||||
inStream.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private final File mDictionaryBinaryFile;
|
||||
private FusionDictionaryBufferInterface mFusionDictionaryBuffer;
|
||||
|
||||
public BinaryDictReader(final File file) {
|
||||
mDictionaryBinaryFile = file;
|
||||
mFusionDictionaryBuffer = null;
|
||||
}
|
||||
|
||||
public void openBuffer(final FusionDictionaryBufferFactory factory)
|
||||
throws FileNotFoundException, IOException {
|
||||
mFusionDictionaryBuffer = factory.getFusionDictionaryBuffer(mDictionaryBinaryFile);
|
||||
}
|
||||
|
||||
public FusionDictionaryBufferInterface getBuffer() {
|
||||
return mFusionDictionaryBuffer;
|
||||
}
|
||||
|
||||
@UsedForTesting
|
||||
public FusionDictionaryBufferInterface openAndGetBuffer(
|
||||
final FusionDictionaryBufferFactory factory)
|
||||
throws FileNotFoundException, IOException {
|
||||
openBuffer(factory);
|
||||
return getBuffer();
|
||||
}
|
||||
}
|
|
@ -28,9 +28,9 @@ import com.android.inputmethod.latin.ExpandableDictionary;
|
|||
import com.android.inputmethod.latin.LatinImeLogger;
|
||||
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
|
||||
import com.android.inputmethod.latin.WordComposer;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||
import com.android.inputmethod.latin.settings.Settings;
|
||||
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
|
||||
import com.android.inputmethod.latin.utils.CollectionUtils;
|
||||
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
|
||||
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
|
||||
|
@ -39,7 +39,6 @@ import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils;
|
|||
import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
|
@ -234,27 +233,17 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
|
|||
};
|
||||
|
||||
// Load the dictionary from binary file
|
||||
FileInputStream inStream = null;
|
||||
final BinaryDictReader reader = new BinaryDictReader(
|
||||
new File(getContext().getFilesDir(), fileName));
|
||||
try {
|
||||
final File file = new File(getContext().getFilesDir(), fileName);
|
||||
final byte[] buffer = new byte[(int)file.length()];
|
||||
inStream = new FileInputStream(file);
|
||||
inStream.read(buffer);
|
||||
UserHistoryDictIOUtils.readDictionaryBinary(
|
||||
new ByteArrayWrapper(buffer), listener);
|
||||
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
|
||||
UserHistoryDictIOUtils.readDictionaryBinary(reader, listener);
|
||||
} catch (FileNotFoundException e) {
|
||||
// This is an expected condition: we don't have a user history dictionary for this
|
||||
// language yet. It will be created sometime later.
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IOException on opening a bytebuffer", e);
|
||||
} finally {
|
||||
if (inStream != null) {
|
||||
try {
|
||||
inStream.close();
|
||||
} catch (IOException e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
if (PROFILE_SAVE_RESTORE) {
|
||||
final long diff = System.currentTimeMillis() - now;
|
||||
Log.d(TAG, "PROF: Load UserHistoryDictionary: "
|
||||
|
|
|
@ -21,7 +21,7 @@ import android.util.Log;
|
|||
import com.android.inputmethod.annotations.UsedForTesting;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
|
||||
|
@ -118,13 +118,13 @@ public final class UserHistoryDictIOUtils {
|
|||
/**
|
||||
* Reads dictionary from file.
|
||||
*/
|
||||
public static void readDictionaryBinary(final FusionDictionaryBufferInterface buffer,
|
||||
public static void readDictionaryBinary(final BinaryDictReader reader,
|
||||
final OnAddWordListener dict) {
|
||||
final Map<Integer, String> unigrams = CollectionUtils.newTreeMap();
|
||||
final Map<Integer, Integer> frequencies = CollectionUtils.newTreeMap();
|
||||
final Map<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap();
|
||||
try {
|
||||
BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, unigrams, frequencies,
|
||||
BinaryDictIOUtils.readUnigramsAndBigramsBinary(reader, unigrams, frequencies,
|
||||
bigrams);
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IO exception while reading file", e);
|
||||
|
|
|
@ -27,15 +27,13 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
|
|||
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
||||
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
|
||||
import com.android.inputmethod.latin.utils.CollectionUtils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
|
@ -120,31 +118,13 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
|||
/**
|
||||
* Makes new buffer according to BUFFER_TYPE.
|
||||
*/
|
||||
private FusionDictionaryBufferInterface getBuffer(final File file, final int bufferType) {
|
||||
FileInputStream inStream = null;
|
||||
try {
|
||||
inStream = new FileInputStream(file);
|
||||
if (bufferType == USE_BYTE_ARRAY) {
|
||||
final byte[] array = new byte[(int)file.length()];
|
||||
inStream.read(array);
|
||||
return new ByteArrayWrapper(array);
|
||||
} else if (bufferType == USE_BYTE_BUFFER){
|
||||
final ByteBuffer buffer = inStream.getChannel().map(
|
||||
FileChannel.MapMode.READ_ONLY, 0, file.length());
|
||||
return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IOException while making buffer", e);
|
||||
} finally {
|
||||
if (inStream != null) {
|
||||
try {
|
||||
inStream.close();
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IOException while closing stream", e);
|
||||
}
|
||||
}
|
||||
private void getBuffer(final BinaryDictReader reader, final int bufferType)
|
||||
throws FileNotFoundException, IOException {
|
||||
if (bufferType == USE_BYTE_BUFFER) {
|
||||
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
|
||||
} else if (bufferType == USE_BYTE_ARRAY) {
|
||||
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -285,13 +265,14 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
|||
final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcutMap,
|
||||
final int bufferType) {
|
||||
long now, diff = -1;
|
||||
final FusionDictionaryBufferInterface buffer = getBuffer(file, bufferType);
|
||||
assertNotNull(buffer);
|
||||
final BinaryDictReader reader = new BinaryDictReader(file);
|
||||
|
||||
FusionDictionary dict = null;
|
||||
try {
|
||||
getBuffer(reader, bufferType);
|
||||
assertNotNull(reader.getBuffer());
|
||||
now = System.currentTimeMillis();
|
||||
dict = BinaryDictInputOutput.readDictionaryBinary(buffer, null);
|
||||
dict = BinaryDictInputOutput.readDictionaryBinary(reader, null);
|
||||
diff = System.currentTimeMillis() - now;
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IOException while reading dictionary", e);
|
||||
|
@ -421,11 +402,12 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
|||
final Map<Integer, Integer> resultFreqs = CollectionUtils.newTreeMap();
|
||||
|
||||
long now = -1, diff = -1;
|
||||
final FusionDictionaryBufferInterface buffer = getBuffer(file, bufferType);
|
||||
assertNotNull("Can't get buffer.", buffer);
|
||||
final BinaryDictReader reader = new BinaryDictReader(file);
|
||||
try {
|
||||
getBuffer(reader, bufferType);
|
||||
assertNotNull("Can't get buffer.", reader.getBuffer());
|
||||
now = System.currentTimeMillis();
|
||||
BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, resultWords, resultFreqs,
|
||||
BinaryDictIOUtils.readUnigramsAndBigramsBinary(reader, resultWords, resultFreqs,
|
||||
resultBigrams);
|
||||
diff = System.currentTimeMillis() - now;
|
||||
} catch (IOException e) {
|
||||
|
@ -562,7 +544,16 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
|||
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
|
||||
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
|
||||
|
||||
final FusionDictionaryBufferInterface buffer = getBuffer(file, USE_BYTE_ARRAY);
|
||||
final BinaryDictReader reader = new BinaryDictReader(file);
|
||||
FusionDictionaryBufferInterface buffer = null;
|
||||
try {
|
||||
buffer = reader.openAndGetBuffer(
|
||||
new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
|
||||
} catch (IOException e) {
|
||||
// ignore
|
||||
Log.e(TAG, "IOException while opening the buffer", e);
|
||||
}
|
||||
assertNotNull("Can't get the buffer", buffer);
|
||||
|
||||
try {
|
||||
// too long word
|
||||
|
@ -614,7 +605,16 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
|||
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
|
||||
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
|
||||
|
||||
final FusionDictionaryBufferInterface buffer = getBuffer(file, USE_BYTE_ARRAY);
|
||||
final BinaryDictReader reader = new BinaryDictReader(file);
|
||||
FusionDictionaryBufferInterface buffer = null;
|
||||
try {
|
||||
buffer = reader.openAndGetBuffer(
|
||||
new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
|
||||
} catch (IOException e) {
|
||||
// ignore
|
||||
Log.e(TAG, "IOException while opening the buffer", e);
|
||||
}
|
||||
assertNotNull("Can't get the buffer", buffer);
|
||||
|
||||
try {
|
||||
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
|
||||
|
|
|
@ -0,0 +1,138 @@
|
|||
/*
|
||||
* Copyright (C) 2013 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.android.inputmethod.latin.makedict;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFactory;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory;
|
||||
|
||||
import android.test.AndroidTestCase;
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Unit tests for BinaryDictReader
|
||||
*/
|
||||
public class BinaryDictReaderTests extends AndroidTestCase {
|
||||
private static final String TAG = BinaryDictReaderTests.class.getSimpleName();
|
||||
|
||||
private final byte[] data = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
|
||||
|
||||
// Utilities for testing
|
||||
public void writeDataToFile(final File file) {
|
||||
FileOutputStream outStream = null;
|
||||
try {
|
||||
outStream = new FileOutputStream(file);
|
||||
outStream.write(data);
|
||||
} catch (IOException e) {
|
||||
fail ("Can't write data to the test file");
|
||||
} finally {
|
||||
if (outStream != null) {
|
||||
try {
|
||||
outStream.close();
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "Failed to close the output stream", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("null")
|
||||
public void runTestOpenBuffer(final String testName,
|
||||
final FusionDictionaryBufferFactory factory) {
|
||||
File testFile = null;
|
||||
try {
|
||||
testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir());
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IOException while the creating temporary file", e);
|
||||
}
|
||||
|
||||
assertNotNull(testFile);
|
||||
final BinaryDictReader reader = new BinaryDictReader(testFile);
|
||||
try {
|
||||
reader.openBuffer(factory);
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Failed to open the buffer", e);
|
||||
}
|
||||
|
||||
writeDataToFile(testFile);
|
||||
|
||||
try {
|
||||
reader.openBuffer(factory);
|
||||
} catch (Exception e) {
|
||||
Log.e(TAG, "Raised the exception while opening buffer", e);
|
||||
}
|
||||
|
||||
assertEquals(testFile.length(), reader.getBuffer().capacity());
|
||||
}
|
||||
|
||||
public void testOpenBufferWithByteBuffer() {
|
||||
runTestOpenBuffer("testOpenBufferWithByteBuffer",
|
||||
new FusionDictionaryBufferFromByteBufferFactory());
|
||||
}
|
||||
|
||||
public void testOpenBufferWithByteArray() {
|
||||
runTestOpenBuffer("testOpenBufferWithByteArray",
|
||||
new FusionDictionaryBufferFromByteArrayFactory());
|
||||
}
|
||||
|
||||
@SuppressWarnings("null")
|
||||
public void runTestGetBuffer(final String testName,
|
||||
final FusionDictionaryBufferFactory factory) {
|
||||
File testFile = null;
|
||||
try {
|
||||
testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir());
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IOException while the creating temporary file", e);
|
||||
}
|
||||
|
||||
final BinaryDictReader reader = new BinaryDictReader(testFile);
|
||||
|
||||
// the default return value of getBuffer() must be null.
|
||||
assertNull("the default return value of getBuffer() is not null", reader.getBuffer());
|
||||
|
||||
writeDataToFile(testFile);
|
||||
assertTrue(testFile.exists());
|
||||
Log.d(TAG, "file length = " + testFile.length());
|
||||
|
||||
FusionDictionaryBufferInterface buffer = null;
|
||||
try {
|
||||
buffer = reader.openAndGetBuffer(factory);
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "Failed to open and get the buffer", e);
|
||||
}
|
||||
assertNotNull("the buffer must not be null", buffer);
|
||||
|
||||
for (int i = 0; i < data.length; ++i) {
|
||||
assertEquals(data[i], buffer.readUnsignedByte());
|
||||
}
|
||||
}
|
||||
|
||||
public void testGetBufferWithByteBuffer() {
|
||||
runTestGetBuffer("testGetBufferWithByteBuffer",
|
||||
new FusionDictionaryBufferFromByteBufferFactory());
|
||||
}
|
||||
|
||||
public void testGetBufferWithByteArray() {
|
||||
runTestGetBuffer("testGetBufferWithByteArray",
|
||||
new FusionDictionaryBufferFromByteArrayFactory());
|
||||
}
|
||||
}
|
|
@ -21,16 +21,15 @@ import android.test.AndroidTestCase;
|
|||
import android.test.suitebuilder.annotation.LargeTest;
|
||||
import android.util.Log;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
|
||||
import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
|
||||
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
|
||||
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
|
||||
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
|
@ -147,27 +146,15 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
|
|||
}
|
||||
|
||||
private void readDictFromFile(final File file, final OnAddWordListener listener) {
|
||||
FileInputStream inStream = null;
|
||||
|
||||
final BinaryDictReader reader = new BinaryDictReader(file);
|
||||
try {
|
||||
inStream = new FileInputStream(file);
|
||||
final byte[] buffer = new byte[(int)file.length()];
|
||||
inStream.read(buffer);
|
||||
|
||||
UserHistoryDictIOUtils.readDictionaryBinary(new ByteArrayWrapper(buffer), listener);
|
||||
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
|
||||
} catch (FileNotFoundException e) {
|
||||
Log.e(TAG, "file not found", e);
|
||||
} catch (IOException e) {
|
||||
Log.e(TAG, "IOException", e);
|
||||
} finally {
|
||||
if (inStream != null) {
|
||||
try {
|
||||
inStream.close();
|
||||
} catch (IOException e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
UserHistoryDictIOUtils.readDictionaryBinary(reader, listener);
|
||||
}
|
||||
|
||||
public void testGenerateFusionDictionary() {
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
package com.android.inputmethod.latin.dicttool;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
|
||||
|
||||
|
@ -185,16 +186,15 @@ public final class BinaryDictOffdeviceUtils {
|
|||
crash(filename, new RuntimeException(
|
||||
filename + " does not seem to be a dictionary file"));
|
||||
} else {
|
||||
final FileInputStream inStream = new FileInputStream(decodedSpec.mFile);
|
||||
final ByteBuffer buffer = inStream.getChannel().map(
|
||||
FileChannel.MapMode.READ_ONLY, 0, decodedSpec.mFile.length());
|
||||
final BinaryDictReader reader = new BinaryDictReader(decodedSpec.mFile);
|
||||
reader.openBuffer(
|
||||
new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
|
||||
if (report) {
|
||||
System.out.println("Format : Binary dictionary format");
|
||||
System.out.println("Packaging : " + decodedSpec.describeChain());
|
||||
System.out.println("Uncompressed size : " + decodedSpec.mFile.length());
|
||||
}
|
||||
return BinaryDictInputOutput.readDictionaryBinary(
|
||||
new BinaryDictInputOutput.ByteBufferWrapper(buffer), null);
|
||||
return BinaryDictInputOutput.readDictionaryBinary(reader, null);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
package com.android.inputmethod.latin.dicttool;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
import com.android.inputmethod.latin.makedict.MakedictLog;
|
||||
|
@ -265,24 +266,10 @@ public class DictionaryMaker {
|
|||
*/
|
||||
private static FusionDictionary readBinaryFile(final String binaryFilename)
|
||||
throws FileNotFoundException, IOException, UnsupportedFormatException {
|
||||
FileInputStream inStream = null;
|
||||
|
||||
try {
|
||||
final File file = new File(binaryFilename);
|
||||
inStream = new FileInputStream(file);
|
||||
final ByteBuffer buffer = inStream.getChannel().map(
|
||||
FileChannel.MapMode.READ_ONLY, 0, file.length());
|
||||
return BinaryDictInputOutput.readDictionaryBinary(
|
||||
new BinaryDictInputOutput.ByteBufferWrapper(buffer), null);
|
||||
} finally {
|
||||
if (inStream != null) {
|
||||
try {
|
||||
inStream.close();
|
||||
} catch (IOException e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
final File file = new File(binaryFilename);
|
||||
final BinaryDictReader reader = new BinaryDictReader(file);
|
||||
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
|
||||
return BinaryDictInputOutput.readDictionaryBinary(reader, null);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
package com.android.inputmethod.latin.dicttool;
|
||||
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
|
||||
import com.android.inputmethod.latin.makedict.BinaryDictReader;
|
||||
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary;
|
||||
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
|
||||
|
@ -69,11 +70,9 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
|
|||
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
|
||||
}
|
||||
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
|
||||
final FileInputStream inStream = new FileInputStream(decodeSpec.mFile);
|
||||
final ByteBuffer buffer = inStream.getChannel().map(
|
||||
FileChannel.MapMode.READ_ONLY, 0, decodeSpec.mFile.length());
|
||||
final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(
|
||||
new BinaryDictInputOutput.ByteBufferWrapper(buffer),
|
||||
final BinaryDictReader reader = new BinaryDictReader(decodeSpec.mFile);
|
||||
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
|
||||
final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(reader,
|
||||
null /* dict : an optional dictionary to add words to, or null */);
|
||||
assertEquals("Dictionary can't be read back correctly",
|
||||
resultDict.findWordInTree(resultDict.mRoot, "foo").getFrequency(), TEST_FREQ);
|
||||
|
|
Loading…
Reference in New Issue