nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Refactor] Add BinaryDictUtils.

Browse source 
Change-Id: I95c5fb58e2cab3a2c523d2bc3c6fbad06692ac59
This commit is contained in:
Yuichiro Hanada 2013-10-16 16:45:24 +09:00
parent ad5b9bcec2
commit 9c13d52d23
3 changed files with 235 additions and 213 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

230
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View file

@ -26,7 +26,6 @@ import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncodin
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
@ -60,9 +59,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
private static final int NUM_OF_NODES_HAVING_SHORTCUTS = 50;
private static final int NUM_OF_SHORTCUTS = 5;
private static final int USE_BYTE_ARRAY = 1;
private static final int USE_BYTE_BUFFER = 2;
private static final ArrayList<String> sWords = CollectionUtils.newArrayList();
private static final SparseArray<List<Integer>> sEmptyBigrams =
CollectionUtils.newSparseArray();
@ -71,21 +67,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
CollectionUtils.newSparseArray();
private static final HashMap<String, List<String>> sShortcuts = CollectionUtils.newHashMap();
private static final FormatSpec.FormatOptions VERSION2 = new FormatSpec.FormatOptions(2);
private static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(3, false /* supportsDynamicUpdate */);
private static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */);
private static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(4, false /* supportsDynamicUpdate */);
private static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */);
private static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP =
new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */,
true /* hasTimestamp */);
private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
public BinaryDictDecoderEncoderTests() {
this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS);
}
@ -124,17 +105,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
}
}
private DictEncoder getDictEncoder(final File file, final FormatOptions formatOptions) {
if (formatOptions.mVersion == FormatSpec.VERSION4) {
return new Ver4DictEncoder(getContext().getCacheDir());
} else if (formatOptions.mVersion == 3 || formatOptions.mVersion == 2) {
return new Ver3DictEncoder(file);
} else {
throw new RuntimeException("The format option has a wrong version : "
+ formatOptions.mVersion);
}
}
private void generateWords(final int number, final Random random, final int[] codePointSet) {
final Set<String> wordSet = CollectionUtils.newHashSet();
while (wordSet.size() < number) {
@ -186,7 +156,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
long now = -1, diff = -1;
try {
final DictEncoder dictEncoder = getDictEncoder(file, formatOptions);
final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions,
getContext().getCacheDir());
now = System.currentTimeMillis();
// If you need to dump the dict to a textual file, uncomment the line below and the
@ -241,54 +212,21 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
private String outputOptions(final int bufferType,
final FormatSpec.FormatOptions formatOptions) {
String result = " : buffer type = "
+ ((bufferType == USE_BYTE_BUFFER) ? "byte buffer" : "byte array");
+ ((bufferType == BinaryDictUtils.USE_BYTE_BUFFER) ? "byte buffer" : "byte array");
result += " : version = " + formatOptions.mVersion;
return result + ", supportsDynamicUpdate = " + formatOptions.mSupportsDynamicUpdate;
}
private DictionaryOptions getDictionaryOptions(final String id, final String version) {
final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>(),
false, false);
options.mAttributes.put("version", version);
options.mAttributes.put("dictionary", id);
return options;
}
private File setUpDictionaryFile(final String name, final String version) {
File file = null;
try {
file = new File(getContext().getCacheDir(), name + "." + version
+ TEST_DICT_FILE_EXTENSION);
file.createNewFile();
} catch (IOException e) {
// do nothing
}
assertTrue("Failed to create the dictionary file.", file.exists());
return file;
}
private DictDecoder getDictDecoder(final File file, final int bufferType,
final FormatOptions formatOptions, final DictionaryOptions dictOptions) {
if (formatOptions.mVersion == FormatSpec.VERSION4) {
final FileHeader header = new FileHeader(0, dictOptions, formatOptions);
return FormatSpec.getDictDecoder(new File(getContext().getCacheDir(),
header.getId() + "." + header.getVersion()), bufferType);
} else {
return FormatSpec.getDictDecoder(file, bufferType);
}
}
// Tests for readDictionaryBinary and writeDictionaryBinary
private long timeReadingAndCheckDict(final File file, final List<String> words,
final SparseArray<List<Integer>> bigrams,
final HashMap<String, List<String>> shortcutMap, final int bufferType,
final FormatOptions formatOptions, final DictionaryOptions dictOptions) {
final HashMap<String, List<String>> shortcutMap, final int bufferType) {
long now, diff = -1;
FusionDictionary dict = null;
try {
final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions,
dictOptions);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
now = System.currentTimeMillis();
dict = dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */);
diff = System.currentTimeMillis() - now;
@ -310,17 +248,17 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
final String dictName = "runReadAndWrite";
final String dictVersion = Long.toString(System.currentTimeMillis());
final File file = setUpDictionaryFile(dictName, dictVersion);
final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions,
getContext().getCacheDir());
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
getDictionaryOptions(dictName, dictVersion));
BinaryDictUtils.getDictionaryOptions(dictName, dictVersion));
addUnigrams(words.size(), dict, words, shortcuts);
addBigrams(dict, words, bigrams);
checkDictionary(dict, words, bigrams, shortcuts);
final long write = timeWritingDictToFile(file, dict, formatOptions);
final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType,
formatOptions, dict.mOptions);
final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType);
return "PROF: read=" + read + "ms, write=" + write + "ms :" + message
+ " : " + outputOptions(bufferType, formatOptions);
@ -349,7 +287,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
final byte[] buffer = new byte[50 * 3];
final DictBuffer dictBuffer = new ByteArrayDictBuffer(buffer);
for (final String word : sWords) {
Log.d("testReadAndWriteString", "write : " + word);
Arrays.fill(buffer, (byte) 0);
CharEncoding.writeString(buffer, 0, word);
dictBuffer.position(0);
@ -361,12 +298,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
public void testReadAndWriteWithByteBuffer() {
final List<String> results = CollectionUtils.newArrayList();
runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION2);
runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE);
runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE);
runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE);
runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE);
runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION2);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
for (final String result : results) {
Log.d(TAG, result);
@ -376,12 +319,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
public void testReadAndWriteWithByteArray() {
final List<String> results = CollectionUtils.newArrayList();
runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION2);
runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE);
runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE);
runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE);
runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE);
runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION2);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE);
runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
for (final String result : results) {
Log.d(TAG, result);
@ -437,8 +386,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
}
private long timeAndCheckReadUnigramsAndBigramsBinary(final File file, final List<String> words,
final SparseArray<List<Integer>> bigrams, final int bufferType,
final FormatOptions formatOptions, final DictionaryOptions dictOptions) {
final SparseArray<List<Integer>> bigrams, final int bufferType) {
FileInputStream inStream = null;
final TreeMap<Integer, String> resultWords = CollectionUtils.newTreeMap();
@ -448,8 +396,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
long now = -1, diff = -1;
try {
final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions,
dictOptions);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
now = System.currentTimeMillis();
dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams);
diff = System.currentTimeMillis() - now;
@ -476,20 +423,20 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
final FormatSpec.FormatOptions formatOptions, final String message) {
final String dictName = "runReadUnigrams";
final String dictVersion = Long.toString(System.currentTimeMillis());
final File file = setUpDictionaryFile(dictName, dictVersion);
final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions,
getContext().getCacheDir());
// making the dictionary from lists of words.
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
getDictionaryOptions(dictName, dictVersion));
BinaryDictUtils.getDictionaryOptions(dictName, dictVersion));
addUnigrams(words.size(), dict, words, null /* shortcutMap */);
addBigrams(dict, words, bigrams);
timeWritingDictToFile(file, dict, formatOptions);
long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType,
formatOptions, dict.mOptions);
long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType);
long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */,
bufferType, formatOptions, dict.mOptions);
bufferType);
return "readDictionaryBinary=" + fullReading + ", readUnigramsAndBigramsBinary=" + wordMap
+ " : " + message + " : " + outputOptions(bufferType, formatOptions);
@ -508,13 +455,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
public void testReadUnigramsAndBigramsBinaryWithByteBuffer() {
final ArrayList<String> results = CollectionUtils.newArrayList();
runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION2);
runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER,
VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION2);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
for (final String result : results) {
Log.d(TAG, result);
@ -524,13 +476,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
public void testReadUnigramsAndBigramsBinaryWithByteArray() {
final ArrayList<String> results = CollectionUtils.newArrayList();
runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION2);
runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY,
VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION2);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE);
runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
for (final String result : results) {
Log.d(TAG, result);
@ -578,16 +535,16 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
final FormatOptions formatOptions, final String message) {
final String dictName = "testGetTerminalPosition";
final String dictVersion = Long.toString(System.currentTimeMillis());
final File file = setUpDictionaryFile(dictName, dictVersion);
final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions,
getContext().getCacheDir());
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
getDictionaryOptions(dictName, dictVersion));
BinaryDictUtils.getDictionaryOptions(dictName, dictVersion));
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
addBigrams(dict, words, bigrams);
timeWritingDictToFile(file, dict, formatOptions);
final DictDecoder dictDecoder = getDictDecoder(file, DictDecoder.USE_BYTEARRAY,
formatOptions, dict.mOptions);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer();
} catch (IOException e) {
@ -638,19 +595,29 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
public void testGetTerminalPosition() {
final ArrayList<String> results = CollectionUtils.newArrayList();
runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION2);
runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE);
runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE);
runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE);
runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE);
runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, BinaryDictUtils.VERSION2);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION2);
runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE);
runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE);
runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE);
runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE);
runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, BinaryDictUtils.VERSION2);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE);
runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER,
BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP);
for (final String result : results) {
Log.d(TAG, result);
@ -660,7 +627,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
private void runTestDeleteWord(final FormatOptions formatOptions) {
final String dictName = "testDeleteWord";
final String dictVersion = Long.toString(System.currentTimeMillis());
final File file = setUpDictionaryFile(dictName, dictVersion);
final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions,
getContext().getCacheDir());
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(
@ -668,15 +636,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, formatOptions);
final DictUpdater dictUpdater;
if (formatOptions.mVersion == 3) {
dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else if (formatOptions.mVersion == 4) {
dictUpdater = new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else {
throw new RuntimeException("DictUpdater for version " + formatOptions.mVersion
+ " doesn't exist.");
}
final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions);
try {
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
@ -696,7 +656,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
}
public void testDeleteWord() {
runTestDeleteWord(VERSION3_WITH_DYNAMIC_UPDATE);
runTestDeleteWord(VERSION4_WITH_DYNAMIC_UPDATE);
runTestDeleteWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
runTestDeleteWord(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE);
}
}

124
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
View file

@ -23,6 +23,7 @@ import android.util.Log;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.CollectionUtils;
@ -30,24 +31,16 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Random;
@LargeTest
public class BinaryDictIOUtilsTests extends AndroidTestCase {
private static final String TAG = BinaryDictIOUtilsTests.class.getSimpleName();
private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
new FormatSpec.FormatOptions(3, true);
private static final ArrayList<String> sWords = CollectionUtils.newArrayList();
public static final int DEFAULT_MAX_UNIGRAMS = 1500;
private final int mMaxUnigrams;
private static final String TEST_DICT_FILE_EXTENSION = ".testDict";
private static final int VERSION3 = 3;
private static final int VERSION4 = 4;
private static final String[] CHARACTERS = {
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
@ -141,7 +134,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
int position = FormatSpec.NOT_VALID_WORD;
try {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file,
DictDecoder.USE_READONLY_BYTEBUFFER);
position = dictDecoder.getTerminalPosition(word);
} catch (IOException e) {
@ -159,7 +152,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
* @throws IOException
* @throws UnsupportedFormatException
*/
private static PtNodeInfo findWordByBinaryDictReader(final DictDecoder dictDecoder,
private static PtNodeInfo findWordByDictDecoder(final DictDecoder dictDecoder,
final String word) throws IOException, UnsupportedFormatException {
int position = dictDecoder.getTerminalPosition(word);
if (position != FormatSpec.NOT_VALID_WORD) {
@ -176,7 +169,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
PtNodeInfo info = null;
try {
dictDecoder.openDictBuffer();
info = findWordByBinaryDictReader(dictDecoder, word);
info = findWordByDictDecoder(dictDecoder, word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
}
@ -186,16 +179,10 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
// return amount of time to insert a word
private long insertAndCheckWord(final File file, final String word, final int frequency,
final boolean exist, final ArrayList<WeightedString> bigrams,
final ArrayList<WeightedString> shortcuts, final int formatVersion) {
final ArrayList<WeightedString> shortcuts, final FormatOptions formatOptions) {
long amountOfTime = -1;
try {
final DictUpdater dictUpdater;
if (formatVersion == VERSION3) {
dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else {
throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't"
+ " exist.");
}
final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions);
if (!exist) {
assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word));
@ -212,18 +199,14 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
return amountOfTime;
}
private void deleteWord(final File file, final String word, final int formatVersion) {
private void deleteWord(final File file, final String word, final FormatOptions formatOptions) {
try {
final DictUpdater dictUpdater;
if (formatVersion == VERSION3) {
dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else {
throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't"
+ " exist.");
}
final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions);
dictUpdater.deleteWord(word);
} catch (IOException e) {
Log.e(TAG, "Raised an IOException while deleting a word", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "Raised an UnsupportedFormatException while deleting a word", e);
}
}
@ -242,23 +225,21 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
}
private void runTestInsertWord(final int formatVersion) {
File file = null;
try {
file = File.createTempFile("testInsertWord", TEST_DICT_FILE_EXTENSION,
private void runTestInsertWord(final FormatOptions formatOptions) {
final String testName = "testInsertWord";
final String version = Long.toString(System.currentTimeMillis());
final File file = BinaryDictUtils.getDictFile(testName, version, formatOptions,
getContext().getCacheDir());
} catch (IOException e) {
fail("IOException while creating temporary file: " + e);
}
// set an initial dictionary.
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false));
BinaryDictUtils.getDictionaryOptions(testName, version));
dict.add("abcd", 10, null, false);
try {
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions,
getContext().getCacheDir());
dictEncoder.writeDictionary(dict, formatOptions);
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
@ -266,54 +247,52 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd"));
insertAndCheckWord(file, "abcde", 10, false, null, null, formatVersion);
insertAndCheckWord(file, "abcde", 10, false, null, null, formatOptions);
insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null, formatVersion);
insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null, formatOptions);
checkReverseLookup(file, "abcdefghijklmn", getWordPosition(file, "abcdefghijklmn"));
insertAndCheckWord(file, "abcdabcd", 10, false, null, null, formatVersion);
insertAndCheckWord(file, "abcdabcd", 10, false, null, null, formatOptions);
checkReverseLookup(file, "abcdabcd", getWordPosition(file, "abcdabcd"));
// update the existing word.
insertAndCheckWord(file, "abcdabcd", 15, true, null, null, formatVersion);
insertAndCheckWord(file, "abcdabcd", 15, true, null, null, formatOptions);
// split 1
insertAndCheckWord(file, "ab", 20, false, null, null, formatVersion);
insertAndCheckWord(file, "ab", 20, false, null, null, formatOptions);
// split 2
insertAndCheckWord(file, "ami", 30, false, null, null, formatVersion);
insertAndCheckWord(file, "ami", 30, false, null, null, formatOptions);
deleteWord(file, "ami", formatVersion);
deleteWord(file, "ami", formatOptions);
assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "ami"));
insertAndCheckWord(file, "abcdabfg", 30, false, null, null, formatVersion);
insertAndCheckWord(file, "abcdabfg", 30, false, null, null, formatOptions);
deleteWord(file, "abcd", formatVersion);
deleteWord(file, "abcd", formatOptions);
assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd"));
}
public void testInsertWord() {
runTestInsertWord(VERSION3);
runTestInsertWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
}
private void runTestInsertWordWithBigrams(final int formatVersion) {
File file = null;
try {
file = File.createTempFile("testInsertWordWithBigrams", TEST_DICT_FILE_EXTENSION,
private void runTestInsertWordWithBigrams(final FormatOptions formatOptions) {
final String testName = "testInsertWordWithBigrams";
final String version = Long.toString(System.currentTimeMillis());
File file = BinaryDictUtils.getDictFile(testName, version, formatOptions,
getContext().getCacheDir());
} catch (IOException e) {
fail("IOException while creating temporary file: " + e);
}
// set an initial dictionary.
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false));
BinaryDictUtils.getDictionaryOptions(testName, version));
dict.add("abcd", 10, null, false);
dict.add("efgh", 15, null, false);
try {
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions,
getContext().getCacheDir());
dictEncoder.writeDictionary(dict, formatOptions);
} catch (IOException e) {
fail("IOException while writing an initial dictionary : " + e);
} catch (UnsupportedFormatException e) {
@ -323,8 +302,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
final ArrayList<WeightedString> banana = new ArrayList<WeightedString>();
banana.add(new WeightedString("banana", 10));
insertAndCheckWord(file, "banana", 0, false, null, null, formatVersion);
insertAndCheckWord(file, "recursive", 60, true, banana, null, formatVersion);
insertAndCheckWord(file, "banana", 0, false, null, null, formatOptions);
insertAndCheckWord(file, "recursive", 60, true, banana, null, formatOptions);
final PtNodeInfo info = findWordFromFile(file, "recursive");
int bananaPos = getWordPosition(file, "banana");
@ -334,27 +313,24 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
public void testInsertWordWithBigrams() {
runTestInsertWordWithBigrams(VERSION3);
runTestInsertWordWithBigrams(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
}
private void runTestRandomWords(final int formatVersion) {
File file = null;
try {
file = File.createTempFile("testRandomWord", TEST_DICT_FILE_EXTENSION,
private void runTestRandomWords(final FormatOptions formatOptions) {
final String testName = "testRandomWord";
final String version = Long.toString(System.currentTimeMillis());
final File file = BinaryDictUtils.getDictFile(testName, version, formatOptions,
getContext().getCacheDir());
} catch (IOException e) {
}
assertNotNull(file);
// set an initial dictionary.
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false,
false));
BinaryDictUtils.getDictionaryOptions(testName, version));
dict.add("initial", 10, null, false);
try {
final DictEncoder dictEncoder = new Ver3DictEncoder(file);
dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions,
getContext().getCacheDir());
dictEncoder.writeDictionary(dict, formatOptions);
} catch (IOException e) {
assertTrue(false);
} catch (UnsupportedFormatException e) {
@ -366,7 +342,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
int cnt = 0;
for (final String word : sWords) {
final long diff = insertAndCheckWord(file, word,
cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null, formatVersion);
cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null, formatOptions);
maxTimeToInsert = Math.max(maxTimeToInsert, diff);
minTimeToInsert = Math.min(minTimeToInsert, diff);
sum += diff;
@ -377,13 +353,13 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word));
}
Log.d(TAG, "Test version " + formatVersion);
Log.d(TAG, "Test version " + formatOptions);
Log.d(TAG, "max = " + ((double)maxTimeToInsert/1000000) + " ms.");
Log.d(TAG, "min = " + ((double)minTimeToInsert/1000000) + " ms.");
Log.d(TAG, "avg = " + ((double)sum/mMaxUnigrams/1000000) + " ms.");
}
public void testRandomWords() {
runTestRandomWords(VERSION3);
runTestRandomWords(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE);
}
}

86
tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import java.io.File;
import java.util.HashMap;
public class BinaryDictUtils {
public static final int USE_BYTE_ARRAY = 1;
public static final int USE_BYTE_BUFFER = 2;
public static final String TEST_DICT_FILE_EXTENSION = ".testDict";
public static final FormatSpec.FormatOptions VERSION2 = new FormatSpec.FormatOptions(2);
public static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(3, false /* supportsDynamicUpdate */);
public static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */);
public static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(4, false /* supportsDynamicUpdate */);
public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE =
new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */);
public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP =
new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */,
true /* hasTimestamp */);
public static DictionaryOptions getDictionaryOptions(final String id, final String version) {
final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>(),
false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */);
options.mAttributes.put("dictionary", id);
options.mAttributes.put("version", version);
return options;
}
public static File getDictFile(final String name, final String version,
final FormatOptions formatOptions, final File directory) {
if (formatOptions.mVersion == 2 || formatOptions.mVersion == 3) {
return new File(directory, name + "." + version + TEST_DICT_FILE_EXTENSION);
} else if (formatOptions.mVersion == 4) {
return new File(directory, name + "." + version);
} else {
throw new RuntimeException("the format option has a wrong version : "
+ formatOptions.mVersion);
}
}
public static DictEncoder getDictEncoder(final File file, final FormatOptions formatOptions,
final File cacheDir) {
if (formatOptions.mVersion == FormatSpec.VERSION4) {
return new Ver4DictEncoder(cacheDir);
} else if (formatOptions.mVersion == 3 || formatOptions.mVersion == 2) {
return new Ver3DictEncoder(file);
} else {
throw new RuntimeException("The format option has a wrong version : "
+ formatOptions.mVersion);
}
}
public static DictUpdater getDictUpdater(final File file, final FormatOptions formatOptions) {
if (formatOptions.mVersion == FormatSpec.VERSION4) {
return new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else if (formatOptions.mVersion == 3) {
return new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER);
} else {
throw new RuntimeException("The format option has a wrong version : "
+ formatOptions.mVersion);
}
}
}