am fc9ea5be: Merge "Refactor BinaryDictIOTests." into jb-mr1-dev
* commit 'fc9ea5be47132d3e42fc310331ac89d2f42392c3': Refactor BinaryDictIOTests.main
commit
4a29e83576
|
@ -54,14 +54,40 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
||||||
private static final int BIGRAM_FREQ = 50;
|
private static final int BIGRAM_FREQ = 50;
|
||||||
private static final int TOLERANCE_OF_BIGRAM_FREQ = 5;
|
private static final int TOLERANCE_OF_BIGRAM_FREQ = 5;
|
||||||
|
|
||||||
|
private static final List<String> sWords = CollectionUtils.newArrayList();
|
||||||
|
private static final SparseArray<List<Integer>> sEmptyBigrams =
|
||||||
|
CollectionUtils.newSparseArray();
|
||||||
|
private static final SparseArray<List<Integer>> sStarBigrams = CollectionUtils.newSparseArray();
|
||||||
|
private static final SparseArray<List<Integer>> sChainBigrams =
|
||||||
|
CollectionUtils.newSparseArray();
|
||||||
|
|
||||||
private static final BinaryDictInputOutput.FormatOptions VERSION2 =
|
private static final BinaryDictInputOutput.FormatOptions VERSION2 =
|
||||||
new BinaryDictInputOutput.FormatOptions(2);
|
new BinaryDictInputOutput.FormatOptions(2);
|
||||||
|
|
||||||
private static final String[] CHARACTERS =
|
private static final String[] CHARACTERS = {
|
||||||
{
|
|
||||||
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
|
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
|
||||||
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
|
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
|
||||||
};
|
};
|
||||||
|
|
||||||
|
public BinaryDictIOTests() {
|
||||||
|
super();
|
||||||
|
|
||||||
|
final Random random = new Random(123456);
|
||||||
|
sWords.clear();
|
||||||
|
generateWords(MAX_UNIGRAMS, random);
|
||||||
|
|
||||||
|
for (int i = 0; i < sWords.size(); ++i) {
|
||||||
|
sChainBigrams.put(i, new ArrayList<Integer>());
|
||||||
|
if (i > 0) {
|
||||||
|
sChainBigrams.get(i-1).add(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sStarBigrams.put(0, new ArrayList<Integer>());
|
||||||
|
for (int i = 1; i < sWords.size(); ++i) {
|
||||||
|
sStarBigrams.get(0).add(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Utilities for test
|
// Utilities for test
|
||||||
/**
|
/**
|
||||||
|
@ -78,12 +104,12 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
||||||
return builder.toString();
|
return builder.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<String> generateWords(final int number, final Random random) {
|
private void generateWords(final int number, final Random random) {
|
||||||
final Set<String> wordSet = CollectionUtils.newHashSet();
|
final Set<String> wordSet = CollectionUtils.newHashSet();
|
||||||
while (wordSet.size() < number) {
|
while (wordSet.size() < number) {
|
||||||
wordSet.add(generateWord(random.nextInt()));
|
wordSet.add(generateWord(random.nextInt()));
|
||||||
}
|
}
|
||||||
return new ArrayList<String>(wordSet);
|
sWords.addAll(wordSet);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -182,10 +208,8 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
||||||
FileChannel.MapMode.READ_ONLY, 0, file.length());
|
FileChannel.MapMode.READ_ONLY, 0, file.length());
|
||||||
|
|
||||||
now = System.currentTimeMillis();
|
now = System.currentTimeMillis();
|
||||||
|
|
||||||
final FusionDictionary dict =
|
final FusionDictionary dict =
|
||||||
BinaryDictInputOutput.readDictionaryBinary(buffer, null);
|
BinaryDictInputOutput.readDictionaryBinary(buffer, null);
|
||||||
|
|
||||||
diff = System.currentTimeMillis() - now;
|
diff = System.currentTimeMillis() - now;
|
||||||
|
|
||||||
checkDictionary(dict, words, bigrams, shortcutMap);
|
checkDictionary(dict, words, bigrams, shortcutMap);
|
||||||
|
@ -212,22 +236,19 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
||||||
private String runReadAndWrite(final List<String> words,
|
private String runReadAndWrite(final List<String> words,
|
||||||
final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcuts,
|
final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcuts,
|
||||||
final String message) {
|
final String message) {
|
||||||
final FusionDictionary dict = new FusionDictionary(new Node(),
|
|
||||||
new FusionDictionary.DictionaryOptions(
|
|
||||||
new HashMap<String,String>(), false, false));
|
|
||||||
|
|
||||||
File file = null;
|
File file = null;
|
||||||
try {
|
try {
|
||||||
file = File.createTempFile("runReadAndWrite", ".dict");
|
file = File.createTempFile("runReadAndWrite", ".dict");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
Log.e(TAG, "IOException: " + e);
|
Log.e(TAG, "IOException: " + e);
|
||||||
}
|
}
|
||||||
|
|
||||||
assertNotNull(file);
|
assertNotNull(file);
|
||||||
|
|
||||||
|
final FusionDictionary dict = new FusionDictionary(new Node(),
|
||||||
|
new FusionDictionary.DictionaryOptions(
|
||||||
|
new HashMap<String,String>(), false, false));
|
||||||
addUnigrams(words.size(), dict, words, shortcuts);
|
addUnigrams(words.size(), dict, words, shortcuts);
|
||||||
addBigrams(dict, words, bigrams);
|
addBigrams(dict, words, bigrams);
|
||||||
// check original dictionary
|
|
||||||
checkDictionary(dict, words, bigrams, shortcuts);
|
checkDictionary(dict, words, bigrams, shortcuts);
|
||||||
|
|
||||||
final long write = timeWritingDictToFile(file, dict);
|
final long write = timeWritingDictToFile(file, dict);
|
||||||
|
@ -239,22 +260,9 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
||||||
public void testReadAndWrite() {
|
public void testReadAndWrite() {
|
||||||
final List<String> results = new ArrayList<String>();
|
final List<String> results = new ArrayList<String>();
|
||||||
|
|
||||||
final Random random = new Random(123456);
|
results.add(runReadAndWrite(sWords, sEmptyBigrams, null /* shortcuts */ , "unigram"));
|
||||||
final List<String> words = generateWords(MAX_UNIGRAMS, random);
|
results.add(runReadAndWrite(sWords, sChainBigrams, null /* shortcuts */ , "chain"));
|
||||||
final SparseArray<List<Integer>> emptyArray = CollectionUtils.newSparseArray();
|
results.add(runReadAndWrite(sWords, sStarBigrams, null /* shortcuts */ , "star"));
|
||||||
|
|
||||||
final SparseArray<List<Integer>> chain = CollectionUtils.newSparseArray();
|
|
||||||
for (int i = 0; i < words.size(); ++i) chain.put(i, new ArrayList<Integer>());
|
|
||||||
for (int i = 1; i < words.size(); ++i) chain.get(i-1).add(i);
|
|
||||||
|
|
||||||
final SparseArray<List<Integer>> star = CollectionUtils.newSparseArray();
|
|
||||||
final List<Integer> list0 = CollectionUtils.newArrayList();
|
|
||||||
star.put(0, list0);
|
|
||||||
for (int i = 1; i < words.size(); ++i) star.get(0).add(i);
|
|
||||||
|
|
||||||
results.add(runReadAndWrite(words, emptyArray, null /* shortcuts */ , "only unigram"));
|
|
||||||
results.add(runReadAndWrite(words, chain, null /* shortcuts */ , "chain"));
|
|
||||||
results.add(runReadAndWrite(words, star, null /* shortcuts */ , "star"));
|
|
||||||
|
|
||||||
for (final String result : results) {
|
for (final String result : results) {
|
||||||
Log.d(TAG, result);
|
Log.d(TAG, result);
|
||||||
|
@ -347,47 +355,41 @@ public class BinaryDictIOTests extends AndroidTestCase {
|
||||||
return diff;
|
return diff;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void runReadUnigramsAndBigramsBinary(final List<String> words,
|
private String runReadUnigramsAndBigramsBinary(final List<String> words,
|
||||||
final SparseArray<List<Integer>> bigrams) {
|
final SparseArray<List<Integer>> bigrams, final String message) {
|
||||||
|
|
||||||
// making the dictionary from lists of words.
|
|
||||||
final FusionDictionary dict = new FusionDictionary(new Node(),
|
|
||||||
new FusionDictionary.DictionaryOptions(
|
|
||||||
new HashMap<String, String>(), false, false));
|
|
||||||
|
|
||||||
File file = null;
|
File file = null;
|
||||||
try {
|
try {
|
||||||
file = File.createTempFile("runReadUnigrams", ".dict");
|
file = File.createTempFile("runReadUnigrams", ".dict");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
Log.e(TAG, "IOException: " + e);
|
Log.e(TAG, "IOException: " + e);
|
||||||
}
|
}
|
||||||
|
|
||||||
assertNotNull(file);
|
assertNotNull(file);
|
||||||
|
|
||||||
|
// making the dictionary from lists of words.
|
||||||
|
final FusionDictionary dict = new FusionDictionary(new Node(),
|
||||||
|
new FusionDictionary.DictionaryOptions(
|
||||||
|
new HashMap<String, String>(), false, false));
|
||||||
addUnigrams(words.size(), dict, words, null /* shortcutMap */);
|
addUnigrams(words.size(), dict, words, null /* shortcutMap */);
|
||||||
addBigrams(dict, words, bigrams);
|
addBigrams(dict, words, bigrams);
|
||||||
|
|
||||||
timeWritingDictToFile(file, dict);
|
timeWritingDictToFile(file, dict);
|
||||||
|
|
||||||
long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams);
|
long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams);
|
||||||
long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */);
|
long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */);
|
||||||
|
|
||||||
Log.d(TAG, "read=" + fullReading + ", bytearray=" + wordMap);
|
return "readDictionaryBinary=" + fullReading + ", readUnigramsAndBigramsBinary=" + wordMap
|
||||||
|
+ " : " + message;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testReadUnigramsAndBigramsBinary() {
|
public void testReadUnigramsAndBigramsBinary() {
|
||||||
final List<String> results = new ArrayList<String>();
|
final List<String> results = new ArrayList<String>();
|
||||||
|
|
||||||
final Random random = new Random(123456);
|
results.add(runReadUnigramsAndBigramsBinary(sWords, sEmptyBigrams, "unigram"));
|
||||||
final List<String> words = generateWords(MAX_UNIGRAMS, random);
|
results.add(runReadUnigramsAndBigramsBinary(sWords, sChainBigrams, "chain"));
|
||||||
final SparseArray<List<Integer>> emptyArray = CollectionUtils.newSparseArray();
|
results.add(runReadUnigramsAndBigramsBinary(sWords, sStarBigrams, "star"));
|
||||||
|
|
||||||
runReadUnigramsAndBigramsBinary(words, emptyArray);
|
for (final String result : results) {
|
||||||
|
Log.d(TAG, result);
|
||||||
final SparseArray<List<Integer>> star = CollectionUtils.newSparseArray();
|
|
||||||
for (int i = 1; i < words.size(); ++i) {
|
|
||||||
star.put(i-1, new ArrayList<Integer>());
|
|
||||||
star.get(i-1).add(i);
|
|
||||||
}
|
}
|
||||||
runReadUnigramsAndBigramsBinary(words, star);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue