nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "Make WeightedString have ProbabilityInfo."

main
Keisuke Kuroyanagi 2014-02-05 12:53:14 +00:00 committed by Android (Google) Code Review
parent e42380a471 df1d3e733e
commit 239f7daf4a
11 changed files with 80 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
View File

@ -511,7 +511,7 @@ public final class BinaryDictDecoderUtils {
final WeightedString word = getWordAtPosition(dictDecoder, headerSize,
bigram.mAddress, options);
final int reconstructedFrequency =
BinaryDictIOUtils.reconstructBigramFrequency(word.mFrequency,
BinaryDictIOUtils.reconstructBigramFrequency(word.getProbability(),
bigram.mFrequency);
bigrams.add(new WeightedString(word.mWord, reconstructedFrequency));
}
@ -618,7 +618,7 @@ public final class BinaryDictDecoderUtils {
// words that are not also registered as unigrams so we don't have to avoid
// them explicitly here.
for (final WeightedString bigram : w.mBigrams) {
newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency);
newDict.setBigram(w.mWord, bigram.mWord, bigram.getProbability());
}
}
}

41
java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
View File

@ -67,29 +67,40 @@ public final class FusionDictionary implements Iterable<Word> {
}
/**
* A string with a frequency.
* A string with a probability.
*
* This represents an "attribute", that is either a bigram or a shortcut.
*/
public static final class WeightedString {
public final String mWord;
public int mFrequency;
public WeightedString(String word, int frequency) {
public ProbabilityInfo mProbabilityInfo;
public WeightedString(final String word, final int probability) {
mWord = word;
mFrequency = frequency;
mProbabilityInfo = new ProbabilityInfo(probability);
}
public int getProbability() {
return mProbabilityInfo.mProbability;
}
public void setProbability(final int probability) {
mProbabilityInfo = new ProbabilityInfo(probability);
}
@Override
public int hashCode() {
return Arrays.hashCode(new Object[] { mWord, mFrequency });
return Arrays.hashCode(new Object[] { mWord, mProbabilityInfo.mProbability,
mProbabilityInfo.mTimestamp, mProbabilityInfo.mLevel,
mProbabilityInfo.mCount });
}
@Override
public boolean equals(Object o) {
if (o == this) return true;
if (!(o instanceof WeightedString)) return false;
WeightedString w = (WeightedString)o;
return mWord.equals(w.mWord) && mFrequency == w.mFrequency;
final WeightedString w = (WeightedString)o;
return mWord.equals(w.mWord) && mProbabilityInfo.equals(w.mProbabilityInfo);
}
}
@ -200,18 +211,18 @@ public final class FusionDictionary implements Iterable<Word> {
}
/**
* Adds a word to the bigram list. Updates the frequency if the word already
* Adds a word to the bigram list. Updates the probability if the word already
* exists.
*/
public void addBigram(final String word, final int frequency) {
public void addBigram(final String word, final int probability) {
if (mBigrams == null) {
mBigrams = new ArrayList<WeightedString>();
}
WeightedString bigram = getBigram(word);
if (bigram != null) {
bigram.mFrequency = frequency;
bigram.setProbability(probability);
} else {
bigram = new WeightedString(word, frequency);
bigram = new WeightedString(word, probability);
mBigrams.add(bigram);
}
}
@ -273,8 +284,8 @@ public final class FusionDictionary implements Iterable<Word> {
final WeightedString existingShortcut = getShortcut(shortcut.mWord);
if (existingShortcut == null) {
mShortcutTargets.add(shortcut);
} else if (existingShortcut.mFrequency < shortcut.mFrequency) {
existingShortcut.mFrequency = shortcut.mFrequency;
} else if (existingShortcut.getProbability() < shortcut.getProbability()) {
existingShortcut.setProbability(shortcut.getProbability());
}
}
}
@ -289,8 +300,8 @@ public final class FusionDictionary implements Iterable<Word> {
final WeightedString existingBigram = getBigram(bigram.mWord);
if (existingBigram == null) {
mBigrams.add(bigram);
} else if (existingBigram.mFrequency < bigram.mFrequency) {
existingBigram.mFrequency = bigram.mFrequency;
} else if (existingBigram.getProbability() < bigram.getProbability()) {
existingBigram.setProbability(bigram.getProbability());
}
}
}

17
java/src/com/android/inputmethod/latin/makedict/ProbabilityInfo.java
View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
public final class ProbabilityInfo {
public final int mProbability;
@ -39,8 +40,24 @@ public final class ProbabilityInfo {
mCount = count;
}
public boolean hasHistoricalInfo() {
return mTimestamp != BinaryDictionary.NOT_A_VALID_TIMESTAMP;
}
@Override
public String toString() {
return mTimestamp + ":" + mLevel + ":" + mCount;
}
@Override
public boolean equals(Object o) {
if (o == this) return true;
if (!(o instanceof ProbabilityInfo)) return false;
final ProbabilityInfo p = (ProbabilityInfo)o;
if (!hasHistoricalInfo() && !p.hasHistoricalInfo()) {
return mProbability == p.mProbability;
}
return mProbability == p.mProbability && mTimestamp == p.mTimestamp && mLevel == p.mLevel
&& mCount == p.mCount;
}
}

4
java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java
View File

@ -197,7 +197,7 @@ public class Ver2DictEncoder implements DictEncoder {
final WeightedString target = shortcutIterator.next();
final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags(
shortcutIterator.hasNext(),
target.mFrequency);
target.getProbability());
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, shortcutFlags,
FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord);
@ -231,7 +231,7 @@ public class Ver2DictEncoder implements DictEncoder {
final int offset = addressOfBigram
- (mPosition + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(),
offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord);
offset, bigram.getProbability(), unigramFrequencyForThisWord, bigram.mWord);
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, bigramFlags,
FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition,

4
java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
View File

@ -80,7 +80,7 @@ public class Ver4DictEncoder implements DictEncoder {
} else {
for (final WeightedString shortcutTarget : word.mShortcutTargets) {
binaryDict.addUnigramWord(word.mWord, word.mFrequency,
shortcutTarget.mWord, shortcutTarget.mFrequency,
shortcutTarget.mWord, shortcutTarget.getProbability(),
word.mIsNotAWord, word.mIsBlacklistEntry, 0 /* timestamp */);
}
}
@ -91,7 +91,7 @@ public class Ver4DictEncoder implements DictEncoder {
for (final Word word0 : dict) {
if (null == word0.mBigrams) continue;
for (final WeightedString word1 : word0.mBigrams) {
binaryDict.addBigramWords(word0.mWord, word1.mWord, word1.mFrequency,
binaryDict.addBigramWords(word0.mWord, word1.mWord, word1.getProbability(),
0 /* timestamp */);
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
binaryDict.flushWithGC();

4
java/src/com/android/inputmethod/latin/utils/WordProperty.java
View File

@ -108,7 +108,7 @@ public class WordProperty {
for (int i = 0; i < mBigramTargets.size(); i++) {
builder.append(" bigram=" + mBigramTargets.get(i).mWord);
builder.append(",");
builder.append("f=" + mBigramTargets.get(i).mFrequency);
builder.append("f=" + mBigramTargets.get(i).getProbability());
if (mBigramProbabilityInfo.get(i).mTimestamp
!= BinaryDictionary.NOT_A_VALID_TIMESTAMP) {
builder.append(",");
@ -119,7 +119,7 @@ public class WordProperty {
for (int i = 0; i < mShortcutTargets.size(); i++) {
builder.append(" shortcut=" + mShortcutTargets.get(i).mWord);
builder.append(",");
builder.append("f=" + mShortcutTargets.get(i).mFrequency);
builder.append("f=" + mShortcutTargets.get(i).getProbability());
builder.append("\n");
}
return builder.toString();

16
tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
View File

@ -962,7 +962,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (int j = 0; j < unigramProperty.mBigramTargets.size(); j++) {
final String word1 = unigramProperty.mBigramTargets.get(j).mWord;
assertTrue(bigramWord1s.contains(word1));
final int probability = unigramProperty.mBigramTargets.get(j).mFrequency;
final int probability = unigramProperty.mBigramTargets.get(j).getProbability();
assertEquals((int)bigramProbabilities.get(new Pair<String, String>(word0, word1)),
probability);
assertEquals(unigramProperty.mBigramProbabilityInfo.get(j).mProbability,
@ -1053,7 +1053,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (int j = 0; j < wordProperty.mBigramTargets.size(); j++) {
final String word1 = wordProperty.mBigramTargets.get(j).mWord;
assertTrue(bigramWord1s.contains(word1));
final int probability = wordProperty.mBigramTargets.get(j).mFrequency;
final int probability = wordProperty.mBigramTargets.get(j).getProbability();
final Pair<String, String> bigram = new Pair<String, String>(word0, word1);
assertEquals((int)bigramProbabilitiesToCheckLater.get(bigram), probability);
bigramSet.remove(bigram);
@ -1087,7 +1087,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
WordProperty wordProperty = binaryDictionary.getWordProperty("aaa");
assertEquals(1, wordProperty.mShortcutTargets.size());
assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).mFrequency);
assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).getProbability());
final int updatedShortcutProbability = 2;
binaryDictionary.addUnigramWord("aaa", unigramProbability, "zzz",
updatedShortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
@ -1096,7 +1096,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(1, wordProperty.mShortcutTargets.size());
assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
assertEquals(updatedShortcutProbability,
wordProperty.mShortcutTargets.get(0).mFrequency);
wordProperty.mShortcutTargets.get(0).getProbability());
binaryDictionary.addUnigramWord("aaa", unigramProbability, "yyy",
shortcutProbability, false /* isNotAWord */, false /* isBlacklisted */,
0 /* timestamp */);
@ -1107,7 +1107,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(2, wordProperty.mShortcutTargets.size());
for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord));
assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), shortcutTarget.mFrequency);
assertEquals((int)shortcutTargets.get(shortcutTarget.mWord),
shortcutTarget.getProbability());
shortcutTargets.remove(shortcutTarget.mWord);
}
shortcutTargets.put("zzz", updatedShortcutProbability);
@ -1117,7 +1118,8 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(2, wordProperty.mShortcutTargets.size());
for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord));
assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), shortcutTarget.mFrequency);
assertEquals((int)shortcutTargets.get(shortcutTarget.mWord),
shortcutTarget.getProbability());
shortcutTargets.remove(shortcutTarget.mWord);
}
}
@ -1193,7 +1195,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
final String targetCodePonts = shortcutTarget.mWord;
assertEquals((int)shortcutTargets.get(word).get(targetCodePonts),
shortcutTarget.mFrequency);
shortcutTarget.getProbability());
}
}
}

8
tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java
View File

@ -127,7 +127,7 @@ public class CombinedInputOutput {
if (null != word) {
dict.add(word, freq, shortcuts.isEmpty() ? null : shortcuts, isNotAWord);
for (WeightedString s : bigrams) {
dict.setBigram(word, s.mWord, s.mFrequency);
dict.setBigram(word, s.mWord, s.getProbability());
}
}
if (!shortcuts.isEmpty()) shortcuts = new ArrayList<WeightedString>();
@ -185,7 +185,7 @@ public class CombinedInputOutput {
if (null != word) {
dict.add(word, freq, shortcuts.isEmpty() ? null : shortcuts, isNotAWord);
for (WeightedString s : bigrams) {
dict.setBigram(word, s.mWord, s.mFrequency);
dict.setBigram(word, s.mWord, s.getProbability());
}
}
@ -222,13 +222,13 @@ public class CombinedInputOutput {
if (null != word.mShortcutTargets) {
for (WeightedString target : word.mShortcutTargets) {
destination.write(" " + SHORTCUT_TAG + "=" + target.mWord + ","
+ FREQUENCY_TAG + "=" + target.mFrequency + "\n");
+ FREQUENCY_TAG + "=" + target.getProbability() + "\n");
}
}
if (null != word.mBigrams) {
for (WeightedString bigram : word.mBigrams) {
destination.write(" " + BIGRAM_TAG + "=" + bigram.mWord + ","
+ FREQUENCY_TAG + "=" + bigram.mFrequency + "\n");
+ FREQUENCY_TAG + "=" + bigram.getProbability() + "\n");
}
}
}

10
tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java
View File

@ -159,7 +159,7 @@ public class Diff extends Dicttool.Command {
if (null == list0) return false;
for (final WeightedString attribute0 : list0) {
System.out.println(type + " removed: " + word + " " + attribute0.mWord + " "
+ attribute0.mFrequency);
+ attribute0.getProbability());
}
return true;
}
@ -175,8 +175,8 @@ public class Diff extends Dicttool.Command {
for (final WeightedString attribute1 : list1) {
if (attribute0.mWord.equals(attribute1.mWord)) {
System.out.println(type + " freq changed: " + word + " "
+ attribute0.mWord + " " + attribute0.mFrequency + " -> "
+ attribute1.mFrequency);
+ attribute0.mWord + " " + attribute0.getProbability() + " -> "
+ attribute1.getProbability());
list1.remove(attribute1);
foundString = true;
break;
@ -185,7 +185,7 @@ public class Diff extends Dicttool.Command {
if (!foundString) {
// We come here if we haven't found any matching string.
System.out.println(type + " removed: " + word + " " + attribute0.mWord + " "
+ attribute0.mFrequency);
+ attribute0.getProbability());
}
} else {
list1.remove(attribute0);
@ -197,7 +197,7 @@ public class Diff extends Dicttool.Command {
for (final WeightedString attribute1 : list1) {
hasDifferences = true;
System.out.println(type + " added: " + word + " " + attribute1.mWord + " "
+ attribute1.mFrequency);
+ attribute1.getProbability());
}
return hasDifferences;
}

11
tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java
View File

@ -51,7 +51,8 @@ public class Info extends Dicttool.Command {
if (null != w.mShortcutTargets) {
shortcutCount += w.mShortcutTargets.size();
for (WeightedString shortcutTarget : w.mShortcutTargets) {
if (FormatSpec.SHORTCUT_WHITELIST_FREQUENCY == shortcutTarget.mFrequency) {
if (FormatSpec.SHORTCUT_WHITELIST_FREQUENCY
== shortcutTarget.getProbability()) {
++whitelistCount;
}
}
@ -84,8 +85,9 @@ public class Info extends Dicttool.Command {
} else {
for (final WeightedString shortcutTarget : shortcutTargets) {
System.out.println(" Shortcut target: " + shortcutTarget.mWord + " ("
+ (FormatSpec.SHORTCUT_WHITELIST_FREQUENCY == shortcutTarget.mFrequency
? "whitelist" : shortcutTarget.mFrequency) + ")");
+ (FormatSpec.SHORTCUT_WHITELIST_FREQUENCY
== shortcutTarget.getProbability() ?
"whitelist" : shortcutTarget.getProbability()) + ")");
}
}
final ArrayList<WeightedString> bigrams = ptNode.getBigrams();
@ -93,7 +95,8 @@ public class Info extends Dicttool.Command {
System.out.println(" No bigrams");
} else {
for (final WeightedString bigram : bigrams) {
System.out.println(" Bigram: " + bigram.mWord + " (" + bigram.mFrequency + ")");
System.out.println(
" Bigram: " + bigram.mWord + " (" + bigram.getProbability() + ")");
}
}
}

7
tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java
View File

@ -319,7 +319,7 @@ public class XmlDictInputOutput {
final ArrayList<WeightedString> bigramList = bigramMap.get(firstWord);
for (final WeightedString bigram : bigramList) {
if (!dict.hasWord(bigram.mWord)) continue;
dict.setBigram(firstWord, bigram.mWord, bigram.mFrequency);
dict.setBigram(firstWord, bigram.mWord, bigram.getProbability());
}
}
return dict;
@ -369,7 +369,7 @@ public class XmlDictInputOutput {
destination.write("\n");
for (WeightedString target : word.mShortcutTargets) {
destination.write(" <" + SHORTCUT_TAG + " " + FREQUENCY_ATTR + "=\""
+ target.mFrequency + "\">" + target.mWord + "</" + SHORTCUT_TAG
+ target.getProbability() + "\">" + target.mWord + "</" + SHORTCUT_TAG
+ ">\n");
}
destination.write(" ");
@ -378,7 +378,8 @@ public class XmlDictInputOutput {
destination.write("\n");
for (WeightedString bigram : word.mBigrams) {
destination.write(" <" + BIGRAM_TAG + " " + FREQUENCY_ATTR + "=\""
+ bigram.mFrequency + "\">" + bigram.mWord + "</" + BIGRAM_TAG + ">\n");
+ bigram.getProbability() + "\">" + bigram.mWord
+ "</" + BIGRAM_TAG + ">\n");
}
destination.write(" ");
}