nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

am 7547a704: Add writePtNode to DictEncoder.

Browse source 
* commit '7547a7042c2f38adbf880637af07b8d559488799':
  Add writePtNode to DictEncoder.
This commit is contained in:
Yuichiro Hanada 2013-09-11 02:41:18 -07:00 committed by Android Git Automerger
commit a96c59bb1e
3 changed files with 76 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
View file

@ -198,6 +198,27 @@ public class BinaryDictEncoderUtils {
}
}
static int writeUIntToBuffer(final byte[] buffer, int position, final int value,
final int size) {
switch(size) {
case 4:
buffer[position++] = (byte) ((value >> 24) & 0xFF);
/* fall through */
case 3:
buffer[position++] = (byte) ((value >> 16) & 0xFF);
/* fall through */
case 2:
buffer[position++] = (byte) ((value >> 8) & 0xFF);
/* fall through */
case 1:
buffer[position++] = (byte) (value & 0xFF);
break;
default:
/* nop */
}
return position;
}
// End utility methods
// This method is responsible for finding a nice ordering of the nodes that favors run-time
@ -733,7 +754,7 @@ public class BinaryDictEncoderUtils {
}
/**
* Write a PtNodeArray to memory. The PtNodeArray is expected to have its final position cached.
* Write a PtNodeArray. The PtNodeArray is expected to have its final position cached.
*
* @param dict the dictionary the node array is a part of (for relative offsets).
* @param dictEncoder the dictionary encoder.
@ -741,7 +762,7 @@ public class BinaryDictEncoderUtils {
* @param formatOptions file format options.
*/
@SuppressWarnings("unused")
/* package */ static void writePlacedNode(final FusionDictionary dict,
/* package */ static void writePlacedPtNodeArray(final FusionDictionary dict,
final DictEncoder dictEncoder, final PtNodeArray ptNodeArray,
final FormatOptions formatOptions) {
// TODO: Make the code in common with BinaryDictIOUtils#writePtNode
@ -767,13 +788,7 @@ public class BinaryDictEncoderUtils {
+ " : " + ptNode.mFrequency);
}
dictEncoder.writePtNodeFlags(ptNode, parentPosition, formatOptions);
dictEncoder.writeParentPosition(parentPosition, ptNode, formatOptions);
dictEncoder.writeCharacters(ptNode.mChars, ptNode.hasSeveralChars());
dictEncoder.writeFrequency(ptNode.mFrequency);
dictEncoder.writeChildrenPosition(ptNode, formatOptions);
dictEncoder.writeShortcuts(ptNode.mShortcutTargets);
dictEncoder.writeBigrams(ptNode.mBigrams, dict);
dictEncoder.writePtNode(ptNode, parentPosition, formatOptions, dict);
}
if (formatOptions.mSupportsDynamicUpdate) {
dictEncoder.writeForwardLinkAddress(FormatSpec.NO_FORWARD_LINK_ADDRESS);

28
java/src/com/android/inputmethod/latin/makedict/DictEncoder.java
View file

@ -18,10 +18,8 @@ package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.io.IOException;
import java.util.ArrayList;
/**
* An interface of binary dictionary encoder.
@ -33,28 +31,8 @@ public interface DictEncoder {
public void setPosition(final int position);
public int getPosition();
public void writePtNodeCount(final int ptNodeCount);
public void writePtNodeFlags(final PtNode ptNode, final int parentAddress,
final FormatOptions formatOptions);
public void writeParentPosition(final int parentPosition, final PtNode ptNode,
final FormatOptions formatOptions);
public void writeCharacters(final int[] characters, final boolean hasSeveralChars);
public void writeFrequency(final int frequency);
public void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions);
/**
* Write a shortcut attributes list to memory.
*
* @param shortcuts the shortcut attributes list.
*/
public void writeShortcuts(final ArrayList<WeightedString> shortcuts);
/**
* Write a bigram attributes list to memory.
*
* @param bigrams the bigram attributes list.
* @param dict the dictionary the node array is a part of (for relative offsets).
*/
public void writeBigrams(final ArrayList<WeightedString> bigrams, final FusionDictionary dict);
public void writeForwardLinkAddress(final int forwardLinkAddress);
public void writePtNode(final PtNode ptNode, final int parentPosition,
final FormatOptions formatOptions, final FusionDictionary dict);
}

80
java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java
View file

@ -103,7 +103,7 @@ public class Ver3DictEncoder implements DictEncoder {
MakedictLog.i("Writing file...");
for (PtNodeArray nodeArray : flatNodes) {
BinaryDictEncoderUtils.writePlacedNode(dict, this, nodeArray, formatOptions);
BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray, formatOptions);
}
if (MakedictLog.DBG) BinaryDictEncoderUtils.showStatistics(flatNodes);
mOutStream.write(mBuffer, 0, mPosition);
@ -126,26 +126,23 @@ public class Ver3DictEncoder implements DictEncoder {
@Override
public void writePtNodeCount(final int ptNodeCount) {
final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount);
if (1 == countSize) {
mBuffer[mPosition++] = (byte) ptNodeCount;
} else if (2 == countSize) {
mBuffer[mPosition++] = (byte) ((ptNodeCount >> 8) & 0xFF);
mBuffer[mPosition++] = (byte) (ptNodeCount & 0xFF);
} else {
if (countSize != 1 && countSize != 2) {
throw new RuntimeException("Strange size from getGroupCountSize : " + countSize);
}
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, ptNodeCount,
countSize);
}
@Override
public void writePtNodeFlags(final PtNode ptNode, final int parentAddress,
private void writePtNodeFlags(final PtNode ptNode, final int parentAddress,
final FormatOptions formatOptions) {
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
mBuffer[mPosition++] = BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mPosition,
childrenPos, formatOptions);
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition,
BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mPosition, childrenPos,
formatOptions),
FormatSpec.PTNODE_FLAGS_SIZE);
}
@Override
public void writeParentPosition(final int parentPosition, final PtNode ptNode,
private void writeParentPosition(final int parentPosition, final PtNode ptNode,
final FormatOptions formatOptions) {
if (parentPosition == FormatSpec.NO_PARENT_ADDRESS) {
mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition,
@ -156,22 +153,20 @@ public class Ver3DictEncoder implements DictEncoder {
}
}
@Override
public void writeCharacters(final int[] codePoints, final boolean hasSeveralChars) {
private void writeCharacters(final int[] codePoints, final boolean hasSeveralChars) {
mPosition = CharEncoding.writeCharArray(codePoints, mBuffer, mPosition);
if (hasSeveralChars) {
mBuffer[mPosition++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR;
}
}
@Override
public void writeFrequency(final int frequency) {
private void writeFrequency(final int frequency) {
if (frequency >= 0) {
mBuffer[mPosition++] = (byte) frequency;
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, frequency,
FormatSpec.PTNODE_FREQUENCY_SIZE);
}
}
@Override
public void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions) {
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
if (formatOptions.mSupportsDynamicUpdate) {
@ -183,8 +178,12 @@ public class Ver3DictEncoder implements DictEncoder {
}
}
@Override
public void writeShortcuts(final ArrayList<WeightedString> shortcuts) {
/**
* Write a shortcut attributes list to mBuffer.
*
* @param shortcuts the shortcut attributes list.
*/
private void writeShortcuts(final ArrayList<WeightedString> shortcuts) {
if (null == shortcuts || shortcuts.isEmpty()) return;
final int indexOfShortcutByteSize = mPosition;
@ -195,7 +194,8 @@ public class Ver3DictEncoder implements DictEncoder {
final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags(
shortcutIterator.hasNext(),
target.mFrequency);
mBuffer[mPosition++] = (byte)shortcutFlags;
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, shortcutFlags,
FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord);
mPosition += shortcutShift;
}
@ -203,12 +203,18 @@ public class Ver3DictEncoder implements DictEncoder {
if (shortcutByteSize > 0xFFFF) {
throw new RuntimeException("Shortcut list too large");
}
mBuffer[indexOfShortcutByteSize] = (byte)((shortcutByteSize >> 8) & 0xFF);
mBuffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF);
BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, indexOfShortcutByteSize, shortcutByteSize,
FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE);
}
@Override
public void writeBigrams(final ArrayList<WeightedString> bigrams, final FusionDictionary dict) {
/**
* Write a bigram attributes list to mBuffer.
*
* @param bigrams the bigram attributes list.
* @param dict the dictionary the node array is a part of (for relative offsets).
*/
private void writeBigrams(final ArrayList<WeightedString> bigrams,
final FusionDictionary dict) {
if (bigrams == null) return;
final Iterator<WeightedString> bigramIterator = bigrams.iterator();
@ -220,9 +226,10 @@ public class Ver3DictEncoder implements DictEncoder {
final int unigramFrequencyForThisWord = target.mFrequency;
final int offset = addressOfBigram
- (mPosition + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(),
final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(),
offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord);
mBuffer[mPosition++] = (byte) bigramFlags;
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, bigramFlags,
FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition,
Math.abs(offset));
}
@ -230,8 +237,19 @@ public class Ver3DictEncoder implements DictEncoder {
@Override
public void writeForwardLinkAddress(final int forwardLinkAddress) {
mBuffer[mPosition++] = (byte) ((forwardLinkAddress >> 16) & 0xFF);
mBuffer[mPosition++] = (byte) ((forwardLinkAddress >> 8) & 0xFF);
mBuffer[mPosition++] = (byte) (forwardLinkAddress & 0xFF);
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, forwardLinkAddress,
FormatSpec.FORWARD_LINK_ADDRESS_SIZE);
}
@Override
public void writePtNode(final PtNode ptNode, final int parentPosition,
final FormatOptions formatOptions, final FusionDictionary dict) {
writePtNodeFlags(ptNode, parentPosition, formatOptions);
writeParentPosition(parentPosition, ptNode, formatOptions);
writeCharacters(ptNode.mChars, ptNode.hasSeveralChars());
writeFrequency(ptNode.mFrequency);
writeChildrenPosition(ptNode, formatOptions);
writeShortcuts(ptNode.mShortcutTargets);
writeBigrams(ptNode.mBigrams, dict);
}
}