nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

am 65feee12: Make BinaryDictIOUtils.

Browse source 
* commit '65feee12e5889601e375d92dfdf5f8e8fbb05092':
  Make BinaryDictIOUtils.
This commit is contained in:
Yuichiro Hanada 2012-09-19 05:49:03 -07:00 committed by Android Git Automerger
commit ae5d4366b0
4 changed files with 135 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
java/src/com/android/inputmethod/latin/UserHistoryDictIOUtils.java
View file

@ -18,6 +18,7 @@ package com.android.inputmethod.latin;
import android.util.Log;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
@ -167,7 +168,7 @@ public class UserHistoryDictIOUtils {
final Map<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap();
try {
BinaryDictInputOutput.readUnigramsAndBigramsBinary(buffer, unigrams, frequencies,
BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, unigrams, frequencies,
bigrams);
addWordsFromWordMap(unigrams, frequencies, bigrams, dict);
} catch (IOException e) {

127
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java Normal file
View file

@ -0,0 +1,127 @@
/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
import java.util.Stack;
public class BinaryDictIOUtils {
private static final boolean DBG = false;
private static class Position {
public static final int NOT_READ_GROUPCOUNT = -1;
public int mAddress;
public int mNumOfCharGroup;
public int mPosition;
public int mLength;
public Position(int address, int length) {
mAddress = address;
mLength = length;
mNumOfCharGroup = NOT_READ_GROUPCOUNT;
}
}
/**
* Tours all node without recursive call.
*/
private static void readUnigramsAndBigramsBinaryInner(
final FusionDictionaryBufferInterface buffer, final int headerSize,
final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
final Map<Integer, ArrayList<PendingAttribute>> bigrams,
final FormatOptions formatOptions) {
int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1];
Stack<Position> stack = new Stack<Position>();
int index = 0;
Position initPos = new Position(headerSize, 0);
stack.push(initPos);
while (!stack.empty()) {
Position p = stack.peek();
if (DBG) {
MakedictLog.d("read: address=" + p.mAddress + ", numOfCharGroup=" +
p.mNumOfCharGroup + ", position=" + p.mPosition + ", length=" + p.mLength);
}
if (buffer.position() != p.mAddress) buffer.position(p.mAddress);
if (index != p.mLength) index = p.mLength;
if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) {
p.mNumOfCharGroup = BinaryDictInputOutput.readCharGroupCount(buffer);
p.mAddress += BinaryDictInputOutput.getGroupCountSize(p.mNumOfCharGroup);
p.mPosition = 0;
}
CharGroupInfo info = BinaryDictInputOutput.readCharGroup(buffer,
p.mAddress - headerSize, formatOptions);
for (int i = 0; i < info.mCharacters.length; ++i) {
pushedChars[index++] = info.mCharacters[i];
}
p.mPosition++;
if (info.mFrequency != FusionDictionary.CharGroup.NOT_A_TERMINAL) { // found word
words.put(info.mOriginalAddress, new String(pushedChars, 0, index));
frequencies.put(info.mOriginalAddress, info.mFrequency);
if (info.mBigrams != null) bigrams.put(info.mOriginalAddress, info.mBigrams);
}
if (p.mPosition == p.mNumOfCharGroup) {
stack.pop();
} else {
// the node has more groups.
p.mAddress = buffer.position();
}
if (BinaryDictInputOutput.hasChildrenAddress(info.mChildrenAddress)) {
Position childrenPos = new Position(info.mChildrenAddress + headerSize, index);
stack.push(childrenPos);
}
}
}
/**
* Reads unigrams and bigrams from the binary file.
* Doesn't make the memory representation of the dictionary.
*
* @param buffer the buffer to read.
* @param words the map to store the address as a key and the word as a value.
* @param frequencies the map to store the address as a key and the frequency as a value.
* @param bigrams the map to store the address as a key and the list of address as a value.
* @throws IOException
* @throws UnsupportedFormatException
*/
public static void readUnigramsAndBigramsBinary(final FusionDictionaryBufferInterface buffer,
final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
UnsupportedFormatException {
// Read header
final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
readUnigramsAndBigramsBinaryInner(buffer, header.mHeaderSize, words, frequencies, bigrams,
header.mFormatOptions);
}
}

107
java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java
View file

@ -290,7 +290,7 @@ public class BinaryDictInputOutput {
* @param count the group count
* @return the size of the group count, either 1 or 2 bytes.
*/
private static int getGroupCountSize(final int count) {
public static int getGroupCountSize(final int count) {
if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= count) {
return 1;
} else if (FormatSpec.MAX_CHARGROUPS_IN_A_NODE >= count) {
@ -385,7 +385,7 @@ public class BinaryDictInputOutput {
/**
* Helper method to hide the actual value of the no children address.
*/
private static boolean hasChildrenAddress(final int address) {
public static boolean hasChildrenAddress(final int address) {
return FormatSpec.NO_CHILDREN_ADDRESS != address;
}
@ -1105,7 +1105,7 @@ public class BinaryDictInputOutput {
// readDictionaryBinary is the public entry point for them.
private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH];
private static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer,
public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer,
final int originalGroupAddress, final FormatOptions options) {
int addressPointer = originalGroupAddress;
final int flags = buffer.readUnsignedByte();
@ -1218,7 +1218,7 @@ public class BinaryDictInputOutput {
/**
* Reads and returns the char group count out of a buffer and forwards the pointer.
*/
private static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) {
public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) {
final int msb = buffer.readUnsignedByte();
if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) {
return msb;
@ -1408,103 +1408,6 @@ public class BinaryDictInputOutput {
return node;
}
// TODO: move these methods (readUnigramsAndBigramsBinary(|Inner)) and an inner class (Position)
// out of this class.
private static class Position {
public static final int NOT_READ_GROUPCOUNT = -1;
public int mAddress;
public int mNumOfCharGroup;
public int mPosition;
public int mLength;
public Position(int address, int length) {
mAddress = address;
mLength = length;
mNumOfCharGroup = NOT_READ_GROUPCOUNT;
}
}
/**
* Tours all node without recursive call.
*/
private static void readUnigramsAndBigramsBinaryInner(
final FusionDictionaryBufferInterface buffer, final int headerSize,
final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
final Map<Integer, ArrayList<PendingAttribute>> bigrams,
final FormatOptions formatOptions) {
int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1];
Stack<Position> stack = new Stack<Position>();
int index = 0;
Position initPos = new Position(headerSize, 0);
stack.push(initPos);
while (!stack.empty()) {
Position p = stack.peek();
if (DBG) {
MakedictLog.d("read: address=" + p.mAddress + ", numOfCharGroup=" +
p.mNumOfCharGroup + ", position=" + p.mPosition + ", length=" + p.mLength);
}
if (buffer.position() != p.mAddress) buffer.position(p.mAddress);
if (index != p.mLength) index = p.mLength;
if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) {
p.mNumOfCharGroup = readCharGroupCount(buffer);
p.mAddress += getGroupCountSize(p.mNumOfCharGroup);
p.mPosition = 0;
}
CharGroupInfo info = readCharGroup(buffer, p.mAddress - headerSize, formatOptions);
for (int i = 0; i < info.mCharacters.length; ++i) {
pushedChars[index++] = info.mCharacters[i];
}
p.mPosition++;
if (info.mFrequency != FusionDictionary.CharGroup.NOT_A_TERMINAL) { // found word
words.put(info.mOriginalAddress, new String(pushedChars, 0, index));
frequencies.put(info.mOriginalAddress, info.mFrequency);
if (info.mBigrams != null) bigrams.put(info.mOriginalAddress, info.mBigrams);
}
if (p.mPosition == p.mNumOfCharGroup) {
stack.pop();
} else {
// the node has more groups.
p.mAddress = buffer.position();
}
if (hasChildrenAddress(info.mChildrenAddress)) {
Position childrenPos = new Position(info.mChildrenAddress + headerSize, index);
stack.push(childrenPos);
}
}
}
/**
* Reads unigrams and bigrams from the binary file.
* Doesn't make the memory representation of the dictionary.
*
* @param buffer the buffer to read.
* @param words the map to store the address as a key and the word as a value.
* @param frequencies the map to store the address as a key and the frequency as a value.
* @param bigrams the map to store the address as a key and the list of address as a value.
* @throws IOException
* @throws UnsupportedFormatException
*/
public static void readUnigramsAndBigramsBinary(final FusionDictionaryBufferInterface buffer,
final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
UnsupportedFormatException {
// Read header
final FileHeader header = readHeader(buffer);
readUnigramsAndBigramsBinaryInner(buffer, header.mHeaderSize, words, frequencies, bigrams,
header.mFormatOptions);
}
/**
* Helper function to get the binary format version from the header.
* @throws IOException
@ -1541,7 +1444,7 @@ public class BinaryDictInputOutput {
* @throws IOException
* @throws UnsupportedFormatException
*/
private static FileHeader readHeader(final FusionDictionaryBufferInterface buffer)
public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer)
throws IOException, UnsupportedFormatException {
final int version = checkFormatVersion(buffer);
final int optionsFlags = buffer.readUnsignedShort();

2
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java
View file

@ -391,7 +391,7 @@ public class BinaryDictIOTests extends AndroidTestCase {
assertNotNull("Can't get buffer.", buffer);
try {
now = System.currentTimeMillis();
BinaryDictInputOutput.readUnigramsAndBigramsBinary(buffer, resultWords, resultFreqs,
BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, resultWords, resultFreqs,
resultBigrams);
diff = System.currentTimeMillis() - now;
} catch (IOException e) {