nyancrimew/LatinIME
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

am 8e19d4a1: am d6e307a4: Add DictUpdater.

Browse source 
* commit '8e19d4a120bf57f6eebce99bac22a01941400d61':
  Add DictUpdater.
This commit is contained in:
Yuichiro Hanada 2013-10-09 17:55:44 -07:00 committed by Android Git Automerger
commit fd91482b5e
7 changed files with 188 additions and 135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View file

@ -288,42 +288,6 @@ public final class BinaryDictIOUtils {
return BinaryDictEncoderUtils.getByteSize(value); return BinaryDictEncoderUtils.getByteSize(value);
} }
// TODO: Remove this method.
@Deprecated
static void skipPtNode(final DictBuffer dictBuffer, final FormatOptions formatOptions) {
final int flags = dictBuffer.readUnsignedByte();
BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions);
skipString(dictBuffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
BinaryDictDecoderUtils.readChildrenAddress(dictBuffer, flags, formatOptions);
if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) dictBuffer.readUnsignedByte();
if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) {
final int shortcutsSize = dictBuffer.readUnsignedShort();
dictBuffer.position(dictBuffer.position() + shortcutsSize
- FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE);
}
if ((flags & FormatSpec.FLAG_HAS_BIGRAMS) != 0) {
int bigramCount = 0;
while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
final int bigramFlags = dictBuffer.readUnsignedByte();
switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) {
case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE:
dictBuffer.readUnsignedByte();
break;
case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES:
dictBuffer.readUnsignedShort();
break;
case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES:
dictBuffer.readUnsignedInt24();
break;
}
if ((bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) == 0) break;
}
if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
throw new RuntimeException("Too many bigrams in a PtNode.");
}
}
}
static void skipString(final DictBuffer dictBuffer, static void skipString(final DictBuffer dictBuffer,
final boolean hasMultipleChars) { final boolean hasMultipleChars) {
if (hasMultipleChars) { if (hasMultipleChars) {

50
java/src/com/android/inputmethod/latin/makedict/DictUpdater.java Normal file
View file

@ -0,0 +1,50 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.io.IOException;
import java.util.ArrayList;
/**
* An interface of a binary dictionary updater.
*/
public interface DictUpdater {
/**
* Deletes the word from the binary dictionary.
*
* @param word the word to be deleted.
*/
public void deleteWord(final String word) throws IOException, UnsupportedFormatException;
/**
* Inserts a word into a binary dictionary.
*
* @param word the word to be inserted.
* @param frequency the frequency of the new word.
* @param bigramStrings bigram list, or null if none.
* @param shortcuts shortcut list, or null if none.
* @param isBlackListEntry whether this should be a blacklist entry.
*/
// TODO: Support batch insertion.
public void insertWord(final String word, final int frequency,
final ArrayList<WeightedString> bigramStrings,
final ArrayList<WeightedString> shortcuts, final boolean isNotAWord,
final boolean isBlackListEntry) throws IOException, UnsupportedFormatException;
}

104
java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java
View file

@ -42,44 +42,22 @@ public final class DynamicBinaryDictIOUtils {
// This utility class is not publicly instantiable. // This utility class is not publicly instantiable.
} }
private static int markAsDeleted(final int flags) { /* package */ static int markAsDeleted(final int flags) {
return (flags & (~FormatSpec.MASK_CHILDREN_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED; return (flags & (~FormatSpec.MASK_CHILDREN_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED;
} }
/**
* Delete the word from the binary file.
*
* @param dictDecoder the dict decoder.
* @param word the word we delete
* @throws IOException
* @throws UnsupportedFormatException
*/
@UsedForTesting
public static void deleteWord(final Ver3DictDecoder dictDecoder, final String word)
throws IOException, UnsupportedFormatException {
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
dictBuffer.position(0);
dictDecoder.readHeader();
final int wordPosition = dictDecoder.getTerminalPosition(word);
if (wordPosition == FormatSpec.NOT_VALID_WORD) return;
dictBuffer.position(wordPosition);
final int flags = dictBuffer.readUnsignedByte();
dictBuffer.position(wordPosition);
dictBuffer.put((byte)markAsDeleted(flags));
}
/** /**
* Update a parent address in a PtNode that is referred to by ptNodeOriginAddress. * Update a parent address in a PtNode that is referred to by ptNodeOriginAddress.
* *
* @param dictBuffer the DictBuffer to write. * @param dictUpdater the DictUpdater to write.
* @param ptNodeOriginAddress the address of the PtNode. * @param ptNodeOriginAddress the address of the PtNode.
* @param newParentAddress the absolute address of the parent. * @param newParentAddress the absolute address of the parent.
* @param formatOptions file format options. * @param formatOptions file format options.
*/ */
private static void updateParentAddress(final DictBuffer dictBuffer, private static void updateParentAddress(final Ver3DictUpdater dictUpdater,
final int ptNodeOriginAddress, final int newParentAddress, final int ptNodeOriginAddress, final int newParentAddress,
final FormatOptions formatOptions) { final FormatOptions formatOptions) {
final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
final int originalPosition = dictBuffer.position(); final int originalPosition = dictBuffer.position();
dictBuffer.position(ptNodeOriginAddress); dictBuffer.position(ptNodeOriginAddress);
if (!formatOptions.mSupportsDynamicUpdate) { if (!formatOptions.mSupportsDynamicUpdate) {
@ -104,41 +82,41 @@ public final class DynamicBinaryDictIOUtils {
/** /**
* Update parent addresses in a node array stored at ptNodeOriginAddress. * Update parent addresses in a node array stored at ptNodeOriginAddress.
* *
* @param dictBuffer the DictBuffer to be modified. * @param dictUpdater the DictUpdater to be modified.
* @param ptNodeOriginAddress the address of the node array to update. * @param ptNodeOriginAddress the address of the node array to update.
* @param newParentAddress the address to be written. * @param newParentAddress the address to be written.
* @param formatOptions file format options. * @param formatOptions file format options.
*/ */
private static void updateParentAddresses(final DictBuffer dictBuffer, private static void updateParentAddresses(final Ver3DictUpdater dictUpdater,
final int ptNodeOriginAddress, final int newParentAddress, final int ptNodeOriginAddress, final int newParentAddress,
final FormatOptions formatOptions) { final FormatOptions formatOptions) {
final int originalPosition = dictBuffer.position(); final int originalPosition = dictUpdater.getPosition();
dictBuffer.position(ptNodeOriginAddress); dictUpdater.setPosition(ptNodeOriginAddress);
do { do {
final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); final int count = dictUpdater.readPtNodeCount();
for (int i = 0; i < count; ++i) { for (int i = 0; i < count; ++i) {
updateParentAddress(dictBuffer, dictBuffer.position(), newParentAddress, updateParentAddress(dictUpdater, dictUpdater.getPosition(), newParentAddress,
formatOptions); formatOptions);
BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions); dictUpdater.skipPtNode(formatOptions);
} }
final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); if (!dictUpdater.readAndFollowForwardLink()) break;
dictBuffer.position(forwardLinkAddress); if (dictUpdater.getPosition() == FormatSpec.NO_FORWARD_LINK_ADDRESS) break;
} while (formatOptions.mSupportsDynamicUpdate } while (formatOptions.mSupportsDynamicUpdate);
&& dictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); dictUpdater.setPosition(originalPosition);
dictBuffer.position(originalPosition);
} }
/** /**
* Update a children address in a PtNode that is addressed by ptNodeOriginAddress. * Update a children address in a PtNode that is addressed by ptNodeOriginAddress.
* *
* @param dictBuffer the DictBuffer to write. * @param dictUpdater the DictUpdater to write.
* @param ptNodeOriginAddress the address of the PtNode. * @param ptNodeOriginAddress the address of the PtNode.
* @param newChildrenAddress the absolute address of the child. * @param newChildrenAddress the absolute address of the child.
* @param formatOptions file format options. * @param formatOptions file format options.
*/ */
private static void updateChildrenAddress(final DictBuffer dictBuffer, private static void updateChildrenAddress(final Ver3DictUpdater dictUpdater,
final int ptNodeOriginAddress, final int newChildrenAddress, final int ptNodeOriginAddress, final int newChildrenAddress,
final FormatOptions formatOptions) { final FormatOptions formatOptions) {
final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
final int originalPosition = dictBuffer.position(); final int originalPosition = dictBuffer.position();
dictBuffer.position(ptNodeOriginAddress); dictBuffer.position(ptNodeOriginAddress);
final int flags = dictBuffer.readUnsignedByte(); final int flags = dictBuffer.readUnsignedByte();
@ -155,31 +133,33 @@ public final class DynamicBinaryDictIOUtils {
* Helper method to move a PtNode to the tail of the file. * Helper method to move a PtNode to the tail of the file.
*/ */
private static int movePtNode(final OutputStream destination, private static int movePtNode(final OutputStream destination,
final DictBuffer dictBuffer, final PtNodeInfo info, final Ver3DictUpdater dictUpdater, final PtNodeInfo info,
final int nodeArrayOriginAddress, final int oldNodeAddress, final int nodeArrayOriginAddress, final int oldNodeAddress,
final FormatOptions formatOptions) throws IOException { final FormatOptions formatOptions) throws IOException {
updateParentAddress(dictBuffer, oldNodeAddress, dictBuffer.limit() + 1, formatOptions); final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
updateParentAddress(dictUpdater, oldNodeAddress, dictBuffer.limit() + 1, formatOptions);
dictBuffer.position(oldNodeAddress); dictBuffer.position(oldNodeAddress);
final int currentFlags = dictBuffer.readUnsignedByte(); final int currentFlags = dictBuffer.readUnsignedByte();
dictBuffer.position(oldNodeAddress); dictBuffer.position(oldNodeAddress);
dictBuffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags dictBuffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags
& (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG))));
int size = FormatSpec.PTNODE_FLAGS_SIZE; int size = FormatSpec.PTNODE_FLAGS_SIZE;
updateForwardLink(dictBuffer, nodeArrayOriginAddress, dictBuffer.limit(), formatOptions); updateForwardLink(dictUpdater, nodeArrayOriginAddress, dictBuffer.limit(), formatOptions);
size += BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { info }); size += BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { info });
return size; return size;
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
private static void updateForwardLink(final DictBuffer dictBuffer, private static void updateForwardLink(final Ver3DictUpdater dictUpdater,
final int nodeArrayOriginAddress, final int newNodeArrayAddress, final int nodeArrayOriginAddress, final int newNodeArrayAddress,
final FormatOptions formatOptions) { final FormatOptions formatOptions) {
dictBuffer.position(nodeArrayOriginAddress); final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
dictUpdater.setPosition(nodeArrayOriginAddress);
int jumpCount = 0; int jumpCount = 0;
while (jumpCount++ < MAX_JUMPS) { while (jumpCount++ < MAX_JUMPS) {
final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); final int count = dictUpdater.readPtNodeCount();
for (int i = 0; i < count; ++i) { for (int i = 0; i < count; ++i) {
BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions); dictUpdater.readPtNode(dictUpdater.getPosition(), formatOptions);
} }
final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); final int forwardLinkAddress = dictBuffer.readUnsignedInt24();
if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) {
@ -207,7 +187,7 @@ public final class DynamicBinaryDictIOUtils {
* @param shortcutTargets the shortcut targets for this PtNode. * @param shortcutTargets the shortcut targets for this PtNode.
* @param bigrams the bigrams for this PtNode. * @param bigrams the bigrams for this PtNode.
* @param destination the stream representing the tail of the file. * @param destination the stream representing the tail of the file.
* @param dictBuffer the DictBuffer representing the (constant-size) body of the file. * @param dictUpdater the DictUpdater.
* @param oldPtNodeArrayOrigin the origin of the old PtNode array this PtNode was a part of. * @param oldPtNodeArrayOrigin the origin of the old PtNode array this PtNode was a part of.
* @param oldPtNodeOrigin the old origin where this PtNode used to be stored. * @param oldPtNodeOrigin the old origin where this PtNode used to be stored.
* @param formatOptions format options for this dictionary. * @param formatOptions format options for this dictionary.
@ -218,7 +198,7 @@ public final class DynamicBinaryDictIOUtils {
final int length, final int flags, final int frequency, final int parentAddress, final int length, final int flags, final int frequency, final int parentAddress,
final ArrayList<WeightedString> shortcutTargets, final ArrayList<WeightedString> shortcutTargets,
final ArrayList<PendingAttribute> bigrams, final OutputStream destination, final ArrayList<PendingAttribute> bigrams, final OutputStream destination,
final DictBuffer dictBuffer, final int oldPtNodeArrayOrigin, final Ver3DictUpdater dictUpdater, final int oldPtNodeArrayOrigin,
final int oldPtNodeOrigin, final FormatOptions formatOptions) throws IOException { final int oldPtNodeOrigin, final FormatOptions formatOptions) throws IOException {
int size = 0; int size = 0;
final int newPtNodeOrigin = fileEndAddress + 1; final int newPtNodeOrigin = fileEndAddress + 1;
@ -231,7 +211,7 @@ public final class DynamicBinaryDictIOUtils {
flags, writtenCharacters, frequency, parentAddress, flags, writtenCharacters, frequency, parentAddress,
fileEndAddress + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets, fileEndAddress + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets,
bigrams); bigrams);
movePtNode(destination, dictBuffer, newInfo, oldPtNodeArrayOrigin, oldPtNodeOrigin, movePtNode(destination, dictUpdater, newInfo, oldPtNodeArrayOrigin, oldPtNodeOrigin,
formatOptions); formatOptions);
return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
} }
@ -239,7 +219,7 @@ public final class DynamicBinaryDictIOUtils {
/** /**
* Insert a word into a binary dictionary. * Insert a word into a binary dictionary.
* *
* @param dictDecoder the dict decoder. * @param dictUpdater the dict updater.
* @param destination a stream to the underlying file, with the pointer at the end of the file. * @param destination a stream to the underlying file, with the pointer at the end of the file.
* @param word the word to insert. * @param word the word to insert.
* @param frequency the frequency of the new word. * @param frequency the frequency of the new word.
@ -252,17 +232,17 @@ public final class DynamicBinaryDictIOUtils {
// TODO: Support batch insertion. // TODO: Support batch insertion.
// TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary. // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary.
@UsedForTesting @UsedForTesting
public static void insertWord(final Ver3DictDecoder dictDecoder, public static void insertWord(final Ver3DictUpdater dictUpdater,
final OutputStream destination, final String word, final int frequency, final OutputStream destination, final String word, final int frequency,
final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> bigramStrings,
final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, final ArrayList<WeightedString> shortcuts, final boolean isNotAWord,
final boolean isBlackListEntry) final boolean isBlackListEntry)
throws IOException, UnsupportedFormatException { throws IOException, UnsupportedFormatException {
final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>(); final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>();
final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
if (bigramStrings != null) { if (bigramStrings != null) {
for (final WeightedString bigram : bigramStrings) { for (final WeightedString bigram : bigramStrings) {
int position = dictDecoder.getTerminalPosition(bigram.mWord); int position = dictUpdater.getTerminalPosition(bigram.mWord);
if (position == FormatSpec.NOT_VALID_WORD) { if (position == FormatSpec.NOT_VALID_WORD) {
// TODO: figure out what is the correct thing to do here. // TODO: figure out what is the correct thing to do here.
} else { } else {
@ -277,7 +257,7 @@ public final class DynamicBinaryDictIOUtils {
// find the insert position of the word. // find the insert position of the word.
if (dictBuffer.position() != 0) dictBuffer.position(0); if (dictBuffer.position() != 0) dictBuffer.position(0);
final FileHeader fileHeader = dictDecoder.readHeader(); final FileHeader fileHeader = dictUpdater.readHeader();
int wordPos = 0, address = dictBuffer.position(), nodeOriginAddress = dictBuffer.position(); int wordPos = 0, address = dictBuffer.position(), nodeOriginAddress = dictBuffer.position();
final int[] codePoints = FusionDictionary.getCodePoints(word); final int[] codePoints = FusionDictionary.getCodePoints(word);
@ -292,7 +272,7 @@ public final class DynamicBinaryDictIOUtils {
for (int i = 0; i < ptNodeCount; ++i) { for (int i = 0; i < ptNodeCount; ++i) {
address = dictBuffer.position(); address = dictBuffer.position();
final PtNodeInfo currentInfo = dictDecoder.readPtNode(address, final PtNodeInfo currentInfo = dictUpdater.readPtNode(address,
fileHeader.mFormatOptions); fileHeader.mFormatOptions);
final boolean isMovedNode = BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, final boolean isMovedNode = BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags,
fileHeader.mFormatOptions); fileHeader.mFormatOptions);
@ -318,12 +298,12 @@ public final class DynamicBinaryDictIOUtils {
false /* isBlackListEntry */, fileHeader.mFormatOptions); false /* isBlackListEntry */, fileHeader.mFormatOptions);
int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, flags, int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, flags,
frequency, nodeParentAddress, shortcuts, bigrams, destination, frequency, nodeParentAddress, shortcuts, bigrams, destination,
dictBuffer, nodeOriginAddress, address, fileHeader.mFormatOptions); dictUpdater, nodeOriginAddress, address, fileHeader.mFormatOptions);
final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p, final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p,
currentInfo.mCharacters.length); currentInfo.mCharacters.length);
if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, updateParentAddresses(dictUpdater, currentInfo.mChildrenAddress,
newNodeAddress + written + 1, fileHeader.mFormatOptions); newNodeAddress + written + 1, fileHeader.mFormatOptions);
} }
final PtNodeInfo newInfo2 = new PtNodeInfo( final PtNodeInfo newInfo2 = new PtNodeInfo(
@ -359,13 +339,13 @@ public final class DynamicBinaryDictIOUtils {
fileHeader.mFormatOptions); fileHeader.mFormatOptions);
int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p,
prefixFlags, -1 /* frequency */, nodeParentAddress, null, null, prefixFlags, -1 /* frequency */, nodeParentAddress, null, null,
destination, dictBuffer, nodeOriginAddress, address, destination, dictUpdater, nodeOriginAddress, address,
fileHeader.mFormatOptions); fileHeader.mFormatOptions);
final int[] suffixCharacters = Arrays.copyOfRange( final int[] suffixCharacters = Arrays.copyOfRange(
currentInfo.mCharacters, p, currentInfo.mCharacters.length); currentInfo.mCharacters, p, currentInfo.mCharacters.length);
if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, updateParentAddresses(dictUpdater, currentInfo.mChildrenAddress,
newNodeAddress + written + 1, fileHeader.mFormatOptions); newNodeAddress + written + 1, fileHeader.mFormatOptions);
} }
final int suffixFlags = BinaryDictEncoderUtils.makePtNodeFlags( final int suffixFlags = BinaryDictEncoderUtils.makePtNodeFlags(
@ -416,7 +396,7 @@ public final class DynamicBinaryDictIOUtils {
-1 /* endAddress */, flags, currentInfo.mCharacters, frequency, -1 /* endAddress */, flags, currentInfo.mCharacters, frequency,
nodeParentAddress, currentInfo.mChildrenAddress, shortcuts, nodeParentAddress, currentInfo.mChildrenAddress, shortcuts,
bigrams); bigrams);
movePtNode(destination, dictBuffer, newInfo, nodeOriginAddress, address, movePtNode(destination, dictUpdater, newInfo, nodeOriginAddress, address,
fileHeader.mFormatOptions); fileHeader.mFormatOptions);
return; return;
} }
@ -435,7 +415,7 @@ public final class DynamicBinaryDictIOUtils {
* ab - cd - e * ab - cd - e
*/ */
final int newNodeArrayAddress = dictBuffer.limit(); final int newNodeArrayAddress = dictBuffer.limit();
updateChildrenAddress(dictBuffer, address, newNodeArrayAddress, updateChildrenAddress(dictUpdater, address, newNodeArrayAddress,
fileHeader.mFormatOptions); fileHeader.mFormatOptions);
final int newNodeAddress = newNodeArrayAddress + 1; final int newNodeAddress = newNodeArrayAddress + 1;
final boolean hasMultipleChars = (wordLen - wordPos) > 1; final boolean hasMultipleChars = (wordLen - wordPos) > 1;

4
java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java
View file

@ -53,9 +53,9 @@ public class Ver3DictDecoder extends DictDecoder {
} }
} }
private final File mDictionaryBinaryFile; protected final File mDictionaryBinaryFile;
private final DictionaryBufferFactory mBufferFactory; private final DictionaryBufferFactory mBufferFactory;
private DictBuffer mDictBuffer; protected DictBuffer mDictBuffer;
/* package */ Ver3DictDecoder(final File file, final int factoryFlag) { /* package */ Ver3DictDecoder(final File file, final int factoryFlag) {
mDictionaryBinaryFile = file; mDictionaryBinaryFile = file;

82
java/src/com/android/inputmethod/latin/makedict/Ver3DictUpdater.java Normal file
View file

@ -0,0 +1,82 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
/**
* An implementation of DictUpdater for version 3 binary dictionary.
*/
@UsedForTesting
public class Ver3DictUpdater extends Ver3DictDecoder implements DictUpdater {
private OutputStream mOutStream;
@UsedForTesting
public Ver3DictUpdater(final File dictFile, final int factoryType) {
// DictUpdater must have an updatable DictBuffer.
super(dictFile, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY)
? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER);
mOutStream = null;
}
private void openStreamAndBuffer() throws FileNotFoundException, IOException {
super.openDictBuffer();
mOutStream = new FileOutputStream(mDictionaryBinaryFile, true /* append */);
}
private void close() throws IOException {
if (mOutStream != null) {
mOutStream.close();
mOutStream = null;
}
}
@Override @UsedForTesting
public void deleteWord(final String word) throws IOException, UnsupportedFormatException {
if (mOutStream == null) openStreamAndBuffer();
mDictBuffer.position(0);
super.readHeader();
final int wordPos = getTerminalPosition(word);
if (wordPos != FormatSpec.NOT_VALID_WORD) {
mDictBuffer.position(wordPos);
final int flags = mDictBuffer.readUnsignedByte();
mDictBuffer.position(wordPos);
mDictBuffer.put((byte) DynamicBinaryDictIOUtils.markAsDeleted(flags));
}
close();
}
@Override @UsedForTesting
public void insertWord(final String word, final int frequency,
final ArrayList<WeightedString> bigramStrings,
final ArrayList<WeightedString> shortcuts,
final boolean isNotAWord, final boolean isBlackListEntry)
throws IOException, UnsupportedFormatException {
if (mOutStream == null) openStreamAndBuffer();
DynamicBinaryDictIOUtils.insertWord(this, mOutStream, word, frequency, bigramStrings,
shortcuts, isNotAWord, isBlackListEntry);
close();
}
}

22
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View file

@ -657,27 +657,21 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY); final Ver3DictUpdater dictUpdater = new Ver3DictUpdater(file,
try { DictDecoder.USE_WRITABLE_BYTEBUFFER);
dictDecoder.openDictBuffer();
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
}
assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen());
try { try {
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
dictDecoder.getTerminalPosition(sWords.get(0))); dictUpdater.getTerminalPosition(sWords.get(0)));
DynamicBinaryDictIOUtils.deleteWord(dictDecoder, sWords.get(0)); dictUpdater.deleteWord(sWords.get(0));
assertEquals(FormatSpec.NOT_VALID_WORD, assertEquals(FormatSpec.NOT_VALID_WORD,
dictDecoder.getTerminalPosition(sWords.get(0))); dictUpdater.getTerminalPosition(sWords.get(0)));
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
dictDecoder.getTerminalPosition(sWords.get(5))); dictUpdater.getTerminalPosition(sWords.get(5)));
DynamicBinaryDictIOUtils.deleteWord(dictDecoder, sWords.get(5)); dictUpdater.deleteWord(sWords.get(5));
assertEquals(FormatSpec.NOT_VALID_WORD, assertEquals(FormatSpec.NOT_VALID_WORD,
dictDecoder.getTerminalPosition(sWords.get(5))); dictUpdater.getTerminalPosition(sWords.get(5)));
} catch (IOException e) { } catch (IOException e) {
} catch (UnsupportedFormatException e) { } catch (UnsupportedFormatException e) {
} }

25
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
View file

@ -27,9 +27,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.BufferedOutputStream;
import java.io.File; import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@ -186,46 +184,31 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
private long insertAndCheckWord(final File file, final String word, final int frequency, private long insertAndCheckWord(final File file, final String word, final int frequency,
final boolean exist, final ArrayList<WeightedString> bigrams, final boolean exist, final ArrayList<WeightedString> bigrams,
final ArrayList<WeightedString> shortcuts) { final ArrayList<WeightedString> shortcuts) {
BufferedOutputStream outStream = null;
long amountOfTime = -1; long amountOfTime = -1;
try { try {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, final Ver3DictUpdater dictUpdater = new Ver3DictUpdater(file,
DictDecoder.USE_WRITABLE_BYTEBUFFER); DictDecoder.USE_WRITABLE_BYTEBUFFER);
dictDecoder.openDictBuffer();
outStream = new BufferedOutputStream(new FileOutputStream(file, true));
if (!exist) { if (!exist) {
assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word));
} }
final long now = System.nanoTime(); final long now = System.nanoTime();
DynamicBinaryDictIOUtils.insertWord(dictDecoder, outStream, word, frequency, bigrams, dictUpdater.insertWord(word, frequency, bigrams, shortcuts, false, false);
shortcuts, false, false);
amountOfTime = System.nanoTime() - now; amountOfTime = System.nanoTime() - now;
outStream.flush();
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word));
outStream.close();
} catch (IOException e) { } catch (IOException e) {
Log.e(TAG, "Raised an IOException while inserting a word", e); Log.e(TAG, "Raised an IOException while inserting a word", e);
} catch (UnsupportedFormatException e) { } catch (UnsupportedFormatException e) {
Log.e(TAG, "Raised an UnsupportedFormatException error while inserting a word", e); Log.e(TAG, "Raised an UnsupportedFormatException error while inserting a word", e);
} finally {
if (outStream != null) {
try {
outStream.close();
} catch (IOException e) {
Log.e(TAG, "Failed to close the output stream", e);
}
}
} }
return amountOfTime; return amountOfTime;
} }
private void deleteWord(final File file, final String word) { private void deleteWord(final File file, final String word) {
try { try {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, final Ver3DictUpdater dictUpdater = new Ver3DictUpdater(file,
DictDecoder.USE_WRITABLE_BYTEBUFFER); DictDecoder.USE_WRITABLE_BYTEBUFFER);
dictDecoder.openDictBuffer(); dictUpdater.deleteWord(word);
DynamicBinaryDictIOUtils.deleteWord(dictDecoder, word);
} catch (IOException e) { } catch (IOException e) {
} catch (UnsupportedFormatException e) { } catch (UnsupportedFormatException e) {
} }