am 8e19d4a1: am d6e307a4: Add DictUpdater.
* commit '8e19d4a120bf57f6eebce99bac22a01941400d61': Add DictUpdater.main
commit
fd91482b5e
|
@ -288,42 +288,6 @@ public final class BinaryDictIOUtils {
|
||||||
return BinaryDictEncoderUtils.getByteSize(value);
|
return BinaryDictEncoderUtils.getByteSize(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: Remove this method.
|
|
||||||
@Deprecated
|
|
||||||
static void skipPtNode(final DictBuffer dictBuffer, final FormatOptions formatOptions) {
|
|
||||||
final int flags = dictBuffer.readUnsignedByte();
|
|
||||||
BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions);
|
|
||||||
skipString(dictBuffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0);
|
|
||||||
BinaryDictDecoderUtils.readChildrenAddress(dictBuffer, flags, formatOptions);
|
|
||||||
if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) dictBuffer.readUnsignedByte();
|
|
||||||
if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) {
|
|
||||||
final int shortcutsSize = dictBuffer.readUnsignedShort();
|
|
||||||
dictBuffer.position(dictBuffer.position() + shortcutsSize
|
|
||||||
- FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE);
|
|
||||||
}
|
|
||||||
if ((flags & FormatSpec.FLAG_HAS_BIGRAMS) != 0) {
|
|
||||||
int bigramCount = 0;
|
|
||||||
while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
|
|
||||||
final int bigramFlags = dictBuffer.readUnsignedByte();
|
|
||||||
switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) {
|
|
||||||
case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE:
|
|
||||||
dictBuffer.readUnsignedByte();
|
|
||||||
break;
|
|
||||||
case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES:
|
|
||||||
dictBuffer.readUnsignedShort();
|
|
||||||
break;
|
|
||||||
case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES:
|
|
||||||
dictBuffer.readUnsignedInt24();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if ((bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) == 0) break;
|
|
||||||
}
|
|
||||||
if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
|
|
||||||
throw new RuntimeException("Too many bigrams in a PtNode.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void skipString(final DictBuffer dictBuffer,
|
static void skipString(final DictBuffer dictBuffer,
|
||||||
final boolean hasMultipleChars) {
|
final boolean hasMultipleChars) {
|
||||||
if (hasMultipleChars) {
|
if (hasMultipleChars) {
|
||||||
|
|
|
@ -0,0 +1,50 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2013 The Android Open Source Project
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.android.inputmethod.latin.makedict;
|
||||||
|
|
||||||
|
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An interface of a binary dictionary updater.
|
||||||
|
*/
|
||||||
|
public interface DictUpdater {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes the word from the binary dictionary.
|
||||||
|
*
|
||||||
|
* @param word the word to be deleted.
|
||||||
|
*/
|
||||||
|
public void deleteWord(final String word) throws IOException, UnsupportedFormatException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Inserts a word into a binary dictionary.
|
||||||
|
*
|
||||||
|
* @param word the word to be inserted.
|
||||||
|
* @param frequency the frequency of the new word.
|
||||||
|
* @param bigramStrings bigram list, or null if none.
|
||||||
|
* @param shortcuts shortcut list, or null if none.
|
||||||
|
* @param isBlackListEntry whether this should be a blacklist entry.
|
||||||
|
*/
|
||||||
|
// TODO: Support batch insertion.
|
||||||
|
public void insertWord(final String word, final int frequency,
|
||||||
|
final ArrayList<WeightedString> bigramStrings,
|
||||||
|
final ArrayList<WeightedString> shortcuts, final boolean isNotAWord,
|
||||||
|
final boolean isBlackListEntry) throws IOException, UnsupportedFormatException;
|
||||||
|
}
|
|
@ -42,44 +42,22 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
// This utility class is not publicly instantiable.
|
// This utility class is not publicly instantiable.
|
||||||
}
|
}
|
||||||
|
|
||||||
private static int markAsDeleted(final int flags) {
|
/* package */ static int markAsDeleted(final int flags) {
|
||||||
return (flags & (~FormatSpec.MASK_CHILDREN_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED;
|
return (flags & (~FormatSpec.MASK_CHILDREN_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete the word from the binary file.
|
|
||||||
*
|
|
||||||
* @param dictDecoder the dict decoder.
|
|
||||||
* @param word the word we delete
|
|
||||||
* @throws IOException
|
|
||||||
* @throws UnsupportedFormatException
|
|
||||||
*/
|
|
||||||
@UsedForTesting
|
|
||||||
public static void deleteWord(final Ver3DictDecoder dictDecoder, final String word)
|
|
||||||
throws IOException, UnsupportedFormatException {
|
|
||||||
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
|
|
||||||
dictBuffer.position(0);
|
|
||||||
dictDecoder.readHeader();
|
|
||||||
final int wordPosition = dictDecoder.getTerminalPosition(word);
|
|
||||||
if (wordPosition == FormatSpec.NOT_VALID_WORD) return;
|
|
||||||
|
|
||||||
dictBuffer.position(wordPosition);
|
|
||||||
final int flags = dictBuffer.readUnsignedByte();
|
|
||||||
dictBuffer.position(wordPosition);
|
|
||||||
dictBuffer.put((byte)markAsDeleted(flags));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Update a parent address in a PtNode that is referred to by ptNodeOriginAddress.
|
* Update a parent address in a PtNode that is referred to by ptNodeOriginAddress.
|
||||||
*
|
*
|
||||||
* @param dictBuffer the DictBuffer to write.
|
* @param dictUpdater the DictUpdater to write.
|
||||||
* @param ptNodeOriginAddress the address of the PtNode.
|
* @param ptNodeOriginAddress the address of the PtNode.
|
||||||
* @param newParentAddress the absolute address of the parent.
|
* @param newParentAddress the absolute address of the parent.
|
||||||
* @param formatOptions file format options.
|
* @param formatOptions file format options.
|
||||||
*/
|
*/
|
||||||
private static void updateParentAddress(final DictBuffer dictBuffer,
|
private static void updateParentAddress(final Ver3DictUpdater dictUpdater,
|
||||||
final int ptNodeOriginAddress, final int newParentAddress,
|
final int ptNodeOriginAddress, final int newParentAddress,
|
||||||
final FormatOptions formatOptions) {
|
final FormatOptions formatOptions) {
|
||||||
|
final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
|
||||||
final int originalPosition = dictBuffer.position();
|
final int originalPosition = dictBuffer.position();
|
||||||
dictBuffer.position(ptNodeOriginAddress);
|
dictBuffer.position(ptNodeOriginAddress);
|
||||||
if (!formatOptions.mSupportsDynamicUpdate) {
|
if (!formatOptions.mSupportsDynamicUpdate) {
|
||||||
|
@ -104,41 +82,41 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
/**
|
/**
|
||||||
* Update parent addresses in a node array stored at ptNodeOriginAddress.
|
* Update parent addresses in a node array stored at ptNodeOriginAddress.
|
||||||
*
|
*
|
||||||
* @param dictBuffer the DictBuffer to be modified.
|
* @param dictUpdater the DictUpdater to be modified.
|
||||||
* @param ptNodeOriginAddress the address of the node array to update.
|
* @param ptNodeOriginAddress the address of the node array to update.
|
||||||
* @param newParentAddress the address to be written.
|
* @param newParentAddress the address to be written.
|
||||||
* @param formatOptions file format options.
|
* @param formatOptions file format options.
|
||||||
*/
|
*/
|
||||||
private static void updateParentAddresses(final DictBuffer dictBuffer,
|
private static void updateParentAddresses(final Ver3DictUpdater dictUpdater,
|
||||||
final int ptNodeOriginAddress, final int newParentAddress,
|
final int ptNodeOriginAddress, final int newParentAddress,
|
||||||
final FormatOptions formatOptions) {
|
final FormatOptions formatOptions) {
|
||||||
final int originalPosition = dictBuffer.position();
|
final int originalPosition = dictUpdater.getPosition();
|
||||||
dictBuffer.position(ptNodeOriginAddress);
|
dictUpdater.setPosition(ptNodeOriginAddress);
|
||||||
do {
|
do {
|
||||||
final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer);
|
final int count = dictUpdater.readPtNodeCount();
|
||||||
for (int i = 0; i < count; ++i) {
|
for (int i = 0; i < count; ++i) {
|
||||||
updateParentAddress(dictBuffer, dictBuffer.position(), newParentAddress,
|
updateParentAddress(dictUpdater, dictUpdater.getPosition(), newParentAddress,
|
||||||
formatOptions);
|
formatOptions);
|
||||||
BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions);
|
dictUpdater.skipPtNode(formatOptions);
|
||||||
}
|
}
|
||||||
final int forwardLinkAddress = dictBuffer.readUnsignedInt24();
|
if (!dictUpdater.readAndFollowForwardLink()) break;
|
||||||
dictBuffer.position(forwardLinkAddress);
|
if (dictUpdater.getPosition() == FormatSpec.NO_FORWARD_LINK_ADDRESS) break;
|
||||||
} while (formatOptions.mSupportsDynamicUpdate
|
} while (formatOptions.mSupportsDynamicUpdate);
|
||||||
&& dictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS);
|
dictUpdater.setPosition(originalPosition);
|
||||||
dictBuffer.position(originalPosition);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Update a children address in a PtNode that is addressed by ptNodeOriginAddress.
|
* Update a children address in a PtNode that is addressed by ptNodeOriginAddress.
|
||||||
*
|
*
|
||||||
* @param dictBuffer the DictBuffer to write.
|
* @param dictUpdater the DictUpdater to write.
|
||||||
* @param ptNodeOriginAddress the address of the PtNode.
|
* @param ptNodeOriginAddress the address of the PtNode.
|
||||||
* @param newChildrenAddress the absolute address of the child.
|
* @param newChildrenAddress the absolute address of the child.
|
||||||
* @param formatOptions file format options.
|
* @param formatOptions file format options.
|
||||||
*/
|
*/
|
||||||
private static void updateChildrenAddress(final DictBuffer dictBuffer,
|
private static void updateChildrenAddress(final Ver3DictUpdater dictUpdater,
|
||||||
final int ptNodeOriginAddress, final int newChildrenAddress,
|
final int ptNodeOriginAddress, final int newChildrenAddress,
|
||||||
final FormatOptions formatOptions) {
|
final FormatOptions formatOptions) {
|
||||||
|
final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
|
||||||
final int originalPosition = dictBuffer.position();
|
final int originalPosition = dictBuffer.position();
|
||||||
dictBuffer.position(ptNodeOriginAddress);
|
dictBuffer.position(ptNodeOriginAddress);
|
||||||
final int flags = dictBuffer.readUnsignedByte();
|
final int flags = dictBuffer.readUnsignedByte();
|
||||||
|
@ -155,31 +133,33 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
* Helper method to move a PtNode to the tail of the file.
|
* Helper method to move a PtNode to the tail of the file.
|
||||||
*/
|
*/
|
||||||
private static int movePtNode(final OutputStream destination,
|
private static int movePtNode(final OutputStream destination,
|
||||||
final DictBuffer dictBuffer, final PtNodeInfo info,
|
final Ver3DictUpdater dictUpdater, final PtNodeInfo info,
|
||||||
final int nodeArrayOriginAddress, final int oldNodeAddress,
|
final int nodeArrayOriginAddress, final int oldNodeAddress,
|
||||||
final FormatOptions formatOptions) throws IOException {
|
final FormatOptions formatOptions) throws IOException {
|
||||||
updateParentAddress(dictBuffer, oldNodeAddress, dictBuffer.limit() + 1, formatOptions);
|
final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
|
||||||
|
updateParentAddress(dictUpdater, oldNodeAddress, dictBuffer.limit() + 1, formatOptions);
|
||||||
dictBuffer.position(oldNodeAddress);
|
dictBuffer.position(oldNodeAddress);
|
||||||
final int currentFlags = dictBuffer.readUnsignedByte();
|
final int currentFlags = dictBuffer.readUnsignedByte();
|
||||||
dictBuffer.position(oldNodeAddress);
|
dictBuffer.position(oldNodeAddress);
|
||||||
dictBuffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags
|
dictBuffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags
|
||||||
& (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG))));
|
& (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG))));
|
||||||
int size = FormatSpec.PTNODE_FLAGS_SIZE;
|
int size = FormatSpec.PTNODE_FLAGS_SIZE;
|
||||||
updateForwardLink(dictBuffer, nodeArrayOriginAddress, dictBuffer.limit(), formatOptions);
|
updateForwardLink(dictUpdater, nodeArrayOriginAddress, dictBuffer.limit(), formatOptions);
|
||||||
size += BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { info });
|
size += BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { info });
|
||||||
return size;
|
return size;
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
private static void updateForwardLink(final DictBuffer dictBuffer,
|
private static void updateForwardLink(final Ver3DictUpdater dictUpdater,
|
||||||
final int nodeArrayOriginAddress, final int newNodeArrayAddress,
|
final int nodeArrayOriginAddress, final int newNodeArrayAddress,
|
||||||
final FormatOptions formatOptions) {
|
final FormatOptions formatOptions) {
|
||||||
dictBuffer.position(nodeArrayOriginAddress);
|
final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
|
||||||
|
dictUpdater.setPosition(nodeArrayOriginAddress);
|
||||||
int jumpCount = 0;
|
int jumpCount = 0;
|
||||||
while (jumpCount++ < MAX_JUMPS) {
|
while (jumpCount++ < MAX_JUMPS) {
|
||||||
final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer);
|
final int count = dictUpdater.readPtNodeCount();
|
||||||
for (int i = 0; i < count; ++i) {
|
for (int i = 0; i < count; ++i) {
|
||||||
BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions);
|
dictUpdater.readPtNode(dictUpdater.getPosition(), formatOptions);
|
||||||
}
|
}
|
||||||
final int forwardLinkAddress = dictBuffer.readUnsignedInt24();
|
final int forwardLinkAddress = dictBuffer.readUnsignedInt24();
|
||||||
if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) {
|
if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) {
|
||||||
|
@ -207,7 +187,7 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
* @param shortcutTargets the shortcut targets for this PtNode.
|
* @param shortcutTargets the shortcut targets for this PtNode.
|
||||||
* @param bigrams the bigrams for this PtNode.
|
* @param bigrams the bigrams for this PtNode.
|
||||||
* @param destination the stream representing the tail of the file.
|
* @param destination the stream representing the tail of the file.
|
||||||
* @param dictBuffer the DictBuffer representing the (constant-size) body of the file.
|
* @param dictUpdater the DictUpdater.
|
||||||
* @param oldPtNodeArrayOrigin the origin of the old PtNode array this PtNode was a part of.
|
* @param oldPtNodeArrayOrigin the origin of the old PtNode array this PtNode was a part of.
|
||||||
* @param oldPtNodeOrigin the old origin where this PtNode used to be stored.
|
* @param oldPtNodeOrigin the old origin where this PtNode used to be stored.
|
||||||
* @param formatOptions format options for this dictionary.
|
* @param formatOptions format options for this dictionary.
|
||||||
|
@ -218,7 +198,7 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
final int length, final int flags, final int frequency, final int parentAddress,
|
final int length, final int flags, final int frequency, final int parentAddress,
|
||||||
final ArrayList<WeightedString> shortcutTargets,
|
final ArrayList<WeightedString> shortcutTargets,
|
||||||
final ArrayList<PendingAttribute> bigrams, final OutputStream destination,
|
final ArrayList<PendingAttribute> bigrams, final OutputStream destination,
|
||||||
final DictBuffer dictBuffer, final int oldPtNodeArrayOrigin,
|
final Ver3DictUpdater dictUpdater, final int oldPtNodeArrayOrigin,
|
||||||
final int oldPtNodeOrigin, final FormatOptions formatOptions) throws IOException {
|
final int oldPtNodeOrigin, final FormatOptions formatOptions) throws IOException {
|
||||||
int size = 0;
|
int size = 0;
|
||||||
final int newPtNodeOrigin = fileEndAddress + 1;
|
final int newPtNodeOrigin = fileEndAddress + 1;
|
||||||
|
@ -231,7 +211,7 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
flags, writtenCharacters, frequency, parentAddress,
|
flags, writtenCharacters, frequency, parentAddress,
|
||||||
fileEndAddress + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets,
|
fileEndAddress + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets,
|
||||||
bigrams);
|
bigrams);
|
||||||
movePtNode(destination, dictBuffer, newInfo, oldPtNodeArrayOrigin, oldPtNodeOrigin,
|
movePtNode(destination, dictUpdater, newInfo, oldPtNodeArrayOrigin, oldPtNodeOrigin,
|
||||||
formatOptions);
|
formatOptions);
|
||||||
return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
|
return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
|
||||||
}
|
}
|
||||||
|
@ -239,7 +219,7 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
/**
|
/**
|
||||||
* Insert a word into a binary dictionary.
|
* Insert a word into a binary dictionary.
|
||||||
*
|
*
|
||||||
* @param dictDecoder the dict decoder.
|
* @param dictUpdater the dict updater.
|
||||||
* @param destination a stream to the underlying file, with the pointer at the end of the file.
|
* @param destination a stream to the underlying file, with the pointer at the end of the file.
|
||||||
* @param word the word to insert.
|
* @param word the word to insert.
|
||||||
* @param frequency the frequency of the new word.
|
* @param frequency the frequency of the new word.
|
||||||
|
@ -252,17 +232,17 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
// TODO: Support batch insertion.
|
// TODO: Support batch insertion.
|
||||||
// TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary.
|
// TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary.
|
||||||
@UsedForTesting
|
@UsedForTesting
|
||||||
public static void insertWord(final Ver3DictDecoder dictDecoder,
|
public static void insertWord(final Ver3DictUpdater dictUpdater,
|
||||||
final OutputStream destination, final String word, final int frequency,
|
final OutputStream destination, final String word, final int frequency,
|
||||||
final ArrayList<WeightedString> bigramStrings,
|
final ArrayList<WeightedString> bigramStrings,
|
||||||
final ArrayList<WeightedString> shortcuts, final boolean isNotAWord,
|
final ArrayList<WeightedString> shortcuts, final boolean isNotAWord,
|
||||||
final boolean isBlackListEntry)
|
final boolean isBlackListEntry)
|
||||||
throws IOException, UnsupportedFormatException {
|
throws IOException, UnsupportedFormatException {
|
||||||
final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>();
|
final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>();
|
||||||
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
|
final DictBuffer dictBuffer = dictUpdater.getDictBuffer();
|
||||||
if (bigramStrings != null) {
|
if (bigramStrings != null) {
|
||||||
for (final WeightedString bigram : bigramStrings) {
|
for (final WeightedString bigram : bigramStrings) {
|
||||||
int position = dictDecoder.getTerminalPosition(bigram.mWord);
|
int position = dictUpdater.getTerminalPosition(bigram.mWord);
|
||||||
if (position == FormatSpec.NOT_VALID_WORD) {
|
if (position == FormatSpec.NOT_VALID_WORD) {
|
||||||
// TODO: figure out what is the correct thing to do here.
|
// TODO: figure out what is the correct thing to do here.
|
||||||
} else {
|
} else {
|
||||||
|
@ -277,7 +257,7 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
|
|
||||||
// find the insert position of the word.
|
// find the insert position of the word.
|
||||||
if (dictBuffer.position() != 0) dictBuffer.position(0);
|
if (dictBuffer.position() != 0) dictBuffer.position(0);
|
||||||
final FileHeader fileHeader = dictDecoder.readHeader();
|
final FileHeader fileHeader = dictUpdater.readHeader();
|
||||||
|
|
||||||
int wordPos = 0, address = dictBuffer.position(), nodeOriginAddress = dictBuffer.position();
|
int wordPos = 0, address = dictBuffer.position(), nodeOriginAddress = dictBuffer.position();
|
||||||
final int[] codePoints = FusionDictionary.getCodePoints(word);
|
final int[] codePoints = FusionDictionary.getCodePoints(word);
|
||||||
|
@ -292,7 +272,7 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
|
|
||||||
for (int i = 0; i < ptNodeCount; ++i) {
|
for (int i = 0; i < ptNodeCount; ++i) {
|
||||||
address = dictBuffer.position();
|
address = dictBuffer.position();
|
||||||
final PtNodeInfo currentInfo = dictDecoder.readPtNode(address,
|
final PtNodeInfo currentInfo = dictUpdater.readPtNode(address,
|
||||||
fileHeader.mFormatOptions);
|
fileHeader.mFormatOptions);
|
||||||
final boolean isMovedNode = BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags,
|
final boolean isMovedNode = BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags,
|
||||||
fileHeader.mFormatOptions);
|
fileHeader.mFormatOptions);
|
||||||
|
@ -318,12 +298,12 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
false /* isBlackListEntry */, fileHeader.mFormatOptions);
|
false /* isBlackListEntry */, fileHeader.mFormatOptions);
|
||||||
int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, flags,
|
int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, flags,
|
||||||
frequency, nodeParentAddress, shortcuts, bigrams, destination,
|
frequency, nodeParentAddress, shortcuts, bigrams, destination,
|
||||||
dictBuffer, nodeOriginAddress, address, fileHeader.mFormatOptions);
|
dictUpdater, nodeOriginAddress, address, fileHeader.mFormatOptions);
|
||||||
|
|
||||||
final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p,
|
final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p,
|
||||||
currentInfo.mCharacters.length);
|
currentInfo.mCharacters.length);
|
||||||
if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
|
if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
|
||||||
updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress,
|
updateParentAddresses(dictUpdater, currentInfo.mChildrenAddress,
|
||||||
newNodeAddress + written + 1, fileHeader.mFormatOptions);
|
newNodeAddress + written + 1, fileHeader.mFormatOptions);
|
||||||
}
|
}
|
||||||
final PtNodeInfo newInfo2 = new PtNodeInfo(
|
final PtNodeInfo newInfo2 = new PtNodeInfo(
|
||||||
|
@ -359,13 +339,13 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
fileHeader.mFormatOptions);
|
fileHeader.mFormatOptions);
|
||||||
int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p,
|
int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p,
|
||||||
prefixFlags, -1 /* frequency */, nodeParentAddress, null, null,
|
prefixFlags, -1 /* frequency */, nodeParentAddress, null, null,
|
||||||
destination, dictBuffer, nodeOriginAddress, address,
|
destination, dictUpdater, nodeOriginAddress, address,
|
||||||
fileHeader.mFormatOptions);
|
fileHeader.mFormatOptions);
|
||||||
|
|
||||||
final int[] suffixCharacters = Arrays.copyOfRange(
|
final int[] suffixCharacters = Arrays.copyOfRange(
|
||||||
currentInfo.mCharacters, p, currentInfo.mCharacters.length);
|
currentInfo.mCharacters, p, currentInfo.mCharacters.length);
|
||||||
if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
|
if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
|
||||||
updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress,
|
updateParentAddresses(dictUpdater, currentInfo.mChildrenAddress,
|
||||||
newNodeAddress + written + 1, fileHeader.mFormatOptions);
|
newNodeAddress + written + 1, fileHeader.mFormatOptions);
|
||||||
}
|
}
|
||||||
final int suffixFlags = BinaryDictEncoderUtils.makePtNodeFlags(
|
final int suffixFlags = BinaryDictEncoderUtils.makePtNodeFlags(
|
||||||
|
@ -416,7 +396,7 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
-1 /* endAddress */, flags, currentInfo.mCharacters, frequency,
|
-1 /* endAddress */, flags, currentInfo.mCharacters, frequency,
|
||||||
nodeParentAddress, currentInfo.mChildrenAddress, shortcuts,
|
nodeParentAddress, currentInfo.mChildrenAddress, shortcuts,
|
||||||
bigrams);
|
bigrams);
|
||||||
movePtNode(destination, dictBuffer, newInfo, nodeOriginAddress, address,
|
movePtNode(destination, dictUpdater, newInfo, nodeOriginAddress, address,
|
||||||
fileHeader.mFormatOptions);
|
fileHeader.mFormatOptions);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -435,7 +415,7 @@ public final class DynamicBinaryDictIOUtils {
|
||||||
* ab - cd - e
|
* ab - cd - e
|
||||||
*/
|
*/
|
||||||
final int newNodeArrayAddress = dictBuffer.limit();
|
final int newNodeArrayAddress = dictBuffer.limit();
|
||||||
updateChildrenAddress(dictBuffer, address, newNodeArrayAddress,
|
updateChildrenAddress(dictUpdater, address, newNodeArrayAddress,
|
||||||
fileHeader.mFormatOptions);
|
fileHeader.mFormatOptions);
|
||||||
final int newNodeAddress = newNodeArrayAddress + 1;
|
final int newNodeAddress = newNodeArrayAddress + 1;
|
||||||
final boolean hasMultipleChars = (wordLen - wordPos) > 1;
|
final boolean hasMultipleChars = (wordLen - wordPos) > 1;
|
||||||
|
|
|
@ -53,9 +53,9 @@ public class Ver3DictDecoder extends DictDecoder {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private final File mDictionaryBinaryFile;
|
protected final File mDictionaryBinaryFile;
|
||||||
private final DictionaryBufferFactory mBufferFactory;
|
private final DictionaryBufferFactory mBufferFactory;
|
||||||
private DictBuffer mDictBuffer;
|
protected DictBuffer mDictBuffer;
|
||||||
|
|
||||||
/* package */ Ver3DictDecoder(final File file, final int factoryFlag) {
|
/* package */ Ver3DictDecoder(final File file, final int factoryFlag) {
|
||||||
mDictionaryBinaryFile = file;
|
mDictionaryBinaryFile = file;
|
||||||
|
|
|
@ -0,0 +1,82 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2013 The Android Open Source Project
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.android.inputmethod.latin.makedict;
|
||||||
|
|
||||||
|
import com.android.inputmethod.annotations.UsedForTesting;
|
||||||
|
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An implementation of DictUpdater for version 3 binary dictionary.
|
||||||
|
*/
|
||||||
|
@UsedForTesting
|
||||||
|
public class Ver3DictUpdater extends Ver3DictDecoder implements DictUpdater {
|
||||||
|
private OutputStream mOutStream;
|
||||||
|
|
||||||
|
@UsedForTesting
|
||||||
|
public Ver3DictUpdater(final File dictFile, final int factoryType) {
|
||||||
|
// DictUpdater must have an updatable DictBuffer.
|
||||||
|
super(dictFile, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY)
|
||||||
|
? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER);
|
||||||
|
mOutStream = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void openStreamAndBuffer() throws FileNotFoundException, IOException {
|
||||||
|
super.openDictBuffer();
|
||||||
|
mOutStream = new FileOutputStream(mDictionaryBinaryFile, true /* append */);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void close() throws IOException {
|
||||||
|
if (mOutStream != null) {
|
||||||
|
mOutStream.close();
|
||||||
|
mOutStream = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override @UsedForTesting
|
||||||
|
public void deleteWord(final String word) throws IOException, UnsupportedFormatException {
|
||||||
|
if (mOutStream == null) openStreamAndBuffer();
|
||||||
|
mDictBuffer.position(0);
|
||||||
|
super.readHeader();
|
||||||
|
final int wordPos = getTerminalPosition(word);
|
||||||
|
if (wordPos != FormatSpec.NOT_VALID_WORD) {
|
||||||
|
mDictBuffer.position(wordPos);
|
||||||
|
final int flags = mDictBuffer.readUnsignedByte();
|
||||||
|
mDictBuffer.position(wordPos);
|
||||||
|
mDictBuffer.put((byte) DynamicBinaryDictIOUtils.markAsDeleted(flags));
|
||||||
|
}
|
||||||
|
close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override @UsedForTesting
|
||||||
|
public void insertWord(final String word, final int frequency,
|
||||||
|
final ArrayList<WeightedString> bigramStrings,
|
||||||
|
final ArrayList<WeightedString> shortcuts,
|
||||||
|
final boolean isNotAWord, final boolean isBlackListEntry)
|
||||||
|
throws IOException, UnsupportedFormatException {
|
||||||
|
if (mOutStream == null) openStreamAndBuffer();
|
||||||
|
DynamicBinaryDictIOUtils.insertWord(this, mOutStream, word, frequency, bigramStrings,
|
||||||
|
shortcuts, isNotAWord, isBlackListEntry);
|
||||||
|
close();
|
||||||
|
}
|
||||||
|
}
|
|
@ -657,27 +657,21 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
|
||||||
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
|
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
|
||||||
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
|
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
|
||||||
|
|
||||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
|
final Ver3DictUpdater dictUpdater = new Ver3DictUpdater(file,
|
||||||
try {
|
DictDecoder.USE_WRITABLE_BYTEBUFFER);
|
||||||
dictDecoder.openDictBuffer();
|
|
||||||
} catch (IOException e) {
|
|
||||||
// ignore
|
|
||||||
Log.e(TAG, "IOException while opening the buffer", e);
|
|
||||||
}
|
|
||||||
assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen());
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
|
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
|
||||||
dictDecoder.getTerminalPosition(sWords.get(0)));
|
dictUpdater.getTerminalPosition(sWords.get(0)));
|
||||||
DynamicBinaryDictIOUtils.deleteWord(dictDecoder, sWords.get(0));
|
dictUpdater.deleteWord(sWords.get(0));
|
||||||
assertEquals(FormatSpec.NOT_VALID_WORD,
|
assertEquals(FormatSpec.NOT_VALID_WORD,
|
||||||
dictDecoder.getTerminalPosition(sWords.get(0)));
|
dictUpdater.getTerminalPosition(sWords.get(0)));
|
||||||
|
|
||||||
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
|
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,
|
||||||
dictDecoder.getTerminalPosition(sWords.get(5)));
|
dictUpdater.getTerminalPosition(sWords.get(5)));
|
||||||
DynamicBinaryDictIOUtils.deleteWord(dictDecoder, sWords.get(5));
|
dictUpdater.deleteWord(sWords.get(5));
|
||||||
assertEquals(FormatSpec.NOT_VALID_WORD,
|
assertEquals(FormatSpec.NOT_VALID_WORD,
|
||||||
dictDecoder.getTerminalPosition(sWords.get(5)));
|
dictUpdater.getTerminalPosition(sWords.get(5)));
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
} catch (UnsupportedFormatException e) {
|
} catch (UnsupportedFormatException e) {
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,9 +27,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
|
||||||
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
|
||||||
import com.android.inputmethod.latin.utils.CollectionUtils;
|
import com.android.inputmethod.latin.utils.CollectionUtils;
|
||||||
|
|
||||||
import java.io.BufferedOutputStream;
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileOutputStream;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
@ -186,46 +184,31 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
|
||||||
private long insertAndCheckWord(final File file, final String word, final int frequency,
|
private long insertAndCheckWord(final File file, final String word, final int frequency,
|
||||||
final boolean exist, final ArrayList<WeightedString> bigrams,
|
final boolean exist, final ArrayList<WeightedString> bigrams,
|
||||||
final ArrayList<WeightedString> shortcuts) {
|
final ArrayList<WeightedString> shortcuts) {
|
||||||
BufferedOutputStream outStream = null;
|
|
||||||
long amountOfTime = -1;
|
long amountOfTime = -1;
|
||||||
try {
|
try {
|
||||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
|
final Ver3DictUpdater dictUpdater = new Ver3DictUpdater(file,
|
||||||
DictDecoder.USE_WRITABLE_BYTEBUFFER);
|
DictDecoder.USE_WRITABLE_BYTEBUFFER);
|
||||||
dictDecoder.openDictBuffer();
|
|
||||||
outStream = new BufferedOutputStream(new FileOutputStream(file, true));
|
|
||||||
|
|
||||||
if (!exist) {
|
if (!exist) {
|
||||||
assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word));
|
assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word));
|
||||||
}
|
}
|
||||||
final long now = System.nanoTime();
|
final long now = System.nanoTime();
|
||||||
DynamicBinaryDictIOUtils.insertWord(dictDecoder, outStream, word, frequency, bigrams,
|
dictUpdater.insertWord(word, frequency, bigrams, shortcuts, false, false);
|
||||||
shortcuts, false, false);
|
|
||||||
amountOfTime = System.nanoTime() - now;
|
amountOfTime = System.nanoTime() - now;
|
||||||
outStream.flush();
|
|
||||||
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word));
|
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word));
|
||||||
outStream.close();
|
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
Log.e(TAG, "Raised an IOException while inserting a word", e);
|
Log.e(TAG, "Raised an IOException while inserting a word", e);
|
||||||
} catch (UnsupportedFormatException e) {
|
} catch (UnsupportedFormatException e) {
|
||||||
Log.e(TAG, "Raised an UnsupportedFormatException error while inserting a word", e);
|
Log.e(TAG, "Raised an UnsupportedFormatException error while inserting a word", e);
|
||||||
} finally {
|
|
||||||
if (outStream != null) {
|
|
||||||
try {
|
|
||||||
outStream.close();
|
|
||||||
} catch (IOException e) {
|
|
||||||
Log.e(TAG, "Failed to close the output stream", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return amountOfTime;
|
return amountOfTime;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void deleteWord(final File file, final String word) {
|
private void deleteWord(final File file, final String word) {
|
||||||
try {
|
try {
|
||||||
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
|
final Ver3DictUpdater dictUpdater = new Ver3DictUpdater(file,
|
||||||
DictDecoder.USE_WRITABLE_BYTEBUFFER);
|
DictDecoder.USE_WRITABLE_BYTEBUFFER);
|
||||||
dictDecoder.openDictBuffer();
|
dictUpdater.deleteWord(word);
|
||||||
DynamicBinaryDictIOUtils.deleteWord(dictDecoder, word);
|
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
} catch (UnsupportedFormatException e) {
|
} catch (UnsupportedFormatException e) {
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue