am 257f59fb: am b2956ef5: am 399422f4: Merge "Make Ver4DictEncoder write an address table of terminal nodes."
* commit '257f59fb024c120c789b8d3b097062a64f07a0f2': Make Ver4DictEncoder write an address table of terminal nodes.main
commit
7988da3797
|
@ -882,8 +882,9 @@ public class BinaryDictEncoderUtils {
|
||||||
* @param destination the stream to write the file header to.
|
* @param destination the stream to write the file header to.
|
||||||
* @param dict the dictionary to write.
|
* @param dict the dictionary to write.
|
||||||
* @param formatOptions file format options.
|
* @param formatOptions file format options.
|
||||||
|
* @return the size of the header.
|
||||||
*/
|
*/
|
||||||
/* package */ static void writeDictionaryHeader(final OutputStream destination,
|
/* package */ static int writeDictionaryHeader(final OutputStream destination,
|
||||||
final FusionDictionary dict, final FormatOptions formatOptions)
|
final FusionDictionary dict, final FormatOptions formatOptions)
|
||||||
throws IOException, UnsupportedFormatException {
|
throws IOException, UnsupportedFormatException {
|
||||||
final int version = formatOptions.mVersion;
|
final int version = formatOptions.mVersion;
|
||||||
|
@ -932,5 +933,6 @@ public class BinaryDictEncoderUtils {
|
||||||
destination.write(bytes);
|
destination.write(bytes);
|
||||||
|
|
||||||
headerBuffer.close();
|
headerBuffer.close();
|
||||||
|
return size;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -263,7 +263,10 @@ public final class FormatSpec {
|
||||||
// These values are used only by version 4 or later.
|
// These values are used only by version 4 or later.
|
||||||
static final String TRIE_FILE_EXTENSION = ".trie";
|
static final String TRIE_FILE_EXTENSION = ".trie";
|
||||||
static final String FREQ_FILE_EXTENSION = ".freq";
|
static final String FREQ_FILE_EXTENSION = ".freq";
|
||||||
|
// tat = Terminal Address Table
|
||||||
|
static final String TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat";
|
||||||
static final int FREQUENCY_AND_FLAGS_SIZE = 2;
|
static final int FREQUENCY_AND_FLAGS_SIZE = 2;
|
||||||
|
static final int TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE = 3;
|
||||||
|
|
||||||
static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE;
|
static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE;
|
||||||
static final int NO_PARENT_ADDRESS = 0;
|
static final int NO_PARENT_ADDRESS = 0;
|
||||||
|
|
|
@ -41,11 +41,13 @@ public class Ver4DictDecoder extends DictDecoder {
|
||||||
|
|
||||||
private static final int FILETYPE_TRIE = 1;
|
private static final int FILETYPE_TRIE = 1;
|
||||||
private static final int FILETYPE_FREQUENCY = 2;
|
private static final int FILETYPE_FREQUENCY = 2;
|
||||||
|
private static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3;
|
||||||
|
|
||||||
private final File mDictDirectory;
|
private final File mDictDirectory;
|
||||||
private final DictionaryBufferFactory mBufferFactory;
|
private final DictionaryBufferFactory mBufferFactory;
|
||||||
private DictBuffer mDictBuffer;
|
private DictBuffer mDictBuffer;
|
||||||
private DictBuffer mFrequencyBuffer;
|
private DictBuffer mFrequencyBuffer;
|
||||||
|
private DictBuffer mTerminalAddressTableBuffer;
|
||||||
|
|
||||||
@UsedForTesting
|
@UsedForTesting
|
||||||
/* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) {
|
/* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) {
|
||||||
|
@ -77,6 +79,9 @@ public class Ver4DictDecoder extends DictDecoder {
|
||||||
} else if (fileType == FILETYPE_FREQUENCY) {
|
} else if (fileType == FILETYPE_FREQUENCY) {
|
||||||
return new File(mDictDirectory,
|
return new File(mDictDirectory,
|
||||||
mDictDirectory.getName() + FormatSpec.FREQ_FILE_EXTENSION);
|
mDictDirectory.getName() + FormatSpec.FREQ_FILE_EXTENSION);
|
||||||
|
} else if (fileType == FILETYPE_TERMINAL_ADDRESS_TABLE) {
|
||||||
|
return new File(mDictDirectory,
|
||||||
|
mDictDirectory.getName() + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
|
||||||
} else {
|
} else {
|
||||||
throw new RuntimeException("Unsupported kind of file : " + fileType);
|
throw new RuntimeException("Unsupported kind of file : " + fileType);
|
||||||
}
|
}
|
||||||
|
@ -87,6 +92,8 @@ public class Ver4DictDecoder extends DictDecoder {
|
||||||
final String filename = mDictDirectory.getName();
|
final String filename = mDictDirectory.getName();
|
||||||
mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE));
|
mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE));
|
||||||
mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY));
|
mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY));
|
||||||
|
mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer(
|
||||||
|
getFile(FILETYPE_TERMINAL_ADDRESS_TABLE));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -41,10 +41,11 @@ import java.util.Iterator;
|
||||||
public class Ver4DictEncoder implements DictEncoder {
|
public class Ver4DictEncoder implements DictEncoder {
|
||||||
private final File mDictPlacedDir;
|
private final File mDictPlacedDir;
|
||||||
private byte[] mTrieBuf;
|
private byte[] mTrieBuf;
|
||||||
private byte[] mFreqBuf;
|
|
||||||
private int mTriePos;
|
private int mTriePos;
|
||||||
|
private int mHeaderSize;
|
||||||
private OutputStream mTrieOutStream;
|
private OutputStream mTrieOutStream;
|
||||||
private OutputStream mFreqOutStream;
|
private OutputStream mFreqOutStream;
|
||||||
|
private OutputStream mTerminalAddressTableOutStream;
|
||||||
|
|
||||||
@UsedForTesting
|
@UsedForTesting
|
||||||
public Ver4DictEncoder(final File dictPlacedDir) {
|
public Ver4DictEncoder(final File dictPlacedDir) {
|
||||||
|
@ -58,14 +59,18 @@ public class Ver4DictEncoder implements DictEncoder {
|
||||||
final File mDictDir = new File(mDictPlacedDir, filename);
|
final File mDictDir = new File(mDictPlacedDir, filename);
|
||||||
final File trieFile = new File(mDictDir, filename + FormatSpec.TRIE_FILE_EXTENSION);
|
final File trieFile = new File(mDictDir, filename + FormatSpec.TRIE_FILE_EXTENSION);
|
||||||
final File freqFile = new File(mDictDir, filename + FormatSpec.FREQ_FILE_EXTENSION);
|
final File freqFile = new File(mDictDir, filename + FormatSpec.FREQ_FILE_EXTENSION);
|
||||||
|
final File terminalAddressTableFile = new File(mDictDir,
|
||||||
|
filename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION);
|
||||||
if (!mDictDir.isDirectory()) {
|
if (!mDictDir.isDirectory()) {
|
||||||
if (mDictDir.exists()) mDictDir.delete();
|
if (mDictDir.exists()) mDictDir.delete();
|
||||||
mDictDir.mkdirs();
|
mDictDir.mkdirs();
|
||||||
}
|
}
|
||||||
if (!trieFile.exists()) trieFile.createNewFile();
|
if (!trieFile.exists()) trieFile.createNewFile();
|
||||||
if (!freqFile.exists()) freqFile.createNewFile();
|
if (!freqFile.exists()) freqFile.createNewFile();
|
||||||
|
if (!terminalAddressTableFile.exists()) terminalAddressTableFile.createNewFile();
|
||||||
mTrieOutStream = new FileOutputStream(trieFile);
|
mTrieOutStream = new FileOutputStream(trieFile);
|
||||||
mFreqOutStream = new FileOutputStream(freqFile);
|
mFreqOutStream = new FileOutputStream(freqFile);
|
||||||
|
mTerminalAddressTableOutStream = new FileOutputStream(terminalAddressTableFile);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void close() throws IOException {
|
private void close() throws IOException {
|
||||||
|
@ -76,9 +81,13 @@ public class Ver4DictEncoder implements DictEncoder {
|
||||||
if (mFreqOutStream != null) {
|
if (mFreqOutStream != null) {
|
||||||
mFreqOutStream.close();
|
mFreqOutStream.close();
|
||||||
}
|
}
|
||||||
|
if (mTerminalAddressTableOutStream != null) {
|
||||||
|
mTerminalAddressTableOutStream.close();
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
mTrieOutStream = null;
|
mTrieOutStream = null;
|
||||||
mFreqOutStream = null;
|
mFreqOutStream = null;
|
||||||
|
mTerminalAddressTableOutStream = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -97,7 +106,8 @@ public class Ver4DictEncoder implements DictEncoder {
|
||||||
openStreams(formatOptions, dict.mOptions);
|
openStreams(formatOptions, dict.mOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict, formatOptions);
|
mHeaderSize = BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict,
|
||||||
|
formatOptions);
|
||||||
|
|
||||||
MakedictLog.i("Flattening the tree...");
|
MakedictLog.i("Flattening the tree...");
|
||||||
ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray);
|
ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray);
|
||||||
|
@ -112,10 +122,11 @@ public class Ver4DictEncoder implements DictEncoder {
|
||||||
BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions);
|
BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions);
|
||||||
if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes);
|
if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes);
|
||||||
|
|
||||||
|
writeTerminalData(flatNodes, terminalCount);
|
||||||
|
|
||||||
final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1);
|
final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1);
|
||||||
final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize;
|
final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize;
|
||||||
mTrieBuf = new byte[bufferSize];
|
mTrieBuf = new byte[bufferSize];
|
||||||
mFreqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE];
|
|
||||||
|
|
||||||
MakedictLog.i("Writing file...");
|
MakedictLog.i("Writing file...");
|
||||||
for (PtNodeArray nodeArray : flatNodes) {
|
for (PtNodeArray nodeArray : flatNodes) {
|
||||||
|
@ -126,7 +137,6 @@ public class Ver4DictEncoder implements DictEncoder {
|
||||||
MakedictLog.i("has " + terminalCount + " terminals.");
|
MakedictLog.i("has " + terminalCount + " terminals.");
|
||||||
}
|
}
|
||||||
mTrieOutStream.write(mTrieBuf);
|
mTrieOutStream.write(mTrieBuf);
|
||||||
mFreqOutStream.write(mFreqBuf);
|
|
||||||
|
|
||||||
MakedictLog.i("Done");
|
MakedictLog.i("Done");
|
||||||
close();
|
close();
|
||||||
|
@ -185,12 +195,6 @@ public class Ver4DictEncoder implements DictEncoder {
|
||||||
FormatSpec.PTNODE_TERMINAL_ID_SIZE);
|
FormatSpec.PTNODE_TERMINAL_ID_SIZE);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void writeFrequency(final int frequency, final int terminalId) {
|
|
||||||
final int freqPos = terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE;
|
|
||||||
BinaryDictEncoderUtils.writeUIntToBuffer(mFreqBuf, freqPos, frequency,
|
|
||||||
FormatSpec.FREQUENCY_AND_FLAGS_SIZE);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) {
|
private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) {
|
||||||
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
|
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
|
||||||
if (formatOptions.mSupportsDynamicUpdate) {
|
if (formatOptions.mSupportsDynamicUpdate) {
|
||||||
|
@ -260,10 +264,31 @@ public class Ver4DictEncoder implements DictEncoder {
|
||||||
writeCharacters(ptNode.mChars, ptNode.hasSeveralChars());
|
writeCharacters(ptNode.mChars, ptNode.hasSeveralChars());
|
||||||
if (ptNode.isTerminal()) {
|
if (ptNode.isTerminal()) {
|
||||||
writeTerminalId(ptNode.mTerminalId);
|
writeTerminalId(ptNode.mTerminalId);
|
||||||
writeFrequency(ptNode.mFrequency, ptNode.mTerminalId);
|
|
||||||
}
|
}
|
||||||
writeChildrenPosition(ptNode, formatOptions);
|
writeChildrenPosition(ptNode, formatOptions);
|
||||||
writeShortcuts(ptNode.mShortcutTargets);
|
writeShortcuts(ptNode.mShortcutTargets);
|
||||||
writeBigrams(ptNode.mBigrams, dict);
|
writeBigrams(ptNode.mBigrams, dict);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void writeTerminalData(final ArrayList<PtNodeArray> flatNodes,
|
||||||
|
final int terminalCount) throws IOException {
|
||||||
|
final byte[] freqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE];
|
||||||
|
final byte[] terminalAddressTableBuf =
|
||||||
|
new byte[terminalCount * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE];
|
||||||
|
for (final PtNodeArray nodeArray : flatNodes) {
|
||||||
|
for (final PtNode ptNode : nodeArray.mData) {
|
||||||
|
if (ptNode.isTerminal()) {
|
||||||
|
BinaryDictEncoderUtils.writeUIntToBuffer(freqBuf,
|
||||||
|
ptNode.mTerminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE,
|
||||||
|
ptNode.mFrequency, FormatSpec.FREQUENCY_AND_FLAGS_SIZE);
|
||||||
|
BinaryDictEncoderUtils.writeUIntToBuffer(terminalAddressTableBuf,
|
||||||
|
ptNode.mTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE,
|
||||||
|
ptNode.mCachedAddressAfterUpdate + mHeaderSize,
|
||||||
|
FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mFreqOutStream.write(freqBuf);
|
||||||
|
mTerminalAddressTableOutStream.write(terminalAddressTableBuf);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue