am 09129f58: am 11cccb56: am d188af70: Add SparseTable.
* commit '09129f58f260edfded5cdd9b9f793a9e7c81acdb': Add SparseTable.main
Yuichiro Hanada
Android Git Automerger
commit
bfdc5d6462
|
|
@ -225,6 +225,26 @@ public class BinaryDictEncoderUtils {
|
|||
return position;
|
||||
}
|
||||
|
||||
static void writeUIntToStream(final OutputStream stream, final int value, final int size)
|
||||
throws IOException {
|
||||
switch(size) {
|
||||
case 4:
|
||||
stream.write((value >> 24) & 0xFF);
|
||||
/* fall through */
|
||||
case 3:
|
||||
stream.write((value >> 16) & 0xFF);
|
||||
/* fall through */
|
||||
case 2:
|
||||
stream.write((value >> 8) & 0xFF);
|
||||
/* fall through */
|
||||
case 1:
|
||||
stream.write(value & 0xFF);
|
||||
break;
|
||||
default:
|
||||
/* nop */
|
||||
}
|
||||
}
|
||||
|
||||
// End utility methods
|
||||
|
||||
// This method is responsible for finding a nice ordering of the nodes that favors run-time
|
||||
|
|
|
|||
|
|
@ -0,0 +1,150 @@
|
|||
/*
|
||||
* Copyright (C) 2013 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.android.inputmethod.latin.makedict;
|
||||
|
||||
import com.android.inputmethod.annotations.UsedForTesting;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
|
||||
/**
|
||||
* SparseTable is an extensible map from integer to integer.
|
||||
* This holds one value for every mBlockSize keys, so it uses 1/mBlockSize'th of the full index
|
||||
* memory.
|
||||
*/
|
||||
@UsedForTesting
|
||||
public class SparseTable {
|
||||
|
||||
/**
|
||||
* mLookupTable is indexed by terminal ID, containing exactly one entry for every mBlockSize
|
||||
* terminals.
|
||||
* It contains at index i = j / mBlockSize the index in mContentsTable where the values for
|
||||
* terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized integer array.
|
||||
*/
|
||||
private final ArrayList<Integer> mLookupTable;
|
||||
private final ArrayList<Integer> mContentTable;
|
||||
|
||||
private final int mBlockSize;
|
||||
public static final int NOT_EXIST = -1;
|
||||
|
||||
@UsedForTesting
|
||||
public SparseTable(final int initialCapacity, final int blockSize) {
|
||||
mBlockSize = blockSize;
|
||||
final int lookupTableSize = initialCapacity / mBlockSize
|
||||
+ (initialCapacity % mBlockSize > 0 ? 1 : 0);
|
||||
mLookupTable = new ArrayList<Integer>(Collections.nCopies(lookupTableSize, NOT_EXIST));
|
||||
mContentTable = new ArrayList<Integer>();
|
||||
}
|
||||
|
||||
@UsedForTesting
|
||||
public SparseTable(final int[] lookupTable, final int[] contentTable, final int blockSize) {
|
||||
mBlockSize = blockSize;
|
||||
mLookupTable = new ArrayList<Integer>(lookupTable.length);
|
||||
for (int i = 0; i < lookupTable.length; ++i) {
|
||||
mLookupTable.add(lookupTable[i]);
|
||||
}
|
||||
mContentTable = new ArrayList<Integer>(contentTable.length);
|
||||
for (int i = 0; i < contentTable.length; ++i) {
|
||||
mContentTable.add(contentTable[i]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts an byte array to an int array considering each set of 4 bytes is an int stored in
|
||||
* big-endian.
|
||||
* The length of byteArray must be a multiple of four.
|
||||
* Otherwise, IndexOutOfBoundsException will be raised.
|
||||
*/
|
||||
@UsedForTesting
|
||||
private static void convertByteArrayToIntegerArray(final byte[] byteArray,
|
||||
final ArrayList<Integer> integerArray) {
|
||||
for (int i = 0; i < byteArray.length; i += 4) {
|
||||
int value = 0;
|
||||
for (int j = i; j < i + 4; ++j) {
|
||||
value <<= 8;
|
||||
value |= byteArray[j] & 0xFF;
|
||||
}
|
||||
integerArray.add(value);
|
||||
}
|
||||
}
|
||||
|
||||
@UsedForTesting
|
||||
public SparseTable(final byte[] lookupTable, final byte[] contentTable, final int blockSize) {
|
||||
mBlockSize = blockSize;
|
||||
mLookupTable = new ArrayList<Integer>(lookupTable.length / 4);
|
||||
mContentTable = new ArrayList<Integer>(contentTable.length / 4);
|
||||
convertByteArrayToIntegerArray(lookupTable, mLookupTable);
|
||||
convertByteArrayToIntegerArray(contentTable, mContentTable);
|
||||
}
|
||||
|
||||
@UsedForTesting
|
||||
public int get(final int index) {
|
||||
if (index < 0 || index / mBlockSize >= mLookupTable.size()
|
||||
|| mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
|
||||
return NOT_EXIST;
|
||||
}
|
||||
return mContentTable.get(mLookupTable.get(index / mBlockSize) + (index % mBlockSize));
|
||||
}
|
||||
|
||||
@UsedForTesting
|
||||
public void set(final int index, final int value) {
|
||||
if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) {
|
||||
mLookupTable.set(index / mBlockSize, mContentTable.size());
|
||||
for (int i = 0; i < mBlockSize; ++i) {
|
||||
mContentTable.add(NOT_EXIST);
|
||||
}
|
||||
}
|
||||
mContentTable.set(mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value);
|
||||
}
|
||||
|
||||
public void remove(final int index) {
|
||||
set(index, NOT_EXIST);
|
||||
}
|
||||
|
||||
@UsedForTesting
|
||||
public int size() {
|
||||
return mLookupTable.size() * mBlockSize;
|
||||
}
|
||||
|
||||
@UsedForTesting
|
||||
/* package */ int getContentTableSize() {
|
||||
return mContentTable.size();
|
||||
}
|
||||
|
||||
@UsedForTesting
|
||||
/* package */ int getLookupTableSize() {
|
||||
return mLookupTable.size();
|
||||
}
|
||||
|
||||
public boolean contains(final int index) {
|
||||
return get(index) != NOT_EXIST;
|
||||
}
|
||||
|
||||
@UsedForTesting
|
||||
public void write(final OutputStream lookupOutStream, final OutputStream contentOutStream)
|
||||
throws IOException {
|
||||
for (final int index : mLookupTable) {
|
||||
BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, 4);
|
||||
}
|
||||
|
||||
for (final int index : mContentTable) {
|
||||
BinaryDictEncoderUtils.writeUIntToStream(contentOutStream, index, 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
/*
|
||||
* Copyright (C) 2013 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.android.inputmethod.latin.makedict;
|
||||
|
||||
import android.test.AndroidTestCase;
|
||||
import android.test.suitebuilder.annotation.LargeTest;
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Random;
|
||||
|
||||
/**
|
||||
* Unit tests for SparseTable.
|
||||
*/
|
||||
@LargeTest
|
||||
public class SparseTableTests extends AndroidTestCase {
|
||||
private static final String TAG = SparseTableTests.class.getSimpleName();
|
||||
|
||||
private static final int[] SMALL_INDEX = { SparseTable.NOT_EXIST, 0 };
|
||||
private static final int[] BIG_INDEX = { SparseTable.NOT_EXIST, 1, 2, 3, 4, 5, 6, 7};
|
||||
|
||||
private final Random mRandom;
|
||||
private final ArrayList<Integer> mRandomIndex;
|
||||
|
||||
private static final int DEFAULT_SIZE = 10000;
|
||||
private static final int BLOCK_SIZE = 8;
|
||||
|
||||
public SparseTableTests() {
|
||||
this(System.currentTimeMillis(), DEFAULT_SIZE);
|
||||
}
|
||||
|
||||
public SparseTableTests(final long seed, final int tableSize) {
|
||||
super();
|
||||
Log.d(TAG, "Seed for test is " + seed + ", size is " + tableSize);
|
||||
mRandom = new Random(seed);
|
||||
mRandomIndex = new ArrayList<Integer>(tableSize);
|
||||
for (int i = 0; i < tableSize; ++i) {
|
||||
mRandomIndex.add(SparseTable.NOT_EXIST);
|
||||
}
|
||||
}
|
||||
|
||||
public void testInitializeWithArray() {
|
||||
final SparseTable table = new SparseTable(SMALL_INDEX, BIG_INDEX, BLOCK_SIZE);
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
assertEquals(SparseTable.NOT_EXIST, table.get(i));
|
||||
}
|
||||
assertEquals(SparseTable.NOT_EXIST, table.get(8));
|
||||
for (int i = 9; i < 16; ++i) {
|
||||
assertEquals(i - 8, table.get(i));
|
||||
}
|
||||
}
|
||||
|
||||
public void testSet() {
|
||||
final SparseTable table = new SparseTable(16, BLOCK_SIZE);
|
||||
table.set(3, 6);
|
||||
table.set(8, 16);
|
||||
for (int i = 0; i < 16; ++i) {
|
||||
if (i == 3 || i == 8) {
|
||||
assertEquals(i * 2, table.get(i));
|
||||
} else {
|
||||
assertEquals(SparseTable.NOT_EXIST, table.get(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void generateRandomIndex(final int size, final int prop) {
|
||||
for (int i = 0; i < DEFAULT_SIZE; ++i) {
|
||||
if (mRandom.nextInt(100) < prop) {
|
||||
mRandomIndex.set(i, mRandom.nextInt());
|
||||
} else {
|
||||
mRandomIndex.set(i, SparseTable.NOT_EXIST);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void runTestRandomSet() {
|
||||
final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE);
|
||||
int elementCount = 0;
|
||||
for (int i = 0; i < DEFAULT_SIZE; ++i) {
|
||||
if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) {
|
||||
table.set(i, mRandomIndex.get(i));
|
||||
elementCount++;
|
||||
}
|
||||
}
|
||||
|
||||
Log.d(TAG, "table size = " + table.getLookupTableSize() + " + "
|
||||
+ table.getContentTableSize());
|
||||
Log.d(TAG, "the table has " + elementCount + " elements");
|
||||
for (int i = 0; i < DEFAULT_SIZE; ++i) {
|
||||
assertEquals(table.get(i), (int)mRandomIndex.get(i));
|
||||
}
|
||||
|
||||
// flush and reload
|
||||
OutputStream lookupOutStream = null;
|
||||
OutputStream contentOutStream = null;
|
||||
InputStream lookupInStream = null;
|
||||
InputStream contentInStream = null;
|
||||
try {
|
||||
final File lookupIndexFile = File.createTempFile("testRandomSet", ".small");
|
||||
final File contentFile = File.createTempFile("testRandomSet", ".big");
|
||||
lookupOutStream = new FileOutputStream(lookupIndexFile);
|
||||
contentOutStream = new FileOutputStream(contentFile);
|
||||
table.write(lookupOutStream, contentOutStream);
|
||||
lookupInStream = new FileInputStream(lookupIndexFile);
|
||||
contentInStream = new FileInputStream(contentFile);
|
||||
final byte[] lookupArray = new byte[(int) lookupIndexFile.length()];
|
||||
final byte[] contentArray = new byte[(int) contentFile.length()];
|
||||
lookupInStream.read(lookupArray);
|
||||
contentInStream.read(contentArray);
|
||||
final SparseTable newTable = new SparseTable(lookupArray, contentArray, BLOCK_SIZE);
|
||||
for (int i = 0; i < DEFAULT_SIZE; ++i) {
|
||||
assertEquals(table.get(i), newTable.get(i));
|
||||
}
|
||||
} catch (IOException e) {
|
||||
Log.d(TAG, "IOException while flushing and realoding", e);
|
||||
} finally {
|
||||
if (lookupOutStream != null) {
|
||||
try {
|
||||
lookupOutStream.close();
|
||||
} catch (IOException e) {
|
||||
Log.d(TAG, "IOException while closing the stream", e);
|
||||
}
|
||||
}
|
||||
if (contentOutStream != null) {
|
||||
try {
|
||||
contentOutStream.close();
|
||||
} catch (IOException e) {
|
||||
Log.d(TAG, "IOException while closing contentStream.", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void testRandomSet() {
|
||||
for (int i = 0; i <= 100; i += 10) {
|
||||
generateRandomIndex(DEFAULT_SIZE, i);
|
||||
runTestRandomSet();
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue