nyancrimew
/
LatinIME
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

AI 143659: am: CL 143472 Reduce dictionary size. 

Changed the tree structure to have variable length nodes to save an average of 21% on the dictionary size.
  Created a shortened English dictionary for Dream - 50K words.
  Added a shortened Spanish dictionary for Dream - 32K words.
  Original author: yamasani
  Merged from: //branches/cupcake/...

Automated import of CL 143659
main
Amith Yamasani 2009-03-31 10:51:17 -07:00 committed by The Android Open Source Project
parent 152d8a7957
commit cc3e5c7c10
2 changed files with 29 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dictionary/src/dictionary.cpp
View File

@ -85,10 +85,14 @@ int
Dictionary::getAddress(int *pos) Dictionary::getAddress(int *pos)
{ {
int address = 0; int address = 0;
address += (mDict[*pos] & 0x7F) << 16; if ((mDict[*pos] & FLAG_ADDRESS_MASK) == 0) {
*pos += 1;
} else {
address += (mDict[*pos] & (ADDRESS_MASK >> 16)) << 16;
address += (mDict[*pos + 1] & 0xFF) << 8; address += (mDict[*pos + 1] & 0xFF) << 8;
address += (mDict[*pos + 2] & 0xFF); address += (mDict[*pos + 2] & 0xFF);
*pos += 3; *pos += 3;
}
return address; return address;
} }
@ -193,7 +197,8 @@ Dictionary::getWordsRec(int pos, int depth, int maxDepth, bool completion, int s
unsigned short lowerC = toLowerCase(c, depth); unsigned short lowerC = toLowerCase(c, depth);
bool terminal = getTerminal(&pos); bool terminal = getTerminal(&pos);
int childrenAddress = getAddress(&pos); int childrenAddress = getAddress(&pos);
int freq = getFreq(&pos); int freq = 1;
if (terminal) freq = getFreq(&pos);
// If we are only doing completions, no need to look at the typed characters. // If we are only doing completions, no need to look at the typed characters.
if (completion) { if (completion) {
mWord[depth] = c; mWord[depth] = c;
@ -266,7 +271,9 @@ Dictionary::isValidWordRec(int pos, unsigned short *word, int offset, int length
} }
} }
} }
if (terminal) {
getFreq(&pos); getFreq(&pos);
}
// There could be two instances of each alphabet - upper and lower case. So continue // There could be two instances of each alphabet - upper and lower case. So continue
// looking ... // looking ...
} }

11
dictionary/src/dictionary.h
View File

@ -19,6 +19,15 @@
namespace latinime { namespace latinime {
// 22-bit address = ~4MB dictionary size limit, which on average would be about 200k-300k words
#define ADDRESS_MASK 0x3FFFFF
// The bit that decides if an address follows in the next 22 bits
#define FLAG_ADDRESS_MASK 0x40
// The bit that decides if this is a terminal node for a word. The node could still have children,
// if the word has other endings.
#define FLAG_TERMINAL_MASK 0x80
class Dictionary { class Dictionary {
public: public:
Dictionary(void *dict, int typedLetterMultipler, int fullWordMultiplier); Dictionary(void *dict, int typedLetterMultipler, int fullWordMultiplier);
@ -32,7 +41,7 @@ public:
private: private:
int getAddress(int *pos); int getAddress(int *pos);
bool getTerminal(int *pos) { return (mDict[*pos] & 0x80) > 0; } bool getTerminal(int *pos) { return (mDict[*pos] & FLAG_TERMINAL_MASK) > 0; }
int getFreq(int *pos) { return mDict[(*pos)++] & 0xFF; } int getFreq(int *pos) { return mDict[(*pos)++] & 0xFF; }
int getCount(int *pos) { return mDict[(*pos)++] & 0xFF; } int getCount(int *pos) { return mDict[(*pos)++] & 0xFF; }
unsigned short getChar(int *pos); unsigned short getChar(int *pos);