a380cfd8e0
* update bleve to master b17287a86f6cac923a5d886e10618df994eeb54b6724eac2e3b8dde89cfbe3a2 * remove unused pkg from dep file * change bleve from master to recent revision
349 lines
8.4 KiB
Go
349 lines
8.4 KiB
Go
// Copyright (c) 2018 Couchbase, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package levenshtein2
|
|
|
|
import (
|
|
"crypto/md5"
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
)
|
|
|
|
type ParametricState struct {
|
|
shapeID uint32
|
|
offset uint32
|
|
}
|
|
|
|
func newParametricState() ParametricState {
|
|
return ParametricState{}
|
|
}
|
|
|
|
func (ps *ParametricState) isDeadEnd() bool {
|
|
return ps.shapeID == 0
|
|
}
|
|
|
|
type Transition struct {
|
|
destShapeID uint32
|
|
deltaOffset uint32
|
|
}
|
|
|
|
func (t *Transition) apply(state ParametricState) ParametricState {
|
|
ps := ParametricState{
|
|
shapeID: t.destShapeID}
|
|
// don't need any offset if we are in the dead state,
|
|
// this ensures we have only one dead state.
|
|
if t.destShapeID != 0 {
|
|
ps.offset = state.offset + t.deltaOffset
|
|
}
|
|
|
|
return ps
|
|
}
|
|
|
|
type ParametricStateIndex struct {
|
|
stateIndex []uint32
|
|
stateQueue []ParametricState
|
|
numOffsets uint32
|
|
}
|
|
|
|
func newParametricStateIndex(queryLen,
|
|
numParamState uint32) ParametricStateIndex {
|
|
numOffsets := queryLen + 1
|
|
if numParamState == 0 {
|
|
numParamState = numOffsets
|
|
}
|
|
maxNumStates := numParamState * numOffsets
|
|
psi := ParametricStateIndex{
|
|
stateIndex: make([]uint32, maxNumStates),
|
|
stateQueue: make([]ParametricState, 0, 150),
|
|
numOffsets: numOffsets,
|
|
}
|
|
|
|
for i := uint32(0); i < maxNumStates; i++ {
|
|
psi.stateIndex[i] = math.MaxUint32
|
|
}
|
|
return psi
|
|
}
|
|
|
|
func (psi *ParametricStateIndex) numStates() int {
|
|
return len(psi.stateQueue)
|
|
}
|
|
|
|
func (psi *ParametricStateIndex) maxNumStates() int {
|
|
return len(psi.stateIndex)
|
|
}
|
|
|
|
func (psi *ParametricStateIndex) get(stateID uint32) ParametricState {
|
|
return psi.stateQueue[stateID]
|
|
}
|
|
|
|
func (psi *ParametricStateIndex) getOrAllocate(ps ParametricState) uint32 {
|
|
bucket := ps.shapeID*psi.numOffsets + ps.offset
|
|
if bucket < uint32(len(psi.stateIndex)) &&
|
|
psi.stateIndex[bucket] != math.MaxUint32 {
|
|
return psi.stateIndex[bucket]
|
|
}
|
|
nState := uint32(len(psi.stateQueue))
|
|
psi.stateQueue = append(psi.stateQueue, ps)
|
|
|
|
psi.stateIndex[bucket] = nState
|
|
return nState
|
|
}
|
|
|
|
type ParametricDFA struct {
|
|
distance []uint8
|
|
transitions []Transition
|
|
maxDistance uint8
|
|
transitionStride uint32
|
|
diameter uint32
|
|
}
|
|
|
|
func (pdfa *ParametricDFA) initialState() ParametricState {
|
|
return ParametricState{shapeID: 1}
|
|
}
|
|
|
|
// Returns true iff whatever characters come afterward,
|
|
// we will never reach a shorter distance
|
|
func (pdfa *ParametricDFA) isPrefixSink(state ParametricState, queryLen uint32) bool {
|
|
if state.isDeadEnd() {
|
|
return true
|
|
}
|
|
|
|
remOffset := queryLen - state.offset
|
|
if remOffset < pdfa.diameter {
|
|
stateDistances := pdfa.distance[pdfa.diameter*state.shapeID:]
|
|
prefixDistance := stateDistances[remOffset]
|
|
if prefixDistance > pdfa.maxDistance {
|
|
return false
|
|
}
|
|
|
|
for _, d := range stateDistances {
|
|
if d < prefixDistance {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (pdfa *ParametricDFA) numStates() int {
|
|
return len(pdfa.transitions) / int(pdfa.transitionStride)
|
|
}
|
|
|
|
func min(x, y uint32) uint32 {
|
|
if x < y {
|
|
return x
|
|
}
|
|
return y
|
|
}
|
|
|
|
func (pdfa *ParametricDFA) transition(state ParametricState,
|
|
chi uint32) Transition {
|
|
return pdfa.transitions[pdfa.transitionStride*state.shapeID+chi]
|
|
}
|
|
|
|
func (pdfa *ParametricDFA) getDistance(state ParametricState,
|
|
qLen uint32) Distance {
|
|
remainingOffset := qLen - state.offset
|
|
if state.isDeadEnd() || remainingOffset >= pdfa.diameter {
|
|
return Atleast{d: pdfa.maxDistance + 1}
|
|
}
|
|
dist := pdfa.distance[int(pdfa.diameter*state.shapeID)+int(remainingOffset)]
|
|
if dist > pdfa.maxDistance {
|
|
return Atleast{d: dist}
|
|
}
|
|
return Exact{d: dist}
|
|
}
|
|
|
|
func (pdfa *ParametricDFA) computeDistance(left, right string) Distance {
|
|
state := pdfa.initialState()
|
|
leftChars := []rune(left)
|
|
for _, chr := range []rune(right) {
|
|
start := state.offset
|
|
stop := min(start+pdfa.diameter, uint32(len(leftChars)))
|
|
chi := characteristicVector(leftChars[start:stop], chr)
|
|
transition := pdfa.transition(state, uint32(chi))
|
|
state = transition.apply(state)
|
|
if state.isDeadEnd() {
|
|
return Atleast{d: pdfa.maxDistance + 1}
|
|
}
|
|
}
|
|
return pdfa.getDistance(state, uint32(len(left)))
|
|
}
|
|
|
|
func (pdfa *ParametricDFA) buildDfa(query string, distance uint8,
|
|
prefix bool) (*DFA, error) {
|
|
qLen := uint32(len([]rune(query)))
|
|
alphabet := queryChars(query)
|
|
|
|
psi := newParametricStateIndex(qLen, uint32(pdfa.numStates()))
|
|
maxNumStates := psi.maxNumStates()
|
|
deadEndStateID := psi.getOrAllocate(newParametricState())
|
|
if deadEndStateID != 0 {
|
|
return nil, fmt.Errorf("Invalid dead end state")
|
|
}
|
|
|
|
initialStateID := psi.getOrAllocate(pdfa.initialState())
|
|
dfaBuilder := withMaxStates(uint32(maxNumStates))
|
|
mask := uint32((1 << pdfa.diameter) - 1)
|
|
|
|
var stateID int
|
|
for stateID = 0; stateID < StateLimit; stateID++ {
|
|
if stateID == psi.numStates() {
|
|
break
|
|
}
|
|
state := psi.get(uint32(stateID))
|
|
if prefix && pdfa.isPrefixSink(state, qLen) {
|
|
distance := pdfa.getDistance(state, qLen)
|
|
dfaBuilder.addState(uint32(stateID), uint32(stateID), distance)
|
|
} else {
|
|
transition := pdfa.transition(state, 0)
|
|
defSuccessor := transition.apply(state)
|
|
defSuccessorID := psi.getOrAllocate(defSuccessor)
|
|
distance := pdfa.getDistance(state, qLen)
|
|
stateBuilder, err := dfaBuilder.addState(uint32(stateID), defSuccessorID, distance)
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parametric_dfa: buildDfa, err: %v", err)
|
|
}
|
|
|
|
alphabet.resetNext()
|
|
chr, cv, err := alphabet.next()
|
|
for err == nil {
|
|
chi := cv.shiftAndMask(state.offset, mask)
|
|
|
|
transition := pdfa.transition(state, chi)
|
|
|
|
destState := transition.apply(state)
|
|
|
|
destStateID := psi.getOrAllocate(destState)
|
|
|
|
stateBuilder.addTransition(chr, destStateID)
|
|
|
|
chr, cv, err = alphabet.next()
|
|
}
|
|
}
|
|
}
|
|
|
|
if stateID == StateLimit {
|
|
return nil, ErrTooManyStates
|
|
}
|
|
|
|
dfaBuilder.setInitialState(initialStateID)
|
|
return dfaBuilder.build(distance), nil
|
|
}
|
|
|
|
func fromNfa(nfa *LevenshteinNFA) (*ParametricDFA, error) {
|
|
lookUp := newHash()
|
|
lookUp.getOrAllocate(*newMultiState())
|
|
initialState := nfa.initialStates()
|
|
lookUp.getOrAllocate(*initialState)
|
|
|
|
maxDistance := nfa.maxDistance()
|
|
msDiameter := nfa.msDiameter()
|
|
|
|
numChi := 1 << msDiameter
|
|
chiValues := make([]uint64, numChi)
|
|
for i := 0; i < numChi; i++ {
|
|
chiValues[i] = uint64(i)
|
|
}
|
|
|
|
transitions := make([]Transition, 0, numChi*int(msDiameter))
|
|
var stateID int
|
|
for stateID = 0; stateID < StateLimit; stateID++ {
|
|
if stateID == len(lookUp.items) {
|
|
break
|
|
}
|
|
|
|
for _, chi := range chiValues {
|
|
destMs := newMultiState()
|
|
|
|
ms := lookUp.getFromID(stateID)
|
|
|
|
nfa.transition(ms, destMs, chi)
|
|
|
|
translation := destMs.normalize()
|
|
|
|
destID := lookUp.getOrAllocate(*destMs)
|
|
|
|
transitions = append(transitions, Transition{
|
|
destShapeID: uint32(destID),
|
|
deltaOffset: translation,
|
|
})
|
|
}
|
|
}
|
|
|
|
if stateID == StateLimit {
|
|
return nil, ErrTooManyStates
|
|
}
|
|
|
|
ns := len(lookUp.items)
|
|
diameter := int(msDiameter)
|
|
|
|
distances := make([]uint8, 0, diameter*ns)
|
|
for stateID := 0; stateID < ns; stateID++ {
|
|
ms := lookUp.getFromID(stateID)
|
|
for offset := 0; offset < diameter; offset++ {
|
|
dist := nfa.multistateDistance(ms, uint32(offset))
|
|
distances = append(distances, dist.distance())
|
|
}
|
|
}
|
|
|
|
return &ParametricDFA{
|
|
diameter: uint32(msDiameter),
|
|
transitions: transitions,
|
|
maxDistance: maxDistance,
|
|
transitionStride: uint32(numChi),
|
|
distance: distances,
|
|
}, nil
|
|
}
|
|
|
|
type hash struct {
|
|
index map[[16]byte]int
|
|
items []MultiState
|
|
}
|
|
|
|
func newHash() *hash {
|
|
return &hash{
|
|
index: make(map[[16]byte]int, 100),
|
|
items: make([]MultiState, 0, 100),
|
|
}
|
|
}
|
|
|
|
func (h *hash) getOrAllocate(m MultiState) int {
|
|
size := len(h.items)
|
|
var exists bool
|
|
var pos int
|
|
md5 := getHash(&m)
|
|
if pos, exists = h.index[md5]; !exists {
|
|
h.index[md5] = size
|
|
pos = size
|
|
h.items = append(h.items, m)
|
|
}
|
|
return pos
|
|
}
|
|
|
|
func (h *hash) getFromID(id int) *MultiState {
|
|
return &h.items[id]
|
|
}
|
|
|
|
func getHash(ms *MultiState) [16]byte {
|
|
msBytes := []byte{}
|
|
for _, state := range ms.states {
|
|
jsonBytes, _ := json.Marshal(&state)
|
|
msBytes = append(msBytes, jsonBytes...)
|
|
}
|
|
return md5.Sum(msBytes)
|
|
}
|