gitea/vendor/github.com/couchbase/vellum/levenshtein2/parametric_dfa.go
Lunny Xiao a380cfd8e0 Update bleve dependency to latest master revision ()
* update bleve to master b17287a86f6cac923a5d886e10618df994eeb54b6724eac2e3b8dde89cfbe3a2

* remove unused pkg from dep file

* change bleve from master to recent revision
2019-02-17 19:50:26 -05:00

349 lines
8.4 KiB
Go

// Copyright (c) 2018 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package levenshtein2
import (
"crypto/md5"
"encoding/json"
"fmt"
"math"
)
type ParametricState struct {
shapeID uint32
offset uint32
}
func newParametricState() ParametricState {
return ParametricState{}
}
func (ps *ParametricState) isDeadEnd() bool {
return ps.shapeID == 0
}
type Transition struct {
destShapeID uint32
deltaOffset uint32
}
func (t *Transition) apply(state ParametricState) ParametricState {
ps := ParametricState{
shapeID: t.destShapeID}
// don't need any offset if we are in the dead state,
// this ensures we have only one dead state.
if t.destShapeID != 0 {
ps.offset = state.offset + t.deltaOffset
}
return ps
}
type ParametricStateIndex struct {
stateIndex []uint32
stateQueue []ParametricState
numOffsets uint32
}
func newParametricStateIndex(queryLen,
numParamState uint32) ParametricStateIndex {
numOffsets := queryLen + 1
if numParamState == 0 {
numParamState = numOffsets
}
maxNumStates := numParamState * numOffsets
psi := ParametricStateIndex{
stateIndex: make([]uint32, maxNumStates),
stateQueue: make([]ParametricState, 0, 150),
numOffsets: numOffsets,
}
for i := uint32(0); i < maxNumStates; i++ {
psi.stateIndex[i] = math.MaxUint32
}
return psi
}
func (psi *ParametricStateIndex) numStates() int {
return len(psi.stateQueue)
}
func (psi *ParametricStateIndex) maxNumStates() int {
return len(psi.stateIndex)
}
func (psi *ParametricStateIndex) get(stateID uint32) ParametricState {
return psi.stateQueue[stateID]
}
func (psi *ParametricStateIndex) getOrAllocate(ps ParametricState) uint32 {
bucket := ps.shapeID*psi.numOffsets + ps.offset
if bucket < uint32(len(psi.stateIndex)) &&
psi.stateIndex[bucket] != math.MaxUint32 {
return psi.stateIndex[bucket]
}
nState := uint32(len(psi.stateQueue))
psi.stateQueue = append(psi.stateQueue, ps)
psi.stateIndex[bucket] = nState
return nState
}
type ParametricDFA struct {
distance []uint8
transitions []Transition
maxDistance uint8
transitionStride uint32
diameter uint32
}
func (pdfa *ParametricDFA) initialState() ParametricState {
return ParametricState{shapeID: 1}
}
// Returns true iff whatever characters come afterward,
// we will never reach a shorter distance
func (pdfa *ParametricDFA) isPrefixSink(state ParametricState, queryLen uint32) bool {
if state.isDeadEnd() {
return true
}
remOffset := queryLen - state.offset
if remOffset < pdfa.diameter {
stateDistances := pdfa.distance[pdfa.diameter*state.shapeID:]
prefixDistance := stateDistances[remOffset]
if prefixDistance > pdfa.maxDistance {
return false
}
for _, d := range stateDistances {
if d < prefixDistance {
return false
}
}
return true
}
return false
}
func (pdfa *ParametricDFA) numStates() int {
return len(pdfa.transitions) / int(pdfa.transitionStride)
}
func min(x, y uint32) uint32 {
if x < y {
return x
}
return y
}
func (pdfa *ParametricDFA) transition(state ParametricState,
chi uint32) Transition {
return pdfa.transitions[pdfa.transitionStride*state.shapeID+chi]
}
func (pdfa *ParametricDFA) getDistance(state ParametricState,
qLen uint32) Distance {
remainingOffset := qLen - state.offset
if state.isDeadEnd() || remainingOffset >= pdfa.diameter {
return Atleast{d: pdfa.maxDistance + 1}
}
dist := pdfa.distance[int(pdfa.diameter*state.shapeID)+int(remainingOffset)]
if dist > pdfa.maxDistance {
return Atleast{d: dist}
}
return Exact{d: dist}
}
func (pdfa *ParametricDFA) computeDistance(left, right string) Distance {
state := pdfa.initialState()
leftChars := []rune(left)
for _, chr := range []rune(right) {
start := state.offset
stop := min(start+pdfa.diameter, uint32(len(leftChars)))
chi := characteristicVector(leftChars[start:stop], chr)
transition := pdfa.transition(state, uint32(chi))
state = transition.apply(state)
if state.isDeadEnd() {
return Atleast{d: pdfa.maxDistance + 1}
}
}
return pdfa.getDistance(state, uint32(len(left)))
}
func (pdfa *ParametricDFA) buildDfa(query string, distance uint8,
prefix bool) (*DFA, error) {
qLen := uint32(len([]rune(query)))
alphabet := queryChars(query)
psi := newParametricStateIndex(qLen, uint32(pdfa.numStates()))
maxNumStates := psi.maxNumStates()
deadEndStateID := psi.getOrAllocate(newParametricState())
if deadEndStateID != 0 {
return nil, fmt.Errorf("Invalid dead end state")
}
initialStateID := psi.getOrAllocate(pdfa.initialState())
dfaBuilder := withMaxStates(uint32(maxNumStates))
mask := uint32((1 << pdfa.diameter) - 1)
var stateID int
for stateID = 0; stateID < StateLimit; stateID++ {
if stateID == psi.numStates() {
break
}
state := psi.get(uint32(stateID))
if prefix && pdfa.isPrefixSink(state, qLen) {
distance := pdfa.getDistance(state, qLen)
dfaBuilder.addState(uint32(stateID), uint32(stateID), distance)
} else {
transition := pdfa.transition(state, 0)
defSuccessor := transition.apply(state)
defSuccessorID := psi.getOrAllocate(defSuccessor)
distance := pdfa.getDistance(state, qLen)
stateBuilder, err := dfaBuilder.addState(uint32(stateID), defSuccessorID, distance)
if err != nil {
return nil, fmt.Errorf("parametric_dfa: buildDfa, err: %v", err)
}
alphabet.resetNext()
chr, cv, err := alphabet.next()
for err == nil {
chi := cv.shiftAndMask(state.offset, mask)
transition := pdfa.transition(state, chi)
destState := transition.apply(state)
destStateID := psi.getOrAllocate(destState)
stateBuilder.addTransition(chr, destStateID)
chr, cv, err = alphabet.next()
}
}
}
if stateID == StateLimit {
return nil, ErrTooManyStates
}
dfaBuilder.setInitialState(initialStateID)
return dfaBuilder.build(distance), nil
}
func fromNfa(nfa *LevenshteinNFA) (*ParametricDFA, error) {
lookUp := newHash()
lookUp.getOrAllocate(*newMultiState())
initialState := nfa.initialStates()
lookUp.getOrAllocate(*initialState)
maxDistance := nfa.maxDistance()
msDiameter := nfa.msDiameter()
numChi := 1 << msDiameter
chiValues := make([]uint64, numChi)
for i := 0; i < numChi; i++ {
chiValues[i] = uint64(i)
}
transitions := make([]Transition, 0, numChi*int(msDiameter))
var stateID int
for stateID = 0; stateID < StateLimit; stateID++ {
if stateID == len(lookUp.items) {
break
}
for _, chi := range chiValues {
destMs := newMultiState()
ms := lookUp.getFromID(stateID)
nfa.transition(ms, destMs, chi)
translation := destMs.normalize()
destID := lookUp.getOrAllocate(*destMs)
transitions = append(transitions, Transition{
destShapeID: uint32(destID),
deltaOffset: translation,
})
}
}
if stateID == StateLimit {
return nil, ErrTooManyStates
}
ns := len(lookUp.items)
diameter := int(msDiameter)
distances := make([]uint8, 0, diameter*ns)
for stateID := 0; stateID < ns; stateID++ {
ms := lookUp.getFromID(stateID)
for offset := 0; offset < diameter; offset++ {
dist := nfa.multistateDistance(ms, uint32(offset))
distances = append(distances, dist.distance())
}
}
return &ParametricDFA{
diameter: uint32(msDiameter),
transitions: transitions,
maxDistance: maxDistance,
transitionStride: uint32(numChi),
distance: distances,
}, nil
}
type hash struct {
index map[[16]byte]int
items []MultiState
}
func newHash() *hash {
return &hash{
index: make(map[[16]byte]int, 100),
items: make([]MultiState, 0, 100),
}
}
func (h *hash) getOrAllocate(m MultiState) int {
size := len(h.items)
var exists bool
var pos int
md5 := getHash(&m)
if pos, exists = h.index[md5]; !exists {
h.index[md5] = size
pos = size
h.items = append(h.items, m)
}
return pos
}
func (h *hash) getFromID(id int) *MultiState {
return &h.items[id]
}
func getHash(ms *MultiState) [16]byte {
msBytes := []byte{}
for _, state := range ms.states {
jsonBytes, _ := json.Marshal(&state)
msBytes = append(msBytes, jsonBytes...)
}
return md5.Sum(msBytes)
}