Add more tests and docs for issue indexer, add db indexer type for searching from database (#6144)

* add more tests and docs for issue indexer, add db indexer type for searching from database

* fix typo

* fix typo

* fix lint

* improve docs
release/v1.15
Lunny Xiao 2019-02-21 13:01:28 +08:00 committed by GitHub
parent 0751153613
commit 477ef46251
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 174 additions and 11 deletions

View File

@ -253,9 +253,19 @@ DB_RETRIES = 10
; Backoff time per DB retry (time.Duration) ; Backoff time per DB retry (time.Duration)
DB_RETRY_BACKOFF = 3s DB_RETRY_BACKOFF = 3s
[indexer] [indexer]
; Issue indexer type, currently support: bleve or db, default is bleve
ISSUE_INDEXER_TYPE = bleve
; Issue indexer storage path, available when ISSUE_INDEXER_TYPE is bleve
ISSUE_INDEXER_PATH = indexers/issues.bleve ISSUE_INDEXER_PATH = indexers/issues.bleve
; Issue indexer queue, currently support: channel or levelqueue, default is levelqueue
ISSUE_INDEXER_QUEUE_TYPE = levelqueue
; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the queue will be saved path,
; default is indexers/issues.queue
ISSUE_INDEXER_QUEUE_DIR = indexers/issues.queue
; Batch queue number, default is 20
ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20
; repo indexer by default disabled, since it uses a lot of disk space ; repo indexer by default disabled, since it uses a lot of disk space
REPO_INDEXER_ENABLED = false REPO_INDEXER_ENABLED = false
REPO_INDEXER_PATH = indexers/repos.bleve REPO_INDEXER_PATH = indexers/repos.bleve

View File

@ -154,7 +154,12 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
## Indexer (`indexer`) ## Indexer (`indexer`)
- `ISSUE_INDEXER_TYPE`: **bleve**: Issue indexer type, currently support: bleve or db, if it's db, below issue indexer item will be invalid.
- `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search. - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search.
- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently support: channel or levelqueue
- `ISSUE_INDEXER_QUEUE_DIR`: **indexers/issues.queue**: When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the queue will be saved path
- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number
- `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space). - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space).
- `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search. - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search.
- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. - `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.

View File

@ -82,6 +82,20 @@ menu:
- `PATH`: Tidb 或者 SQLite3 数据文件存放路径。 - `PATH`: Tidb 或者 SQLite3 数据文件存放路径。
- `LOG_SQL`: **true**: 显示生成的SQL默认为真。 - `LOG_SQL`: **true**: 显示生成的SQL默认为真。
## Indexer (`indexer`)
- `ISSUE_INDEXER_TYPE`: **bleve**: 工单索引类型,当前支持 `bleve``db`,当为 `db` 时其它工单索引项可不用设置。
- `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: 工单索引文件存放路径,当索引类型为 `bleve` 时有效。
- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: 工单索引队列类型,当前支持 `channel``levelqueue`
- `ISSUE_INDEXER_QUEUE_DIR`: **indexers/issues.queue**: 当 `ISSUE_INDEXER_QUEUE_TYPE``levelqueue` 时,保存索引队列的磁盘路径。
- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: 队列处理中批量提交数量。
- `REPO_INDEXER_ENABLED`: **false**: 是否启用代码搜索(启用后会占用比较大的磁盘空间)。
- `REPO_INDEXER_PATH`: **indexers/repos.bleve**: 用于代码搜索的索引文件路径。
- `UPDATE_BUFFER_LEN`: **20**: 代码索引请求的缓冲区长度。
- `MAX_FILE_SIZE`: **1048576**: 进行解析的源代码文件的最大长度,小于该值时才会索引。
## Security (`security`) ## Security (`security`)
- `INSTALL_LOCK`: 是否允许运行安装向导,(跟管理员账号有关,十分重要)。 - `INSTALL_LOCK`: 是否允许运行安装向导,(跟管理员账号有关,十分重要)。

View File

@ -1684,6 +1684,40 @@ func GetRepoIssueStats(repoID, uid int64, filterMode int, isPull bool) (numOpen
return openResult, closedResult return openResult, closedResult
} }
// SearchIssueIDsByKeyword search issues on database
func SearchIssueIDsByKeyword(kw string, repoID int64, limit, start int) (int64, []int64, error) {
var repoCond = builder.Eq{"repo_id": repoID}
var subQuery = builder.Select("id").From("issue").Where(repoCond)
var cond = builder.And(
repoCond,
builder.Or(
builder.Like{"name", kw},
builder.Like{"content", kw},
builder.In("id", builder.Select("issue_id").
From("comment").
Where(builder.And(
builder.Eq{"type": CommentTypeComment},
builder.In("issue_id", subQuery),
builder.Like{"content", kw},
)),
),
),
)
var ids = make([]int64, 0, limit)
err := x.Distinct("id").Table("issue").Where(cond).Limit(limit, start).Find(&ids)
if err != nil {
return 0, nil, err
}
total, err := x.Distinct("id").Table("issue").Where(cond).Count()
if err != nil {
return 0, nil, err
}
return total, ids, nil
}
func updateIssue(e Engine, issue *Issue) error { func updateIssue(e Engine, issue *Issue) error {
_, err := e.ID(issue.ID).AllCols().Update(issue) _, err := e.ID(issue.ID).AllCols().Update(issue)
if err != nil { if err != nil {

View File

@ -295,3 +295,28 @@ func TestIssue_loadTotalTimes(t *testing.T) {
assert.NoError(t, ms.loadTotalTimes(x)) assert.NoError(t, ms.loadTotalTimes(x))
assert.Equal(t, int64(3662), ms.TotalTrackedTime) assert.Equal(t, int64(3662), ms.TotalTrackedTime)
} }
func TestIssue_SearchIssueIDsByKeyword(t *testing.T) {
assert.NoError(t, PrepareTestDatabase())
total, ids, err := SearchIssueIDsByKeyword("issue2", 1, 10, 0)
assert.NoError(t, err)
assert.EqualValues(t, 1, total)
assert.EqualValues(t, []int64{2}, ids)
total, ids, err = SearchIssueIDsByKeyword("first", 1, 10, 0)
assert.NoError(t, err)
assert.EqualValues(t, 1, total)
assert.EqualValues(t, []int64{1}, ids)
total, ids, err = SearchIssueIDsByKeyword("for", 1, 10, 0)
assert.NoError(t, err)
assert.EqualValues(t, 4, total)
assert.EqualValues(t, []int64{1, 2, 3, 5}, ids)
// issue1's comment id 2
total, ids, err = SearchIssueIDsByKeyword("good", 1, 10, 0)
assert.NoError(t, err)
assert.EqualValues(t, 1, total)
assert.EqualValues(t, []int64{1}, ids)
}

View File

@ -0,0 +1,45 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package issues
import "code.gitea.io/gitea/models"
// DBIndexer implements Indexer inteface to use database's like search
type DBIndexer struct {
}
// Init dummy function
func (db *DBIndexer) Init() (bool, error) {
return false, nil
}
// Index dummy function
func (db *DBIndexer) Index(issue []*IndexerData) error {
return nil
}
// Delete dummy function
func (db *DBIndexer) Delete(ids ...int64) error {
return nil
}
// Search dummy function
func (db *DBIndexer) Search(kw string, repoID int64, limit, start int) (*SearchResult, error) {
total, ids, err := models.SearchIssueIDsByKeyword(kw, repoID, limit, start)
if err != nil {
return nil, err
}
var result = SearchResult{
Total: total,
Hits: make([]Match, 0, limit),
}
for _, id := range ids {
result.Hits = append(result.Hits, Match{
ID: id,
RepoID: repoID,
})
}
return &result, nil
}

View File

@ -33,7 +33,8 @@ type Match struct {
// SearchResult represents search results // SearchResult represents search results
type SearchResult struct { type SearchResult struct {
Hits []Match Total int64
Hits []Match
} }
// Indexer defines an inteface to indexer issues contents // Indexer defines an inteface to indexer issues contents
@ -54,6 +55,7 @@ var (
// all issue index done. // all issue index done.
func InitIssueIndexer(syncReindex bool) error { func InitIssueIndexer(syncReindex bool) error {
var populate bool var populate bool
var dummyQueue bool
switch setting.Indexer.IssueType { switch setting.Indexer.IssueType {
case "bleve": case "bleve":
issueIndexer = NewBleveIndexer(setting.Indexer.IssuePath) issueIndexer = NewBleveIndexer(setting.Indexer.IssuePath)
@ -62,10 +64,17 @@ func InitIssueIndexer(syncReindex bool) error {
return err return err
} }
populate = !exist populate = !exist
case "db":
issueIndexer = &DBIndexer{}
dummyQueue = true
default: default:
return fmt.Errorf("unknow issue indexer type: %s", setting.Indexer.IssueType) return fmt.Errorf("unknow issue indexer type: %s", setting.Indexer.IssueType)
} }
if dummyQueue {
return nil
}
var err error var err error
switch setting.Indexer.IssueIndexerQueueType { switch setting.Indexer.IssueIndexerQueueType {
case setting.LevelQueueType: case setting.LevelQueueType:

View File

@ -48,4 +48,8 @@ func TestSearchIssues(t *testing.T) {
ids, err = SearchIssuesByKeyword(1, "for") ids, err = SearchIssuesByKeyword(1, "for")
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, []int64{1, 2, 3, 5}, ids) assert.EqualValues(t, []int64{1, 2, 3, 5}, ids)
ids, err = SearchIssuesByKeyword(1, "good")
assert.NoError(t, err)
assert.EqualValues(t, []int64{1}, ids)
} }

View File

@ -7,5 +7,19 @@ package issues
// Queue defines an interface to save an issue indexer queue // Queue defines an interface to save an issue indexer queue
type Queue interface { type Queue interface {
Run() error Run() error
Push(*IndexerData) Push(*IndexerData) error
}
// DummyQueue represents an empty queue
type DummyQueue struct {
}
// Run starts to run the queue
func (b *DummyQueue) Run() error {
return nil
}
// Push pushes data to indexer
func (b *DummyQueue) Push(*IndexerData) error {
return nil
} }

View File

@ -33,6 +33,11 @@ func (c *ChannelQueue) Run() error {
for { for {
select { select {
case data := <-c.queue: case data := <-c.queue:
if data.IsDelete {
c.indexer.Delete(data.IDs...)
continue
}
datas = append(datas, data) datas = append(datas, data)
if len(datas) >= c.batchNumber { if len(datas) >= c.batchNumber {
c.indexer.Index(datas) c.indexer.Index(datas)
@ -51,6 +56,7 @@ func (c *ChannelQueue) Run() error {
} }
// Push will push the indexer data to queue // Push will push the indexer data to queue
func (c *ChannelQueue) Push(data *IndexerData) { func (c *ChannelQueue) Push(data *IndexerData) error {
c.queue <- data c.queue <- data
return nil
} }

View File

@ -94,14 +94,10 @@ func (l *LevelQueue) Run() error {
} }
// Push will push the indexer data to queue // Push will push the indexer data to queue
func (l *LevelQueue) Push(data *IndexerData) { func (l *LevelQueue) Push(data *IndexerData) error {
bs, err := json.Marshal(data) bs, err := json.Marshal(data)
if err != nil { if err != nil {
log.Error(4, "Marshal: %v", err) return err
return
}
err = l.queue.LPush(bs)
if err != nil {
log.Error(4, "LPush: %v", err)
} }
return l.queue.LPush(bs)
} }

View File

@ -38,6 +38,7 @@ var (
func newIndexerService() { func newIndexerService() {
sec := Cfg.Section("indexer") sec := Cfg.Section("indexer")
Indexer.IssueType = sec.Key("ISSUE_INDEXER_TYPE").MustString("bleve")
Indexer.IssuePath = sec.Key("ISSUE_INDEXER_PATH").MustString(path.Join(AppDataPath, "indexers/issues.bleve")) Indexer.IssuePath = sec.Key("ISSUE_INDEXER_PATH").MustString(path.Join(AppDataPath, "indexers/issues.bleve"))
if !filepath.IsAbs(Indexer.IssuePath) { if !filepath.IsAbs(Indexer.IssuePath) {
Indexer.IssuePath = path.Join(AppWorkPath, Indexer.IssuePath) Indexer.IssuePath = path.Join(AppWorkPath, Indexer.IssuePath)