5e6a008fba
This PR adds basic repository LFS management UI including the ability to find all possible pointers within the repository. Locks are not managed at present but would be addable through some simple additions. * Add basic repository lfs management * add auto-associate function * Add functionality to find commits with this lfs file * Add link to find commits on the lfs file view * Adjust commit view to state the likely branch causing the commit * Only read Oid from database
551 lines
15 KiB
Go
551 lines
15 KiB
Go
// Copyright 2019 The Gitea Authors. All rights reserved.
|
|
// Use of this source code is governed by a MIT-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package repo
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"fmt"
|
|
gotemplate "html/template"
|
|
"io"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"code.gitea.io/gitea/models"
|
|
"code.gitea.io/gitea/modules/base"
|
|
"code.gitea.io/gitea/modules/charset"
|
|
"code.gitea.io/gitea/modules/context"
|
|
"code.gitea.io/gitea/modules/git"
|
|
"code.gitea.io/gitea/modules/git/pipeline"
|
|
"code.gitea.io/gitea/modules/lfs"
|
|
"code.gitea.io/gitea/modules/log"
|
|
"code.gitea.io/gitea/modules/setting"
|
|
|
|
"github.com/mcuadros/go-version"
|
|
"github.com/unknwon/com"
|
|
gogit "gopkg.in/src-d/go-git.v4"
|
|
"gopkg.in/src-d/go-git.v4/plumbing"
|
|
"gopkg.in/src-d/go-git.v4/plumbing/object"
|
|
)
|
|
|
|
const (
|
|
tplSettingsLFS base.TplName = "repo/settings/lfs"
|
|
tplSettingsLFSFile base.TplName = "repo/settings/lfs_file"
|
|
tplSettingsLFSFileFind base.TplName = "repo/settings/lfs_file_find"
|
|
tplSettingsLFSPointers base.TplName = "repo/settings/lfs_pointers"
|
|
)
|
|
|
|
// LFSFiles shows a repository's LFS files
|
|
func LFSFiles(ctx *context.Context) {
|
|
if !setting.LFS.StartServer {
|
|
ctx.NotFound("LFSFiles", nil)
|
|
return
|
|
}
|
|
page := ctx.QueryInt("page")
|
|
if page <= 1 {
|
|
page = 1
|
|
}
|
|
total, err := ctx.Repo.Repository.CountLFSMetaObjects()
|
|
if err != nil {
|
|
ctx.ServerError("LFSFiles", err)
|
|
return
|
|
}
|
|
|
|
pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5)
|
|
ctx.Data["Title"] = ctx.Tr("repo.settings.lfs")
|
|
ctx.Data["PageIsSettingsLFS"] = true
|
|
lfsMetaObjects, err := ctx.Repo.Repository.GetLFSMetaObjects(pager.Paginater.Current(), setting.UI.ExplorePagingNum)
|
|
if err != nil {
|
|
ctx.ServerError("LFSFiles", err)
|
|
return
|
|
}
|
|
ctx.Data["LFSFiles"] = lfsMetaObjects
|
|
ctx.Data["Page"] = pager
|
|
ctx.HTML(200, tplSettingsLFS)
|
|
}
|
|
|
|
// LFSFileGet serves a single LFS file
|
|
func LFSFileGet(ctx *context.Context) {
|
|
if !setting.LFS.StartServer {
|
|
ctx.NotFound("LFSFileGet", nil)
|
|
return
|
|
}
|
|
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
|
|
oid := ctx.Params("oid")
|
|
ctx.Data["Title"] = oid
|
|
ctx.Data["PageIsSettingsLFS"] = true
|
|
meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(oid)
|
|
if err != nil {
|
|
if err == models.ErrLFSObjectNotExist {
|
|
ctx.NotFound("LFSFileGet", nil)
|
|
return
|
|
}
|
|
ctx.ServerError("LFSFileGet", err)
|
|
return
|
|
}
|
|
ctx.Data["LFSFile"] = meta
|
|
dataRc, err := lfs.ReadMetaObject(meta)
|
|
if err != nil {
|
|
ctx.ServerError("LFSFileGet", err)
|
|
return
|
|
}
|
|
defer dataRc.Close()
|
|
buf := make([]byte, 1024)
|
|
n, err := dataRc.Read(buf)
|
|
if err != nil {
|
|
ctx.ServerError("Data", err)
|
|
return
|
|
}
|
|
buf = buf[:n]
|
|
|
|
isTextFile := base.IsTextFile(buf)
|
|
ctx.Data["IsTextFile"] = isTextFile
|
|
|
|
fileSize := meta.Size
|
|
ctx.Data["FileSize"] = meta.Size
|
|
ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, "direct")
|
|
switch {
|
|
case isTextFile:
|
|
if fileSize >= setting.UI.MaxDisplayFileSize {
|
|
ctx.Data["IsFileTooLarge"] = true
|
|
break
|
|
}
|
|
|
|
d, _ := ioutil.ReadAll(dataRc)
|
|
buf = charset.ToUTF8WithFallback(append(buf, d...))
|
|
|
|
// Building code view blocks with line number on server side.
|
|
var fileContent string
|
|
if content, err := charset.ToUTF8WithErr(buf); err != nil {
|
|
log.Error("ToUTF8WithErr: %v", err)
|
|
fileContent = string(buf)
|
|
} else {
|
|
fileContent = content
|
|
}
|
|
|
|
var output bytes.Buffer
|
|
lines := strings.Split(fileContent, "\n")
|
|
//Remove blank line at the end of file
|
|
if len(lines) > 0 && lines[len(lines)-1] == "" {
|
|
lines = lines[:len(lines)-1]
|
|
}
|
|
for index, line := range lines {
|
|
line = gotemplate.HTMLEscapeString(line)
|
|
if index != len(lines)-1 {
|
|
line += "\n"
|
|
}
|
|
output.WriteString(fmt.Sprintf(`<li class="L%d" rel="L%d">%s</li>`, index+1, index+1, line))
|
|
}
|
|
ctx.Data["FileContent"] = gotemplate.HTML(output.String())
|
|
|
|
output.Reset()
|
|
for i := 0; i < len(lines); i++ {
|
|
output.WriteString(fmt.Sprintf(`<span id="L%d">%d</span>`, i+1, i+1))
|
|
}
|
|
ctx.Data["LineNums"] = gotemplate.HTML(output.String())
|
|
|
|
case base.IsPDFFile(buf):
|
|
ctx.Data["IsPDFFile"] = true
|
|
case base.IsVideoFile(buf):
|
|
ctx.Data["IsVideoFile"] = true
|
|
case base.IsAudioFile(buf):
|
|
ctx.Data["IsAudioFile"] = true
|
|
case base.IsImageFile(buf):
|
|
ctx.Data["IsImageFile"] = true
|
|
}
|
|
ctx.HTML(200, tplSettingsLFSFile)
|
|
}
|
|
|
|
// LFSDelete disassociates the provided oid from the repository and if the lfs file is no longer associated with any repositories - deletes it
|
|
func LFSDelete(ctx *context.Context) {
|
|
if !setting.LFS.StartServer {
|
|
ctx.NotFound("LFSDelete", nil)
|
|
return
|
|
}
|
|
oid := ctx.Params("oid")
|
|
count, err := ctx.Repo.Repository.RemoveLFSMetaObjectByOid(oid)
|
|
if err != nil {
|
|
ctx.ServerError("LFSDelete", err)
|
|
return
|
|
}
|
|
// FIXME: Warning: the LFS store is not locked - and can't be locked - there could be a race condition here
|
|
// Please note a similar condition happens in models/repo.go DeleteRepository
|
|
if count == 0 {
|
|
oidPath := filepath.Join(oid[0:2], oid[2:4], oid[4:])
|
|
err = os.Remove(filepath.Join(setting.LFS.ContentPath, oidPath))
|
|
if err != nil {
|
|
ctx.ServerError("LFSDelete", err)
|
|
return
|
|
}
|
|
}
|
|
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
|
|
}
|
|
|
|
type lfsResult struct {
|
|
Name string
|
|
SHA string
|
|
Summary string
|
|
When time.Time
|
|
ParentHashes []plumbing.Hash
|
|
BranchName string
|
|
FullCommitName string
|
|
}
|
|
|
|
type lfsResultSlice []*lfsResult
|
|
|
|
func (a lfsResultSlice) Len() int { return len(a) }
|
|
func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
|
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
|
|
|
|
// LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha
|
|
func LFSFileFind(ctx *context.Context) {
|
|
if !setting.LFS.StartServer {
|
|
ctx.NotFound("LFSFind", nil)
|
|
return
|
|
}
|
|
oid := ctx.Query("oid")
|
|
size := ctx.QueryInt64("size")
|
|
if len(oid) == 0 || size == 0 {
|
|
ctx.NotFound("LFSFind", nil)
|
|
return
|
|
}
|
|
sha := ctx.Query("sha")
|
|
ctx.Data["Title"] = oid
|
|
ctx.Data["PageIsSettingsLFS"] = true
|
|
var hash plumbing.Hash
|
|
if len(sha) == 0 {
|
|
meta := models.LFSMetaObject{Oid: oid, Size: size}
|
|
pointer := meta.Pointer()
|
|
hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer))
|
|
sha = hash.String()
|
|
} else {
|
|
hash = plumbing.NewHash(sha)
|
|
}
|
|
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
|
|
ctx.Data["Oid"] = oid
|
|
ctx.Data["Size"] = size
|
|
ctx.Data["SHA"] = sha
|
|
|
|
resultsMap := map[string]*lfsResult{}
|
|
results := make([]*lfsResult, 0)
|
|
|
|
basePath := ctx.Repo.Repository.RepoPath()
|
|
gogitRepo := ctx.Repo.GitRepo.GoGitRepo()
|
|
|
|
commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
|
|
Order: gogit.LogOrderCommitterTime,
|
|
All: true,
|
|
})
|
|
if err != nil {
|
|
log.Error("Failed to get GoGit CommitsIter: %v", err)
|
|
ctx.ServerError("LFSFind: Iterate Commits", err)
|
|
return
|
|
}
|
|
|
|
err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
|
|
tree, err := gitCommit.Tree()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
treeWalker := object.NewTreeWalker(tree, true, nil)
|
|
defer treeWalker.Close()
|
|
for {
|
|
name, entry, err := treeWalker.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if entry.Hash == hash {
|
|
result := lfsResult{
|
|
Name: name,
|
|
SHA: gitCommit.Hash.String(),
|
|
Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0],
|
|
When: gitCommit.Author.When,
|
|
ParentHashes: gitCommit.ParentHashes,
|
|
}
|
|
resultsMap[gitCommit.Hash.String()+":"+name] = &result
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil && err != io.EOF {
|
|
log.Error("Failure in CommitIter.ForEach: %v", err)
|
|
ctx.ServerError("LFSFind: IterateCommits ForEach", err)
|
|
return
|
|
}
|
|
|
|
for _, result := range resultsMap {
|
|
hasParent := false
|
|
for _, parentHash := range result.ParentHashes {
|
|
if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
|
|
break
|
|
}
|
|
}
|
|
if !hasParent {
|
|
results = append(results, result)
|
|
}
|
|
}
|
|
|
|
sort.Sort(lfsResultSlice(results))
|
|
|
|
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
|
|
shasToNameReader, shasToNameWriter := io.Pipe()
|
|
nameRevStdinReader, nameRevStdinWriter := io.Pipe()
|
|
errChan := make(chan error, 1)
|
|
wg := sync.WaitGroup{}
|
|
wg.Add(3)
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
scanner := bufio.NewScanner(nameRevStdinReader)
|
|
i := 0
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
if len(line) == 0 {
|
|
continue
|
|
}
|
|
result := results[i]
|
|
result.FullCommitName = line
|
|
result.BranchName = strings.Split(line, "~")[0]
|
|
i++
|
|
}
|
|
}()
|
|
go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
|
|
go func() {
|
|
defer wg.Done()
|
|
defer shasToNameWriter.Close()
|
|
for _, result := range results {
|
|
i := 0
|
|
if i < len(result.SHA) {
|
|
n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
|
|
if err != nil {
|
|
errChan <- err
|
|
break
|
|
}
|
|
i += n
|
|
}
|
|
n := 0
|
|
for n < 1 {
|
|
n, err = shasToNameWriter.Write([]byte{'\n'})
|
|
if err != nil {
|
|
errChan <- err
|
|
break
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
}()
|
|
|
|
wg.Wait()
|
|
|
|
select {
|
|
case err, has := <-errChan:
|
|
if has {
|
|
ctx.ServerError("LFSPointerFiles", err)
|
|
}
|
|
default:
|
|
}
|
|
|
|
ctx.Data["Results"] = results
|
|
ctx.HTML(200, tplSettingsLFSFileFind)
|
|
}
|
|
|
|
// LFSPointerFiles will search the repository for pointer files and report which are missing LFS files in the content store
|
|
func LFSPointerFiles(ctx *context.Context) {
|
|
if !setting.LFS.StartServer {
|
|
ctx.NotFound("LFSFileGet", nil)
|
|
return
|
|
}
|
|
ctx.Data["PageIsSettingsLFS"] = true
|
|
binVersion, err := git.BinVersion()
|
|
if err != nil {
|
|
log.Fatal("Error retrieving git version: %v", err)
|
|
}
|
|
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
|
|
|
|
basePath := ctx.Repo.Repository.RepoPath()
|
|
|
|
pointerChan := make(chan pointerResult)
|
|
|
|
catFileCheckReader, catFileCheckWriter := io.Pipe()
|
|
shasToBatchReader, shasToBatchWriter := io.Pipe()
|
|
catFileBatchReader, catFileBatchWriter := io.Pipe()
|
|
errChan := make(chan error, 1)
|
|
wg := sync.WaitGroup{}
|
|
wg.Add(5)
|
|
|
|
var numPointers, numAssociated, numNoExist, numAssociatable int
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
pointers := make([]pointerResult, 0, 50)
|
|
for pointer := range pointerChan {
|
|
pointers = append(pointers, pointer)
|
|
if pointer.InRepo {
|
|
numAssociated++
|
|
}
|
|
if !pointer.Exists {
|
|
numNoExist++
|
|
}
|
|
if !pointer.InRepo && pointer.Accessible {
|
|
numAssociatable++
|
|
}
|
|
}
|
|
numPointers = len(pointers)
|
|
ctx.Data["Pointers"] = pointers
|
|
ctx.Data["NumPointers"] = numPointers
|
|
ctx.Data["NumAssociated"] = numAssociated
|
|
ctx.Data["NumAssociatable"] = numAssociatable
|
|
ctx.Data["NumNoExist"] = numNoExist
|
|
ctx.Data["NumNotAssociated"] = numPointers - numAssociated
|
|
}()
|
|
go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User)
|
|
go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath)
|
|
go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
|
|
if !version.Compare(binVersion, "2.6.0", ">=") {
|
|
revListReader, revListWriter := io.Pipe()
|
|
shasToCheckReader, shasToCheckWriter := io.Pipe()
|
|
wg.Add(2)
|
|
go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath)
|
|
go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
|
|
go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan)
|
|
} else {
|
|
go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan)
|
|
}
|
|
wg.Wait()
|
|
|
|
select {
|
|
case err, has := <-errChan:
|
|
if has {
|
|
ctx.ServerError("LFSPointerFiles", err)
|
|
}
|
|
default:
|
|
}
|
|
ctx.HTML(200, tplSettingsLFSPointers)
|
|
}
|
|
|
|
type pointerResult struct {
|
|
SHA string
|
|
Oid string
|
|
Size int64
|
|
InRepo bool
|
|
Exists bool
|
|
Accessible bool
|
|
}
|
|
|
|
func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
|
|
defer wg.Done()
|
|
defer catFileBatchReader.Close()
|
|
contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath}
|
|
|
|
bufferedReader := bufio.NewReader(catFileBatchReader)
|
|
buf := make([]byte, 1025)
|
|
for {
|
|
// File descriptor line: sha
|
|
sha, err := bufferedReader.ReadString(' ')
|
|
if err != nil {
|
|
_ = catFileBatchReader.CloseWithError(err)
|
|
break
|
|
}
|
|
// Throw away the blob
|
|
if _, err := bufferedReader.ReadString(' '); err != nil {
|
|
_ = catFileBatchReader.CloseWithError(err)
|
|
break
|
|
}
|
|
sizeStr, err := bufferedReader.ReadString('\n')
|
|
if err != nil {
|
|
_ = catFileBatchReader.CloseWithError(err)
|
|
break
|
|
}
|
|
size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
|
|
if err != nil {
|
|
_ = catFileBatchReader.CloseWithError(err)
|
|
break
|
|
}
|
|
pointerBuf := buf[:size+1]
|
|
if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
|
|
_ = catFileBatchReader.CloseWithError(err)
|
|
break
|
|
}
|
|
pointerBuf = pointerBuf[:size]
|
|
// Now we need to check if the pointerBuf is an LFS pointer
|
|
pointer := lfs.IsPointerFile(&pointerBuf)
|
|
if pointer == nil {
|
|
continue
|
|
}
|
|
|
|
result := pointerResult{
|
|
SHA: strings.TrimSpace(sha),
|
|
Oid: pointer.Oid,
|
|
Size: pointer.Size,
|
|
}
|
|
|
|
// Then we need to check that this pointer is in the db
|
|
if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil {
|
|
if err != models.ErrLFSObjectNotExist {
|
|
_ = catFileBatchReader.CloseWithError(err)
|
|
break
|
|
}
|
|
} else {
|
|
result.InRepo = true
|
|
}
|
|
|
|
result.Exists = contentStore.Exists(pointer)
|
|
|
|
if result.Exists {
|
|
if !result.InRepo {
|
|
// Can we fix?
|
|
// OK well that's "simple"
|
|
// - we need to check whether current user has access to a repo that has access to the file
|
|
result.Accessible, err = models.LFSObjectAccessible(user, result.Oid)
|
|
if err != nil {
|
|
_ = catFileBatchReader.CloseWithError(err)
|
|
break
|
|
}
|
|
} else {
|
|
result.Accessible = true
|
|
}
|
|
}
|
|
pointerChan <- result
|
|
}
|
|
close(pointerChan)
|
|
}
|
|
|
|
// LFSAutoAssociate auto associates accessible lfs files
|
|
func LFSAutoAssociate(ctx *context.Context) {
|
|
if !setting.LFS.StartServer {
|
|
ctx.NotFound("LFSAutoAssociate", nil)
|
|
return
|
|
}
|
|
oids := ctx.QueryStrings("oid")
|
|
metas := make([]*models.LFSMetaObject, len(oids))
|
|
for i, oid := range oids {
|
|
idx := strings.IndexRune(oid, ' ')
|
|
if idx < 0 || idx+1 > len(oid) {
|
|
ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s", oid))
|
|
return
|
|
}
|
|
var err error
|
|
metas[i] = &models.LFSMetaObject{}
|
|
metas[i].Size, err = com.StrTo(oid[idx+1:]).Int64()
|
|
if err != nil {
|
|
ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s %v", oid, err))
|
|
return
|
|
}
|
|
metas[i].Oid = oid[:idx]
|
|
//metas[i].RepositoryID = ctx.Repo.Repository.ID
|
|
}
|
|
if err := models.LFSAutoAssociate(metas, ctx.User, ctx.Repo.Repository.ID); err != nil {
|
|
ctx.ServerError("LFSAutoAssociate", err)
|
|
return
|
|
}
|
|
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
|
|
}
|