gitea/routers/repo/download.go
KN4CK3R c03e488e14
Add LFS Migration and Mirror (#14726)
* Implemented LFS client.

* Implemented scanning for pointer files.

* Implemented downloading of lfs files.

* Moved model-dependent code into services.

* Removed models dependency. Added TryReadPointerFromBuffer.

* Migrated code from service to module.

* Centralised storage creation.

* Removed dependency from models.

* Moved ContentStore into modules.

* Share structs between server and client.

* Moved method to services.

* Implemented lfs download on clone.

* Implemented LFS sync on clone and mirror update.

* Added form fields.

* Updated templates.

* Fixed condition.

* Use alternate endpoint.

* Added missing methods.

* Fixed typo and make linter happy.

* Detached pointer parser from gogit dependency.

* Fixed TestGetLFSRange test.

* Added context to support cancellation.

* Use ReadFull to probably read more data.

* Removed duplicated code from models.

* Moved scan implementation into pointer_scanner_nogogit.

* Changed method name.

* Added comments.

* Added more/specific log/error messages.

* Embedded lfs.Pointer into models.LFSMetaObject.

* Moved code from models to module.

* Moved code from models to module.

* Moved code from models to module.

* Reduced pointer usage.

* Embedded type.

* Use promoted fields.

* Fixed unexpected eof.

* Added unit tests.

* Implemented migration of local file paths.

* Show an error on invalid LFS endpoints.

* Hide settings if not used.

* Added LFS info to mirror struct.

* Fixed comment.

* Check LFS endpoint.

* Manage LFS settings from mirror page.

* Fixed selector.

* Adjusted selector.

* Added more tests.

* Added local filesystem migration test.

* Fixed typo.

* Reset settings.

* Added special windows path handling.

* Added unit test for HTTPClient.

* Added unit test for BasicTransferAdapter.

* Moved into util package.

* Test if LFS endpoint is allowed.

* Added support for git://

* Just use a static placeholder as the displayed url may be invalid.

* Reverted to original code.

* Added "Advanced Settings".

* Updated wording.

* Added discovery info link.

* Implemented suggestion.

* Fixed missing format parameter.

* Added Pointer.IsValid().

* Always remove model on error.

* Added suggestions.

* Use channel instead of array.

* Update routers/repo/migrate.go

* fmt

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: zeripath <art27@cantab.net>
2021-04-08 18:25:57 -04:00

182 lines
4.9 KiB
Go

// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repo
import (
"fmt"
"io"
"path"
"strings"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
)
// ServeData download file from io.Reader
func ServeData(ctx *context.Context, name string, size int64, reader io.Reader) error {
buf := make([]byte, 1024)
n, err := reader.Read(buf)
if err != nil && err != io.EOF {
return err
}
if n >= 0 {
buf = buf[:n]
}
ctx.Resp.Header().Set("Cache-Control", "public,max-age=86400")
if size >= 0 {
ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", size))
} else {
log.Error("ServeData called to serve data: %s with size < 0: %d", name, size)
}
name = path.Base(name)
// Google Chrome dislike commas in filenames, so let's change it to a space
name = strings.ReplaceAll(name, ",", " ")
if base.IsTextFile(buf) || ctx.QueryBool("render") {
cs, err := charset.DetectEncoding(buf)
if err != nil {
log.Error("Detect raw file %s charset failed: %v, using by default utf-8", name, err)
cs = "utf-8"
}
ctx.Resp.Header().Set("Content-Type", "text/plain; charset="+strings.ToLower(cs))
} else if base.IsImageFile(buf) || base.IsPDFFile(buf) {
ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"`, name))
ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
if base.IsSVGImageFile(buf) {
ctx.Resp.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox")
ctx.Resp.Header().Set("X-Content-Type-Options", "nosniff")
ctx.Resp.Header().Set("Content-Type", base.SVGMimeType)
}
} else {
ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name))
ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
}
_, err = ctx.Resp.Write(buf)
if err != nil {
return err
}
_, err = io.Copy(ctx.Resp, reader)
return err
}
// ServeBlob download a git.Blob
func ServeBlob(ctx *context.Context, blob *git.Blob) error {
dataRc, err := blob.DataAsync()
if err != nil {
return err
}
defer func() {
if err = dataRc.Close(); err != nil {
log.Error("ServeBlob: Close: %v", err)
}
}()
return ServeData(ctx, ctx.Repo.TreePath, blob.Size(), dataRc)
}
// ServeBlobOrLFS download a git.Blob redirecting to LFS if necessary
func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error {
dataRc, err := blob.DataAsync()
if err != nil {
return err
}
defer func() {
if err = dataRc.Close(); err != nil {
log.Error("ServeBlobOrLFS: Close: %v", err)
}
}()
pointer, _ := lfs.ReadPointer(dataRc)
if pointer.IsValid() {
meta, _ := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid)
if meta == nil {
return ServeBlob(ctx, blob)
}
lfsDataRc, err := lfs.ReadMetaObject(meta.Pointer)
if err != nil {
return err
}
defer func() {
if err = lfsDataRc.Close(); err != nil {
log.Error("ServeBlobOrLFS: Close: %v", err)
}
}()
return ServeData(ctx, ctx.Repo.TreePath, meta.Size, lfsDataRc)
}
return ServeBlob(ctx, blob)
}
// SingleDownload download a file by repos path
func SingleDownload(ctx *context.Context) {
blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath)
if err != nil {
if git.IsErrNotExist(err) {
ctx.NotFound("GetBlobByPath", nil)
} else {
ctx.ServerError("GetBlobByPath", err)
}
return
}
if err = ServeBlob(ctx, blob); err != nil {
ctx.ServerError("ServeBlob", err)
}
}
// SingleDownloadOrLFS download a file by repos path redirecting to LFS if necessary
func SingleDownloadOrLFS(ctx *context.Context) {
blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath)
if err != nil {
if git.IsErrNotExist(err) {
ctx.NotFound("GetBlobByPath", nil)
} else {
ctx.ServerError("GetBlobByPath", err)
}
return
}
if err = ServeBlobOrLFS(ctx, blob); err != nil {
ctx.ServerError("ServeBlobOrLFS", err)
}
}
// DownloadByID download a file by sha1 ID
func DownloadByID(ctx *context.Context) {
blob, err := ctx.Repo.GitRepo.GetBlob(ctx.Params("sha"))
if err != nil {
if git.IsErrNotExist(err) {
ctx.NotFound("GetBlob", nil)
} else {
ctx.ServerError("GetBlob", err)
}
return
}
if err = ServeBlob(ctx, blob); err != nil {
ctx.ServerError("ServeBlob", err)
}
}
// DownloadByIDOrLFS download a file by sha1 ID taking account of LFS
func DownloadByIDOrLFS(ctx *context.Context) {
blob, err := ctx.Repo.GitRepo.GetBlob(ctx.Params("sha"))
if err != nil {
if git.IsErrNotExist(err) {
ctx.NotFound("GetBlob", nil)
} else {
ctx.ServerError("GetBlob", err)
}
return
}
if err = ServeBlobOrLFS(ctx, blob); err != nil {
ctx.ServerError("ServeBlob", err)
}
}