* Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
		
			
				
	
	
		
			254 lines
		
	
	
	
		
			8.7 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
			
		
		
	
	
			254 lines
		
	
	
	
		
			8.7 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
| // Copyright 2014 The Gogs Authors. All rights reserved.
 | |
| // Copyright 2020 The Gitea Authors. All rights reserved.
 | |
| // Use of this source code is governed by a MIT-style
 | |
| // license that can be found in the LICENSE file.
 | |
| 
 | |
| package repo
 | |
| 
 | |
| import (
 | |
| 	"net/http"
 | |
| 	"strings"
 | |
| 
 | |
| 	"code.gitea.io/gitea/models"
 | |
| 	"code.gitea.io/gitea/modules/base"
 | |
| 	"code.gitea.io/gitea/modules/context"
 | |
| 	"code.gitea.io/gitea/modules/lfs"
 | |
| 	"code.gitea.io/gitea/modules/log"
 | |
| 	"code.gitea.io/gitea/modules/migrations"
 | |
| 	"code.gitea.io/gitea/modules/setting"
 | |
| 	"code.gitea.io/gitea/modules/structs"
 | |
| 	"code.gitea.io/gitea/modules/task"
 | |
| 	"code.gitea.io/gitea/modules/util"
 | |
| 	"code.gitea.io/gitea/modules/web"
 | |
| 	"code.gitea.io/gitea/services/forms"
 | |
| )
 | |
| 
 | |
| const (
 | |
| 	tplMigrate base.TplName = "repo/migrate/migrate"
 | |
| )
 | |
| 
 | |
| // Migrate render migration of repository page
 | |
| func Migrate(ctx *context.Context) {
 | |
| 	if setting.Repository.DisableMigrations {
 | |
| 		ctx.Error(http.StatusForbidden, "Migrate: the site administrator has disabled migrations")
 | |
| 		return
 | |
| 	}
 | |
| 
 | |
| 	serviceType := structs.GitServiceType(ctx.QueryInt("service_type"))
 | |
| 
 | |
| 	setMigrationContextData(ctx, serviceType)
 | |
| 
 | |
| 	if serviceType == 0 {
 | |
| 		ctx.Data["Org"] = ctx.Query("org")
 | |
| 		ctx.Data["Mirror"] = ctx.Query("mirror")
 | |
| 
 | |
| 		ctx.HTML(http.StatusOK, tplMigrate)
 | |
| 		return
 | |
| 	}
 | |
| 
 | |
| 	ctx.Data["private"] = getRepoPrivate(ctx)
 | |
| 	ctx.Data["mirror"] = ctx.Query("mirror") == "1"
 | |
| 	ctx.Data["lfs"] = ctx.Query("lfs") == "1"
 | |
| 	ctx.Data["wiki"] = ctx.Query("wiki") == "1"
 | |
| 	ctx.Data["milestones"] = ctx.Query("milestones") == "1"
 | |
| 	ctx.Data["labels"] = ctx.Query("labels") == "1"
 | |
| 	ctx.Data["issues"] = ctx.Query("issues") == "1"
 | |
| 	ctx.Data["pull_requests"] = ctx.Query("pull_requests") == "1"
 | |
| 	ctx.Data["releases"] = ctx.Query("releases") == "1"
 | |
| 
 | |
| 	ctxUser := checkContextUser(ctx, ctx.QueryInt64("org"))
 | |
| 	if ctx.Written() {
 | |
| 		return
 | |
| 	}
 | |
| 	ctx.Data["ContextUser"] = ctxUser
 | |
| 
 | |
| 	ctx.HTML(http.StatusOK, base.TplName("repo/migrate/"+serviceType.Name()))
 | |
| }
 | |
| 
 | |
| func handleMigrateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form *forms.MigrateRepoForm) {
 | |
| 	if setting.Repository.DisableMigrations {
 | |
| 		ctx.Error(http.StatusForbidden, "MigrateError: the site administrator has disabled migrations")
 | |
| 		return
 | |
| 	}
 | |
| 
 | |
| 	switch {
 | |
| 	case migrations.IsRateLimitError(err):
 | |
| 		ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form)
 | |
| 	case migrations.IsTwoFactorAuthError(err):
 | |
| 		ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tpl, form)
 | |
| 	case models.IsErrReachLimitOfRepo(err):
 | |
| 		ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form)
 | |
| 	case models.IsErrRepoAlreadyExist(err):
 | |
| 		ctx.Data["Err_RepoName"] = true
 | |
| 		ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tpl, form)
 | |
| 	case models.IsErrRepoFilesAlreadyExist(err):
 | |
| 		ctx.Data["Err_RepoName"] = true
 | |
| 		switch {
 | |
| 		case ctx.IsUserSiteAdmin() || (setting.Repository.AllowAdoptionOfUnadoptedRepositories && setting.Repository.AllowDeleteOfUnadoptedRepositories):
 | |
| 			ctx.RenderWithErr(ctx.Tr("form.repository_files_already_exist.adopt_or_delete"), tpl, form)
 | |
| 		case setting.Repository.AllowAdoptionOfUnadoptedRepositories:
 | |
| 			ctx.RenderWithErr(ctx.Tr("form.repository_files_already_exist.adopt"), tpl, form)
 | |
| 		case setting.Repository.AllowDeleteOfUnadoptedRepositories:
 | |
| 			ctx.RenderWithErr(ctx.Tr("form.repository_files_already_exist.delete"), tpl, form)
 | |
| 		default:
 | |
| 			ctx.RenderWithErr(ctx.Tr("form.repository_files_already_exist"), tpl, form)
 | |
| 		}
 | |
| 	case models.IsErrNameReserved(err):
 | |
| 		ctx.Data["Err_RepoName"] = true
 | |
| 		ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tpl, form)
 | |
| 	case models.IsErrNamePatternNotAllowed(err):
 | |
| 		ctx.Data["Err_RepoName"] = true
 | |
| 		ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form)
 | |
| 	default:
 | |
| 		remoteAddr, _ := forms.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword)
 | |
| 		err = util.URLSanitizedError(err, remoteAddr)
 | |
| 		if strings.Contains(err.Error(), "Authentication failed") ||
 | |
| 			strings.Contains(err.Error(), "Bad credentials") ||
 | |
| 			strings.Contains(err.Error(), "could not read Username") {
 | |
| 			ctx.Data["Err_Auth"] = true
 | |
| 			ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tpl, form)
 | |
| 		} else if strings.Contains(err.Error(), "fatal:") {
 | |
| 			ctx.Data["Err_CloneAddr"] = true
 | |
| 			ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tpl, form)
 | |
| 		} else {
 | |
| 			ctx.ServerError(name, err)
 | |
| 		}
 | |
| 	}
 | |
| }
 | |
| 
 | |
| func handleMigrateRemoteAddrError(ctx *context.Context, err error, tpl base.TplName, form *forms.MigrateRepoForm) {
 | |
| 	if models.IsErrInvalidCloneAddr(err) {
 | |
| 		addrErr := err.(*models.ErrInvalidCloneAddr)
 | |
| 		switch {
 | |
| 		case addrErr.IsProtocolInvalid:
 | |
| 			ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tpl, form)
 | |
| 		case addrErr.IsURLError:
 | |
| 			ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
 | |
| 		case addrErr.IsPermissionDenied:
 | |
| 			if addrErr.LocalPath {
 | |
| 				ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tpl, form)
 | |
| 			} else if len(addrErr.PrivateNet) == 0 {
 | |
| 				ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, form)
 | |
| 			} else {
 | |
| 				ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tpl, form)
 | |
| 			}
 | |
| 		case addrErr.IsInvalidPath:
 | |
| 			ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tpl, form)
 | |
| 		default:
 | |
| 			log.Error("Error whilst updating url: %v", err)
 | |
| 			ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
 | |
| 		}
 | |
| 	} else {
 | |
| 		log.Error("Error whilst updating url: %v", err)
 | |
| 		ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
 | |
| 	}
 | |
| }
 | |
| 
 | |
| // MigratePost response for migrating from external git repository
 | |
| func MigratePost(ctx *context.Context) {
 | |
| 	form := web.GetForm(ctx).(*forms.MigrateRepoForm)
 | |
| 	if setting.Repository.DisableMigrations {
 | |
| 		ctx.Error(http.StatusForbidden, "MigratePost: the site administrator has disabled migrations")
 | |
| 		return
 | |
| 	}
 | |
| 
 | |
| 	serviceType := structs.GitServiceType(form.Service)
 | |
| 
 | |
| 	setMigrationContextData(ctx, serviceType)
 | |
| 
 | |
| 	ctxUser := checkContextUser(ctx, form.UID)
 | |
| 	if ctx.Written() {
 | |
| 		return
 | |
| 	}
 | |
| 	ctx.Data["ContextUser"] = ctxUser
 | |
| 
 | |
| 	tpl := base.TplName("repo/migrate/" + serviceType.Name())
 | |
| 
 | |
| 	if ctx.HasError() {
 | |
| 		ctx.HTML(http.StatusOK, tpl)
 | |
| 		return
 | |
| 	}
 | |
| 
 | |
| 	remoteAddr, err := forms.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword)
 | |
| 	if err == nil {
 | |
| 		err = migrations.IsMigrateURLAllowed(remoteAddr, ctx.User)
 | |
| 	}
 | |
| 	if err != nil {
 | |
| 		ctx.Data["Err_CloneAddr"] = true
 | |
| 		handleMigrateRemoteAddrError(ctx, err, tpl, form)
 | |
| 		return
 | |
| 	}
 | |
| 
 | |
| 	form.LFS = form.LFS && setting.LFS.StartServer
 | |
| 
 | |
| 	if form.LFS && len(form.LFSEndpoint) > 0 {
 | |
| 		ep := lfs.DetermineEndpoint("", form.LFSEndpoint)
 | |
| 		if ep == nil {
 | |
| 			ctx.Data["Err_LFSEndpoint"] = true
 | |
| 			ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_lfs_endpoint"), tpl, &form)
 | |
| 			return
 | |
| 		}
 | |
| 		err = migrations.IsMigrateURLAllowed(ep.String(), ctx.User)
 | |
| 		if err != nil {
 | |
| 			ctx.Data["Err_LFSEndpoint"] = true
 | |
| 			handleMigrateRemoteAddrError(ctx, err, tpl, form)
 | |
| 			return
 | |
| 		}
 | |
| 	}
 | |
| 
 | |
| 	var opts = migrations.MigrateOptions{
 | |
| 		OriginalURL:    form.CloneAddr,
 | |
| 		GitServiceType: serviceType,
 | |
| 		CloneAddr:      remoteAddr,
 | |
| 		RepoName:       form.RepoName,
 | |
| 		Description:    form.Description,
 | |
| 		Private:        form.Private || setting.Repository.ForcePrivate,
 | |
| 		Mirror:         form.Mirror && !setting.Repository.DisableMirrors,
 | |
| 		LFS:            form.LFS,
 | |
| 		LFSEndpoint:    form.LFSEndpoint,
 | |
| 		AuthUsername:   form.AuthUsername,
 | |
| 		AuthPassword:   form.AuthPassword,
 | |
| 		AuthToken:      form.AuthToken,
 | |
| 		Wiki:           form.Wiki,
 | |
| 		Issues:         form.Issues,
 | |
| 		Milestones:     form.Milestones,
 | |
| 		Labels:         form.Labels,
 | |
| 		Comments:       form.Issues || form.PullRequests,
 | |
| 		PullRequests:   form.PullRequests,
 | |
| 		Releases:       form.Releases,
 | |
| 	}
 | |
| 	if opts.Mirror {
 | |
| 		opts.Issues = false
 | |
| 		opts.Milestones = false
 | |
| 		opts.Labels = false
 | |
| 		opts.Comments = false
 | |
| 		opts.PullRequests = false
 | |
| 		opts.Releases = false
 | |
| 	}
 | |
| 
 | |
| 	err = models.CheckCreateRepository(ctx.User, ctxUser, opts.RepoName, false)
 | |
| 	if err != nil {
 | |
| 		handleMigrateError(ctx, ctxUser, err, "MigratePost", tpl, form)
 | |
| 		return
 | |
| 	}
 | |
| 
 | |
| 	err = task.MigrateRepository(ctx.User, ctxUser, opts)
 | |
| 	if err == nil {
 | |
| 		ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + opts.RepoName)
 | |
| 		return
 | |
| 	}
 | |
| 
 | |
| 	handleMigrateError(ctx, ctxUser, err, "MigratePost", tpl, form)
 | |
| }
 | |
| 
 | |
| func setMigrationContextData(ctx *context.Context, serviceType structs.GitServiceType) {
 | |
| 	ctx.Data["Title"] = ctx.Tr("new_migrate")
 | |
| 
 | |
| 	ctx.Data["LFSActive"] = setting.LFS.StartServer
 | |
| 	ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate
 | |
| 	ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors
 | |
| 
 | |
| 	// Plain git should be first
 | |
| 	ctx.Data["Services"] = append([]structs.GitServiceType{structs.PlainGitService}, structs.SupportedFullGitService...)
 | |
| 	ctx.Data["service"] = serviceType
 | |
| }
 |