lavender-legacy
/
gitea
7
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[API] Migration: Change ServiceType String (#12672) 

* use different structs for MigrateRepoOptions on UI and API

* Fix TokenAuth and rename UID to an understandable Name

* fix swagger doc

* simplify & mk redable

* R E F A C T O R:
migration has now internal 3 structs to store its options:
 * the Options for WebUI: modules/auth/repo_form.go
 * the Options for API: modules/structs/repo.go
 * the option struct with after validation for internal prossessing: modules/migrations/base/options.go

* Copyright Header

* Deprecate UID - add RepoOwner

* adopt repo.go -> migrate.go

* add comment about each struct purpose

* lint
release/v1.15
6543 2020-09-11 00:29:19 +02:00 committed by GitHub
parent daefdd1385
commit fd60ebfe14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 230 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
integrations/api_repo_test.go
View File

@ -316,9 +316,9 @@ func TestAPIRepoMigrate(t *testing.T) {
user := models.AssertExistsAndLoadBean(t, &models.User{ID: testCase.ctxUserID}).(*models.User)
session := loginUser(t, user.Name)
token := getTokenForLoggedInUser(t, session)
req := NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate?token="+token, &api.MigrateRepoOption{
req := NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate?token="+token, &api.MigrateRepoOptions{
CloneAddr: testCase.cloneURL,
UID: int(testCase.userID),
RepoOwnerID: testCase.userID,
RepoName: testCase.repoName,
})
resp := MakeRequest(t, req, NoExpectedStatus)
@ -360,9 +360,9 @@ func testAPIRepoMigrateConflict(t *testing.T, u *url.URL) {
cloneURL := "https://github.com/go-gitea/test_repo.git"
req := NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate?token="+httpContext.Token,
&api.MigrateRepoOption{
&api.MigrateRepoOptions{
CloneAddr: cloneURL,
UID: int(userID),
RepoOwnerID: userID,
RepoName: httpContext.Reponame,
})
resp := httpContext.Session.MakeRequest(t, req, http.StatusConflict)

5
models/task.go
View File

@ -8,6 +8,7 @@ import (
"encoding/json"
"fmt"
migration "code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
@ -101,9 +102,9 @@ func (task *Task) UpdateCols(cols ...string) error {
}
// MigrateConfig returns task config when migrate repository
func (task *Task) MigrateConfig() (*structs.MigrateRepoOption, error) {
func (task *Task) MigrateConfig() (*migration.MigrateOptions, error) {
if task.Type == structs.TaskTypeMigrateRepo {
var opts structs.MigrateRepoOption
var opts migration.MigrateOptions
err := json.Unmarshal([]byte(task.PayloadContent), &opts)
if err != nil {
return nil, err

10
modules/auth/repo_form.go
View File

@ -53,6 +53,7 @@ func (f *CreateRepoForm) Validate(ctx *macaron.Context, errs binding.Errors) bin
}
// MigrateRepoForm form for migrating repository
// this is used to interact with web ui
type MigrateRepoForm struct {
// required: true
CloneAddr string `json:"clone_addr" binding:"Required"`
@ -84,9 +85,8 @@ func (f *MigrateRepoForm) Validate(ctx *macaron.Context, errs binding.Errors) bi
// and returns composed URL with needed username and password.
// It also checks if given user has permission when remote address
// is actually a local path.
func (f MigrateRepoForm) ParseRemoteAddr(user *models.User) (string, error) {
remoteAddr := strings.TrimSpace(f.CloneAddr)
func ParseRemoteAddr(remoteAddr, authUsername, authPassword string, user *models.User) (string, error) {
remoteAddr = strings.TrimSpace(remoteAddr)
// Remote address can be HTTP/HTTPS/Git URL or local path.
if strings.HasPrefix(remoteAddr, "http://") ||
strings.HasPrefix(remoteAddr, "https://") ||
@ -95,8 +95,8 @@ func (f MigrateRepoForm) ParseRemoteAddr(user *models.User) (string, error) {
if err != nil {
return "", models.ErrInvalidCloneAddr{IsURLError: true}
}
if len(f.AuthUsername)+len(f.AuthPassword) > 0 {
u.User = url.UserPassword(f.AuthUsername, f.AuthPassword)
if len(authUsername)+len(authPassword) > 0 {
u.User = url.UserPassword(authUsername, authPassword)
}
remoteAddr = u.String()
} else if !user.CanImportLocal() {

20
modules/convert/utils.go
View File

@ -1,3 +1,4 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Copyright 2016 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
@ -5,7 +6,10 @@
package convert
import (
"strings"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
)
// ToCorrectPageSize makes sure page size is in allowed range.
@ -17,3 +21,19 @@ func ToCorrectPageSize(size int) int {
}
return size
}
// ToGitServiceType return GitServiceType based on string
func ToGitServiceType(value string) structs.GitServiceType {
switch strings.ToLower(value) {
case "github":
return structs.GithubService
case "gitea":
return structs.GiteaService
case "gitlab":
return structs.GitlabService
case "gogs":
return structs.GogsService
default:
return structs.PlainGitService
}
}

26
modules/migrations/base/options.go
View File

@ -8,4 +8,28 @@ package base
import "code.gitea.io/gitea/modules/structs"
// MigrateOptions defines the way a repository gets migrated
type MigrateOptions = structs.MigrateRepoOption
// this is for internal usage by migrations module and func who interact with it
type MigrateOptions struct {
// required: true
CloneAddr string `json:"clone_addr" binding:"Required"`
AuthUsername string `json:"auth_username"`
AuthPassword string `json:"auth_password"`
AuthToken string `json:"auth_token"`
// required: true
UID int `json:"uid" binding:"Required"`
// required: true
RepoName string `json:"repo_name" binding:"Required"`
Mirror bool `json:"mirror"`
Private bool `json:"private"`
Description string `json:"description"`
OriginalURL string
GitServiceType structs.GitServiceType
Wiki bool
Issues bool
Milestones bool
Labels bool
Releases bool
Comments bool
PullRequests bool
MigrateToRepoID int64
}

2
modules/migrations/gitea.go
View File

@ -123,7 +123,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
return err
}
r, err = repository.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{
r, err = repository.MigrateRepositoryGitData(g.doer, owner, r, base.MigrateOptions{
RepoName: g.repoName,
Description: repo.Description,
OriginalURL: repo.OriginalURL,

3
modules/migrations/gitea_test.go
View File

@ -12,6 +12,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
@ -32,7 +33,7 @@ func TestGiteaUploadRepo(t *testing.T) {
uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
)
err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{
err := migrateRepository(downloader, uploader, base.MigrateOptions{
CloneAddr: "https://github.com/go-xorm/builder",
RepoName: repoName,
AuthUsername: "",

4
modules/repository/repo.go
View File

@ -13,8 +13,8 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
migration "code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
@ -41,7 +41,7 @@ func WikiRemoteURL(remote string) string {
}
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opts api.MigrateRepoOption) (*models.Repository, error) {
func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opts migration.MigrateOptions) (*models.Repository, error) {
repoPath := models.RepoPath(u.Name, opts.RepoName)
if u.IsOrganization() {

55
modules/structs/repo.go
View File

@ -226,6 +226,35 @@ func (gt GitServiceType) Title() string {
return ""
}
// MigrateRepoOptions options for migrating repository's
// this is used to interact with api v1
type MigrateRepoOptions struct {
// required: true
CloneAddr string `json:"clone_addr" binding:"Required"`
// deprecated (only for backwards compatibility)
RepoOwnerID int64 `json:"uid"`
// Name of User or Organisation who will own Repo after migration
RepoOwner string `json:"repo_owner"`
// required: true
RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
// enum: git,github,gitea,gitlab
Service string `json:"service"`
AuthUsername string `json:"auth_username"`
AuthPassword string `json:"auth_password"`
AuthToken string `json:"auth_token"`
Mirror bool `json:"mirror"`
Private bool `json:"private"`
Description string `json:"description" binding:"MaxSize(255)"`
Wiki bool `json:"wiki"`
Milestones bool `json:"milestones"`
Labels bool `json:"labels"`
Issues bool `json:"issues"`
PullRequests bool `json:"pull_requests"`
Releases bool `json:"releases"`
}
// TokenAuth represents whether a service type supports token-based auth
func (gt GitServiceType) TokenAuth() bool {
switch gt {
@ -243,29 +272,3 @@ var (
GitlabService,
}
)
// MigrateRepoOption options for migrating a repository from an external service
type MigrateRepoOption struct {
// required: true
CloneAddr string `json:"clone_addr" binding:"Required"`
AuthUsername string `json:"auth_username"`
AuthPassword string `json:"auth_password"`
AuthToken string `json:"auth_token"`
// required: true
UID int `json:"uid" binding:"Required"`
// required: true
RepoName string `json:"repo_name" binding:"Required"`
Mirror bool `json:"mirror"`
Private bool `json:"private"`
Description string `json:"description"`
OriginalURL string
GitServiceType GitServiceType
Wiki bool
Issues bool
Milestones bool
Labels bool
Releases bool
Comments bool
PullRequests bool
MigrateToRepoID int64
}

3
modules/task/migrate.go
View File

@ -14,6 +14,7 @@ import (
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
migration "code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/notification"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
@ -89,7 +90,7 @@ func runMigrateTask(t *models.Task) (err error) {
return err
}
var opts *structs.MigrateRepoOption
var opts *migration.MigrateOptions
opts, err = t.MigrateConfig()
if err != nil {
return err

2
routers/api/v1/api.go
View File

@ -636,7 +636,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/issues/search", repo.SearchIssues)
m.Post("/migrate", reqToken(), bind(auth.MigrateRepoForm{}), repo.Migrate)
m.Post("/migrate", reqToken(), bind(api.MigrateRepoOptions{}), repo.Migrate)
m.Group("/:username/:reponame", func() {
m.Combo("").Get(reqAnyRepoReader(), repo.Get).

56
routers/api/v1/repo/migrate.go
View File

@ -9,12 +9,12 @@ import (
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/auth"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/convert"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
@ -26,7 +26,7 @@ import (
)
// Migrate migrate remote git repository to gitea
func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
func Migrate(ctx *context.APIContext, form api.MigrateRepoOptions) {
// swagger:operation POST /repos/migrate repository repoMigrate
// ---
// summary: Migrate a remote git repository
@ -38,7 +38,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
// - name: body
// in: body
// schema:
// "$ref": "#/definitions/MigrateRepoForm"
// "$ref": "#/definitions/MigrateRepoOptions"
// responses:
// "201":
// "$ref": "#/responses/Repository"
@ -47,21 +47,26 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
// "422":
// "$ref": "#/responses/validationError"
ctxUser := ctx.User
// Not equal means context user is an organization,
// or is another user/organization if current user is admin.
if form.UID != ctxUser.ID {
org, err := models.GetUserByID(form.UID)
//get repoOwner
var (
repoOwner *models.User
err error
)
if len(form.RepoOwner) != 0 {
repoOwner, err = models.GetUserByName(form.RepoOwner)
} else if form.RepoOwnerID != 0 {
repoOwner, err = models.GetUserByID(form.RepoOwnerID)
} else {
repoOwner = ctx.User
}
if err != nil {
if models.IsErrUserNotExist(err) {
ctx.Error(http.StatusUnprocessableEntity, "", err)
} else {
ctx.Error(http.StatusInternalServerError, "GetUserByID", err)
ctx.Error(http.StatusInternalServerError, "GetUser", err)
}
return
}
ctxUser = org
}
if ctx.HasError() {
ctx.Error(http.StatusUnprocessableEntity, "", ctx.GetErrMsg())
@ -69,14 +74,14 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
}
if !ctx.User.IsAdmin {
if !ctxUser.IsOrganization() && ctx.User.ID != ctxUser.ID {
if !repoOwner.IsOrganization() && ctx.User.ID != repoOwner.ID {
ctx.Error(http.StatusForbidden, "", "Given user is not an organization.")
return
}
if ctxUser.IsOrganization() {
if repoOwner.IsOrganization() {
// Check ownership of organization.
isOwner, err := ctxUser.IsOwnedBy(ctx.User.ID)
isOwner, err := repoOwner.IsOwnedBy(ctx.User.ID)
if err != nil {
ctx.Error(http.StatusInternalServerError, "IsOwnedBy", err)
return
@ -87,7 +92,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
}
}
remoteAddr, err := form.ParseRemoteAddr(ctx.User)
remoteAddr, err := auth.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword, ctx.User)
if err != nil {
if models.IsErrInvalidCloneAddr(err) {
addrErr := err.(models.ErrInvalidCloneAddr)
@ -107,11 +112,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
return
}
var gitServiceType = api.PlainGitService
u, err := url.Parse(remoteAddr)
if err == nil && strings.EqualFold(u.Host, "github.com") {
gitServiceType = api.GithubService
}
gitServiceType := convert.ToGitServiceType(form.Service)
if form.Mirror && setting.Repository.DisableMirrors {
ctx.Error(http.StatusForbidden, "MirrorsGlobalDisabled", fmt.Errorf("the site administrator has disabled mirrors"))
@ -126,6 +127,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
Mirror: form.Mirror,
AuthUsername: form.AuthUsername,
AuthPassword: form.AuthPassword,
AuthToken: form.AuthToken,
Wiki: form.Wiki,
Issues: form.Issues,
Milestones: form.Milestones,
@ -144,7 +146,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
opts.Releases = false
}
repo, err := repo_module.CreateRepository(ctx.User, ctxUser, models.CreateRepoOptions{
repo, err := repo_module.CreateRepository(ctx.User, repoOwner, models.CreateRepoOptions{
Name: opts.RepoName,
Description: opts.Description,
OriginalURL: form.CloneAddr,
@ -154,7 +156,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
Status: models.RepositoryBeingMigrated,
})
if err != nil {
handleMigrateError(ctx, ctxUser, remoteAddr, err)
handleMigrateError(ctx, repoOwner, remoteAddr, err)
return
}
@ -171,24 +173,24 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
if err == nil {
repo.Status = models.RepositoryReady
if err := models.UpdateRepositoryCols(repo, "status"); err == nil {
notification.NotifyMigrateRepository(ctx.User, ctxUser, repo)
notification.NotifyMigrateRepository(ctx.User, repoOwner, repo)
return
}
}
if repo != nil {
if errDelete := models.DeleteRepository(ctx.User, ctxUser.ID, repo.ID); errDelete != nil {
if errDelete := models.DeleteRepository(ctx.User, repoOwner.ID, repo.ID); errDelete != nil {
log.Error("DeleteRepository: %v", errDelete)
}
}
}()
if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, ctxUser.Name, opts); err != nil {
handleMigrateError(ctx, ctxUser, remoteAddr, err)
if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, repoOwner.Name, opts); err != nil {
handleMigrateError(ctx, repoOwner, remoteAddr, err)
return
}
log.Trace("Repository migrated: %s/%s", ctxUser.Name, form.RepoName)
log.Trace("Repository migrated: %s/%s", repoOwner.Name, form.RepoName)
ctx.JSON(http.StatusCreated, repo.APIFormat(models.AccessModeAdmin))
}

3
routers/api/v1/swagger/options.go
View File

@ -149,4 +149,7 @@ type swaggerParameterBodies struct {
// in:body
SubmitPullReviewOptions api.SubmitPullReviewOptions
// in:body
MigrateRepoOptions api.MigrateRepoOptions
}

4
routers/repo/migrate.go
View File

@ -74,7 +74,7 @@ func handleMigrateError(ctx *context.Context, owner *models.User, err error, nam
ctx.Data["Err_RepoName"] = true
ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form)
default:
remoteAddr, _ := form.ParseRemoteAddr(owner)
remoteAddr, _ := auth.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword, owner)
err = util.URLSanitizedError(err, remoteAddr)
if strings.Contains(err.Error(), "Authentication failed") ||
strings.Contains(err.Error(), "Bad credentials") ||
@ -108,7 +108,7 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) {
return
}
remoteAddr, err := form.ParseRemoteAddr(ctx.User)
remoteAddr, err := auth.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword, ctx.User)
if err != nil {
if models.IsErrInvalidCloneAddr(err) {
ctx.Data["Err_CloneAddr"] = true

4
services/mirror/mirror_test.go
View File

@ -10,8 +10,8 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
migration "code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/structs"
release_service "code.gitea.io/gitea/services/release"
"github.com/stretchr/testify/assert"
@ -28,7 +28,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
repoPath := models.RepoPath(user.Name, repo.Name)
opts := structs.MigrateRepoOption{
opts := migration.MigrateOptions{
RepoName: "test_mirror",
Description: "Test mirror",
Private: false,

94
templates/swagger/v1_json.tmpl
View File

@ -1798,7 +1798,7 @@
"name": "body",
"in": "body",
"schema": {
"$ref": "#/definitions/MigrateRepoForm"
"$ref": "#/definitions/MigrateRepoOptions"
}
}
],
@ -13522,7 +13522,7 @@
"x-go-package": "code.gitea.io/gitea/modules/auth"
},
"MigrateRepoForm": {
"description": "MigrateRepoForm form for migrating repository",
"description": "MigrateRepoForm form for migrating repository\nthis is used to interact with web ui",
"type": "object",
"required": [
"clone_addr",
@ -13599,6 +13599,94 @@
},
"x-go-package": "code.gitea.io/gitea/modules/auth"
},
"MigrateRepoOptions": {
"description": "MigrateRepoOptions options for migrating repository's\nthis is used to interact with api v1",
"type": "object",
"required": [
"clone_addr",
"repo_name"
],
"properties": {
"auth_password": {
"type": "string",
"x-go-name": "AuthPassword"
},
"auth_token": {
"type": "string",
"x-go-name": "AuthToken"
},
"auth_username": {
"type": "string",
"x-go-name": "AuthUsername"
},
"clone_addr": {
"type": "string",
"x-go-name": "CloneAddr"
},
"description": {
"type": "string",
"x-go-name": "Description"
},
"issues": {
"type": "boolean",
"x-go-name": "Issues"
},
"labels": {
"type": "boolean",
"x-go-name": "Labels"
},
"milestones": {
"type": "boolean",
"x-go-name": "Milestones"
},
"mirror": {
"type": "boolean",
"x-go-name": "Mirror"
},
"private": {
"type": "boolean",
"x-go-name": "Private"
},
"pull_requests": {
"type": "boolean",
"x-go-name": "PullRequests"
},
"releases": {
"type": "boolean",
"x-go-name": "Releases"
},
"repo_name": {
"type": "string",
"x-go-name": "RepoName"
},
"repo_owner": {
"description": "Name of User or Organisation who will own Repo after migration",
"type": "string",
"x-go-name": "RepoOwner"
},
"service": {
"type": "string",
"enum": [
"git",
"github",
"gitea",
"gitlab"
],
"x-go-name": "Service"
},
"uid": {
"description": "deprecated (only for backwards compatibility)",
"type": "integer",
"format": "int64",
"x-go-name": "RepoOwnerID"
},
"wiki": {
"type": "boolean",
"x-go-name": "Wiki"
}
},
"x-go-package": "code.gitea.io/gitea/modules/structs"
},
"Milestone": {
"description": "Milestone milestone is a collection of issues on one repository",
"type": "object",
@ -15795,7 +15883,7 @@
"parameterBodies": {
"description": "parameterBodies",
"schema": {
"$ref": "#/definitions/SubmitPullReviewOptions"
"$ref": "#/definitions/MigrateRepoOptions"
}
},
"redirect": {