gitea/modules/migrations/gitlab.go

540 lines
14 KiB
Go
Raw Normal View History

Add support for migrating from Gitlab (#9084) * First stab at a Gitlab migrations interface. * Modify JS to show migration for Gitlab * Properly strip out #gitlab tag from repo name * Working Gitlab migrations! Still need to figure out how to hide tokens/etc from showing up in opts.CloneAddr * Try #2 at trying to hide credentials. CloneAddr was being used as OriginalURL. Now passing OriginalURL through from the form and saving it. * Add go-gitlab dependency * Vendor go-gitlab * Use gitlab.BasicAuthClient Correct CloneURL. This should be functioning! Previous commits fixed "Migrated from" from including the migration credentials. * Replaced repoPath with repoID globally. RepoID is grabbed in NewGitlabDownloader * Logging touchup * Properly set private repo status. Properly set milestone deadline time. Consistently use Gitlab username for 'Name'. * Add go-gitlab vendor cache * Fix PR migrations: - Count of issues is kept to set a non-conflicting PR.ID - Bool is used to tell whether to fetch Issue or PR comments * Ensure merged PRs are closed and set with the proper time * Remove copyright and some commented code * Rip out '#gitlab' based self-hosted Gitlab support * Hide given credentials for migrated repos. CloneAddr was being saved as OriginalURL. Now passing OriginalURL through from the form and saving it in it's place * Use asset.URL directly, no point in parsing. Opened PRs should fall through to false. * Fix importing Milestones. Allow importing using Personal Tokens or anonymous access. * Fix Gitlab Milestone migration if DueDate isn't set * Empty Milestone due dates properly return nil, not zero time * Add GITLAB_READ_TOKEN to drone unit-test step * Add working gitlab_test.go. A Personal Access Token, given in env variable GITLAB_READ_TOKEN is required to run the test. * Fix linting issues * Add modified JS files * Remove pre-build JS files * Only merged PRs are marged as merged/closed * Test topics * Skip test if gitlab is inaccessible * Grab personal token from username, not password. Matches Github migration implementation * Add SetContext() to GitlabDownloader. * Checking Updated field in Issues. * Actually fetch Issue Updated time from Gitlab * Add Gitlab migration GetReviews() stub * Fix Patch and Clone URLs * check Updated too * fix mod * make vendor with go1.14 Co-authored-by: techknowlogick <techknowlogick@gitea.io> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2020-04-19 15:44:11 +00:00
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/structs"
"github.com/xanzy/go-gitlab"
)
var (
_ base.Downloader = &GitlabDownloader{}
_ base.DownloaderFactory = &GitlabDownloaderFactory{}
)
func init() {
RegisterDownloaderFactory(&GitlabDownloaderFactory{})
}
// GitlabDownloaderFactory defines a gitlab downloader factory
type GitlabDownloaderFactory struct {
}
// Match returns ture if the migration remote URL matched this downloader factory
func (f *GitlabDownloaderFactory) Match(opts base.MigrateOptions) (bool, error) {
var matched bool
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return false, err
}
if strings.EqualFold(u.Host, "gitlab.com") && opts.AuthUsername != "" {
matched = true
}
return matched, nil
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GitlabDownloaderFactory) New(opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return nil, err
}
baseURL := u.Scheme + "://" + u.Host
repoNameSpace := strings.TrimPrefix(u.Path, "/")
log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
return NewGitlabDownloader(baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword), nil
}
// GitServiceType returns the type of git service
func (f *GitlabDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.GitlabService
}
// GitlabDownloader implements a Downloader interface to get repository informations
// from gitlab via go-gitlab
// - issueCount is incremented in GetIssues() to ensure PR and Issue numbers do not overlap,
// because Gitlab has individual Issue and Pull Request numbers.
// - issueSeen, working alongside issueCount, is checked in GetComments() to see whether we
// need to fetch the Issue or PR comments, as Gitlab stores them separately.
type GitlabDownloader struct {
ctx context.Context
client *gitlab.Client
repoID int
repoName string
issueCount int64
fetchPRcomments bool
}
// NewGitlabDownloader creates a gitlab Downloader via gitlab API
// Use either a username/password, personal token entered into the username field, or anonymous/public access
// Note: Public access only allows very basic access
func NewGitlabDownloader(baseURL, repoPath, username, password string) *GitlabDownloader {
var client *http.Client
var gitlabClient *gitlab.Client
var err error
if username != "" {
if password == "" {
gitlabClient = gitlab.NewClient(client, username)
} else {
gitlabClient, err = gitlab.NewBasicAuthClient(client, baseURL, username, password)
}
}
if err != nil {
log.Trace("Error logging into gitlab: %v", err)
return nil
}
// Grab and store project/repo ID here, due to issues using the URL escaped path
gr, _, err := gitlabClient.Projects.GetProject(repoPath, nil, nil)
if err != nil {
log.Trace("Error retrieving project: %v", err)
return nil
}
if gr == nil {
log.Trace("Error getting project, project is nil")
return nil
}
return &GitlabDownloader{
ctx: context.Background(),
client: gitlabClient,
repoID: gr.ID,
repoName: gr.Name,
}
}
// SetContext set context
func (g *GitlabDownloader) SetContext(ctx context.Context) {
g.ctx = ctx
}
// GetRepoInfo returns a repository information
func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) {
if g == nil {
return nil, errors.New("error: GitlabDownloader is nil")
}
gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil)
if err != nil {
return nil, err
}
var private bool
switch gr.Visibility {
case gitlab.InternalVisibility:
private = true
case gitlab.PrivateVisibility:
private = true
}
var owner string
if gr.Owner == nil {
log.Trace("gr.Owner is nil, trying to get owner from Namespace")
if gr.Namespace != nil && gr.Namespace.Kind == "user" {
owner = gr.Namespace.Path
}
} else {
owner = gr.Owner.Username
}
// convert gitlab repo to stand Repo
return &base.Repository{
Owner: owner,
Name: gr.Name,
IsPrivate: private,
Description: gr.Description,
OriginalURL: gr.WebURL,
CloneURL: gr.HTTPURLToRepo,
}, nil
}
// GetTopics return gitlab topics
func (g *GitlabDownloader) GetTopics() ([]string, error) {
if g == nil {
return nil, errors.New("error: GitlabDownloader is nil")
}
gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil)
if err != nil {
return nil, err
}
return gr.TagList, err
}
// GetMilestones returns milestones
func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
if g == nil {
return nil, errors.New("error: GitlabDownloader is nil")
}
var perPage = 100
var state = "all"
var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
ms, _, err := g.client.Milestones.ListMilestones(g.repoID, &gitlab.ListMilestonesOptions{
State: &state,
ListOptions: gitlab.ListOptions{
Page: i,
PerPage: perPage,
}}, nil)
if err != nil {
return nil, err
}
for _, m := range ms {
var desc string
if m.Description != "" {
desc = m.Description
}
var state = "open"
var closedAt *time.Time
if m.State != "" {
state = m.State
if state == "closed" {
closedAt = m.UpdatedAt
}
}
var deadline *time.Time
if m.DueDate != nil {
deadlineParsed, err := time.Parse("2006-01-02", m.DueDate.String())
if err != nil {
log.Trace("Error parsing Milestone DueDate time")
deadline = nil
} else {
deadline = &deadlineParsed
}
}
milestones = append(milestones, &base.Milestone{
Title: m.Title,
Description: desc,
Deadline: deadline,
State: state,
Created: *m.CreatedAt,
Updated: m.UpdatedAt,
Closed: closedAt,
})
}
if len(ms) < perPage {
break
}
}
return milestones, nil
}
// GetLabels returns labels
func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
if g == nil {
return nil, errors.New("error: GitlabDownloader is nil")
}
var perPage = 100
var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{
Page: i,
PerPage: perPage,
}, nil)
if err != nil {
return nil, err
}
for _, label := range ls {
baseLabel := &base.Label{
Name: label.Name,
Color: strings.TrimLeft(label.Color, "#)"),
Description: label.Description,
}
labels = append(labels, baseLabel)
}
if len(ls) < perPage {
break
}
}
return labels, nil
}
func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release {
r := &base.Release{
TagName: rel.TagName,
TargetCommitish: rel.Commit.ID,
Name: rel.Name,
Body: rel.Description,
Created: *rel.CreatedAt,
PublisherID: int64(rel.Author.ID),
PublisherName: rel.Author.Username,
}
for k, asset := range rel.Assets.Links {
r.Assets = append(r.Assets, base.ReleaseAsset{
URL: asset.URL,
Name: asset.Name,
ContentType: &rel.Assets.Sources[k].Format,
})
}
return r
}
// GetReleases returns releases
func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
var perPage = 100
var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{
Page: i,
PerPage: perPage,
}, nil)
if err != nil {
return nil, err
}
for _, release := range ls {
releases = append(releases, g.convertGitlabRelease(release))
}
if len(ls) < perPage {
break
}
}
return releases, nil
}
// GetIssues returns issues according start and limit
// Note: issue label description and colors are not supported by the go-gitlab library at this time
// TODO: figure out how to transfer issue reactions
func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
state := "all"
sort := "asc"
opt := &gitlab.ListProjectIssuesOptions{
State: &state,
Sort: &sort,
ListOptions: gitlab.ListOptions{
PerPage: perPage,
Page: page,
},
}
var allIssues = make([]*base.Issue, 0, perPage)
issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil)
if err != nil {
return nil, false, fmt.Errorf("error while listing issues: %v", err)
}
for _, issue := range issues {
var labels = make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, &base.Label{
Name: l,
})
}
var milestone string
if issue.Milestone != nil {
milestone = issue.Milestone.Title
}
allIssues = append(allIssues, &base.Issue{
Title: issue.Title,
Number: int64(issue.IID),
PosterID: int64(issue.Author.ID),
PosterName: issue.Author.Username,
Content: issue.Description,
Milestone: milestone,
State: issue.State,
Created: *issue.CreatedAt,
Labels: labels,
Closed: issue.ClosedAt,
IsLocked: issue.DiscussionLocked,
Updated: *issue.UpdatedAt,
})
// increment issueCount, to be used in GetPullRequests()
g.issueCount++
}
return allIssues, len(issues) < perPage, nil
}
// GetComments returns comments according issueNumber
func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
var allComments = make([]*base.Comment, 0, 100)
var page = 1
var realIssueNumber int64
for {
var comments []*gitlab.Discussion
var resp *gitlab.Response
var err error
// fetchPRcomments decides whether to fetch Issue or PR comments
if !g.fetchPRcomments {
realIssueNumber = issueNumber
comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListIssueDiscussionsOptions{
Page: page,
PerPage: 100,
}, nil)
} else {
// If this is a PR, we need to figure out the Gitlab/original PR ID to be passed below
realIssueNumber = issueNumber - g.issueCount
comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListMergeRequestDiscussionsOptions{
Page: page,
PerPage: 100,
}, nil)
}
if err != nil {
return nil, fmt.Errorf("error while listing comments: %v %v", g.repoID, err)
}
for _, comment := range comments {
// Flatten comment threads
if !comment.IndividualNote {
for _, note := range comment.Notes {
allComments = append(allComments, &base.Comment{
IssueIndex: realIssueNumber,
PosterID: int64(note.Author.ID),
PosterName: note.Author.Username,
PosterEmail: note.Author.Email,
Content: note.Body,
Created: *note.CreatedAt,
})
}
} else {
c := comment.Notes[0]
allComments = append(allComments, &base.Comment{
IssueIndex: realIssueNumber,
PosterID: int64(c.Author.ID),
PosterName: c.Author.Username,
PosterEmail: c.Author.Email,
Content: c.Body,
Created: *c.CreatedAt,
})
}
}
if resp.NextPage == 0 {
break
}
page = resp.NextPage
}
return allComments, nil
}
// GetPullRequests returns pull requests according page and perPage
func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) {
opt := &gitlab.ListProjectMergeRequestsOptions{
ListOptions: gitlab.ListOptions{
PerPage: perPage,
Page: page,
},
}
// Set fetchPRcomments to true here, so PR comments are fetched instead of Issue comments
g.fetchPRcomments = true
var allPRs = make([]*base.PullRequest, 0, perPage)
prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil)
if err != nil {
return nil, fmt.Errorf("error while listing merge requests: %v", err)
}
for _, pr := range prs {
var labels = make([]*base.Label, 0, len(pr.Labels))
for _, l := range pr.Labels {
labels = append(labels, &base.Label{
Name: l,
})
}
var merged bool
if pr.State == "merged" {
merged = true
pr.State = "closed"
}
var mergeTime = pr.MergedAt
if merged && pr.MergedAt == nil {
mergeTime = pr.UpdatedAt
}
var closeTime = pr.ClosedAt
if merged && pr.ClosedAt == nil {
closeTime = pr.UpdatedAt
}
var locked bool
if pr.State == "locked" {
locked = true
}
var milestone string
if pr.Milestone != nil {
milestone = pr.Milestone.Title
}
// Add the PR ID to the Issue Count because PR and Issues share ID space in Gitea
newPRnumber := g.issueCount + int64(pr.IID)
allPRs = append(allPRs, &base.PullRequest{
Title: pr.Title,
Number: int64(newPRnumber),
PosterName: pr.Author.Username,
PosterID: int64(pr.Author.ID),
Content: pr.Description,
Milestone: milestone,
State: pr.State,
Created: *pr.CreatedAt,
Closed: closeTime,
Labels: labels,
Merged: merged,
MergeCommitSHA: pr.MergeCommitSHA,
MergedTime: mergeTime,
IsLocked: locked,
Head: base.PullRequestBranch{
Ref: pr.SourceBranch,
SHA: pr.SHA,
RepoName: g.repoName,
OwnerName: pr.Author.Username,
CloneURL: pr.WebURL,
},
Base: base.PullRequestBranch{
Ref: pr.TargetBranch,
SHA: pr.DiffRefs.BaseSha,
RepoName: g.repoName,
OwnerName: pr.Author.Username,
},
PatchURL: pr.WebURL + ".patch",
})
}
return allPRs, nil
}
// GetReviews returns pull requests review
func (g *GitlabDownloader) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
return nil, nil
}