Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244)

* Dump github/gitlab repository data to a local directory

* Fix lint

* Adjust directory structure

* Allow migration special units

* Allow migration ignore release assets

* Fix lint

* Add restore repository

* stage the changes

* Merge

* Fix lint

* Update the interface

* Add some restore methods

* Finish restore

* Add comments

* Fix restore

* Add a token flag

* Fix bug

* Fix test

* Fix test

* Fix bug

* Fix bug

* Fix lint

* Fix restore

* refactor downloader

* fmt

* Fix bug isEnd detection on getIssues

* Refactor maxPerPage

* Remove unused codes

* Remove unused codes

* Fix bug

* Fix restore

* Fix dump

* Uploader should not depend downloader

* use release attachment name but not id

* Fix restore bug

* Fix lint

* Fix restore bug

* Add a method of DownloadFunc for base.Release to make uploader not depend on downloader

* fix Release yml marshal

* Fix trace information

* Fix bug when dump & restore

* Save relative path on yml file

* Fix bug

* Use relative path

* Update docs

* Use git service string but not int

* Recognize clone addr to service type

cmd/dump_repo.go

@@ -0,0 +1,162 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"strings"
+
+	"code.gitea.io/gitea/modules/convert"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations"
+	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
+
+	"github.com/urfave/cli"
+)
+
+// CmdDumpRepository represents the available dump repository sub-command.
+var CmdDumpRepository = cli.Command{
+	Name:        "dump-repo",
+	Usage:       "Dump the repository from git/github/gitea/gitlab",
+	Description: "This is a command for dumping the repository data.",
+	Action:      runDumpRepository,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "git_service",
+			Value: "",
+			Usage: "Git service, git, github, gitea, gitlab. If clone_addr could be recognized, this could be ignored.",
+		},
+		cli.StringFlag{
+			Name:  "repo_dir, r",
+			Value: "./data",
+			Usage: "Repository dir path to store the data",
+		},
+		cli.StringFlag{
+			Name:  "clone_addr",
+			Value: "",
+			Usage: "The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL",
+		},
+		cli.StringFlag{
+			Name:  "auth_username",
+			Value: "",
+			Usage: "The username to visit the clone_addr",
+		},
+		cli.StringFlag{
+			Name:  "auth_password",
+			Value: "",
+			Usage: "The password to visit the clone_addr",
+		},
+		cli.StringFlag{
+			Name:  "auth_token",
+			Value: "",
+			Usage: "The personal token to visit the clone_addr",
+		},
+		cli.StringFlag{
+			Name:  "owner_name",
+			Value: "",
+			Usage: "The data will be stored on a directory with owner name if not empty",
+		},
+		cli.StringFlag{
+			Name:  "repo_name",
+			Value: "",
+			Usage: "The data will be stored on a directory with repository name if not empty",
+		},
+		cli.StringFlag{
+			Name:  "units",
+			Value: "",
+			Usage: `Which items will be migrated, one or more units should be separated as comma. 
+wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`,
+		},
+	},
+}
+
+func runDumpRepository(ctx *cli.Context) error {
+	if err := initDB(); err != nil {
+		return err
+	}
+
+	log.Trace("AppPath: %s", setting.AppPath)
+	log.Trace("AppWorkPath: %s", setting.AppWorkPath)
+	log.Trace("Custom path: %s", setting.CustomPath)
+	log.Trace("Log path: %s", setting.LogRootPath)
+	setting.InitDBConfig()
+
+	var (
+		serviceType structs.GitServiceType
+		cloneAddr   = ctx.String("clone_addr")
+		serviceStr  = ctx.String("git_service")
+	)
+
+	if strings.HasPrefix(strings.ToLower(cloneAddr), "https://github.com/") {
+		serviceStr = "github"
+	} else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitlab.com/") {
+		serviceStr = "gitlab"
+	} else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitea.com/") {
+		serviceStr = "gitea"
+	}
+	if serviceStr == "" {
+		return errors.New("git_service missed or clone_addr cannot be recognized")
+	}
+	serviceType = convert.ToGitServiceType(serviceStr)
+
+	var opts = base.MigrateOptions{
+		GitServiceType: serviceType,
+		CloneAddr:      cloneAddr,
+		AuthUsername:   ctx.String("auth_username"),
+		AuthPassword:   ctx.String("auth_password"),
+		AuthToken:      ctx.String("auth_token"),
+		RepoName:       ctx.String("repo_name"),
+	}
+
+	if len(ctx.String("units")) == 0 {
+		opts.Wiki = true
+		opts.Issues = true
+		opts.Milestones = true
+		opts.Labels = true
+		opts.Releases = true
+		opts.Comments = true
+		opts.PullRequests = true
+		opts.ReleaseAssets = true
+	} else {
+		units := strings.Split(ctx.String("units"), ",")
+		for _, unit := range units {
+			switch strings.ToLower(unit) {
+			case "wiki":
+				opts.Wiki = true
+			case "issues":
+				opts.Issues = true
+			case "milestones":
+				opts.Milestones = true
+			case "labels":
+				opts.Labels = true
+			case "releases":
+				opts.Releases = true
+			case "release_assets":
+				opts.ReleaseAssets = true
+			case "comments":
+				opts.Comments = true
+			case "pull_requests":
+				opts.PullRequests = true
+			}
+		}
+	}
+
+	if err := migrations.DumpRepository(
+		context.Background(),
+		ctx.String("repo_dir"),
+		ctx.String("owner_name"),
+		opts,
+	); err != nil {
+		log.Fatal("Failed to dump repository: %v", err)
+		return err
+	}
+
+	log.Trace("Dump finished!!!")
+
+	return nil
+}

cmd/restore_repo.go

@@ -0,0 +1,119 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"context"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations"
+	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
+	pull_service "code.gitea.io/gitea/services/pull"
+
+	"github.com/urfave/cli"
+)
+
+// CmdRestoreRepository represents the available restore a repository sub-command.
+var CmdRestoreRepository = cli.Command{
+	Name:        "restore-repo",
+	Usage:       "Restore the repository from disk",
+	Description: "This is a command for restoring the repository data.",
+	Action:      runRestoreRepository,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "repo_dir, r",
+			Value: "./data",
+			Usage: "Repository dir path to restore from",
+		},
+		cli.StringFlag{
+			Name:  "owner_name",
+			Value: "",
+			Usage: "Restore destination owner name",
+		},
+		cli.StringFlag{
+			Name:  "repo_name",
+			Value: "",
+			Usage: "Restore destination repository name",
+		},
+		cli.StringFlag{
+			Name:  "units",
+			Value: "",
+			Usage: `Which items will be restored, one or more units should be separated as comma. 
+wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`,
+		},
+	},
+}
+
+func runRestoreRepository(ctx *cli.Context) error {
+	if err := initDB(); err != nil {
+		return err
+	}
+
+	log.Trace("AppPath: %s", setting.AppPath)
+	log.Trace("AppWorkPath: %s", setting.AppWorkPath)
+	log.Trace("Custom path: %s", setting.CustomPath)
+	log.Trace("Log path: %s", setting.LogRootPath)
+	setting.InitDBConfig()
+
+	if err := storage.Init(); err != nil {
+		return err
+	}
+
+	if err := pull_service.Init(); err != nil {
+		return err
+	}
+
+	var opts = base.MigrateOptions{
+		RepoName: ctx.String("repo_name"),
+	}
+
+	if len(ctx.String("units")) == 0 {
+		opts.Wiki = true
+		opts.Issues = true
+		opts.Milestones = true
+		opts.Labels = true
+		opts.Releases = true
+		opts.Comments = true
+		opts.PullRequests = true
+		opts.ReleaseAssets = true
+	} else {
+		units := strings.Split(ctx.String("units"), ",")
+		for _, unit := range units {
+			switch strings.ToLower(unit) {
+			case "wiki":
+				opts.Wiki = true
+			case "issues":
+				opts.Issues = true
+			case "milestones":
+				opts.Milestones = true
+			case "labels":
+				opts.Labels = true
+			case "releases":
+				opts.Releases = true
+			case "release_assets":
+				opts.ReleaseAssets = true
+			case "comments":
+				opts.Comments = true
+			case "pull_requests":
+				opts.PullRequests = true
+			}
+		}
+	}
+
+	if err := migrations.RestoreRepository(
+		context.Background(),
+		ctx.String("repo_dir"),
+		ctx.String("owner_name"),
+		ctx.String("repo_name"),
+	); err != nil {
+		log.Fatal("Failed to restore repository: %v", err)
+		return err
+	}
+
+	return nil
+}

docs/content/doc/usage/command-line.en-us.md

@@ -441,3 +441,28 @@ Manage running server operations:
               - `--host value`, `-H value`: Mail server host (defaults to: 127.0.0.1:25)
               - `--send-to value`, `-s value`: Email address(es) to send to
               - `--subject value`, `-S value`: Subject header of sent emails
+
+### dump-repo
+
+Dump-repo dumps repository data from git/github/gitea/gitlab:
+
+- Options:
+  - `--git_service service` : Git service, it could be `git`, `github`, `gitea`, `gitlab`, If clone_addr could be recognized, this could be ignored.
+  - `--repo_dir dir`, `-r dir`: Repository dir path to store the data 
+  - `--clone_addr addr`: The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL. i.e. https://github.com/lunny/tango.git
+  - `--auth_username lunny`: The username to visit the clone_addr
+  - `--auth_password <password>`: The password to visit the clone_addr
+  - `--auth_token <token>`: The personal token to visit the clone_addr
+  - `--owner_name lunny`: The data will be stored on a directory with owner name if not empty
+  - `--repo_name tango`: The data will be stored on a directory with repository name if not empty
+  - `--units <units>`: Which items will be migrated, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.
+
+### restore-repo
+
+Restore-repo restore repository data from disk dir:
+
+- Options:
+  - `--repo_dir dir`, `-r dir`: Repository dir path to restore from
+  - `--owner_name lunny`: Restore destination owner name
+  - `--repo_name tango`: Restore destination repository name
+  - `--units <units>`: Which items will be restored, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.

main.go

@@ -72,6 +72,8 @@ arguments - which can alternatively be run by running the subcommand web.`
 		cmd.Cmdembedded,
 		cmd.CmdMigrateStorage,
 		cmd.CmdDocs,
+		cmd.CmdDumpRepository,
+		cmd.CmdRestoreRepository,
 	}
 	// Now adjust these commands to add our global configuration options
 

models/admin.go

@@ -132,3 +132,16 @@ func DeleteNoticesByIDs(ids []int64) error {
 		Delete(new(Notice))
 	return err
 }
+
+// GetAdminUser returns the first administrator
+func GetAdminUser() (*User, error) {
+	var admin User
+	has, err := x.Where("is_admin=?", true).Get(&admin)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, ErrUserNotExist{}
+	}
+
+	return &admin, nil
+}

models/task.go

@@ -211,10 +211,6 @@ func FinishMigrateTask(task *Task) error {
 	if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil {
 		return err
 	}
-	task.Repo.Status = RepositoryReady
-	if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil {
-		return err
-	}
 
 	return sess.Commit()
 }

modules/migrations/base/comment.go

@@ -9,10 +9,10 @@ import "time"
 
 // Comment is a standard comment information
 type Comment struct {
-	IssueIndex  int64
+	IssueIndex  int64  `yaml:"issue_index"`
-	PosterID    int64
+	PosterID    int64  `yaml:"poster_id"`
-	PosterName  string
+	PosterName  string `yaml:"poster_name"`
-	PosterEmail string
+	PosterEmail string `yaml:"poster_email"`
 	Created     time.Time
 	Updated     time.Time
 	Content     string

modules/migrations/base/downloader.go

@@ -7,20 +7,13 @@ package base
 
 import (
 	"context"
-	"io"
 	"time"
 
 	"code.gitea.io/gitea/modules/structs"
 )
 
-// AssetDownloader downloads an asset (attachment) for a release
-type AssetDownloader interface {
-	GetAsset(relTag string, relID, id int64) (io.ReadCloser, error)
-}
-
 // Downloader downloads the site repo informations
 type Downloader interface {
-	AssetDownloader
 	SetContext(context.Context)
 	GetRepoInfo() (*Repository, error)
 	GetTopics() ([]string, error)

modules/migrations/base/issue.go

@@ -10,15 +10,15 @@ import "time"
 // Issue is a standard issue information
 type Issue struct {
 	Number      int64
-	PosterID    int64
+	PosterID    int64  `yaml:"poster_id"`
-	PosterName  string
+	PosterName  string `yaml:"poster_name"`
-	PosterEmail string
+	PosterEmail string `yaml:"poster_email"`
 	Title       string
 	Content     string
 	Ref         string
 	Milestone   string
 	State       string // closed, open
-	IsLocked    bool
+	IsLocked    bool   `yaml:"is_locked"`
 	Created     time.Time
 	Updated     time.Time
 	Closed      *time.Time

modules/migrations/base/options.go

@@ -31,5 +31,6 @@ type MigrateOptions struct {
 	Releases        bool
 	Comments        bool
 	PullRequests    bool
+	ReleaseAssets   bool
 	MigrateToRepoID int64
 }

modules/migrations/base/pullrequest.go

@@ -13,11 +13,11 @@ import (
 // PullRequest defines a standard pull request information
 type PullRequest struct {
 	Number         int64
-	OriginalNumber int64
+	OriginalNumber int64 `yaml:"original_number"`
 	Title          string
-	PosterName     string
+	PosterName     string `yaml:"poster_name"`
-	PosterID       int64
+	PosterID       int64  `yaml:"poster_id"`
-	PosterEmail    string
+	PosterEmail    string `yaml:"poster_email"`
 	Content        string
 	Milestone      string
 	State          string
@@ -25,14 +25,14 @@ type PullRequest struct {
 	Updated        time.Time
 	Closed         *time.Time
 	Labels         []*Label
-	PatchURL       string
+	PatchURL       string `yaml:"patch_url"`
 	Merged         bool
-	MergedTime     *time.Time
+	MergedTime     *time.Time `yaml:"merged_time"`
-	MergeCommitSHA string
+	MergeCommitSHA string     `yaml:"merge_commit_sha"`
 	Head           PullRequestBranch
 	Base           PullRequestBranch
 	Assignees      []string
-	IsLocked       bool
+	IsLocked       bool `yaml:"is_locked"`
 	Reactions      []*Reaction
 }
 
@@ -43,11 +43,11 @@ func (p *PullRequest) IsForkPullRequest() bool {
 
 // PullRequestBranch represents a pull request branch
 type PullRequestBranch struct {
-	CloneURL  string
+	CloneURL  string `yaml:"clone_url"`
 	Ref       string
 	SHA       string
-	RepoName  string
+	RepoName  string `yaml:"repo_name"`
-	OwnerName string
+	OwnerName string `yaml:"owner_name"`
 }
 
 // RepoPath returns pull request repo path

modules/migrations/base/reaction.go

@@ -6,7 +6,7 @@ package base
 
 // Reaction represents a reaction to an issue/pr/comment.
 type Reaction struct {
-	UserID   int64
+	UserID   int64  `yaml:"user_id"`
-	UserName string
+	UserName string `yaml:"user_name"`
 	Content  string
 }

modules/migrations/base/release.go

@@ -4,32 +4,37 @@
 
 package base
 
-import "time"
+import (
+	"io"
+	"time"
+)
 
 // ReleaseAsset represents a release asset
 type ReleaseAsset struct {
 	ID            int64
 	Name          string
-	ContentType   *string
+	ContentType   *string `yaml:"content_type"`
 	Size          *int
-	DownloadCount *int
+	DownloadCount *int `yaml:"download_count"`
 	Created       time.Time
 	Updated       time.Time
-	DownloadURL   *string
+	DownloadURL   *string `yaml:"download_url"`
+	// if DownloadURL is nil, the function should be invoked
+	DownloadFunc func() (io.ReadCloser, error) `yaml:"-"`
 }
 
 // Release represents a release
 type Release struct {
-	TagName         string
+	TagName         string `yaml:"tag_name"`
-	TargetCommitish string
+	TargetCommitish string `yaml:"target_commitish"`
 	Name            string
 	Body            string
 	Draft           bool
 	Prerelease      bool
-	PublisherID     int64
+	PublisherID     int64  `yaml:"publisher_id"`
-	PublisherName   string
+	PublisherName   string `yaml:"publisher_name"`
-	PublisherEmail  string
+	PublisherEmail  string `yaml:"publisher_email"`
-	Assets          []ReleaseAsset
+	Assets          []*ReleaseAsset
 	Created         time.Time
 	Published       time.Time
 }

modules/migrations/base/repo.go

@@ -9,10 +9,10 @@ package base
 type Repository struct {
 	Name          string
 	Owner         string
-	IsPrivate     bool
+	IsPrivate     bool `yaml:"is_private"`
-	IsMirror      bool
+	IsMirror      bool `yaml:"is_mirror"`
 	Description   string
-	CloneURL      string
+	CloneURL      string `yaml:"clone_url"`
-	OriginalURL   string
+	OriginalURL   string `yaml:"original_url"`
 	DefaultBranch string
 }

modules/migrations/base/review.go

@@ -17,13 +17,13 @@ const (
 // Review is a standard review information
 type Review struct {
 	ID           int64
-	IssueIndex   int64
+	IssueIndex   int64  `yaml:"issue_index"`
-	ReviewerID   int64
+	ReviewerID   int64  `yaml:"reviewer_id"`
-	ReviewerName string
+	ReviewerName string `yaml:"reviewer_name"`
 	Official     bool
-	CommitID     string
+	CommitID     string `yaml:"commit_id"`
 	Content      string
-	CreatedAt    time.Time
+	CreatedAt    time.Time `yaml:"created_at"`
 	State        string    // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
 	Comments     []*ReviewComment
 }
@@ -31,15 +31,15 @@ type Review struct {
 // ReviewComment represents a review comment
 type ReviewComment struct {
 	ID        int64
-	InReplyTo int64
+	InReplyTo int64 `yaml:"in_reply_to"`
 	Content   string
-	TreePath  string
+	TreePath  string `yaml:"tree_path"`
-	DiffHunk  string
+	DiffHunk  string `yaml:"diff_hunk"`
 	Position  int
 	Line      int
-	CommitID  string
+	CommitID  string `yaml:"commit_id"`
-	PosterID  int64
+	PosterID  int64  `yaml:"poster_id"`
 	Reactions []*Reaction
-	CreatedAt time.Time
+	CreatedAt time.Time `yaml:"created_at"`
-	UpdatedAt time.Time
+	UpdatedAt time.Time `yaml:"updated_at"`
 }

modules/migrations/base/uploader.go

@@ -11,7 +11,7 @@ type Uploader interface {
 	CreateRepo(repo *Repository, opts MigrateOptions) error
 	CreateTopics(topic ...string) error
 	CreateMilestones(milestones ...*Milestone) error
-	CreateReleases(downloader Downloader, releases ...*Release) error
+	CreateReleases(releases ...*Release) error
 	SyncTags() error
 	CreateLabels(labels ...*Label) error
 	CreateIssues(issues ...*Issue) error
@@ -19,5 +19,6 @@ type Uploader interface {
 	CreatePullRequests(prs ...*PullRequest) error
 	CreateReviews(reviews ...*Review) error
 	Rollback() error
+	Finish() error
 	Close()
 }

modules/migrations/dump.go

@@ -0,0 +1,591 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/repository"
+
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	_ base.Uploader = &RepositoryDumper{}
+)
+
+// RepositoryDumper implements an Uploader to the local directory
+type RepositoryDumper struct {
+	ctx             context.Context
+	baseDir         string
+	repoOwner       string
+	repoName        string
+	opts            base.MigrateOptions
+	milestoneFile   *os.File
+	labelFile       *os.File
+	releaseFile     *os.File
+	issueFile       *os.File
+	commentFiles    map[int64]*os.File
+	pullrequestFile *os.File
+	reviewFiles     map[int64]*os.File
+
+	gitRepo     *git.Repository
+	prHeadCache map[string]struct{}
+}
+
+// NewRepositoryDumper creates an gitea Uploader
+func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) (*RepositoryDumper, error) {
+	baseDir = filepath.Join(baseDir, repoOwner, repoName)
+	if err := os.MkdirAll(baseDir, os.ModePerm); err != nil {
+		return nil, err
+	}
+	return &RepositoryDumper{
+		ctx:          ctx,
+		opts:         opts,
+		baseDir:      baseDir,
+		repoOwner:    repoOwner,
+		repoName:     repoName,
+		prHeadCache:  make(map[string]struct{}),
+		commentFiles: make(map[int64]*os.File),
+		reviewFiles:  make(map[int64]*os.File),
+	}, nil
+}
+
+// MaxBatchInsertSize returns the table's max batch insert size
+func (g *RepositoryDumper) MaxBatchInsertSize(tp string) int {
+	return 1000
+}
+
+func (g *RepositoryDumper) gitPath() string {
+	return filepath.Join(g.baseDir, "git")
+}
+
+func (g *RepositoryDumper) wikiPath() string {
+	return filepath.Join(g.baseDir, "wiki")
+}
+
+func (g *RepositoryDumper) commentDir() string {
+	return filepath.Join(g.baseDir, "comments")
+}
+
+func (g *RepositoryDumper) reviewDir() string {
+	return filepath.Join(g.baseDir, "reviews")
+}
+
+func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) {
+	if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 {
+		u, err := url.Parse(remoteAddr)
+		if err != nil {
+			return "", err
+		}
+		u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword)
+		if len(g.opts.AuthToken) > 0 {
+			u.User = url.UserPassword("oauth2", g.opts.AuthToken)
+		}
+		remoteAddr = u.String()
+	}
+
+	return remoteAddr, nil
+}
+
+// CreateRepo creates a repository
+func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
+	f, err := os.Create(filepath.Join(g.baseDir, "repo.yml"))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	bs, err := yaml.Marshal(map[string]interface{}{
+		"name":         repo.Name,
+		"owner":        repo.Owner,
+		"description":  repo.Description,
+		"clone_addr":   opts.CloneAddr,
+		"original_url": repo.OriginalURL,
+		"is_private":   opts.Private,
+		"service_type": opts.GitServiceType,
+		"wiki":         opts.Wiki,
+		"issues":       opts.Issues,
+		"milestones":   opts.Milestones,
+		"labels":       opts.Labels,
+		"releases":     opts.Releases,
+		"comments":     opts.Comments,
+		"pulls":        opts.PullRequests,
+		"assets":       opts.ReleaseAssets,
+	})
+	if err != nil {
+		return err
+	}
+
+	if _, err := f.Write(bs); err != nil {
+		return err
+	}
+
+	repoPath := g.gitPath()
+	if err := os.MkdirAll(repoPath, os.ModePerm); err != nil {
+		return err
+	}
+
+	migrateTimeout := 2 * time.Hour
+
+	remoteAddr, err := g.setURLToken(repo.CloneURL)
+	if err != nil {
+		return err
+	}
+
+	err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{
+		Mirror:  true,
+		Quiet:   true,
+		Timeout: migrateTimeout,
+	})
+	if err != nil {
+		return fmt.Errorf("Clone: %v", err)
+	}
+
+	if opts.Wiki {
+		wikiPath := g.wikiPath()
+		wikiRemotePath := repository.WikiRemoteURL(remoteAddr)
+		if len(wikiRemotePath) > 0 {
+			if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil {
+				return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
+			}
+
+			if err := git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
+				Mirror:  true,
+				Quiet:   true,
+				Timeout: migrateTimeout,
+				Branch:  "master",
+			}); err != nil {
+				log.Warn("Clone wiki: %v", err)
+				if err := os.RemoveAll(wikiPath); err != nil {
+					return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
+				}
+			}
+		}
+	}
+
+	g.gitRepo, err = git.OpenRepository(g.gitPath())
+	return err
+}
+
+// Close closes this uploader
+func (g *RepositoryDumper) Close() {
+	if g.gitRepo != nil {
+		g.gitRepo.Close()
+	}
+	if g.milestoneFile != nil {
+		g.milestoneFile.Close()
+	}
+	if g.labelFile != nil {
+		g.labelFile.Close()
+	}
+	if g.releaseFile != nil {
+		g.releaseFile.Close()
+	}
+	if g.issueFile != nil {
+		g.issueFile.Close()
+	}
+	for _, f := range g.commentFiles {
+		f.Close()
+	}
+	if g.pullrequestFile != nil {
+		g.pullrequestFile.Close()
+	}
+	for _, f := range g.reviewFiles {
+		f.Close()
+	}
+}
+
+// CreateTopics creates topics
+func (g *RepositoryDumper) CreateTopics(topics ...string) error {
+	f, err := os.Create(filepath.Join(g.baseDir, "topic.yml"))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	bs, err := yaml.Marshal(map[string]interface{}{
+		"topics": topics,
+	})
+	if err != nil {
+		return err
+	}
+
+	if _, err := f.Write(bs); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// CreateMilestones creates milestones
+func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error {
+	var err error
+	if g.milestoneFile == nil {
+		g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml"))
+		if err != nil {
+			return err
+		}
+	}
+
+	bs, err := yaml.Marshal(milestones)
+	if err != nil {
+		return err
+	}
+
+	if _, err := g.milestoneFile.Write(bs); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// CreateLabels creates labels
+func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error {
+	var err error
+	if g.labelFile == nil {
+		g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml"))
+		if err != nil {
+			return err
+		}
+	}
+
+	bs, err := yaml.Marshal(labels)
+	if err != nil {
+		return err
+	}
+
+	if _, err := g.labelFile.Write(bs); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// CreateReleases creates releases
+func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error {
+	if g.opts.ReleaseAssets {
+		for _, release := range releases {
+			attachDir := filepath.Join("release_assets", release.TagName)
+			if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil {
+				return err
+			}
+			for _, asset := range release.Assets {
+				attachLocalPath := filepath.Join(attachDir, asset.Name)
+				// download attachment
+
+				err := func(attachPath string) error {
+					var rc io.ReadCloser
+					var err error
+					if asset.DownloadURL == nil {
+						rc, err = asset.DownloadFunc()
+						if err != nil {
+							return err
+						}
+					} else {
+						resp, err := http.Get(*asset.DownloadURL)
+						if err != nil {
+							return err
+						}
+						rc = resp.Body
+					}
+					defer rc.Close()
+
+					fw, err := os.Create(attachPath)
+					if err != nil {
+						return fmt.Errorf("Create: %v", err)
+					}
+					defer fw.Close()
+
+					_, err = io.Copy(fw, rc)
+					return err
+				}(filepath.Join(g.baseDir, attachLocalPath))
+				if err != nil {
+					return err
+				}
+				asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source
+			}
+		}
+	}
+
+	var err error
+	if g.releaseFile == nil {
+		g.releaseFile, err = os.Create(filepath.Join(g.baseDir, "release.yml"))
+		if err != nil {
+			return err
+		}
+	}
+
+	bs, err := yaml.Marshal(releases)
+	if err != nil {
+		return err
+	}
+
+	if _, err := g.releaseFile.Write(bs); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SyncTags syncs releases with tags in the database
+func (g *RepositoryDumper) SyncTags() error {
+	return nil
+}
+
+// CreateIssues creates issues
+func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error {
+	var err error
+	if g.issueFile == nil {
+		g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml"))
+		if err != nil {
+			return err
+		}
+	}
+
+	bs, err := yaml.Marshal(issues)
+	if err != nil {
+		return err
+	}
+
+	if _, err := g.issueFile.Write(bs); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File, itemsMap map[int64][]interface{}) error {
+	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		return err
+	}
+
+	for number, items := range itemsMap {
+		var err error
+		itemFile := itemFiles[number]
+		if itemFile == nil {
+			itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number)))
+			if err != nil {
+				return err
+			}
+			itemFiles[number] = itemFile
+		}
+
+		bs, err := yaml.Marshal(items)
+		if err != nil {
+			return err
+		}
+
+		if _, err := itemFile.Write(bs); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// CreateComments creates comments of issues
+func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error {
+	var commentsMap = make(map[int64][]interface{}, len(comments))
+	for _, comment := range comments {
+		commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment)
+	}
+
+	return g.createItems(g.commentDir(), g.commentFiles, commentsMap)
+}
+
+// CreatePullRequests creates pull requests
+func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
+	for _, pr := range prs {
+		// download patch file
+		err := func() error {
+			u, err := g.setURLToken(pr.PatchURL)
+			if err != nil {
+				return err
+			}
+			resp, err := http.Get(u)
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+			pullDir := filepath.Join(g.gitPath(), "pulls")
+			if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
+				return err
+			}
+			fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))
+			f, err := os.Create(fPath)
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+			if _, err = io.Copy(f, resp.Body); err != nil {
+				return err
+			}
+			pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number)
+
+			return nil
+		}()
+		if err != nil {
+			return err
+		}
+
+		// set head information
+		pullHead := filepath.Join(g.gitPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
+		if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
+			return err
+		}
+		p, err := os.Create(filepath.Join(pullHead, "head"))
+		if err != nil {
+			return err
+		}
+		_, err = p.WriteString(pr.Head.SHA)
+		p.Close()
+		if err != nil {
+			return err
+		}
+
+		if pr.IsForkPullRequest() && pr.State != "closed" {
+			if pr.Head.OwnerName != "" {
+				remote := pr.Head.OwnerName
+				_, ok := g.prHeadCache[remote]
+				if !ok {
+					// git remote add
+					// TODO: how to handle private CloneURL?
+					err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
+					if err != nil {
+						log.Error("AddRemote failed: %s", err)
+					} else {
+						g.prHeadCache[remote] = struct{}{}
+						ok = true
+					}
+				}
+
+				if ok {
+					_, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.gitPath())
+					if err != nil {
+						log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
+					} else {
+						headBranch := filepath.Join(g.gitPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
+						if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
+							return err
+						}
+						b, err := os.Create(headBranch)
+						if err != nil {
+							return err
+						}
+						_, err = b.WriteString(pr.Head.SHA)
+						b.Close()
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+	}
+
+	var err error
+	if g.pullrequestFile == nil {
+		if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil {
+			return err
+		}
+		g.pullrequestFile, err = os.Create(filepath.Join(g.baseDir, "pull_request.yml"))
+		if err != nil {
+			return err
+		}
+	}
+
+	bs, err := yaml.Marshal(prs)
+	if err != nil {
+		return err
+	}
+
+	if _, err := g.pullrequestFile.Write(bs); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// CreateReviews create pull request reviews
+func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error {
+	var reviewsMap = make(map[int64][]interface{}, len(reviews))
+	for _, review := range reviews {
+		reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review)
+	}
+
+	return g.createItems(g.reviewDir(), g.reviewFiles, reviewsMap)
+}
+
+// Rollback when migrating failed, this will rollback all the changes.
+func (g *RepositoryDumper) Rollback() error {
+	g.Close()
+	return os.RemoveAll(g.baseDir)
+}
+
+// Finish when migrating succeed, this will update something.
+func (g *RepositoryDumper) Finish() error {
+	return nil
+}
+
+// DumpRepository dump repository according MigrateOptions to a local directory
+func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error {
+	downloader, err := newDownloader(ctx, ownerName, opts)
+	if err != nil {
+		return err
+	}
+	uploader, err := NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts)
+	if err != nil {
+		return err
+	}
+
+	if err := migrateRepository(downloader, uploader, opts); err != nil {
+		if err1 := uploader.Rollback(); err1 != nil {
+			log.Error("rollback failed: %v", err1)
+		}
+		return err
+	}
+	return nil
+}
+
+// RestoreRepository restore a repository from the disk directory
+func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string) error {
+	doer, err := models.GetAdminUser()
+	if err != nil {
+		return err
+	}
+	var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName)
+	downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName)
+	if err != nil {
+		return err
+	}
+	if err = migrateRepository(downloader, uploader, base.MigrateOptions{
+		Wiki:          true,
+		Issues:        true,
+		Milestones:    true,
+		Labels:        true,
+		Releases:      true,
+		Comments:      true,
+		PullRequests:  true,
+		ReleaseAssets: true,
+	}); err != nil {
+		if err1 := uploader.Rollback(); err1 != nil {
+			log.Error("rollback failed: %v", err1)
+		}
+		return err
+	}
+	return nil
+}

modules/migrations/error.go

@@ -14,6 +14,9 @@ import (
 var (
 	// ErrNotSupported returns the error not supported
 	ErrNotSupported = errors.New("not supported")
+
+	// ErrRepoNotCreated returns the error that repository not created
+	ErrRepoNotCreated = errors.New("repository is not created yet")
 )
 
 // IsRateLimitError returns true if the err is github.RateLimitError

modules/migrations/git.go

@@ -6,7 +6,6 @@ package migrations
 
 import (
 	"context"
-	"io"
 
 	"code.gitea.io/gitea/modules/migrations/base"
 )
@@ -65,11 +64,6 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
 	return nil, ErrNotSupported
 }
 
-// GetAsset returns an asset
-func (g *PlainGitDownloader) GetAsset(_ string, _, _ int64) (io.ReadCloser, error) {
-	return nil, ErrNotSupported
-}
-
 // GetIssues returns issues according page and perPage
 func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
 	return nil, false, ErrNotSupported

modules/migrations/gitea_downloader.go

@@ -268,13 +268,27 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
 	for _, asset := range rel.Attachments {
 		size := int(asset.Size)
 		dlCount := int(asset.DownloadCount)
-		r.Assets = append(r.Assets, base.ReleaseAsset{
+		r.Assets = append(r.Assets, &base.ReleaseAsset{
 			ID:            asset.ID,
 			Name:          asset.Name,
 			Size:          &size,
 			DownloadCount: &dlCount,
 			Created:       asset.Created,
 			DownloadURL:   &asset.DownloadURL,
+			DownloadFunc: func() (io.ReadCloser, error) {
+				asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, rel.ID, asset.ID)
+				if err != nil {
+					return nil, err
+				}
+				// FIXME: for a private download?
+				resp, err := http.Get(asset.DownloadURL)
+				if err != nil {
+					return nil, err
+				}
+
+				// resp.Body is closed by the uploader
+				return resp.Body, nil
+			},
 		})
 	}
 	return r
@@ -310,21 +324,6 @@ func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) {
 	return releases, nil
 }
 
-// GetAsset returns an asset
-func (g *GiteaDownloader) GetAsset(_ string, relID, id int64) (io.ReadCloser, error) {
-	asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, relID, id)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := http.Get(asset.DownloadURL)
-	if err != nil {
-		return nil, err
-	}
-
-	// resp.Body is closed by the uploader
-	return resp.Body, nil
-}
-
 func (g *GiteaDownloader) getIssueReactions(index int64) ([]*base.Reaction, error) {
 	var reactions []*base.Reaction
 	if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {

modules/migrations/gitea_uploader.go

@@ -10,7 +10,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -28,6 +27,7 @@ import (
 	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/uri"
 	"code.gitea.io/gitea/services/pull"
 
 	gouuid "github.com/google/uuid"
@@ -86,6 +86,22 @@ func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
 	return 10
 }
 
+func fullURL(opts base.MigrateOptions, remoteAddr string) (string, error) {
+	var fullRemoteAddr = remoteAddr
+	if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
+		u, err := url.Parse(remoteAddr)
+		if err != nil {
+			return "", err
+		}
+		u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
+		if len(opts.AuthToken) > 0 {
+			u.User = url.UserPassword("oauth2", opts.AuthToken)
+		}
+		fullRemoteAddr = u.String()
+	}
+	return fullRemoteAddr, nil
+}
+
 // CreateRepo creates a repository
 func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
 	owner, err := models.GetUserByName(g.repoOwner)
@@ -93,19 +109,10 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
 		return err
 	}
 
-	var remoteAddr = repo.CloneURL
+	remoteAddr, err := fullURL(opts, repo.CloneURL)
-	if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
-		u, err := url.Parse(repo.CloneURL)
 	if err != nil {
 		return err
 	}
-		u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
-		if len(opts.AuthToken) > 0 {
-			u.User = url.UserPassword("oauth2", opts.AuthToken)
-		}
-		remoteAddr = u.String()
-	}
-
 	var r *models.Repository
 	if opts.MigrateToRepoID <= 0 {
 		r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{
@@ -224,7 +231,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
 }
 
 // CreateReleases creates releases
-func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases ...*base.Release) error {
+func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 	var rels = make([]*models.Release, 0, len(releases))
 	for _, release := range releases {
 		var rel = models.Release{
@@ -283,25 +290,27 @@ func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases
 
 			// download attachment
 			err = func() error {
+				// asset.DownloadURL maybe a local file
 				var rc io.ReadCloser
 				if asset.DownloadURL == nil {
-					rc, err = downloader.GetAsset(rel.TagName, rel.ID, asset.ID)
+					rc, err = asset.DownloadFunc()
 					if err != nil {
 						return err
 					}
 				} else {
-					resp, err := http.Get(*asset.DownloadURL)
+					rc, err = uri.Open(*asset.DownloadURL)
 					if err != nil {
 						return err
 					}
-					rc = resp.Body
 				}
+				defer rc.Close()
 				_, err = storage.Attachments.Save(attach.RelativePath(), rc)
 				return err
 			}()
 			if err != nil {
 				return err
 			}
+
 			rel.Attachments = append(rel.Attachments, &attach)
 		}
 
@@ -559,11 +568,12 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 
 	// download patch file
 	err := func() error {
-		resp, err := http.Get(pr.PatchURL)
+		// pr.PatchURL maybe a local file
+		ret, err := uri.Open(pr.PatchURL)
 		if err != nil {
 			return err
 		}
-		defer resp.Body.Close()
+		defer ret.Close()
 		pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
 		if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
 			return err
@@ -573,7 +583,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 			return err
 		}
 		defer f.Close()
-		_, err = io.Copy(f, resp.Body)
+		_, err = io.Copy(f, ret)
 		return err
 	}()
 	if err != nil {
@@ -859,3 +869,13 @@ func (g *GiteaLocalUploader) Rollback() error {
 	}
 	return nil
 }
+
+// Finish when migrating success, this will do some status update things.
+func (g *GiteaLocalUploader) Finish() error {
+	if g.repo == nil || g.repo.ID <= 0 {
+		return ErrRepoNotCreated
+	}
+
+	g.repo.Status = models.RepositoryReady
+	return models.UpdateRepositoryCols(g.repo, "status")
+}

modules/migrations/gitea_uploader_test.go

@@ -52,6 +52,7 @@ func TestGiteaUploadRepo(t *testing.T) {
 
 	repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)
 	assert.True(t, repo.HasWiki())
+	assert.EqualValues(t, models.RepositoryReady, repo.Status)
 
 	milestones, err := models.GetMilestones(models.GetMilestonesOption{
 		RepoID: repo.ID,

modules/migrations/github.go

@@ -291,7 +291,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
 	}
 
 	for _, asset := range rel.Assets {
-		r.Assets = append(r.Assets, base.ReleaseAsset{
+		r.Assets = append(r.Assets, &base.ReleaseAsset{
 			ID:            *asset.ID,
 			Name:          *asset.Name,
 			ContentType:   asset.ContentType,
@@ -299,6 +299,16 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
 			DownloadCount: asset.DownloadCount,
 			Created:       asset.CreatedAt.Time,
 			Updated:       asset.UpdatedAt.Time,
+			DownloadFunc: func() (io.ReadCloser, error) {
+				asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, *asset.ID, http.DefaultClient)
+				if err != nil {
+					return nil, err
+				}
+				if asset == nil {
+					return ioutil.NopCloser(bytes.NewBufferString(redir)), nil
+				}
+				return asset, nil
+			},
 		})
 	}
 	return r
@@ -330,18 +340,6 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
 	return releases, nil
 }
 
-// GetAsset returns an asset
-func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, error) {
-	asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, id, http.DefaultClient)
-	if err != nil {
-		return nil, err
-	}
-	if asset == nil {
-		return ioutil.NopCloser(bytes.NewBufferString(redir)), nil
-	}
-	return asset, nil
-}
-
 // GetIssues returns issues according start and limit
 func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
 	if perPage > g.maxPerPage {
@@ -363,6 +361,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
 	if err != nil {
 		return nil, false, fmt.Errorf("error while listing repos: %v", err)
 	}
+	log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))
 	g.rate = &resp.Rate
 	for _, issue := range issues {
 		if issue.IsPullRequest() {

modules/migrations/gitlab.go

@@ -295,12 +295,32 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
 	}
 
 	for k, asset := range rel.Assets.Links {
-		r.Assets = append(r.Assets, base.ReleaseAsset{
+		r.Assets = append(r.Assets, &base.ReleaseAsset{
 			ID:            int64(asset.ID),
 			Name:          asset.Name,
 			ContentType:   &rel.Assets.Sources[k].Format,
 			Size:          &zero,
 			DownloadCount: &zero,
+			DownloadFunc: func() (io.ReadCloser, error) {
+				link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, asset.ID, gitlab.WithContext(g.ctx))
+				if err != nil {
+					return nil, err
+				}
+
+				req, err := http.NewRequest("GET", link.URL, nil)
+				if err != nil {
+					return nil, err
+				}
+				req = req.WithContext(g.ctx)
+
+				resp, err := http.DefaultClient.Do(req)
+				if err != nil {
+					return nil, err
+				}
+
+				// resp.Body is closed by the uploader
+				return resp.Body, nil
+			},
 		})
 	}
 	return r
@@ -329,28 +349,6 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
 	return releases, nil
 }
 
-// GetAsset returns an asset
-func (g *GitlabDownloader) GetAsset(tag string, _, id int64) (io.ReadCloser, error) {
-	link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, tag, int(id), gitlab.WithContext(g.ctx))
-	if err != nil {
-		return nil, err
-	}
-
-	req, err := http.NewRequest("GET", link.URL, nil)
-	if err != nil {
-		return nil, err
-	}
-	req = req.WithContext(g.ctx)
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-
-	// resp.Body is closed by the uploader
-	return resp.Body, nil
-}
-
 // GetIssues returns issues according start and limit
 //   Note: issue label description and colors are not supported by the go-gitlab library at this time
 func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {

modules/migrations/migrate.go

@@ -73,10 +73,30 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
 	if err != nil {
 		return nil, err
 	}
+	downloader, err := newDownloader(ctx, ownerName, opts)
+	if err != nil {
+		return nil, err
+	}
 
+	var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
+	uploader.gitServiceType = opts.GitServiceType
+
+	if err := migrateRepository(downloader, uploader, opts); err != nil {
+		if err1 := uploader.Rollback(); err1 != nil {
+			log.Error("rollback failed: %v", err1)
+		}
+		if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
+			log.Error("create respotiry notice failed: ", err2)
+		}
+		return nil, err
+	}
+	return uploader.repo, nil
+}
+
+func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) {
 	var (
 		downloader base.Downloader
-		uploader   = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
+		err        error
 	)
 
 	for _, factory := range factories {
@@ -101,24 +121,10 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
 		log.Trace("Will migrate from git: %s", opts.OriginalURL)
 	}
 
-	uploader.gitServiceType = opts.GitServiceType
-
 	if setting.Migrations.MaxAttempts > 1 {
 		downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
 	}
-
+	return downloader, nil
-	if err := migrateRepository(downloader, uploader, opts); err != nil {
-		if err1 := uploader.Rollback(); err1 != nil {
-			log.Error("rollback failed: %v", err1)
-		}
-
-		if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
-			log.Error("create repository notice failed: ", err2)
-		}
-		return nil, err
-	}
-
-	return uploader.repo, nil
 }
 
 // migrateRepository will download information and then upload it to Uploader, this is a simple
@@ -204,7 +210,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 				relBatchSize = len(releases)
 			}
 
-			if err := uploader.CreateReleases(downloader, releases[:relBatchSize]...); err != nil {
+			if err := uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
 				return err
 			}
 			releases = releases[relBatchSize:]
@@ -235,12 +241,10 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 				return err
 			}
 
-			if !opts.Comments {
+			if opts.Comments {
-				continue
-			}
-
 				var allComments = make([]*base.Comment, 0, commentBatchSize)
 				for _, issue := range issues {
+					log.Trace("migrating issue %d's comments", issue.Number)
 					comments, err := downloader.GetComments(issue.Number)
 					if err != nil {
 						return err
@@ -262,6 +266,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 						return err
 					}
 				}
+			}
 
 			if isEnd {
 				break
@@ -282,13 +287,11 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 				return err
 			}
 
-			if !opts.Comments {
+			if opts.Comments {
-				continue
-			}
-
 				// plain comments
 				var allComments = make([]*base.Comment, 0, commentBatchSize)
 				for _, pr := range prs {
+					log.Trace("migrating pull request %d's comments", pr.Number)
 					comments, err := downloader.GetComments(pr.Number)
 					if err != nil {
 						return err
@@ -343,6 +346,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 						return err
 					}
 				}
+			}
 
 			if isEnd {
 				break
@@ -350,7 +354,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 		}
 	}
 
-	return nil
+	return uploader.Finish()
 }
 
 // Init migrations service

modules/migrations/restore.go

@@ -0,0 +1,276 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"code.gitea.io/gitea/modules/migrations/base"
+
+	"gopkg.in/yaml.v2"
+)
+
+// RepositoryRestorer implements an Downloader from the local directory
+type RepositoryRestorer struct {
+	ctx       context.Context
+	baseDir   string
+	repoOwner string
+	repoName  string
+}
+
+// NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder
+func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) (*RepositoryRestorer, error) {
+	baseDir, err := filepath.Abs(baseDir)
+	if err != nil {
+		return nil, err
+	}
+	return &RepositoryRestorer{
+		ctx:       ctx,
+		baseDir:   baseDir,
+		repoOwner: owner,
+		repoName:  repoName,
+	}, nil
+}
+
+func (r *RepositoryRestorer) commentDir() string {
+	return filepath.Join(r.baseDir, "comments")
+}
+
+func (r *RepositoryRestorer) reviewDir() string {
+	return filepath.Join(r.baseDir, "reviews")
+}
+
+// SetContext set context
+func (r *RepositoryRestorer) SetContext(ctx context.Context) {
+	r.ctx = ctx
+}
+
+// GetRepoInfo returns a repository information
+func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) {
+	p := filepath.Join(r.baseDir, "repo.yml")
+	bs, err := ioutil.ReadFile(p)
+	if err != nil {
+		return nil, err
+	}
+
+	var opts = make(map[string]string)
+	err = yaml.Unmarshal(bs, &opts)
+	if err != nil {
+		return nil, err
+	}
+
+	isPrivate, _ := strconv.ParseBool(opts["is_private"])
+
+	return &base.Repository{
+		Owner:         r.repoOwner,
+		Name:          r.repoName,
+		IsPrivate:     isPrivate,
+		Description:   opts["description"],
+		OriginalURL:   opts["original_url"],
+		CloneURL:      opts["clone_addr"],
+		DefaultBranch: opts["default_branch"],
+	}, nil
+}
+
+// GetTopics return github topics
+func (r *RepositoryRestorer) GetTopics() ([]string, error) {
+	p := filepath.Join(r.baseDir, "topic.yml")
+
+	var topics = struct {
+		Topics []string `yaml:"topics"`
+	}{}
+
+	bs, err := ioutil.ReadFile(p)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(bs, &topics)
+	if err != nil {
+		return nil, err
+	}
+	return topics.Topics, nil
+}
+
+// GetMilestones returns milestones
+func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) {
+	var milestones = make([]*base.Milestone, 0, 10)
+	p := filepath.Join(r.baseDir, "milestone.yml")
+	_, err := os.Stat(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	bs, err := ioutil.ReadFile(p)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(bs, &milestones)
+	if err != nil {
+		return nil, err
+	}
+	return milestones, nil
+}
+
+// GetReleases returns releases
+func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) {
+	var releases = make([]*base.Release, 0, 10)
+	p := filepath.Join(r.baseDir, "release.yml")
+	_, err := os.Stat(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	bs, err := ioutil.ReadFile(p)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(bs, &releases)
+	if err != nil {
+		return nil, err
+	}
+	for _, rel := range releases {
+		for _, asset := range rel.Assets {
+			*asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL)
+		}
+	}
+	return releases, nil
+}
+
+// GetLabels returns labels
+func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) {
+	var labels = make([]*base.Label, 0, 10)
+	p := filepath.Join(r.baseDir, "label.yml")
+	_, err := os.Stat(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	bs, err := ioutil.ReadFile(p)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(bs, &labels)
+	if err != nil {
+		return nil, err
+	}
+	return labels, nil
+}
+
+// GetIssues returns issues according start and limit
+func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
+	var issues = make([]*base.Issue, 0, 10)
+	p := filepath.Join(r.baseDir, "issue.yml")
+	_, err := os.Stat(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, true, nil
+		}
+		return nil, false, err
+	}
+
+	bs, err := ioutil.ReadFile(p)
+	if err != nil {
+		return nil, false, err
+	}
+
+	err = yaml.Unmarshal(bs, &issues)
+	if err != nil {
+		return nil, false, err
+	}
+	return issues, true, nil
+}
+
+// GetComments returns comments according issueNumber
+func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) {
+	var comments = make([]*base.Comment, 0, 10)
+	p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber))
+	_, err := os.Stat(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	bs, err := ioutil.ReadFile(p)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(bs, &comments)
+	if err != nil {
+		return nil, err
+	}
+	return comments, nil
+}
+
+// GetPullRequests returns pull requests according page and perPage
+func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
+	var pulls = make([]*base.PullRequest, 0, 10)
+	p := filepath.Join(r.baseDir, "pull_request.yml")
+	_, err := os.Stat(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, true, nil
+		}
+		return nil, false, err
+	}
+
+	bs, err := ioutil.ReadFile(p)
+	if err != nil {
+		return nil, false, err
+	}
+
+	err = yaml.Unmarshal(bs, &pulls)
+	if err != nil {
+		return nil, false, err
+	}
+	for _, pr := range pulls {
+		pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL)
+	}
+	return pulls, true, nil
+}
+
+// GetReviews returns pull requests review
+func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
+	var reviews = make([]*base.Review, 0, 10)
+	p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", pullRequestNumber))
+	_, err := os.Stat(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	bs, err := ioutil.ReadFile(p)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(bs, &reviews)
+	if err != nil {
+		return nil, err
+	}
+	return reviews, nil
+}

modules/uri/uri.go

@@ -0,0 +1,40 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uri
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+)
+
+// ErrURISchemeNotSupported represents a scheme error
+type ErrURISchemeNotSupported struct {
+	Scheme string
+}
+
+func (e ErrURISchemeNotSupported) Error() string {
+	return fmt.Sprintf("Unsupported scheme: %v", e.Scheme)
+}
+
+// Open open a local file or a remote file
+func Open(uriStr string) (io.ReadCloser, error) {
+	u, err := url.Parse(uriStr)
+	if err != nil {
+		return nil, err
+	}
+	switch strings.ToLower(u.Scheme) {
+	case "http", "https":
+		f, err := http.Get(uriStr)
+		return f.Body, err
+	case "file":
+		return os.Open(u.Path)
+	default:
+		return nil, ErrURISchemeNotSupported{Scheme: u.Scheme}
+	}
+}

modules/uri/uri_test.go

@@ -0,0 +1,20 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uri
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadURI(t *testing.T) {
+	p, err := filepath.Abs("./uri.go")
+	assert.NoError(t, err)
+	f, err := Open("file://" + p)
+	assert.NoError(t, err)
+	defer f.Close()
+}

routers/api/v1/repo/migrate.go

@@ -176,12 +176,9 @@ func Migrate(ctx *context.APIContext, form api.MigrateRepoOptions) {
 		}
 
 		if err == nil {
-			repo.Status = models.RepositoryReady
-			if err := models.UpdateRepositoryCols(repo, "status"); err == nil {
 			notification.NotifyMigrateRepository(ctx.User, repoOwner, repo)
 			return
 		}
-		}
 
 		if repo != nil {
 			if errDelete := models.DeleteRepository(ctx.User, repoOwner.ID, repo.ID); errDelete != nil {