lavender-legacy/gitea
7
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Refactor process package and introduce ProcessManager{} with tests (#75)

Browse source 
* Add a process.Manager singleton with process.GetManager()

* Use process.GetManager everywhere

* Fix godoc comments for process module

* Increment process counter id after locking the mutex
This commit is contained in:
Matthias Loibl 2017-01-17 06:58:58 +01:00 committed by Lunny Xiao
parent 6dd096b7f0
commit d1006150fb
10 changed files with 161 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
models/git_diff.go
View file

@ -483,8 +483,8 @@ func GetDiffRange(repoPath, beforeCommitID, afterCommitID string, maxLines, maxL
return nil, fmt.Errorf("Start: %v", err)
}
pid := process.Add(fmt.Sprintf("GetDiffRange [repo_path: %s]", repoPath), cmd)
defer process.Remove(pid)
pid := process.GetManager().Add(fmt.Sprintf("GetDiffRange [repo_path: %s]", repoPath), cmd)
defer process.GetManager().Remove(pid)
diff, err := ParsePatch(maxLines, maxLineCharacters, maxFiles, stdout)
if err != nil {

18
models/pull.go
View file

@ -281,41 +281,41 @@ func (pr *PullRequest) Merge(doer *User, baseGitRepo *git.Repository) (err error
defer os.RemoveAll(path.Dir(tmpBasePath))
var stderr string
if _, stderr, err = process.ExecTimeout(5*time.Minute,
if _, stderr, err = process.GetManager().ExecTimeout(5*time.Minute,
fmt.Sprintf("PullRequest.Merge (git clone): %s", tmpBasePath),
"git", "clone", baseGitRepo.Path, tmpBasePath); err != nil {
return fmt.Errorf("git clone: %s", stderr)
}
// Check out base branch.
if _, stderr, err = process.ExecDir(-1, tmpBasePath,
if _, stderr, err = process.GetManager().ExecDir(-1, tmpBasePath,
fmt.Sprintf("PullRequest.Merge (git checkout): %s", tmpBasePath),
"git", "checkout", pr.BaseBranch); err != nil {
return fmt.Errorf("git checkout: %s", stderr)
}
// Add head repo remote.
if _, stderr, err = process.ExecDir(-1, tmpBasePath,
if _, stderr, err = process.GetManager().ExecDir(-1, tmpBasePath,
fmt.Sprintf("PullRequest.Merge (git remote add): %s", tmpBasePath),
"git", "remote", "add", "head_repo", headRepoPath); err != nil {
return fmt.Errorf("git remote add [%s -> %s]: %s", headRepoPath, tmpBasePath, stderr)
}
// Merge commits.
if _, stderr, err = process.ExecDir(-1, tmpBasePath,
if _, stderr, err = process.GetManager().ExecDir(-1, tmpBasePath,
fmt.Sprintf("PullRequest.Merge (git fetch): %s", tmpBasePath),
"git", "fetch", "head_repo"); err != nil {
return fmt.Errorf("git fetch [%s -> %s]: %s", headRepoPath, tmpBasePath, stderr)
}
if _, stderr, err = process.ExecDir(-1, tmpBasePath,
if _, stderr, err = process.GetManager().ExecDir(-1, tmpBasePath,
fmt.Sprintf("PullRequest.Merge (git merge --no-ff --no-commit): %s", tmpBasePath),
"git", "merge", "--no-ff", "--no-commit", "head_repo/"+pr.HeadBranch); err != nil {
return fmt.Errorf("git merge --no-ff --no-commit [%s]: %v - %s", tmpBasePath, err, stderr)
}
sig := doer.NewGitSig()
if _, stderr, err = process.ExecDir(-1, tmpBasePath,
if _, stderr, err = process.GetManager().ExecDir(-1, tmpBasePath,
fmt.Sprintf("PullRequest.Merge (git merge): %s", tmpBasePath),
"git", "commit", fmt.Sprintf("--author='%s <%s>'", sig.Name, sig.Email),
"-m", fmt.Sprintf("Merge branch '%s' of %s/%s into %s", pr.HeadBranch, pr.HeadUserName, pr.HeadRepo.Name, pr.BaseBranch)); err != nil {
@ -323,7 +323,7 @@ func (pr *PullRequest) Merge(doer *User, baseGitRepo *git.Repository) (err error
}
// Push back to upstream.
if _, stderr, err = process.ExecDir(-1, tmpBasePath,
if _, stderr, err = process.GetManager().ExecDir(-1, tmpBasePath,
fmt.Sprintf("PullRequest.Merge (git push): %s", tmpBasePath),
"git", "push", baseGitRepo.Path, pr.BaseBranch); err != nil {
return fmt.Errorf("git push: %s", stderr)
@ -437,14 +437,14 @@ func (pr *PullRequest) testPatch() (err error) {
defer os.Remove(indexTmpPath)
var stderr string
_, stderr, err = process.ExecDirEnv(-1, "", fmt.Sprintf("testPatch (git read-tree): %d", pr.BaseRepo.ID),
_, stderr, err = process.GetManager().ExecDirEnv(-1, "", fmt.Sprintf("testPatch (git read-tree): %d", pr.BaseRepo.ID),
[]string{"GIT_DIR=" + pr.BaseRepo.RepoPath(), "GIT_INDEX_FILE=" + indexTmpPath},
"git", "read-tree", pr.BaseBranch)
if err != nil {
return fmt.Errorf("git read-tree --index-output=%s %s: %v - %s", indexTmpPath, pr.BaseBranch, err, stderr)
}
_, stderr, err = process.ExecDirEnv(-1, "", fmt.Sprintf("testPatch (git apply --check): %d", pr.BaseRepo.ID),
_, stderr, err = process.GetManager().ExecDirEnv(-1, "", fmt.Sprintf("testPatch (git apply --check): %d", pr.BaseRepo.ID),
[]string{"GIT_INDEX_FILE=" + indexTmpPath, "GIT_DIR=" + pr.BaseRepo.RepoPath()},
"git", "apply", "--check", "--cached", patchPath)
if err != nil {

8
models/release.go
View file

@ -10,14 +10,11 @@ import (
"strings"
"time"
"github.com/go-xorm/xorm"
"code.gitea.io/git"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/sdk/gitea"
"github.com/go-xorm/xorm"
)
// Release represents a release of repository.
@ -260,6 +257,7 @@ type releaseMetaSearch struct {
ID []int64
Rel []*Release
}
func (s releaseMetaSearch) Len() int {
return len(s.ID)
}
@ -375,7 +373,7 @@ func DeleteReleaseByID(id int64, u *User, delTag bool) error {
}
if delTag {
_, stderr, err := process.ExecDir(-1, repo.RepoPath(),
_, stderr, err := process.GetManager().ExecDir(-1, repo.RepoPath(),
fmt.Sprintf("DeleteReleaseByID (git tag -d): %d", rel.ID),
"git", "tag", "-d", rel.TagName)
if err != nil && !strings.Contains(stderr, "not found") {

26
models/repo.go
View file

@ -147,10 +147,10 @@ func NewRepoContext() {
// Git requires setting user.name and user.email in order to commit changes.
for configKey, defaultValue := range map[string]string{"user.name": "Gitea", "user.email": "gitea@fake.local"} {
if stdout, stderr, err := process.Exec("NewRepoContext(get setting)", "git", "config", "--get", configKey); err != nil || strings.TrimSpace(stdout) == "" {
if stdout, stderr, err := process.GetManager().Exec("NewRepoContext(get setting)", "git", "config", "--get", configKey); err != nil || strings.TrimSpace(stdout) == "" {
// ExitError indicates this config is not set
if _, ok := err.(*exec.ExitError); ok || strings.TrimSpace(stdout) == "" {
if _, stderr, gerr := process.Exec("NewRepoContext(set "+configKey+")", "git", "config", "--global", configKey, defaultValue); gerr != nil {
if _, stderr, gerr := process.GetManager().Exec("NewRepoContext(set "+configKey+")", "git", "config", "--global", configKey, defaultValue); gerr != nil {
log.Fatal(4, "Fail to set git %s(%s): %s", configKey, gerr, stderr)
}
log.Info("Git config %s set to %s", configKey, defaultValue)
@ -161,7 +161,7 @@ func NewRepoContext() {
}
// Set git some configurations.
if _, stderr, err := process.Exec("NewRepoContext(git config --global core.quotepath false)",
if _, stderr, err := process.GetManager().Exec("NewRepoContext(git config --global core.quotepath false)",
"git", "config", "--global", "core.quotepath", "false"); err != nil {
log.Fatal(4, "Fail to execute 'git config --global core.quotepath false': %s", stderr)
}
@ -797,20 +797,20 @@ func CleanUpMigrateInfo(repo *Repository) (*Repository, error) {
// initRepoCommit temporarily changes with work directory.
func initRepoCommit(tmpPath string, sig *git.Signature) (err error) {
var stderr string
if _, stderr, err = process.ExecDir(-1,
if _, stderr, err = process.GetManager().ExecDir(-1,
tmpPath, fmt.Sprintf("initRepoCommit (git add): %s", tmpPath),
"git", "add", "--all"); err != nil {
return fmt.Errorf("git add: %s", stderr)
}
if _, stderr, err = process.ExecDir(-1,
if _, stderr, err = process.GetManager().ExecDir(-1,
tmpPath, fmt.Sprintf("initRepoCommit (git commit): %s", tmpPath),
"git", "commit", fmt.Sprintf("--author='%s <%s>'", sig.Name, sig.Email),
"-m", "Initial commit"); err != nil {
return fmt.Errorf("git commit: %s", stderr)
}
if _, stderr, err = process.ExecDir(-1,
if _, stderr, err = process.GetManager().ExecDir(-1,
tmpPath, fmt.Sprintf("initRepoCommit (git push): %s", tmpPath),
"git", "push", "origin", "master"); err != nil {
return fmt.Errorf("git push: %s", stderr)
@ -856,8 +856,10 @@ func getRepoInitFile(tp, name string) ([]byte, error) {
func prepareRepoCommit(repo *Repository, tmpDir, repoPath string, opts CreateRepoOptions) error {
// Clone to temporary path and do the init commit.
_, stderr, err := process.Exec(
fmt.Sprintf("initRepository(git clone): %s", repoPath), "git", "clone", repoPath, tmpDir)
_, stderr, err := process.GetManager().Exec(
fmt.Sprintf("initRepository(git clone): %s", repoPath),
"git", "clone", repoPath, tmpDir,
)
if err != nil {
return fmt.Errorf("git clone: %v - %s", err, stderr)
}
@ -1066,7 +1068,7 @@ func CreateRepository(u *User, opts CreateRepoOptions) (_ *Repository, err error
return nil, fmt.Errorf("initRepository: %v", err)
}
_, stderr, err := process.ExecDir(-1,
_, stderr, err := process.GetManager().ExecDir(-1,
repoPath, fmt.Sprintf("CreateRepository(git update-server-info): %s", repoPath),
"git", "update-server-info")
if err != nil {
@ -1839,7 +1841,7 @@ func GitGcRepos() error {
if err := repo.GetOwner(); err != nil {
return err
}
_, stderr, err := process.ExecDir(
_, stderr, err := process.GetManager().ExecDir(
time.Duration(setting.Git.Timeout.GC)*time.Second,
RepoPath(repo.Owner.Name, repo.Name), "Repository garbage collection",
"git", args...)
@ -2192,14 +2194,14 @@ func ForkRepository(u *User, oldRepo *Repository, name, desc string) (_ *Reposit
}
repoPath := RepoPath(u.Name, repo.Name)
_, stderr, err := process.ExecTimeout(10*time.Minute,
_, stderr, err := process.GetManager().ExecTimeout(10*time.Minute,
fmt.Sprintf("ForkRepository(git clone): %s/%s", u.Name, repo.Name),
"git", "clone", "--bare", oldRepo.RepoPath(), repoPath)
if err != nil {
return nil, fmt.Errorf("git clone: %v", stderr)
}
_, stderr, err = process.ExecDir(-1,
_, stderr, err = process.GetManager().ExecDir(-1,
repoPath, fmt.Sprintf("ForkRepository(git update-server-info): %s", repoPath),
"git", "update-server-info")
if err != nil {

4
models/repo_editor.go
View file

@ -212,8 +212,8 @@ func (repo *Repository) GetDiffPreview(branch, treePath, content string) (diff *
return nil, fmt.Errorf("Start: %v", err)
}
pid := process.Add(fmt.Sprintf("GetDiffPreview [repo_path: %s]", repo.RepoPath()), cmd)
defer process.Remove(pid)
pid := process.GetManager().Add(fmt.Sprintf("GetDiffPreview [repo_path: %s]", repo.RepoPath()), cmd)
defer process.GetManager().Remove(pid)
diff, err = ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout)
if err != nil {

4
models/repo_mirror.go
View file

@ -137,7 +137,7 @@ func (m *Mirror) runSync() bool {
gitArgs = append(gitArgs, "--prune")
}
if _, stderr, err := process.ExecDir(
if _, stderr, err := process.GetManager().ExecDir(
timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
"git", gitArgs...); err != nil {
desc := fmt.Sprintf("Fail to update mirror repository '%s': %s", repoPath, stderr)
@ -148,7 +148,7 @@ func (m *Mirror) runSync() bool {
return false
}
if m.Repo.HasWiki() {
if _, stderr, err := process.ExecDir(
if _, stderr, err := process.GetManager().ExecDir(
timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath),
"git", "remote", "update", "--prune"); err != nil {
desc := fmt.Sprintf("Fail to update mirror wiki repository '%s': %s", wikiPath, stderr)

4
models/ssh_key.go
View file

@ -197,7 +197,7 @@ func SSHKeyGenParsePublicKey(key string) (string, int, error) {
}
defer os.Remove(tmpName)
stdout, stderr, err := process.Exec("SSHKeyGenParsePublicKey", setting.SSH.KeygenPath, "-lf", tmpName)
stdout, stderr, err := process.GetManager().Exec("SSHKeyGenParsePublicKey", setting.SSH.KeygenPath, "-lf", tmpName)
if err != nil {
return "", 0, fmt.Errorf("fail to parse public key: %s - %s", err, stderr)
}
@ -382,7 +382,7 @@ func addKey(e Engine, key *PublicKey) (err error) {
if err = ioutil.WriteFile(tmpPath, []byte(key.Content), 0644); err != nil {
return err
}
stdout, stderr, err := process.Exec("AddPublicKey", "ssh-keygen", "-lf", tmpPath)
stdout, stderr, err := process.GetManager().Exec("AddPublicKey", "ssh-keygen", "-lf", tmpPath)
if err != nil {
return fmt.Errorf("'ssh-keygen -lf %s' failed with error '%s': %s", tmpPath, err, stderr)
} else if len(stdout) < 2 {

159
modules/process/manager.go
View file

@ -9,71 +9,108 @@ import (
"errors"
"fmt"
"os/exec"
"sync"
"time"
"code.gitea.io/gitea/modules/log"
)
// TODO: This packages still uses a singleton for the Manager.
// Once there's a decent web framework and dependencies are passed around like they should,
// then we delete the singleton.
var (
// ErrExecTimeout represent a timeout error
ErrExecTimeout = errors.New("Process execution timeout")
// DefaultTimeout is the timeout used by Exec* family
// of function when timeout parameter is omitted or
// passed as -1
// NOTE: could be custom in config file for default.
DefaultTimeout = 60 * time.Second
manager *Manager
)
// Process represents a working process inherit from Gogs.
type Process struct {
Pid int64 // Process ID, not system one.
PID int64 // Process ID, not system one.
Description string
Start time.Time
Cmd *exec.Cmd
}
// List of existing processes.
var (
curPid int64 = 1
Processes []*Process
)
// Manager knows about all processes and counts PIDs.
type Manager struct {
mutex sync.Mutex
// Add adds a existing process and returns its PID.
func Add(desc string, cmd *exec.Cmd) int64 {
pid := curPid
Processes = append(Processes, &Process{
Pid: pid,
Description: desc,
counter int64
Processes map[int64]*Process
}
// GetManager returns a Manager and initializes one as singleton if there's none yet
func GetManager() *Manager {
if manager == nil {
manager = &Manager{
Processes: make(map[int64]*Process),
}
}
return manager
}
// Add a process to the ProcessManager and returns its PID.
func (pm *Manager) Add(description string, cmd *exec.Cmd) int64 {
pm.mutex.Lock()
pid := pm.counter + 1
pm.Processes[pid] = &Process{
PID: pid,
Description: description,
Start: time.Now(),
Cmd: cmd,
})
curPid++
}
pm.counter = pid
pm.mutex.Unlock()
return pid
}
// Remove a process from the ProcessManager.
func (pm *Manager) Remove(pid int64) {
pm.mutex.Lock()
delete(pm.Processes, pid)
pm.mutex.Unlock()
}
// Exec a command and use the default timeout.
func (pm *Manager) Exec(desc, cmdName string, args ...string) (string, string, error) {
return pm.ExecDir(-1, "", desc, cmdName, args...)
}
// ExecTimeout a command and use a specific timeout duration.
func (pm *Manager) ExecTimeout(timeout time.Duration, desc, cmdName string, args ...string) (string, string, error) {
return pm.ExecDir(timeout, "", desc, cmdName, args...)
}
// ExecDir a command and use the default timeout.
func (pm *Manager) ExecDir(timeout time.Duration, dir, desc, cmdName string, args ...string) (string, string, error) {
return pm.ExecDirEnv(timeout, dir, desc, nil, cmdName, args...)
}
// ExecDirEnv runs a command in given path and environment variables, and waits for its completion
// up to the given timeout (or DefaultTimeout if -1 is given).
// Returns its complete stdout and stderr
// outputs and an error, if any (including timeout)
func ExecDirEnv(timeout time.Duration, dir, desc string, env []string, cmdName string, args ...string) (string, string, error) {
func (pm *Manager) ExecDirEnv(timeout time.Duration, dir, desc string, env []string, cmdName string, args ...string) (string, string, error) {
if timeout == -1 {
timeout = DefaultTimeout
timeout = 60 * time.Second
}
bufOut := new(bytes.Buffer)
bufErr := new(bytes.Buffer)
stdOut := new(bytes.Buffer)
stdErr := new(bytes.Buffer)
cmd := exec.Command(cmdName, args...)
cmd.Dir = dir
cmd.Env = env
cmd.Stdout = bufOut
cmd.Stderr = bufErr
cmd.Stdout = stdOut
cmd.Stderr = stdErr
if err := cmd.Start(); err != nil {
return "", err.Error(), err
return "", "", err
}
pid := Add(desc, cmd)
pid := pm.Add(desc, cmd)
done := make(chan error)
go func() {
done <- cmd.Wait()
@ -82,61 +119,39 @@ func ExecDirEnv(timeout time.Duration, dir, desc string, env []string, cmdName s
var err error
select {
case <-time.After(timeout):
if errKill := Kill(pid); errKill != nil {
log.Error(4, "Exec(%d:%s): %v", pid, desc, errKill)
if errKill := pm.Kill(pid); errKill != nil {
log.Error(4, "exec(%d:%s) failed to kill: %v", pid, desc, errKill)
}
<-done
return "", ErrExecTimeout.Error(), ErrExecTimeout
return "", "", ErrExecTimeout
case err = <-done:
}
Remove(pid)
return bufOut.String(), bufErr.String(), err
pm.Remove(pid)
if err != nil {
out := fmt.Errorf("exec(%d:%s) failed: %v stdout: %v stderr: %v", pid, desc, err, stdOut, stdErr)
return stdOut.String(), stdErr.String(), out
}
// ExecDir works exactly like ExecDirEnv except no environment variable is provided.
func ExecDir(timeout time.Duration, dir, desc, cmdName string, args ...string) (string, string, error) {
return ExecDirEnv(timeout, dir, desc, nil, cmdName, args...)
return stdOut.String(), stdErr.String(), nil
}
// ExecTimeout runs a command and waits for its completion
// up to the given timeout (or DefaultTimeout if -1 is given).
// Returns its complete stdout and stderr
// outputs and an error, if any (including timeout)
func ExecTimeout(timeout time.Duration, desc, cmdName string, args ...string) (string, string, error) {
return ExecDir(timeout, "", desc, cmdName, args...)
}
// Exec runs a command and waits for its completion
// up to DefaultTimeout. Returns its complete stdout and stderr
// outputs and an error, if any (including timeout)
func Exec(desc, cmdName string, args ...string) (string, string, error) {
return ExecDir(-1, "", desc, cmdName, args...)
}
// Remove removes a process from list.
func Remove(pid int64) {
for i, proc := range Processes {
if proc.Pid == pid {
Processes = append(Processes[:i], Processes[i+1:]...)
return
}
}
}
// Kill kills and removes a process from list.
func Kill(pid int64) error {
for i, proc := range Processes {
if proc.Pid == pid {
if proc.Cmd != nil && proc.Cmd.Process != nil &&
proc.Cmd.ProcessState != nil && !proc.Cmd.ProcessState.Exited() {
// Kill and remove a process from list.
func (pm *Manager) Kill(pid int64) error {
if proc, exists := pm.Processes[pid]; exists {
pm.mutex.Lock()
if proc.Cmd != nil &&
proc.Cmd.Process != nil &&
proc.Cmd.ProcessState != nil &&
!proc.Cmd.ProcessState.Exited() {
if err := proc.Cmd.Process.Kill(); err != nil {
return fmt.Errorf("fail to kill process(%d/%s): %v", proc.Pid, proc.Description, err)
}
}
Processes = append(Processes[:i], Processes[i+1:]...)
return nil
return fmt.Errorf("failed to kill process(%d/%s): %v", pid, proc.Description, err)
}
}
delete(pm.Processes, pid)
pm.mutex.Unlock()
}
return nil
}

33
modules/process/manager_test.go Normal file
View file

@ -0,0 +1,33 @@
package process
import (
"os/exec"
"testing"
"github.com/stretchr/testify/assert"
)
func TestManager_Add(t *testing.T) {
pm := Manager{Processes: make(map[int64]*Process)}
pid := pm.Add("foo", exec.Command("foo"))
assert.Equal(t, int64(1), pid, "expected to get pid 1 got %d", pid)
pid = pm.Add("bar", exec.Command("bar"))
assert.Equal(t, int64(2), pid, "expected to get pid 2 got %d", pid)
}
func TestManager_Remove(t *testing.T) {
pm := Manager{Processes: make(map[int64]*Process)}
pid1 := pm.Add("foo", exec.Command("foo"))
assert.Equal(t, int64(1), pid1, "expected to get pid 1 got %d", pid1)
pid2 := pm.Add("bar", exec.Command("bar"))
assert.Equal(t, int64(2), pid2, "expected to get pid 2 got %d", pid2)
pm.Remove(pid2)
_, exists := pm.Processes[pid2]
assert.False(t, exists, "PID %d is in the list but shouldn't", pid2)
}

2
routers/admin/admin.go
View file

@ -246,7 +246,7 @@ func Monitor(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("admin.monitor")
ctx.Data["PageIsAdmin"] = true
ctx.Data["PageIsAdminMonitor"] = true
ctx.Data["Processes"] = process.Processes
ctx.Data["Processes"] = process.GetManager().Processes
ctx.Data["Entries"] = cron.ListTasks()
ctx.HTML(200, tplMonitor)
}