初始提交: Gitea 项目代码

modules/process/context.go

@@ -0,0 +1,68 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package process
+
+import (
+	"context"
+)
+
+// Context is a wrapper around context.Context and contains the current pid for this context
+type Context struct {
+	context.Context
+	pid IDType
+}
+
+// GetPID returns the PID for this context
+func (c *Context) GetPID() IDType {
+	return c.pid
+}
+
+// GetParent returns the parent process context (if any)
+func (c *Context) GetParent() *Context {
+	return GetContext(c.Context)
+}
+
+// Value is part of the interface for context.Context. We mostly defer to the internal context - but we return this in response to the ProcessContextKey
+func (c *Context) Value(key any) any {
+	if key == ProcessContextKey {
+		return c
+	}
+	return c.Context.Value(key)
+}
+
+// ProcessContextKey is the key under which process contexts are stored
+var ProcessContextKey any = "process_context"
+
+// GetContext will return a process context if one exists
+func GetContext(ctx context.Context) *Context {
+	if pCtx, ok := ctx.(*Context); ok {
+		return pCtx
+	}
+	pCtxInterface := ctx.Value(ProcessContextKey)
+	if pCtxInterface == nil {
+		return nil
+	}
+	if pCtx, ok := pCtxInterface.(*Context); ok {
+		return pCtx
+	}
+	return nil
+}
+
+// GetPID returns the PID for this context
+func GetPID(ctx context.Context) IDType {
+	pCtx := GetContext(ctx)
+	if pCtx == nil {
+		return ""
+	}
+	return pCtx.GetPID()
+}
+
+// GetParentPID returns the ParentPID for this context
+func GetParentPID(ctx context.Context) IDType {
+	var parentPID IDType
+	if parentProcess := GetContext(ctx); parentProcess != nil {
+		parentPID = parentProcess.GetPID()
+	}
+	return parentPID
+}

modules/process/error.go

@@ -0,0 +1,25 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package process
+
+import "fmt"
+
+// Error is a wrapped error describing the error results of Process Execution
+type Error struct {
+	PID         IDType
+	Description string
+	Err         error
+	CtxErr      error
+	Stdout      string
+	Stderr      string
+}
+
+func (err *Error) Error() string {
+	return fmt.Sprintf("exec(%s:%s) failed: %v(%v) stdout: %s stderr: %s", err.PID, err.Description, err.Err, err.CtxErr, err.Stdout, err.Stderr)
+}
+
+// Unwrap implements the unwrappable implicit interface for go1.13 Unwrap()
+func (err *Error) Unwrap() error {
+	return err.Err
+}

modules/process/manager.go

@@ -0,0 +1,245 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package process
+
+import (
+	"context"
+	"runtime/pprof"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"gitea.dev/modules/gtprof"
+	"gitea.dev/modules/util"
+)
+
+// TODO: This packages still uses a singleton for the Manager.
+// Once there's a decent web framework and dependencies are passed around like they should,
+// then we delete the singleton.
+
+var (
+	manager     *Manager
+	managerInit sync.Once
+
+	// DefaultContext is the default context to run processing commands in
+	DefaultContext = context.Background()
+)
+
+type (
+	// IDType is a pid type
+	IDType string
+
+	CancelCauseFunc func(cause ...error)
+	// FinishedFunc is a function that marks that the process is finished and can be removed from the process table
+	FinishedFunc func()
+)
+
+var (
+	traceDisabled atomic.Int64
+	TraceCallback = defaultTraceCallback // this global can be overridden by particular logging packages - thus avoiding import cycles
+)
+
+// defaultTraceCallback is a no-op. Without a proper TraceCallback (provided by the logger system), this "Trace" level messages shouldn't be outputted.
+func defaultTraceCallback(skip int, start bool, pid IDType, description string, parentPID IDType, typ string) {
+}
+
+// TraceLogDisable disables (or revert the disabling) the trace log for the process lifecycle.
+// eg: the logger system shouldn't print the trace log for themselves, that's cycle dependency (Logger -> ProcessManager -> TraceCallback -> Logger ...)
+// Theoretically, such trace log should only be enabled when the logger system is ready with a proper level, so the default TraceCallback is a no-op.
+func TraceLogDisable(v bool) {
+	if v {
+		traceDisabled.Add(1)
+	} else {
+		traceDisabled.Add(-1)
+	}
+}
+
+func Trace(start bool, pid IDType, description string, parentPID IDType, typ string) {
+	if traceDisabled.Load() != 0 {
+		// the traceDisabled counter is mainly for recursive calls, so no concurrency problem.
+		// because the counter can't be 0 since the caller function hasn't returned (decreased the counter) yet.
+		return
+	}
+	TraceCallback(1, start, pid, description, parentPID, typ)
+}
+
+// Manager manages all processes and counts PIDs.
+type Manager struct {
+	mutex sync.Mutex
+
+	next     int64
+	lastTime int64
+
+	processMap map[IDType]*process
+}
+
+// GetManager returns a Manager and initializes one as singleton if there's none yet
+func GetManager() *Manager {
+	managerInit.Do(func() {
+		manager = &Manager{
+			processMap: make(map[IDType]*process),
+			next:       1,
+		}
+	})
+	return manager
+}
+
+func cancelCauseFunc(cancelCause context.CancelCauseFunc) CancelCauseFunc {
+	return func(cause ...error) { cancelCause(util.OptionalArg(cause)) }
+}
+
+// AddContext creates a new context and adds it as a process. Once the process is finished, finished must be called
+// to remove the process from the process table. It should not be called until the process is finished but must always be called.
+//
+// cancel should be used to cancel the returned context, however it will not remove the process from the process table.
+// finished will cancel the returned context and remove it from the process table.
+//
+// Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the
+// process table.
+func (pm *Manager) AddContext(parent context.Context, description string) (context.Context, CancelCauseFunc, FinishedFunc) {
+	ctx, ctxCancel := context.WithCancelCause(parent)
+	cancel := cancelCauseFunc(ctxCancel)
+	ctx, _, finished := pm.Add(ctx, description, cancel, NormalProcessType, true)
+	return ctx, cancel, finished
+}
+
+// AddTypedContext creates a new context and adds it as a process. Once the process is finished, finished must be called
+// to remove the process from the process table. It should not be called until the process is finished but must always be called.
+//
+// cancel should be used to cancel the returned context, however it will not remove the process from the process table.
+// finished will cancel the returned context and remove it from the process table.
+//
+// Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the
+// process table.
+func (pm *Manager) AddTypedContext(parent context.Context, description, processType string, currentlyRunning bool) (context.Context, CancelCauseFunc, FinishedFunc) {
+	ctx, ctxCancel := context.WithCancelCause(parent)
+	cancel := cancelCauseFunc(ctxCancel)
+	ctx, _, finished := pm.Add(ctx, description, cancel, processType, currentlyRunning)
+	return ctx, cancel, finished
+}
+
+// AddContextTimeout creates a new context and add it as a process. Once the process is finished, finished must be called
+// to remove the process from the process table. It should not be called until the process is finished but must always be called.
+//
+// cancel should be used to cancel the returned context, however it will not remove the process from the process table.
+// finished will cancel the returned context and remove it from the process table.
+//
+// Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the
+// process table.
+func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Duration, description string) (context.Context, CancelCauseFunc, FinishedFunc) {
+	if timeout <= 0 {
+		// it's meaningless to use timeout <= 0, and it must be a bug! so we must panic here to tell developers to make the timeout correct
+		panic("the timeout must be greater than zero, otherwise the context will be cancelled immediately")
+	}
+	ctx, ctxCancelTimeout := context.WithTimeout(parent, timeout)
+	ctx, ctxCancelCause := context.WithCancelCause(ctx)
+	cancel := func(cause ...error) {
+		ctxCancelCause(util.OptionalArg(cause))
+		ctxCancelTimeout()
+	}
+	ctx, _, finished := pm.Add(ctx, description, cancel, NormalProcessType, true)
+	return ctx, cancel, finished
+}
+
+// Add create a new process
+func (pm *Manager) Add(ctx context.Context, description string, cancel CancelCauseFunc, processType string, currentlyRunning bool) (context.Context, IDType, FinishedFunc) {
+	parentPID := GetParentPID(ctx)
+
+	pm.mutex.Lock()
+	start, pid := pm.nextPID()
+
+	parent := pm.processMap[parentPID]
+	if parent == nil {
+		parentPID = ""
+	}
+
+	process := &process{
+		PID:         pid,
+		ParentPID:   parentPID,
+		Description: description,
+		Start:       start,
+		Cancel:      cancel,
+		Type:        processType,
+	}
+
+	var finished FinishedFunc
+	if currentlyRunning {
+		finished = func() {
+			cancel()
+			pm.remove(process)
+			pprof.SetGoroutineLabels(ctx)
+		}
+	} else {
+		finished = func() {
+			cancel()
+			pm.remove(process)
+		}
+	}
+
+	pm.processMap[pid] = process
+	pm.mutex.Unlock()
+
+	Trace(true, pid, description, parentPID, processType)
+
+	pprofCtx := pprof.WithLabels(ctx, pprof.Labels(
+		gtprof.LabelProcessDescription, description,
+		gtprof.LabelPpid, string(parentPID),
+		gtprof.LabelPid, string(pid),
+		gtprof.LabelProcessType, processType,
+	))
+	if currentlyRunning {
+		pprof.SetGoroutineLabels(pprofCtx)
+	}
+
+	return &Context{
+		Context: pprofCtx,
+		pid:     pid,
+	}, pid, finished
+}
+
+// nextPID will return the next available PID. pm.mutex should already be locked.
+func (pm *Manager) nextPID() (start time.Time, pid IDType) {
+	start = time.Now()
+	startUnix := start.Unix()
+	if pm.lastTime == startUnix {
+		pm.next++
+	} else {
+		pm.next = 1
+	}
+	pm.lastTime = startUnix
+	pid = IDType(strconv.FormatInt(start.Unix(), 16))
+
+	if pm.next == 1 {
+		return start, pid
+	}
+	pid = IDType(string(pid) + "-" + strconv.FormatInt(pm.next, 10))
+	return start, pid
+}
+
+func (pm *Manager) remove(process *process) {
+	deleted := false
+
+	pm.mutex.Lock()
+	if pm.processMap[process.PID] == process {
+		delete(pm.processMap, process.PID)
+		deleted = true
+	}
+	pm.mutex.Unlock()
+
+	if deleted {
+		Trace(false, process.PID, process.Description, process.ParentPID, process.Type)
+	}
+}
+
+// Cancel a process in the ProcessManager.
+func (pm *Manager) Cancel(pid IDType) {
+	pm.mutex.Lock()
+	process, ok := pm.processMap[pid]
+	pm.mutex.Unlock()
+	if ok && process.Type != SystemProcessType {
+		process.Cancel()
+	}
+}

modules/process/manager_exec.go

@@ -0,0 +1,79 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package process
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"os/exec"
+	"time"
+)
+
+// Exec a command and use the default timeout.
+func (pm *Manager) Exec(desc, cmdName string, args ...string) (string, string, error) {
+	return pm.ExecDir(DefaultContext, -1, "", desc, cmdName, args...)
+}
+
+// ExecTimeout a command and use a specific timeout duration.
+func (pm *Manager) ExecTimeout(timeout time.Duration, desc, cmdName string, args ...string) (string, string, error) {
+	return pm.ExecDir(DefaultContext, timeout, "", desc, cmdName, args...)
+}
+
+// ExecDir a command and use the default timeout.
+func (pm *Manager) ExecDir(ctx context.Context, timeout time.Duration, dir, desc, cmdName string, args ...string) (string, string, error) {
+	return pm.ExecDirEnv(ctx, timeout, dir, desc, nil, cmdName, args...)
+}
+
+// ExecDirEnv runs a command in given path and environment variables, and waits for its completion
+// up to the given timeout (or DefaultTimeout if -1 is given).
+// Returns its complete stdout and stderr
+// outputs and an error, if any (including timeout)
+func (pm *Manager) ExecDirEnv(ctx context.Context, timeout time.Duration, dir, desc string, env []string, cmdName string, args ...string) (string, string, error) {
+	return pm.ExecDirEnvStdIn(ctx, timeout, dir, desc, env, nil, cmdName, args...)
+}
+
+// ExecDirEnvStdIn runs a command in given path and environment variables with provided stdIN, and waits for its completion
+// up to the given timeout (or DefaultTimeout if timeout <= 0 is given).
+// Returns its complete stdout and stderr
+// outputs and an error, if any (including timeout)
+func (pm *Manager) ExecDirEnvStdIn(ctx context.Context, timeout time.Duration, dir, desc string, env []string, stdIn io.Reader, cmdName string, args ...string) (string, string, error) {
+	if timeout <= 0 {
+		timeout = 60 * time.Second
+	}
+
+	stdOut := new(bytes.Buffer)
+	stdErr := new(bytes.Buffer)
+
+	ctx, _, finished := pm.AddContextTimeout(ctx, timeout, desc)
+	defer finished()
+
+	cmd := exec.CommandContext(ctx, cmdName, args...)
+	cmd.Dir = dir
+	cmd.Env = env
+	cmd.Stdout = stdOut
+	cmd.Stderr = stdErr
+	if stdIn != nil {
+		cmd.Stdin = stdIn
+	}
+	SetSysProcAttribute(cmd)
+
+	if err := cmd.Start(); err != nil {
+		return "", "", err
+	}
+
+	err := cmd.Wait()
+	if err != nil {
+		err = &Error{
+			PID:         GetPID(ctx),
+			Description: desc,
+			Err:         err,
+			CtxErr:      ctx.Err(),
+			Stdout:      stdOut.String(),
+			Stderr:      stdErr.String(),
+		}
+	}
+
+	return stdOut.String(), stdErr.String(), err
+}

modules/process/manager_stacktraces.go

@@ -0,0 +1,355 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package process
+
+import (
+	"fmt"
+	"io"
+	"runtime/pprof"
+	"sort"
+	"time"
+
+	"gitea.dev/modules/gtprof"
+
+	"github.com/google/pprof/profile"
+)
+
+// StackEntry is an entry on a stacktrace
+type StackEntry struct {
+	Function string
+	File     string
+	Line     int
+}
+
+// Label represents a pprof label assigned to goroutine stack
+type Label struct {
+	Name  string
+	Value string
+}
+
+// Stack is a stacktrace relating to a goroutine. (Multiple goroutines may have the same stacktrace)
+type Stack struct {
+	Count       int64 // Number of goroutines with this stack trace
+	Description string
+	Labels      []*Label      `json:",omitempty"`
+	Entry       []*StackEntry `json:",omitempty"`
+}
+
+// A Process is a combined representation of a Process and a Stacktrace for the goroutines associated with it
+type Process struct {
+	PID         IDType
+	ParentPID   IDType
+	Description string
+	Start       time.Time
+	Type        string
+
+	Children []*Process `json:",omitempty"`
+	Stacks   []*Stack   `json:",omitempty"`
+}
+
+// Processes gets the processes in a thread safe manner
+func (pm *Manager) Processes(flat, noSystem bool) ([]*Process, int) {
+	pm.mutex.Lock()
+	processCount := len(pm.processMap)
+	processes := make([]*Process, 0, len(pm.processMap))
+	if flat {
+		for _, process := range pm.processMap {
+			if noSystem && process.Type == SystemProcessType {
+				continue
+			}
+			processes = append(processes, process.toProcess())
+		}
+	} else {
+		// We need our own processMap
+		processMap := map[IDType]*Process{}
+		for _, internalProcess := range pm.processMap {
+			process, ok := processMap[internalProcess.PID]
+			if !ok {
+				process = internalProcess.toProcess()
+				processMap[process.PID] = process
+			}
+
+			// Check its parent
+			if process.ParentPID == "" {
+				processes = append(processes, process)
+				continue
+			}
+
+			internalParentProcess, ok := pm.processMap[internalProcess.ParentPID]
+			if ok {
+				parentProcess, ok := processMap[process.ParentPID]
+				if !ok {
+					parentProcess = internalParentProcess.toProcess()
+					processMap[parentProcess.PID] = parentProcess
+				}
+				parentProcess.Children = append(parentProcess.Children, process)
+				continue
+			}
+
+			processes = append(processes, process)
+		}
+	}
+	pm.mutex.Unlock()
+
+	if !flat && noSystem {
+		for i := 0; i < len(processes); i++ {
+			process := processes[i]
+			if process.Type != SystemProcessType {
+				continue
+			}
+			processes[len(processes)-1], processes[i] = processes[i], processes[len(processes)-1]
+			processes = append(processes[:len(processes)-1], process.Children...)
+			i--
+		}
+	}
+
+	// Sort by process' start time. Oldest process appears first.
+	sort.Slice(processes, func(i, j int) bool {
+		left, right := processes[i], processes[j]
+
+		return left.Start.Before(right.Start)
+	})
+
+	return processes, processCount
+}
+
+// ProcessStacktraces gets the processes and stacktraces in a thread safe manner
+func (pm *Manager) ProcessStacktraces(flat, noSystem bool) ([]*Process, int, int64, error) {
+	var stacks *profile.Profile
+	var err error
+
+	// We cannot use the pm.ProcessMap here because we will release the mutex ...
+	processMap := map[IDType]*Process{}
+	var processCount int
+
+	// Lock the manager
+	pm.mutex.Lock()
+	processCount = len(pm.processMap)
+
+	// Add a defer to unlock in case there is a panic
+	unlocked := false
+	defer func() {
+		if !unlocked {
+			pm.mutex.Unlock()
+		}
+	}()
+
+	processes := make([]*Process, 0, len(pm.processMap))
+	if flat {
+		for _, internalProcess := range pm.processMap {
+			process := internalProcess.toProcess()
+			processMap[process.PID] = process
+			if noSystem && internalProcess.Type == SystemProcessType {
+				continue
+			}
+			processes = append(processes, process)
+		}
+	} else {
+		for _, internalProcess := range pm.processMap {
+			process, ok := processMap[internalProcess.PID]
+			if !ok {
+				process = internalProcess.toProcess()
+				processMap[process.PID] = process
+			}
+
+			// Check its parent
+			if process.ParentPID == "" {
+				processes = append(processes, process)
+				continue
+			}
+
+			internalParentProcess, ok := pm.processMap[internalProcess.ParentPID]
+			if ok {
+				parentProcess, ok := processMap[process.ParentPID]
+				if !ok {
+					parentProcess = internalParentProcess.toProcess()
+					processMap[parentProcess.PID] = parentProcess
+				}
+				parentProcess.Children = append(parentProcess.Children, process)
+				continue
+			}
+
+			processes = append(processes, process)
+		}
+	}
+
+	// Now from within the lock we need to get the goroutines.
+	// Why? If we release the lock then between between filling the above map and getting
+	// the stacktraces another process could be created which would then look like a dead process below
+	reader, writer := io.Pipe()
+	defer reader.Close()
+	go func() {
+		err := pprof.Lookup("goroutine").WriteTo(writer, 0)
+		_ = writer.CloseWithError(err)
+	}()
+	stacks, err = profile.Parse(reader)
+	if err != nil {
+		return nil, 0, 0, err
+	}
+
+	// Unlock the mutex
+	pm.mutex.Unlock()
+	unlocked = true
+
+	goroutineCount := int64(0)
+
+	// Now walk through the "Sample" slice in the goroutines stack
+	for _, sample := range stacks.Sample {
+		// In the "goroutine" pprof profile each sample represents one or more goroutines
+		// with the same labels and stacktraces.
+
+		// We will represent each goroutine by a `Stack`
+		stack := &Stack{}
+
+		// Add the non-process associated labels from the goroutine sample to the Stack
+		for name, value := range sample.Label {
+			if name == gtprof.LabelProcessDescription || name == gtprof.LabelPid || (!flat && name == gtprof.LabelPpid) || name == gtprof.LabelProcessType {
+				continue
+			}
+
+			// Labels from the "goroutine" pprof profile only have one value.
+			// This is because the underlying representation is a map[string]string
+			if len(value) != 1 {
+				// Unexpected...
+				return nil, 0, 0, fmt.Errorf("label: %s in goroutine stack with unexpected number of values: %v", name, value)
+			}
+
+			stack.Labels = append(stack.Labels, &Label{Name: name, Value: value[0]})
+		}
+
+		// The number of goroutines that this sample represents is the `stack.Value[0]`
+		stack.Count = sample.Value[0]
+		goroutineCount += stack.Count
+
+		// Now we want to associate this Stack with a Process.
+		var process *Process
+
+		// Try to get the PID from the goroutine labels
+		if pidvalue, ok := sample.Label[gtprof.LabelPid]; ok && len(pidvalue) == 1 {
+			pid := IDType(pidvalue[0])
+
+			// Now try to get the process from our map
+			process, ok = processMap[pid]
+			if !ok && pid != "" {
+				// This means that no process has been found in the process map - but there was a process PID
+				// Therefore this goroutine belongs to a dead process and it has escaped control of the process as it
+				// should have died with the process context cancellation.
+
+				// We need to create a dead process holder for this process and label it appropriately
+
+				// get the parent PID
+				ppid := IDType("")
+				if value, ok := sample.Label[gtprof.LabelPpid]; ok && len(value) == 1 {
+					ppid = IDType(value[0])
+				}
+
+				// format the description
+				description := "(dead process)"
+				if value, ok := sample.Label[gtprof.LabelProcessDescription]; ok && len(value) == 1 {
+					description = value[0] + " " + description
+				}
+
+				// override the type of the process to "code" but add the old type as a label on the first stack
+				ptype := NoneProcessType
+				if value, ok := sample.Label[gtprof.LabelProcessType]; ok && len(value) == 1 {
+					stack.Labels = append(stack.Labels, &Label{Name: gtprof.LabelProcessType, Value: value[0]})
+				}
+				process = &Process{
+					PID:         pid,
+					ParentPID:   ppid,
+					Description: description,
+					Type:        ptype,
+				}
+
+				// Now add the dead process back to the map and tree so we don't go back through this again.
+				processMap[process.PID] = process
+				added := false
+				if process.ParentPID != "" && !flat {
+					if parent, ok := processMap[process.ParentPID]; ok {
+						parent.Children = append(parent.Children, process)
+						added = true
+					}
+				}
+				if !added {
+					processes = append(processes, process)
+				}
+			}
+		}
+
+		if process == nil {
+			// This means that the sample we're looking has no PID label
+			var ok bool
+			process, ok = processMap[""]
+			if !ok {
+				// this is the first time we've come acrross an unassociated goroutine so create a "process" to hold them
+				process = &Process{
+					Description: "(unassociated)",
+					Type:        NoneProcessType,
+				}
+				processMap[process.PID] = process
+				processes = append(processes, process)
+			}
+		}
+
+		// The sample.Location represents a stack trace for this goroutine,
+		// however each Location can represent multiple lines (mostly due to inlining)
+		// so we need to walk the lines too
+		for _, location := range sample.Location {
+			for _, line := range location.Line {
+				entry := &StackEntry{
+					Function: line.Function.Name,
+					File:     line.Function.Filename,
+					Line:     int(line.Line),
+				}
+				stack.Entry = append(stack.Entry, entry)
+			}
+		}
+
+		// Now we need a short-descriptive name to call the stack trace if when it is folded and
+		// assuming the stack trace has some lines we'll choose the bottom of the stack (i.e. the
+		// initial function that started the stack trace.) The top of the stack is unlikely to
+		// be very helpful as a lot of the time it will be runtime.select or some other call into
+		// a std library.
+		stack.Description = "(unknown)"
+		if len(stack.Entry) > 0 {
+			stack.Description = stack.Entry[len(stack.Entry)-1].Function
+		}
+
+		process.Stacks = append(process.Stacks, stack)
+	}
+
+	// restrict to not show system processes
+	if noSystem {
+		for i := 0; i < len(processes); i++ {
+			process := processes[i]
+			if process.Type != SystemProcessType && process.Type != NoneProcessType {
+				continue
+			}
+			processes[len(processes)-1], processes[i] = processes[i], processes[len(processes)-1]
+			processes = append(processes[:len(processes)-1], process.Children...)
+			i--
+		}
+	}
+
+	// Now finally re-sort the processes. Newest process appears first
+	after := func(processes []*Process) func(i, j int) bool {
+		return func(i, j int) bool {
+			left, right := processes[i], processes[j]
+			return left.Start.After(right.Start)
+		}
+	}
+	sort.Slice(processes, after(processes))
+	if !flat {
+		var sortChildren func(process *Process)
+
+		sortChildren = func(process *Process) {
+			sort.Slice(process.Children, after(process.Children))
+			for _, child := range process.Children {
+				sortChildren(child)
+			}
+		}
+	}
+
+	return processes, processCount, goroutineCount, err
+}

modules/process/manager_test.go

@@ -0,0 +1,111 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package process
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetManager(t *testing.T) {
+	go func() {
+		// test race protection
+		_ = GetManager()
+	}()
+	pm := GetManager()
+	assert.NotNil(t, pm)
+}
+
+func TestManager_AddContext(t *testing.T) {
+	pm := Manager{processMap: make(map[IDType]*process), next: 1}
+
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+
+	p1Ctx, _, finished := pm.AddContext(ctx, "foo")
+	defer finished()
+	assert.NotEmpty(t, GetContext(p1Ctx).GetPID(), "expected to get non-empty pid")
+
+	p2Ctx, _, finished := pm.AddContext(p1Ctx, "bar")
+	defer finished()
+
+	assert.NotEmpty(t, GetContext(p2Ctx).GetPID(), "expected to get non-empty pid")
+
+	assert.NotEqual(t, GetContext(p1Ctx).GetPID(), GetContext(p2Ctx).GetPID(), "expected to get different pids %s == %s", GetContext(p2Ctx).GetPID(), GetContext(p1Ctx).GetPID())
+	assert.Equal(t, GetContext(p1Ctx).GetPID(), GetContext(p2Ctx).GetParent().GetPID(), "expected to get pid %s got %s", GetContext(p1Ctx).GetPID(), GetContext(p2Ctx).GetParent().GetPID())
+}
+
+func TestManager_Cancel(t *testing.T) {
+	pm := Manager{processMap: make(map[IDType]*process), next: 1}
+
+	ctx, _, finished := pm.AddContext(t.Context(), "foo")
+	defer finished()
+
+	pm.Cancel(GetPID(ctx))
+
+	select {
+	case <-ctx.Done():
+	default:
+		assert.FailNow(t, "Cancel should cancel the provided context")
+	}
+	finished()
+
+	ctx, cancel, finished := pm.AddContext(t.Context(), "foo")
+	defer finished()
+
+	cancel()
+
+	select {
+	case <-ctx.Done():
+	default:
+		assert.FailNow(t, "Cancel should cancel the provided context")
+	}
+	finished()
+}
+
+func TestManager_Remove(t *testing.T) {
+	pm := Manager{processMap: make(map[IDType]*process), next: 1}
+
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+
+	p1Ctx, _, finished := pm.AddContext(ctx, "foo")
+	defer finished()
+	assert.NotEmpty(t, GetContext(p1Ctx).GetPID(), "expected to have non-empty PID")
+
+	p2Ctx, _, finished := pm.AddContext(p1Ctx, "bar")
+	defer finished()
+
+	assert.NotEqual(t, GetContext(p1Ctx).GetPID(), GetContext(p2Ctx).GetPID(), "expected to get different pids got %s == %s", GetContext(p2Ctx).GetPID(), GetContext(p1Ctx).GetPID())
+
+	finished()
+
+	_, exists := pm.processMap[GetPID(p2Ctx)]
+	assert.False(t, exists, "PID %d is in the list but shouldn't", GetPID(p2Ctx))
+}
+
+func TestExecTimeoutNever(t *testing.T) {
+	// TODO Investigate how to improve the time elapsed per round.
+	maxLoops := 10
+	for i := 1; i < maxLoops; i++ {
+		_, stderr, err := GetManager().ExecTimeout(5*time.Second, "ExecTimeout", "git", "--version")
+		if err != nil {
+			t.Fatalf("git --version: %v(%s)", err, stderr)
+		}
+	}
+}
+
+func TestExecTimeoutAlways(t *testing.T) {
+	maxLoops := 100
+	for i := 1; i < maxLoops; i++ {
+		_, stderr, err := GetManager().ExecTimeout(100*time.Microsecond, "ExecTimeout", "sleep", "5")
+		// TODO Simplify logging and errors to get precise error type. E.g. checking "if err != context.DeadlineExceeded".
+		if err == nil {
+			t.Fatalf("sleep 5 secs: %v(%s)", err, stderr)
+		}
+	}
+}

modules/process/manager_unix.go

@@ -0,0 +1,17 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build !windows
+
+package process
+
+import (
+	"os/exec"
+	"syscall"
+)
+
+// SetSysProcAttribute sets the common SysProcAttrs for commands
+func SetSysProcAttribute(cmd *exec.Cmd) {
+	// When Gitea runs SubProcessA -> SubProcessB and SubProcessA gets killed by context timeout, use setpgid to make sure the sub processes can be reaped instead of leaving defunct(zombie) processes.
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+}

modules/process/manager_windows.go

@@ -0,0 +1,15 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build windows
+
+package process
+
+import (
+	"os/exec"
+)
+
+// SetSysProcAttribute sets the common SysProcAttrs for commands
+func SetSysProcAttribute(cmd *exec.Cmd) {
+	// Do nothing
+}

modules/process/process.go

@@ -0,0 +1,37 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package process
+
+import (
+	"time"
+)
+
+var (
+	SystemProcessType  = "system"
+	RequestProcessType = "request"
+	NormalProcessType  = "normal"
+	NoneProcessType    = "none"
+)
+
+// process represents a working process inheriting from Gitea.
+type process struct {
+	PID         IDType // Process ID, not system one.
+	ParentPID   IDType
+	Description string
+	Start       time.Time
+	Cancel      CancelCauseFunc
+	Type        string
+}
+
+// ToProcess converts a process to a externally usable Process
+func (p *process) toProcess() *Process {
+	process := &Process{
+		PID:         p.PID,
+		ParentPID:   p.ParentPID,
+		Description: p.Description,
+		Start:       p.Start,
+		Type:        p.Type,
+	}
+	return process
+}