初始提交: Gitea 项目代码

This commit is contained in:
root
2026-05-30 22:47:36 +08:00
commit f288f76350
6116 changed files with 776822 additions and 0 deletions
+29
View File
@@ -0,0 +1,29 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
"crypto/hmac"
"crypto/sha256"
"encoding/binary"
"io"
"gitea.dev/modules/setting"
)
type tagType string
// BuildSignature builds a hmac signature for the input values.
// "tag" is an internal pre-defined static string to distinguish the signatures for different purpose.
func BuildSignature(tag tagType, vals ...string) []byte {
m := hmac.New(sha256.New, setting.GetGeneralTokenSigningSecret())
_, _ = io.WriteString(m, string(tag))
var buf8 [8]byte
for _, v := range vals {
binary.LittleEndian.PutUint64(buf8[:], uint64(len(v)))
_, _ = m.Write(buf8[:])
_, _ = io.WriteString(m, v)
}
return m.Sum(nil)
}
+36
View File
@@ -0,0 +1,36 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestBuildSignature(t *testing.T) {
a := BuildSignature("v0", "x")
b := BuildSignature("v0", "x")
assert.Equal(t, a, b)
a = BuildSignature("v0", "x", "yz")
b = BuildSignature("v0", "xy", "z")
assert.NotEqual(t, a, b)
a = BuildSignature("v1", "x")
b = BuildSignature("v2", "x")
assert.NotEqual(t, a, b)
a = BuildSignature("v0", "x")
b = BuildSignature("v0x")
assert.NotEqual(t, a, b)
a = BuildSignature("v0", "", "x")
b = BuildSignature("v0", "x", "")
assert.NotEqual(t, a, b)
a = BuildSignature("v0")
b = BuildSignature("v0")
assert.Equal(t, a, b)
}
+70
View File
@@ -0,0 +1,70 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
"context"
"maps"
"slices"
actions_model "gitea.dev/models/actions"
"gitea.dev/models/db"
git_model "gitea.dev/models/git"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/log"
)
// CommitActionsStatusMap maps CommitStatus.ID to the live ActionRunJob status
// for Gitea Actions rows.
type CommitActionsStatusMap map[int64]actions_model.Status
// IconStatus returns the action status name to route the icon through
// repo/icons/action_status, or "" when the row isn't from Gitea Actions.
func (m CommitActionsStatusMap) IconStatus(s *git_model.CommitStatus) string {
if status, ok := m[s.ID]; ok {
return status.String()
}
return ""
}
// GetCommitActionsStatusMap resolves the live ActionRunJob.Status for every
// CommitStatus row backed by Gitea Actions. Rows from other sources (external
// CIs, API) are left untouched and rendered from their stored State.
func GetCommitActionsStatusMap(ctx context.Context, statuses []*git_model.CommitStatus) CommitActionsStatusMap {
if len(statuses) == 0 {
return nil
}
statusByJobID := make(map[int64]*git_model.CommitStatus)
repoByID := make(map[int64]*repo_model.Repository)
for _, status := range statuses {
if status == nil || status.TargetURL == "" {
continue
}
if status.Repo == nil {
status.Repo = repoByID[status.RepoID]
}
// ParseGiteaActionsTargetURL lazy-loads status.Repo on miss; cache the
// outcome so later entries with the same RepoID skip that load.
_, jobID, ok := status.ParseGiteaActionsTargetURL(ctx)
repoByID[status.RepoID] = status.Repo
if ok {
statusByJobID[jobID] = status
}
}
if len(statusByJobID) == 0 {
return nil
}
jobs := make(map[int64]*actions_model.ActionRunJob, len(statusByJobID))
if err := db.GetEngine(ctx).In("id", slices.Collect(maps.Keys(statusByJobID))).Cols("id", "status").Find(&jobs); err != nil {
log.Error("db.Find: failed to find action run jobs: %v", err)
return nil
}
info := make(CommitActionsStatusMap, len(jobs))
for jobID, status := range statusByJobID {
if job, ok := jobs[jobID]; ok {
info[status.ID] = job.Status
}
}
return info
}
+129
View File
@@ -0,0 +1,129 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
webhook_module "gitea.dev/modules/webhook"
)
const (
GithubEventPullRequest = "pull_request"
GithubEventPullRequestTarget = "pull_request_target"
GithubEventPullRequestReviewComment = "pull_request_review_comment"
GithubEventPullRequestReview = "pull_request_review"
GithubEventRegistryPackage = "registry_package"
GithubEventCreate = "create"
GithubEventDelete = "delete"
GithubEventFork = "fork"
GithubEventPush = "push"
GithubEventIssues = "issues"
GithubEventIssueComment = "issue_comment"
GithubEventRelease = "release"
GithubEventPullRequestComment = "pull_request_comment"
GithubEventGollum = "gollum"
GithubEventSchedule = "schedule"
)
// IsDefaultBranchWorkflow returns true if the event only triggers workflows on the default branch
func IsDefaultBranchWorkflow(triggedEvent webhook_module.HookEventType) bool {
switch triggedEvent {
case webhook_module.HookEventDelete:
// GitHub "delete" event
// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#delete
return true
case webhook_module.HookEventFork:
// GitHub "fork" event
// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#fork
return true
case webhook_module.HookEventIssueComment:
// GitHub "issue_comment" event
// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#issue_comment
return true
case webhook_module.HookEventPullRequestComment:
// GitHub "pull_request_comment" event
// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_comment-use-issue_comment
return true
case webhook_module.HookEventWiki:
// GitHub "gollum" event
// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#gollum
return true
case webhook_module.HookEventSchedule:
// GitHub "schedule" event
// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
return true
case webhook_module.HookEventIssues,
webhook_module.HookEventIssueAssign,
webhook_module.HookEventIssueLabel,
webhook_module.HookEventIssueMilestone:
// Github "issues" event
// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#issues
return true
case webhook_module.HookEventWorkflowRun:
// GitHub "workflow_run" event
// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_run
return true
}
return false
}
// canGithubEventMatch check if the input Github event can match any Gitea event.
func canGithubEventMatch(eventName string, triggedEvent webhook_module.HookEventType) bool {
switch eventName {
case GithubEventRegistryPackage:
return triggedEvent == webhook_module.HookEventPackage
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#gollum
case GithubEventGollum:
return triggedEvent == webhook_module.HookEventWiki
case GithubEventIssues:
switch triggedEvent {
case webhook_module.HookEventIssues,
webhook_module.HookEventIssueAssign,
webhook_module.HookEventIssueLabel,
webhook_module.HookEventIssueMilestone:
return true
default:
return false
}
case GithubEventPullRequest, GithubEventPullRequestTarget:
switch triggedEvent {
case webhook_module.HookEventPullRequest,
webhook_module.HookEventPullRequestSync,
webhook_module.HookEventPullRequestAssign,
webhook_module.HookEventPullRequestLabel,
webhook_module.HookEventPullRequestReviewRequest,
webhook_module.HookEventPullRequestMilestone:
return true
default:
return false
}
case GithubEventPullRequestReview:
switch triggedEvent {
case webhook_module.HookEventPullRequestReviewApproved,
webhook_module.HookEventPullRequestReviewComment,
webhook_module.HookEventPullRequestReviewRejected:
return true
default:
return false
}
case GithubEventSchedule:
return triggedEvent == webhook_module.HookEventSchedule
case GithubEventIssueComment:
// https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_comment-use-issue_comment
return triggedEvent == webhook_module.HookEventIssueComment ||
triggedEvent == webhook_module.HookEventPullRequestComment
default:
return eventName == string(triggedEvent)
}
}
+119
View File
@@ -0,0 +1,119 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
"testing"
webhook_module "gitea.dev/modules/webhook"
"github.com/stretchr/testify/assert"
)
func TestCanGithubEventMatch(t *testing.T) {
testCases := []struct {
desc string
eventName string
triggeredEvent webhook_module.HookEventType
expected bool
}{
// registry_package event
{
"registry_package matches",
GithubEventRegistryPackage,
webhook_module.HookEventPackage,
true,
},
{
"registry_package cannot match",
GithubEventRegistryPackage,
webhook_module.HookEventPush,
false,
},
// issues event
{
"issue matches",
GithubEventIssues,
webhook_module.HookEventIssueLabel,
true,
},
{
"issue cannot match",
GithubEventIssues,
webhook_module.HookEventIssueComment,
false,
},
// issue_comment event
{
"issue_comment matches",
GithubEventIssueComment,
webhook_module.HookEventIssueComment,
true,
},
{
"issue_comment cannot match",
GithubEventIssueComment,
webhook_module.HookEventIssues,
false,
},
// pull_request event
{
"pull_request matches",
GithubEventPullRequest,
webhook_module.HookEventPullRequestSync,
true,
},
{
"pull_request cannot match",
GithubEventPullRequest,
webhook_module.HookEventPullRequestComment,
false,
},
// pull_request_target event
{
"pull_request_target matches",
GithubEventPullRequest,
webhook_module.HookEventPullRequest,
true,
},
{
"pull_request_target cannot match",
GithubEventPullRequest,
webhook_module.HookEventPullRequestComment,
false,
},
// pull_request_review event
{
"pull_request_review matches",
GithubEventPullRequestReview,
webhook_module.HookEventPullRequestReviewComment,
true,
},
{
"pull_request_review cannot match",
GithubEventPullRequestReview,
webhook_module.HookEventPullRequestComment,
false,
},
// other events
{
"create event",
GithubEventCreate,
webhook_module.HookEventCreate,
true,
},
{
"create pull request comment",
GithubEventIssueComment,
webhook_module.HookEventPullRequestComment,
true,
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
assert.Equalf(t, tc.expected, canGithubEventMatch(tc.eventName, tc.triggeredEvent), "canGithubEventMatch(%v, %v)", tc.eventName, tc.triggeredEvent)
})
}
}
+189
View File
@@ -0,0 +1,189 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package jobparser
import (
"errors"
"fmt"
"regexp"
"strings"
"gitea.com/gitea/runner/act/exprparser"
"go.yaml.in/yaml/v4"
)
// ExpressionEvaluator is copied from runner.expressionEvaluator,
// to avoid unnecessary dependencies
type ExpressionEvaluator struct {
interpreter exprparser.Interpreter
}
func NewExpressionEvaluator(interpreter exprparser.Interpreter) *ExpressionEvaluator {
return &ExpressionEvaluator{interpreter: interpreter}
}
func (ee ExpressionEvaluator) evaluate(in string, defaultStatusCheck exprparser.DefaultStatusCheck) (any, error) {
evaluated, err := ee.interpreter.Evaluate(in, defaultStatusCheck)
return evaluated, err
}
func (ee ExpressionEvaluator) evaluateScalarYamlNode(node *yaml.Node) error {
var in string
if err := node.Decode(&in); err != nil {
return err
}
if !strings.Contains(in, "${{") || !strings.Contains(in, "}}") {
return nil
}
expr, _ := rewriteSubExpression(in, false)
res, err := ee.evaluate(expr, exprparser.DefaultStatusCheckNone)
if err != nil {
return err
}
return node.Encode(res)
}
func (ee ExpressionEvaluator) evaluateMappingYamlNode(node *yaml.Node) error {
// GitHub has this undocumented feature to merge maps, called insert directive
insertDirective := regexp.MustCompile(`\${{\s*insert\s*}}`)
for i := 0; i < len(node.Content)/2; {
k := node.Content[i*2]
v := node.Content[i*2+1]
if err := ee.EvaluateYamlNode(v); err != nil {
return err
}
var sk string
// Merge the nested map of the insert directive
if k.Decode(&sk) == nil && insertDirective.MatchString(sk) {
node.Content = append(append(node.Content[:i*2], v.Content...), node.Content[(i+1)*2:]...)
i += len(v.Content) / 2
} else {
if err := ee.EvaluateYamlNode(k); err != nil {
return err
}
i++
}
}
return nil
}
func (ee ExpressionEvaluator) evaluateSequenceYamlNode(node *yaml.Node) error {
for i := 0; i < len(node.Content); {
v := node.Content[i]
// Preserve nested sequences
wasseq := v.Kind == yaml.SequenceNode
if err := ee.EvaluateYamlNode(v); err != nil {
return err
}
// GitHub has this undocumented feature to merge sequences / arrays
// We have a nested sequence via evaluation, merge the arrays
if v.Kind == yaml.SequenceNode && !wasseq {
node.Content = append(append(node.Content[:i], v.Content...), node.Content[i+1:]...)
i += len(v.Content)
} else {
i++
}
}
return nil
}
func (ee ExpressionEvaluator) EvaluateYamlNode(node *yaml.Node) error {
switch node.Kind {
case yaml.ScalarNode:
return ee.evaluateScalarYamlNode(node)
case yaml.MappingNode:
return ee.evaluateMappingYamlNode(node)
case yaml.SequenceNode:
return ee.evaluateSequenceYamlNode(node)
default:
return nil
}
}
func (ee ExpressionEvaluator) Interpolate(in string) string {
if !strings.Contains(in, "${{") || !strings.Contains(in, "}}") {
return in
}
expr, _ := rewriteSubExpression(in, true)
evaluated, err := ee.evaluate(expr, exprparser.DefaultStatusCheckNone)
if err != nil {
return ""
}
value, ok := evaluated.(string)
if !ok {
panic(fmt.Sprintf("Expression %s did not evaluate to a string", expr))
}
return value
}
func escapeFormatString(in string) string {
return strings.ReplaceAll(strings.ReplaceAll(in, "{", "{{"), "}", "}}")
}
func rewriteSubExpression(in string, forceFormat bool) (string, error) {
if !strings.Contains(in, "${{") || !strings.Contains(in, "}}") {
return in, nil
}
strPattern := regexp.MustCompile("(?:''|[^'])*'")
pos := 0
exprStart := -1
strStart := -1
var results []string
var formatOut strings.Builder
for pos < len(in) {
if strStart > -1 {
matches := strPattern.FindStringIndex(in[pos:])
if matches == nil {
return "", errors.New("unclosed string")
}
strStart = -1
pos += matches[1]
} else if exprStart > -1 {
exprEnd := strings.Index(in[pos:], "}}")
strStart = strings.Index(in[pos:], "'")
if exprEnd > -1 && strStart > -1 {
if exprEnd < strStart {
strStart = -1
} else {
exprEnd = -1
}
}
if exprEnd > -1 {
fmt.Fprintf(&formatOut, "{%d}", len(results))
results = append(results, strings.TrimSpace(in[exprStart:pos+exprEnd]))
pos += exprEnd + 2
exprStart = -1
} else if strStart > -1 {
pos += strStart + 1
} else {
panic("unclosed expression.")
}
} else {
exprStart = strings.Index(in[pos:], "${{")
if exprStart != -1 {
formatOut.WriteString(escapeFormatString(in[pos : pos+exprStart]))
exprStart = pos + exprStart + 3
pos = exprStart
} else {
formatOut.WriteString(escapeFormatString(in[pos:]))
pos = len(in)
}
}
}
if len(results) == 1 && formatOut.String() == "{0}" && !forceFormat {
return in, nil
}
out := fmt.Sprintf("format('%s', %s)", strings.ReplaceAll(formatOut.String(), "'", "''"), strings.Join(results, ", "))
return out, nil
}
+87
View File
@@ -0,0 +1,87 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package jobparser
import (
"gitea.com/gitea/runner/act/exprparser"
"gitea.com/gitea/runner/act/model"
"go.yaml.in/yaml/v4"
)
// NewInterpeter returns an interpeter used in the server,
// need github, needs, strategy, matrix, inputs context only,
// see https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability
func NewInterpeter(
jobID string,
job *model.Job,
matrix map[string]any,
gitCtx *model.GithubContext,
results map[string]*JobResult,
vars map[string]string,
inputs map[string]any,
) exprparser.Interpreter {
strategy := make(map[string]any)
if job.Strategy != nil {
strategy["fail-fast"] = job.Strategy.FailFast
strategy["max-parallel"] = job.Strategy.MaxParallel
}
run := &model.Run{
Workflow: &model.Workflow{
Jobs: map[string]*model.Job{},
},
JobID: jobID,
}
for id, result := range results {
need := yaml.Node{}
_ = need.Encode(result.Needs)
run.Workflow.Jobs[id] = &model.Job{
RawNeeds: need,
Result: result.Result,
Outputs: result.Outputs,
}
}
jobs := run.Workflow.Jobs
jobNeeds := run.Job().Needs()
using := map[string]exprparser.Needs{}
for _, need := range jobNeeds {
if v, ok := jobs[need]; ok {
using[need] = exprparser.Needs{
Outputs: v.Outputs,
Result: v.Result,
}
}
}
ee := &exprparser.EvaluationEnvironment{
Github: gitCtx,
Env: nil, // no need
Job: nil, // no need
Steps: nil, // no need
Runner: nil, // no need
Secrets: nil, // no need
Strategy: strategy,
Matrix: matrix,
Needs: using,
Inputs: inputs,
Vars: vars,
}
config := exprparser.Config{
Run: run,
WorkingDir: "", // WorkingDir is used for the function hashFiles, but it's not needed in the server
Context: "job",
}
return exprparser.NewInterpeter(ee, config)
}
// JobResult is the minimum requirement of job results for Interpeter
type JobResult struct {
Needs []string
Result string
Outputs map[string]string
}
+174
View File
@@ -0,0 +1,174 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package jobparser
import (
"bytes"
"fmt"
"sort"
"strings"
"gitea.com/gitea/runner/act/exprparser"
"gitea.com/gitea/runner/act/model"
"go.yaml.in/yaml/v4"
)
func Parse(content []byte, options ...ParseOption) ([]*SingleWorkflow, error) {
origin, err := model.ReadWorkflow(bytes.NewReader(content))
if err != nil {
return nil, fmt.Errorf("model.ReadWorkflow: %w", err)
}
workflow := &SingleWorkflow{}
if err := yaml.Unmarshal(content, workflow); err != nil {
return nil, fmt.Errorf("yaml.Unmarshal: %w", err)
}
pc := &parseContext{}
for _, o := range options {
o(pc)
}
results := map[string]*JobResult{}
for id, job := range origin.Jobs {
if job == nil {
return nil, fmt.Errorf("needed job not found: %q", id)
}
results[id] = &JobResult{
Needs: job.Needs(),
Result: pc.jobResults[id],
Outputs: nil, // not supported yet
}
}
var ret []*SingleWorkflow
ids, jobs, err := workflow.jobs()
if err != nil {
return nil, fmt.Errorf("invalid jobs: %w", err)
}
evaluator := NewExpressionEvaluator(exprparser.NewInterpeter(&exprparser.EvaluationEnvironment{Github: pc.gitContext, Vars: pc.vars, Inputs: pc.inputs}, exprparser.Config{}))
workflow.RunName = evaluator.Interpolate(workflow.RunName)
for i, id := range ids {
job := jobs[i]
matricxes, err := getMatrixes(origin.GetJob(id))
if err != nil {
return nil, fmt.Errorf("getMatrixes: %w", err)
}
for _, matrix := range matricxes {
job := job.Clone()
if job.Name == "" {
job.Name = id
}
job.Strategy.RawMatrix = encodeMatrix(matrix)
evaluator := NewExpressionEvaluator(NewInterpeter(id, origin.GetJob(id), matrix, pc.gitContext, results, pc.vars, pc.inputs))
job.Name = nameWithMatrix(job.Name, matrix, evaluator)
runsOn := origin.GetJob(id).RunsOn()
for i, v := range runsOn {
runsOn[i] = evaluator.Interpolate(v)
}
job.RawRunsOn = encodeRunsOn(runsOn)
swf := &SingleWorkflow{
Name: workflow.Name,
RawOn: workflow.RawOn,
Env: workflow.Env,
Defaults: workflow.Defaults,
RawPermissions: workflow.RawPermissions,
RunName: workflow.RunName,
}
if err := swf.SetJob(id, job); err != nil {
return nil, fmt.Errorf("SetJob: %w", err)
}
ret = append(ret, swf)
}
}
return ret, nil
}
func WithGitContext(context *model.GithubContext) ParseOption {
return func(c *parseContext) {
c.gitContext = context
}
}
func WithVars(vars map[string]string) ParseOption {
return func(c *parseContext) {
c.vars = vars
}
}
func WithInputs(inputs map[string]any) ParseOption {
return func(c *parseContext) {
c.inputs = inputs
}
}
type parseContext struct {
jobResults map[string]string
gitContext *model.GithubContext
vars map[string]string
inputs map[string]any
}
type ParseOption func(c *parseContext)
func getMatrixes(job *model.Job) ([]map[string]any, error) {
ret, err := job.GetMatrixes()
if err != nil {
return nil, fmt.Errorf("GetMatrixes: %w", err)
}
sort.Slice(ret, func(i, j int) bool {
return matrixName(ret[i]) < matrixName(ret[j])
})
return ret, nil
}
func encodeMatrix(matrix map[string]any) yaml.Node {
if len(matrix) == 0 {
return yaml.Node{}
}
value := map[string][]any{}
for k, v := range matrix {
value[k] = []any{v}
}
node := yaml.Node{}
_ = node.Encode(value)
return node
}
func encodeRunsOn(runsOn []string) yaml.Node {
node := yaml.Node{}
if len(runsOn) == 1 {
_ = node.Encode(runsOn[0])
} else {
_ = node.Encode(runsOn)
}
return node
}
func nameWithMatrix(name string, m map[string]any, evaluator *ExpressionEvaluator) string {
if len(m) == 0 {
return name
}
if !strings.Contains(name, "${{") || !strings.Contains(name, "}}") {
return name + " " + matrixName(m)
}
return evaluator.Interpolate(name)
}
func matrixName(m map[string]any) string {
ks := make([]string, 0, len(m))
for k := range m {
ks = append(ks, k)
}
sort.Strings(ks)
vs := make([]string, 0, len(m))
for _, v := range ks {
vs = append(vs, fmt.Sprint(m[v]))
}
return fmt.Sprintf("(%s)", strings.Join(vs, ", "))
}
+104
View File
@@ -0,0 +1,104 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package jobparser
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.yaml.in/yaml/v4"
)
func TestParse(t *testing.T) {
tests := []struct {
name string
options []ParseOption
wantErr bool
}{
{
name: "multiple_jobs",
options: nil,
wantErr: false,
},
{
name: "multiple_matrix",
options: nil,
wantErr: false,
},
{
name: "has_needs",
options: nil,
wantErr: false,
},
{
name: "has_with",
options: nil,
wantErr: false,
},
{
name: "has_secrets",
options: nil,
wantErr: false,
},
{
name: "empty_step",
options: nil,
wantErr: false,
},
{
name: "job_name_with_matrix",
options: nil,
wantErr: false,
},
{
name: "prefixed_newline",
options: nil,
wantErr: false,
},
}
invalidFileTests := []struct {
name string
}{
{name: "null_job_implicit"},
{name: "null_job_explicit"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
content := ReadTestdata(t, tt.name+".in.yaml")
want := ReadTestdata(t, tt.name+".out.yaml")
got, err := Parse(content, tt.options...)
if tt.wantErr {
require.Error(t, err)
}
require.NoError(t, err)
builder := &strings.Builder{}
for _, v := range got {
if builder.Len() > 0 {
builder.WriteString("---\n")
}
encoder := yaml.NewEncoder(builder)
encoder.SetIndent(2)
require.NoError(t, encoder.Encode(v))
id, job := v.Job()
assert.NotEmpty(t, id)
assert.NotNil(t, job)
}
assert.Equal(t, string(want), builder.String())
})
}
for _, tt := range invalidFileTests {
t.Run(tt.name, func(t *testing.T) {
content := ReadTestdata(t, tt.name+".in.yaml")
require.NotPanics(t, func() {
_, err := Parse(content)
require.Error(t, err)
})
})
}
}
+530
View File
@@ -0,0 +1,530 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package jobparser
import (
"bytes"
"errors"
"fmt"
"gitea.com/gitea/runner/act/exprparser"
"gitea.com/gitea/runner/act/model"
"go.yaml.in/yaml/v4"
)
// SingleWorkflow is a workflow with single job and single matrix
type SingleWorkflow struct {
Name string `yaml:"name,omitempty"`
RawOn yaml.Node `yaml:"on,omitempty"`
Env map[string]string `yaml:"env,omitempty"`
RawJobs yaml.Node `yaml:"jobs,omitempty"`
Defaults Defaults `yaml:"defaults,omitempty"`
RawPermissions yaml.Node `yaml:"permissions,omitempty"`
RunName string `yaml:"run-name,omitempty"`
}
func (w *SingleWorkflow) Job() (string, *Job) {
ids, jobs, _ := w.jobs()
if len(ids) >= 1 {
return ids[0], jobs[0]
}
return "", nil
}
func (w *SingleWorkflow) jobs() ([]string, []*Job, error) {
ids, jobs, err := parseMappingNode[*Job](&w.RawJobs)
if err != nil {
return nil, nil, err
}
for _, job := range jobs {
steps := make([]*Step, 0, len(job.Steps))
for _, s := range job.Steps {
if s != nil {
steps = append(steps, s)
}
}
job.Steps = steps
}
return ids, jobs, nil
}
func (w *SingleWorkflow) SetJob(id string, job *Job) error {
m := map[string]*Job{
id: job,
}
var buf bytes.Buffer
encoder := yaml.NewEncoder(&buf)
encoder.SetIndent(2)
if err := encoder.Encode(m); err != nil {
return err
}
encoder.Close()
node := yaml.Node{}
if err := yaml.Unmarshal(buf.Bytes(), &node); err != nil {
return err
}
if len(node.Content) != 1 || node.Content[0].Kind != yaml.MappingNode {
return fmt.Errorf("can not set job: %s", buf.String())
}
w.RawJobs = *node.Content[0]
return nil
}
func (w *SingleWorkflow) Marshal() ([]byte, error) {
return yaml.Marshal(w)
}
type Job struct {
Name string `yaml:"name,omitempty"`
RawNeeds yaml.Node `yaml:"needs,omitempty"`
RawRunsOn yaml.Node `yaml:"runs-on,omitempty"`
Env yaml.Node `yaml:"env,omitempty"`
If yaml.Node `yaml:"if,omitempty"`
Steps []*Step `yaml:"steps,omitempty"`
TimeoutMinutes string `yaml:"timeout-minutes,omitempty"`
Services map[string]*ContainerSpec `yaml:"services,omitempty"`
Strategy Strategy `yaml:"strategy,omitempty"`
RawContainer yaml.Node `yaml:"container,omitempty"`
Defaults Defaults `yaml:"defaults,omitempty"`
Outputs map[string]string `yaml:"outputs,omitempty"`
Uses string `yaml:"uses,omitempty"`
With map[string]any `yaml:"with,omitempty"`
RawSecrets yaml.Node `yaml:"secrets,omitempty"`
RawConcurrency *model.RawConcurrency `yaml:"concurrency,omitempty"`
RawPermissions yaml.Node `yaml:"permissions,omitempty"`
}
func (j *Job) Clone() *Job {
if j == nil {
return nil
}
return &Job{
Name: j.Name,
RawNeeds: j.RawNeeds,
RawRunsOn: j.RawRunsOn,
Env: j.Env,
If: j.If,
Steps: j.Steps,
TimeoutMinutes: j.TimeoutMinutes,
Services: j.Services,
Strategy: j.Strategy,
RawContainer: j.RawContainer,
Defaults: j.Defaults,
Outputs: j.Outputs,
Uses: j.Uses,
With: j.With,
RawSecrets: j.RawSecrets,
RawConcurrency: j.RawConcurrency,
RawPermissions: j.RawPermissions,
}
}
func (j *Job) Needs() []string {
return (&model.Job{RawNeeds: j.RawNeeds}).Needs()
}
func (j *Job) EraseNeeds() *Job {
j.RawNeeds = yaml.Node{}
return j
}
func (j *Job) RunsOn() []string {
return (&model.Job{RawRunsOn: j.RawRunsOn}).RunsOn()
}
type Step struct {
ID string `yaml:"id,omitempty"`
If yaml.Node `yaml:"if,omitempty"`
Name string `yaml:"name,omitempty"`
Uses string `yaml:"uses,omitempty"`
Run string `yaml:"run,omitempty"`
WorkingDirectory string `yaml:"working-directory,omitempty"`
Shell string `yaml:"shell,omitempty"`
Env yaml.Node `yaml:"env,omitempty"`
With map[string]string `yaml:"with,omitempty"`
ContinueOnError bool `yaml:"continue-on-error,omitempty"`
TimeoutMinutes string `yaml:"timeout-minutes,omitempty"`
}
// String gets the name of step
func (s *Step) String() string {
if s == nil {
return ""
}
return (&model.Step{
ID: s.ID,
Name: s.Name,
Uses: s.Uses,
Run: s.Run,
}).String()
}
type ContainerSpec struct {
Image string `yaml:"image,omitempty"`
Env map[string]string `yaml:"env,omitempty"`
Ports []string `yaml:"ports,omitempty"`
Volumes []string `yaml:"volumes,omitempty"`
Options string `yaml:"options,omitempty"`
Credentials map[string]string `yaml:"credentials,omitempty"`
Cmd []string `yaml:"cmd,omitempty"`
}
type Strategy struct {
FailFastString string `yaml:"fail-fast,omitempty"`
MaxParallelString string `yaml:"max-parallel,omitempty"`
RawMatrix yaml.Node `yaml:"matrix,omitempty"`
}
type Defaults struct {
Run RunDefaults `yaml:"run,omitempty"`
}
type RunDefaults struct {
Shell string `yaml:"shell,omitempty"`
WorkingDirectory string `yaml:"working-directory,omitempty"`
}
type WorkflowDispatchInput struct {
Name string `yaml:"name"`
Description string `yaml:"description"`
Required bool `yaml:"required"`
Default string `yaml:"default"`
Type string `yaml:"type"`
Options []string `yaml:"options"`
}
type Event struct {
Name string
acts map[string][]string
schedules []map[string]string
inputs []WorkflowDispatchInput
}
func (evt *Event) IsSchedule() bool {
return evt.schedules != nil
}
func (evt *Event) Acts() map[string][]string {
return evt.acts
}
func (evt *Event) Schedules() []map[string]string {
return evt.schedules
}
func (evt *Event) Inputs() []WorkflowDispatchInput {
return evt.inputs
}
func ReadWorkflowRawConcurrency(content []byte) (*model.RawConcurrency, error) {
w := new(model.Workflow)
err := yaml.NewDecoder(bytes.NewReader(content)).Decode(w)
return w.RawConcurrency, err
}
func EvaluateConcurrency(rc *model.RawConcurrency, jobID string, job *Job, gitCtx map[string]any, results map[string]*JobResult, vars map[string]string, inputs map[string]any) (string, bool, error) {
actJob := &model.Job{}
if job != nil {
actJob.Strategy = &model.Strategy{
FailFastString: job.Strategy.FailFastString,
MaxParallelString: job.Strategy.MaxParallelString,
RawMatrix: job.Strategy.RawMatrix,
}
actJob.Strategy.FailFast = actJob.Strategy.GetFailFast()
actJob.Strategy.MaxParallel = actJob.Strategy.GetMaxParallel()
}
matrix := make(map[string]any)
matrixes, err := actJob.GetMatrixes()
if err != nil {
return "", false, err
}
if len(matrixes) > 0 {
matrix = matrixes[0]
}
evaluator := NewExpressionEvaluator(NewInterpeter(jobID, actJob, matrix, toGitContext(gitCtx), results, vars, inputs))
var node yaml.Node
if err := node.Encode(rc); err != nil {
return "", false, fmt.Errorf("failed to encode concurrency: %w", err)
}
if err := evaluator.EvaluateYamlNode(&node); err != nil {
return "", false, fmt.Errorf("failed to evaluate concurrency: %w", err)
}
var evaluated model.RawConcurrency
if err := node.Decode(&evaluated); err != nil {
return "", false, fmt.Errorf("failed to unmarshal evaluated concurrency: %w", err)
}
if evaluated.RawExpression != "" {
return evaluated.RawExpression, false, nil
}
return evaluated.Group, evaluated.CancelInProgress == "true", nil
}
func toGitContext(input map[string]any) *model.GithubContext {
gitContext := &model.GithubContext{
EventPath: asString(input["event_path"]),
Workflow: asString(input["workflow"]),
RunID: asString(input["run_id"]),
RunNumber: asString(input["run_number"]),
Actor: asString(input["actor"]),
Repository: asString(input["repository"]),
EventName: asString(input["event_name"]),
Sha: asString(input["sha"]),
Ref: asString(input["ref"]),
RefName: asString(input["ref_name"]),
RefType: asString(input["ref_type"]),
HeadRef: asString(input["head_ref"]),
BaseRef: asString(input["base_ref"]),
Token: asString(input["token"]),
Workspace: asString(input["workspace"]),
Action: asString(input["action"]),
ActionPath: asString(input["action_path"]),
ActionRef: asString(input["action_ref"]),
ActionRepository: asString(input["action_repository"]),
Job: asString(input["job"]),
RepositoryOwner: asString(input["repository_owner"]),
RetentionDays: asString(input["retention_days"]),
}
event, ok := input["event"].(map[string]any)
if ok {
gitContext.Event = event
}
return gitContext
}
// workflowCallEvent is only fired by another workflow's `uses:`, so it is excluded from trigger detection.
const workflowCallEvent = "workflow_call"
func ParseRawOn(rawOn *yaml.Node) ([]*Event, error) {
switch rawOn.Kind {
case yaml.ScalarNode:
var val string
err := rawOn.Decode(&val)
if err != nil {
return nil, err
}
if val == workflowCallEvent {
return []*Event{}, nil
}
return []*Event{
{Name: val},
}, nil
case yaml.SequenceNode:
var val []any
err := rawOn.Decode(&val)
if err != nil {
return nil, err
}
res := make([]*Event, 0, len(val))
for _, v := range val {
switch t := v.(type) {
case string:
if t == workflowCallEvent {
continue
}
res = append(res, &Event{Name: t})
default:
return nil, fmt.Errorf("invalid type %T", t)
}
}
return res, nil
case yaml.MappingNode:
events, triggers, err := parseMappingNode[yaml.Node](rawOn)
if err != nil {
return nil, err
}
res := make([]*Event, 0, len(events))
for i, k := range events {
if k == workflowCallEvent {
continue
}
v := triggers[i]
switch v.Kind {
case yaml.ScalarNode:
res = append(res, &Event{
Name: k,
})
case yaml.SequenceNode:
var t []any
err := v.Decode(&t)
if err != nil {
return nil, err
}
schedules := make([]map[string]string, len(t))
if k == "schedule" {
for i, tt := range t {
vv, ok := tt.(map[string]any)
if !ok {
return nil, fmt.Errorf("unknown on type(schedule): %#v", v)
}
schedules[i] = make(map[string]string, len(vv))
for k, vvv := range vv {
var ok bool
if schedules[i][k], ok = vvv.(string); !ok {
return nil, fmt.Errorf("unknown on type(schedule): %#v", v)
}
}
}
}
if len(schedules) == 0 {
schedules = nil
}
res = append(res, &Event{
Name: k,
schedules: schedules,
})
case yaml.MappingNode:
acts := make(map[string][]string, len(v.Content)/2)
var inputs []WorkflowDispatchInput
expectedKey := true
var act string
for _, content := range v.Content {
if expectedKey {
if content.Kind != yaml.ScalarNode {
return nil, fmt.Errorf("key type not string: %#v", content)
}
act = ""
err := content.Decode(&act)
if err != nil {
return nil, err
}
} else {
switch content.Kind {
case yaml.SequenceNode:
var t []string
err := content.Decode(&t)
if err != nil {
return nil, err
}
acts[act] = t
case yaml.ScalarNode:
var t string
err := content.Decode(&t)
if err != nil {
return nil, err
}
acts[act] = []string{t}
case yaml.MappingNode:
if k != "workflow_dispatch" || act != "inputs" {
return nil, fmt.Errorf("map should only for workflow_dispatch but %s: %#v", act, content)
}
var key string
for i, vv := range content.Content {
if i%2 == 0 {
if vv.Kind != yaml.ScalarNode {
return nil, fmt.Errorf("key type not string: %#v", vv)
}
key = ""
if err := vv.Decode(&key); err != nil {
return nil, err
}
} else {
if vv.Kind != yaml.MappingNode {
return nil, fmt.Errorf("key type not map(%s): %#v", key, vv)
}
input := WorkflowDispatchInput{}
if err := vv.Decode(&input); err != nil {
return nil, err
}
input.Name = key
inputs = append(inputs, input)
}
}
default:
return nil, fmt.Errorf("unknown on type: %#v", content)
}
}
expectedKey = !expectedKey
}
if len(inputs) == 0 {
inputs = nil
}
if len(acts) == 0 {
acts = nil
}
res = append(res, &Event{
Name: k,
acts: acts,
inputs: inputs,
})
default:
return nil, fmt.Errorf("unknown on type: %v", v.Kind)
}
}
return res, nil
default:
return nil, fmt.Errorf("unknown on type: %v", rawOn.Kind)
}
}
func EvaluateJobIfExpression(jobID string, job *Job, gitCtx map[string]any, results map[string]*JobResult, vars map[string]string, inputs map[string]any) (bool, error) {
actJob := &model.Job{
Strategy: &model.Strategy{
FailFastString: job.Strategy.FailFastString,
MaxParallelString: job.Strategy.MaxParallelString,
RawMatrix: job.Strategy.RawMatrix,
},
}
evaluator := NewExpressionEvaluator(NewInterpeter(jobID, actJob, nil, toGitContext(gitCtx), results, vars, inputs))
expr, err := rewriteSubExpression(job.If.Value, false)
if err != nil {
return false, err
}
result, err := evaluator.evaluate(expr, exprparser.DefaultStatusCheckSuccess)
if err != nil {
return false, err
}
return exprparser.IsTruthy(result), nil
}
// parseMappingNode parse a mapping node and preserve order.
func parseMappingNode[T any](node *yaml.Node) ([]string, []T, error) {
if node.Kind != yaml.MappingNode {
return nil, nil, errors.New("input node is not a mapping node")
}
var scalars []string
var datas []T
expectKey := true
for _, item := range node.Content {
if expectKey {
if item.Kind != yaml.ScalarNode {
return nil, nil, fmt.Errorf("not a valid scalar node: %v", item.Value)
}
scalars = append(scalars, item.Value)
expectKey = false
} else {
var val T
if err := item.Decode(&val); err != nil {
return nil, nil, err
}
datas = append(datas, val)
expectKey = true
}
}
if len(scalars) != len(datas) {
return nil, nil, fmt.Errorf("invalid definition of on: %v", node.Value)
}
return scalars, datas, nil
}
func asString(v any) string {
if v == nil {
return ""
} else if s, ok := v.(string); ok {
return s
}
return ""
}
+420
View File
@@ -0,0 +1,420 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package jobparser
import (
"strings"
"testing"
"gitea.com/gitea/runner/act/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.yaml.in/yaml/v4"
)
func TestParseRawOn(t *testing.T) {
kases := []struct {
input string
result []*Event
}{
{
input: "on: issue_comment",
result: []*Event{
{
Name: "issue_comment",
},
},
},
{
input: "on:\n push",
result: []*Event{
{
Name: "push",
},
},
},
{
input: "on:\n - push\n - pull_request",
result: []*Event{
{
Name: "push",
},
{
Name: "pull_request",
},
},
},
{
input: "on:\n push:\n branches:\n - master",
result: []*Event{
{
Name: "push",
acts: map[string][]string{
"branches": {
"master",
},
},
},
},
},
{
input: "on:\n push:\n branches: main",
result: []*Event{
{
Name: "push",
acts: map[string][]string{
"branches": {
"main",
},
},
},
},
},
{
input: "on:\n branch_protection_rule:\n types: [created, deleted]",
result: []*Event{
{
Name: "branch_protection_rule",
acts: map[string][]string{
"types": {
"created",
"deleted",
},
},
},
},
},
{
input: "on:\n project:\n types: [created, deleted]\n milestone:\n types: [opened, deleted]",
result: []*Event{
{
Name: "project",
acts: map[string][]string{
"types": {
"created",
"deleted",
},
},
},
{
Name: "milestone",
acts: map[string][]string{
"types": {
"opened",
"deleted",
},
},
},
},
},
{
input: "on:\n pull_request:\n types:\n - opened\n branches:\n - 'releases/**'",
result: []*Event{
{
Name: "pull_request",
acts: map[string][]string{
"types": {
"opened",
},
"branches": {
"releases/**",
},
},
},
},
},
{
input: "on:\n push:\n branches:\n - main\n pull_request:\n types:\n - opened\n branches:\n - '**'",
result: []*Event{
{
Name: "push",
acts: map[string][]string{
"branches": {
"main",
},
},
},
{
Name: "pull_request",
acts: map[string][]string{
"types": {
"opened",
},
"branches": {
"**",
},
},
},
},
},
{
input: "on:\n push:\n branches:\n - 'main'\n - 'releases/**'",
result: []*Event{
{
Name: "push",
acts: map[string][]string{
"branches": {
"main",
"releases/**",
},
},
},
},
},
{
input: "on:\n push:\n tags:\n - v1.**",
result: []*Event{
{
Name: "push",
acts: map[string][]string{
"tags": {
"v1.**",
},
},
},
},
},
{
input: "on: [pull_request, workflow_dispatch]",
result: []*Event{
{
Name: "pull_request",
},
{
Name: "workflow_dispatch",
},
},
},
{
input: "on:\n schedule:\n - cron: '20 6 * * *'",
result: []*Event{
{
Name: "schedule",
schedules: []map[string]string{
{
"cron": "20 6 * * *",
},
},
},
},
},
{
input: `on:
workflow_dispatch:
inputs:
logLevel:
description: 'Log level'
required: true
default: 'warning'
type: choice
options:
- info
- warning
- debug
tags:
description: 'Test scenario tags'
required: false
type: boolean
environment:
description: 'Environment to run tests against'
type: environment
required: true
push:
`,
result: []*Event{
{
Name: "workflow_dispatch",
inputs: []WorkflowDispatchInput{
{
Name: "logLevel",
Description: "Log level",
Required: true,
Default: "warning",
Type: "choice",
Options: []string{"info", "warning", "debug"},
},
{
Name: "tags",
Description: "Test scenario tags",
Required: false,
Type: "boolean",
},
{
Name: "environment",
Description: "Environment to run tests against",
Type: "environment",
Required: true,
},
},
},
{
Name: "push",
},
},
},
{
// `workflow_call` is only fired by another workflow's `uses:`, so ParseRawOn intentionally excludes it from trigger detection.
input: `on:
workflow_call:
inputs:
env:
type: string
required: true
outputs:
sha:
value: ${{ jobs.build.outputs.commit }}
secrets:
DEPLOY_KEY:
required: true
`,
result: []*Event{},
},
{
// Mixed: a workflow that is both callable AND triggered by push. Only the "push" event surfaces.
input: `on:
workflow_call:
inputs:
env:
type: string
push:
branches: [main]
`,
result: []*Event{
{
Name: "push",
acts: map[string][]string{"branches": {"main"}},
},
},
},
{
// Scalar form: a purely reusable workflow has no event triggers.
input: "on: workflow_call",
result: []*Event{},
},
{
// Sequence form: `workflow_call` is excluded while sibling events are kept.
input: "on:\n - push\n - workflow_call\n - pull_request",
result: []*Event{
{Name: "push"},
{Name: "pull_request"},
},
},
}
for _, kase := range kases {
t.Run(kase.input, func(t *testing.T) {
origin, err := model.ReadWorkflow(strings.NewReader(kase.input))
assert.NoError(t, err)
events, err := ParseRawOn(&origin.RawOn)
assert.NoError(t, err)
assert.Equal(t, kase.result, events, events)
})
}
}
func TestSingleWorkflow_SetJob(t *testing.T) {
t.Run("erase needs", func(t *testing.T) {
content := ReadTestdata(t, "erase_needs.in.yaml")
want := ReadTestdata(t, "erase_needs.out.yaml")
swf, err := Parse(content)
require.NoError(t, err)
builder := &strings.Builder{}
for _, v := range swf {
id, job := v.Job()
require.NoError(t, v.SetJob(id, job.EraseNeeds()))
if builder.Len() > 0 {
builder.WriteString("---\n")
}
encoder := yaml.NewEncoder(builder)
encoder.SetIndent(2)
require.NoError(t, encoder.Encode(v))
}
assert.Equal(t, string(want), builder.String())
})
}
func TestParseMappingNode(t *testing.T) {
tests := []struct {
input string
scalars []string
datas []any
}{
{
input: "on:\n push:\n branches:\n - master",
scalars: []string{"push"},
datas: []any{
map[string]any{
"branches": []any{"master"},
},
},
},
{
input: "on:\n branch_protection_rule:\n types: [created, deleted]",
scalars: []string{"branch_protection_rule"},
datas: []any{
map[string]any{
"types": []any{"created", "deleted"},
},
},
},
{
input: "on:\n project:\n types: [created, deleted]\n milestone:\n types: [opened, deleted]",
scalars: []string{"project", "milestone"},
datas: []any{
map[string]any{
"types": []any{"created", "deleted"},
},
map[string]any{
"types": []any{"opened", "deleted"},
},
},
},
{
input: "on:\n pull_request:\n types:\n - opened\n branches:\n - 'releases/**'",
scalars: []string{"pull_request"},
datas: []any{
map[string]any{
"types": []any{"opened"},
"branches": []any{"releases/**"},
},
},
},
{
input: "on:\n push:\n branches:\n - main\n pull_request:\n types:\n - opened\n branches:\n - '**'",
scalars: []string{"push", "pull_request"},
datas: []any{
map[string]any{
"branches": []any{"main"},
},
map[string]any{
"types": []any{"opened"},
"branches": []any{"**"},
},
},
},
{
input: "on:\n schedule:\n - cron: '20 6 * * *'",
scalars: []string{"schedule"},
datas: []any{
[]any{map[string]any{
"cron": "20 6 * * *",
}},
},
},
}
for _, test := range tests {
t.Run(test.input, func(t *testing.T) {
workflow, err := model.ReadWorkflow(strings.NewReader(test.input))
assert.NoError(t, err)
scalars, datas, err := parseMappingNode[any](&workflow.RawOn)
assert.NoError(t, err)
assert.Equal(t, test.scalars, scalars, scalars)
assert.Equal(t, test.datas, datas, datas)
})
}
}
+8
View File
@@ -0,0 +1,8 @@
name: test
jobs:
job1:
name: job1
runs-on: linux
steps:
- run: echo job-1
-
@@ -0,0 +1,7 @@
name: test
jobs:
job1:
name: job1
runs-on: linux
steps:
- run: echo job-1
+16
View File
@@ -0,0 +1,16 @@
name: test
jobs:
job1:
runs-on: linux
steps:
- run: uname -a
job2:
runs-on: linux
steps:
- run: uname -a
needs: job1
job3:
runs-on: linux
steps:
- run: uname -a
needs: [job1, job2]
+23
View File
@@ -0,0 +1,23 @@
name: test
jobs:
job1:
name: job1
runs-on: linux
steps:
- run: uname -a
---
name: test
jobs:
job2:
name: job2
runs-on: linux
steps:
- run: uname -a
---
name: test
jobs:
job3:
name: job3
runs-on: linux
steps:
- run: uname -a
+16
View File
@@ -0,0 +1,16 @@
name: test
jobs:
job1:
runs-on: linux
steps:
- run: uname -a
job2:
runs-on: linux
steps:
- run: uname -a
needs: job1
job3:
runs-on: linux
steps:
- run: uname -a
needs: [job1, job2]
+25
View File
@@ -0,0 +1,25 @@
name: test
jobs:
job1:
name: job1
runs-on: linux
steps:
- run: uname -a
---
name: test
jobs:
job2:
name: job2
needs: job1
runs-on: linux
steps:
- run: uname -a
---
name: test
jobs:
job3:
name: job3
needs: [job1, job2]
runs-on: linux
steps:
- run: uname -a
+14
View File
@@ -0,0 +1,14 @@
name: test
jobs:
job1:
name: job1
runs-on: linux
uses: .gitea/workflows/build.yml
secrets:
secret: hideme
job2:
name: job2
runs-on: linux
uses: .gitea/workflows/build.yml
secrets: inherit
+16
View File
@@ -0,0 +1,16 @@
name: test
jobs:
job1:
name: job1
runs-on: linux
uses: .gitea/workflows/build.yml
secrets:
secret: hideme
---
name: test
jobs:
job2:
name: job2
runs-on: linux
uses: .gitea/workflows/build.yml
secrets: inherit
+15
View File
@@ -0,0 +1,15 @@
name: test
jobs:
job1:
name: job1
runs-on: linux
uses: .gitea/workflows/build.yml
with:
package: service
job2:
name: job2
runs-on: linux
uses: .gitea/workflows/build.yml
with:
package: module
+17
View File
@@ -0,0 +1,17 @@
name: test
jobs:
job1:
name: job1
runs-on: linux
uses: .gitea/workflows/build.yml
with:
package: service
---
name: test
jobs:
job2:
name: job2
runs-on: linux
uses: .gitea/workflows/build.yml
with:
package: module
@@ -0,0 +1,14 @@
name: test
jobs:
job1:
strategy:
matrix:
os: [ubuntu-22.04, ubuntu-20.04]
version: [1.17, 1.18, 1.19]
runs-on: ${{ matrix.os }}
name: test_version_${{ matrix.version }}_on_${{ matrix.os }}
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
@@ -0,0 +1,101 @@
name: test
jobs:
job1:
name: test_version_1.17_on_ubuntu-20.04
runs-on: ubuntu-20.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-20.04
version:
- 1.17
---
name: test
jobs:
job1:
name: test_version_1.18_on_ubuntu-20.04
runs-on: ubuntu-20.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-20.04
version:
- 1.18
---
name: test
jobs:
job1:
name: test_version_1.19_on_ubuntu-20.04
runs-on: ubuntu-20.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-20.04
version:
- 1.19
---
name: test
jobs:
job1:
name: test_version_1.17_on_ubuntu-22.04
runs-on: ubuntu-22.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-22.04
version:
- 1.17
---
name: test
jobs:
job1:
name: test_version_1.18_on_ubuntu-22.04
runs-on: ubuntu-22.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-22.04
version:
- 1.18
---
name: test
jobs:
job1:
name: test_version_1.19_on_ubuntu-22.04
runs-on: ubuntu-22.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-22.04
version:
- 1.19
@@ -0,0 +1,22 @@
name: test
jobs:
zzz:
runs-on: linux
steps:
- run: echo zzz
job1:
runs-on: linux
steps:
- run: uname -a && go version
job2:
runs-on: linux
steps:
- run: uname -a && go version
job3:
runs-on: linux
steps:
- run: uname -a && go version
aaa:
runs-on: linux
steps:
- run: uname -a && go version
@@ -0,0 +1,39 @@
name: test
jobs:
zzz:
name: zzz
runs-on: linux
steps:
- run: echo zzz
---
name: test
jobs:
job1:
name: job1
runs-on: linux
steps:
- run: uname -a && go version
---
name: test
jobs:
job2:
name: job2
runs-on: linux
steps:
- run: uname -a && go version
---
name: test
jobs:
job3:
name: job3
runs-on: linux
steps:
- run: uname -a && go version
---
name: test
jobs:
aaa:
name: aaa
runs-on: linux
steps:
- run: uname -a && go version
@@ -0,0 +1,13 @@
name: test
jobs:
job1:
strategy:
matrix:
os: [ubuntu-22.04, ubuntu-20.04]
version: [1.17, 1.18, 1.19]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
@@ -0,0 +1,101 @@
name: test
jobs:
job1:
name: job1 (ubuntu-20.04, 1.17)
runs-on: ubuntu-20.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-20.04
version:
- 1.17
---
name: test
jobs:
job1:
name: job1 (ubuntu-20.04, 1.18)
runs-on: ubuntu-20.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-20.04
version:
- 1.18
---
name: test
jobs:
job1:
name: job1 (ubuntu-20.04, 1.19)
runs-on: ubuntu-20.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-20.04
version:
- 1.19
---
name: test
jobs:
job1:
name: job1 (ubuntu-22.04, 1.17)
runs-on: ubuntu-22.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-22.04
version:
- 1.17
---
name: test
jobs:
job1:
name: job1 (ubuntu-22.04, 1.18)
runs-on: ubuntu-22.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-22.04
version:
- 1.18
---
name: test
jobs:
job1:
name: job1 (ubuntu-22.04, 1.19)
runs-on: ubuntu-22.04
steps:
- uses: actions/setup-go@v3
with:
go-version: ${{ matrix.version }}
- run: uname -a && go version
strategy:
matrix:
os:
- ubuntu-22.04
version:
- 1.19
@@ -0,0 +1,9 @@
# null_job_explicit.in.yaml
on: push
jobs:
empty: null
notempty:
needs: empty
runs-on: ubuntu-latest
steps:
- run: echo ok
@@ -0,0 +1,9 @@
# null_job_implicit.in.yaml
on: push
jobs:
empty:
notempty:
needs: empty
runs-on: ubuntu-latest
steps:
- run: echo ok
@@ -0,0 +1,14 @@
name: Step with leading new line
on:
push:
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: "\nExtract tag for variant"
id: extract_tag
run: |
echo Test
@@ -0,0 +1,15 @@
name: Step with leading new line
"on":
push:
jobs:
test:
name: test
runs-on: ubuntu-latest
steps:
- id: extract_tag
name: |2-
Extract tag for variant
run: |2
echo Test
@@ -0,0 +1,21 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package jobparser
import (
"embed"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
//go:embed testdata
var testdata embed.FS
func ReadTestdata(t *testing.T, name string) []byte {
content, err := testdata.ReadFile(filepath.Join("testdata", name))
require.NoError(t, err)
return content
}
+75
View File
@@ -0,0 +1,75 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package jobparser
import (
"errors"
"fmt"
"path"
"regexp"
"strings"
)
// UsesKind enumerates the supported forms of a reusable workflow "uses:" value.
type UsesKind int
const (
// UsesKindLocalSameRepo is "./.gitea/workflows/foo.yml" - a path inside the calling repository.
UsesKindLocalSameRepo UsesKind = iota + 1
// UsesKindLocalCrossRepo is "owner/repo/.gitea/workflows/foo.yml@ref" - a workflow in another repo on the same instance.
UsesKindLocalCrossRepo
)
// UsesRef is the parsed form of a reusable workflow "uses:" value.
type UsesRef struct {
Kind UsesKind
Owner string // empty for UsesKindLocalSameRepo
Repo string // empty for UsesKindLocalSameRepo
Path string // workflow file path inside the source repo
Ref string // git ref; empty for UsesKindLocalSameRepo
}
var (
reLocalSameRepo = regexp.MustCompile(`^\./\.(gitea|github)/workflows/([^@]+\.ya?ml)$`)
reLocalCrossRepo = regexp.MustCompile(`^([-.\w]+)/([-.\w]+)/\.(gitea|github)/workflows/([^@]+\.ya?ml)@(.+)$`)
)
// ParseUses parses a reusable workflow "uses:" value.
// Only two forms are supported:
// - "./.gitea/workflows/foo.yml" (UsesKindLocalSameRepo, no @ref)
// - "OWNER/REPO/.gitea/workflows/foo.yml@REF" (UsesKindLocalCrossRepo)
func ParseUses(s string) (*UsesRef, error) {
s = strings.TrimSpace(s)
if s == "" {
return nil, errors.New("empty uses value")
}
if strings.HasPrefix(s, "./") {
m := reLocalSameRepo.FindStringSubmatch(s)
if m == nil {
return nil, fmt.Errorf(`invalid local "uses:" %q (expect ./.gitea/workflows/<file>.yml)`, s)
}
p := fmt.Sprintf(".%s/workflows/%s", m[1], m[2])
if path.Clean(p) != p {
return nil, fmt.Errorf("invalid workflow path %q", s)
}
return &UsesRef{Kind: UsesKindLocalSameRepo, Path: p}, nil
}
m := reLocalCrossRepo.FindStringSubmatch(s)
if m == nil {
return nil, fmt.Errorf(`invalid cross-repo "uses:" %q (expect owner/repo/.gitea/workflows/<file>.yml@ref)`, s)
}
p := fmt.Sprintf(".%s/workflows/%s", m[3], m[4])
if path.Clean(p) != p {
return nil, fmt.Errorf("invalid workflow path %q", s)
}
return &UsesRef{
Kind: UsesKindLocalCrossRepo,
Owner: m[1],
Repo: m[2],
Path: p,
Ref: m[5],
}, nil
}
+170
View File
@@ -0,0 +1,170 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package jobparser
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseUses(t *testing.T) {
t.Run("LocalSameRepo", func(t *testing.T) {
cases := []struct {
name string
in string
want UsesRef
}{
{
name: "gitea dir, .yml",
in: "./.gitea/workflows/build.yml",
want: UsesRef{Kind: UsesKindLocalSameRepo, Path: ".gitea/workflows/build.yml"},
},
{
name: "github dir, .yml",
in: "./.github/workflows/build.yml",
want: UsesRef{Kind: UsesKindLocalSameRepo, Path: ".github/workflows/build.yml"},
},
{
name: "gitea dir, .yaml",
in: "./.gitea/workflows/build.yaml",
want: UsesRef{Kind: UsesKindLocalSameRepo, Path: ".gitea/workflows/build.yaml"},
},
{
name: "filename containing dots is allowed",
in: "./.gitea/workflows/foo..bar.yml",
want: UsesRef{Kind: UsesKindLocalSameRepo, Path: ".gitea/workflows/foo..bar.yml"},
},
{
name: "nested subdirectory",
in: "./.gitea/workflows/sub/build.yml",
want: UsesRef{Kind: UsesKindLocalSameRepo, Path: ".gitea/workflows/sub/build.yml"},
},
{
name: "leading/trailing whitespace is trimmed",
in: " ./.gitea/workflows/build.yml ",
want: UsesRef{Kind: UsesKindLocalSameRepo, Path: ".gitea/workflows/build.yml"},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := ParseUses(c.in)
require.NoError(t, err)
assert.Equal(t, c.want, *got)
})
}
})
t.Run("LocalCrossRepo", func(t *testing.T) {
cases := []struct {
name string
in string
want UsesRef
}{
{
name: "gitea dir, simple ref",
in: "owner/repo/.gitea/workflows/build.yml@v1",
want: UsesRef{
Kind: UsesKindLocalCrossRepo,
Owner: "owner",
Repo: "repo",
Path: ".gitea/workflows/build.yml",
Ref: "v1",
},
},
{
name: "github dir, branch ref",
in: "owner/repo/.github/workflows/build.yml@main",
want: UsesRef{
Kind: UsesKindLocalCrossRepo,
Owner: "owner",
Repo: "repo",
Path: ".github/workflows/build.yml",
Ref: "main",
},
},
{
name: ".yaml extension",
in: "owner/repo/.gitea/workflows/build.yaml@abc123",
want: UsesRef{
Kind: UsesKindLocalCrossRepo,
Owner: "owner",
Repo: "repo",
Path: ".gitea/workflows/build.yaml",
Ref: "abc123",
},
},
{
name: "ref with slashes (refs/heads/feature)",
in: "owner/repo/.gitea/workflows/build.yml@refs/heads/feature",
want: UsesRef{
Kind: UsesKindLocalCrossRepo,
Owner: "owner",
Repo: "repo",
Path: ".gitea/workflows/build.yml",
Ref: "refs/heads/feature",
},
},
{
name: "nested subdirectory under workflows",
in: "owner/repo/.gitea/workflows/sub/build.yml@v1",
want: UsesRef{
Kind: UsesKindLocalCrossRepo,
Owner: "owner",
Repo: "repo",
Path: ".gitea/workflows/sub/build.yml",
Ref: "v1",
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := ParseUses(c.in)
require.NoError(t, err)
assert.Equal(t, c.want, *got)
})
}
})
t.Run("Errors", func(t *testing.T) {
cases := []struct {
name string
in string
}{
{name: "empty string", in: ""},
{name: "whitespace only", in: " "},
// Same-repo malformed
{name: "same-repo with @ref", in: "./.gitea/workflows/build.yml@v1"},
{name: "same-repo wrong directory", in: "./not-workflows/build.yml"},
{name: "same-repo wrong extension", in: "./.gitea/workflows/build.txt"},
{name: "same-repo missing extension", in: "./.gitea/workflows/build"},
{name: "same-repo absolute path", in: "/.gitea/workflows/build.yml"},
{name: "same-repo path traversal", in: "./.gitea/workflows/../escape.yml"},
{name: "same-repo double slash", in: "./.gitea/workflows//build.yml"},
{name: "same-repo redundant ./", in: "./.gitea/workflows/./build.yml"},
{name: "same-repo no filename", in: "./.gitea/workflows/.yml"},
// Cross-repo malformed
{name: "cross-repo missing @ref", in: "owner/repo/.gitea/workflows/build.yml"},
{name: "cross-repo empty ref", in: "owner/repo/.gitea/workflows/build.yml@"},
{name: "cross-repo missing owner", in: "/repo/.gitea/workflows/build.yml@v1"},
{name: "cross-repo missing repo", in: "owner//.gitea/workflows/build.yml@v1"},
{name: "cross-repo wrong workflows dir", in: "owner/repo/workflows/build.yml@v1"},
{name: "cross-repo wrong extension", in: "owner/repo/.gitea/workflows/build.txt@v1"},
{name: "cross-repo path traversal", in: "owner/repo/.gitea/workflows/../escape.yml@v1"},
{name: "cross-repo double slash in path", in: "owner/repo/.gitea/workflows//build.yml@v1"},
// owner/repo with chars Gitea's name validators reject
{name: "cross-repo owner with space", in: "bad owner/repo/.gitea/workflows/build.yml@v1"},
{name: "cross-repo repo with @", in: "owner/re@po/.gitea/workflows/build.yml@v1"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
_, err := ParseUses(c.in)
assert.Error(t, err)
})
}
})
}
+401
View File
@@ -0,0 +1,401 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package jobparser
import (
"errors"
"fmt"
"regexp"
"strings"
"gitea.dev/modules/container"
"gitea.dev/modules/util"
"gitea.com/gitea/runner/act/exprparser"
"gitea.com/gitea/runner/act/model"
"go.yaml.in/yaml/v4"
)
// InputType enumerates the allowed types for a workflow_call input.
type InputType string
const (
InputTypeString InputType = "string"
InputTypeBoolean InputType = "boolean"
InputTypeNumber InputType = "number"
)
// InputSpec describes a single workflow_call input declaration.
type InputSpec struct {
Description string `yaml:"description"`
Required bool `yaml:"required"`
Default yaml.Node `yaml:"default"`
Type InputType `yaml:"type"`
}
// SecretSpec describes a single workflow_call secret declaration.
type SecretSpec struct {
Description string `yaml:"description"`
Required bool `yaml:"required"`
}
// OutputSpec describes a single workflow_call output declaration.
type OutputSpec struct {
Description string `yaml:"description"`
Value string `yaml:"value"`
}
// WorkflowCallSpec is the parsed "on.workflow_call" schema of a called workflow.
type WorkflowCallSpec struct {
Inputs map[string]InputSpec
Secrets map[string]SecretSpec
Outputs map[string]OutputSpec
}
// JobOutputs is the per-job-id outputs map used for evaluating workflow_call outputs.
type JobOutputs map[string]map[string]string
// ParseWorkflowCallSpec extracts on.workflow_call.{inputs,secrets,outputs} from a workflow YAML.
// Returns an error if the workflow does not declare on.workflow_call at all.
func ParseWorkflowCallSpec(content []byte) (*WorkflowCallSpec, error) {
var doc struct {
On yaml.Node `yaml:"on"`
}
if err := yaml.Unmarshal(content, &doc); err != nil {
return nil, fmt.Errorf("parse workflow yaml: %w", err)
}
wcNode, ok := findWorkflowCallNode(&doc.On)
if !ok {
return nil, errors.New("workflow does not declare on.workflow_call")
}
spec := &WorkflowCallSpec{
Inputs: map[string]InputSpec{},
Secrets: map[string]SecretSpec{},
Outputs: map[string]OutputSpec{},
}
if wcNode == nil || wcNode.Kind != yaml.MappingNode {
return spec, nil
}
for i := 0; i+1 < len(wcNode.Content); i += 2 {
key := wcNode.Content[i]
val := wcNode.Content[i+1]
switch key.Value {
case "inputs":
if err := decodeWorkflowCallMapping(val, spec.Inputs); err != nil {
return nil, fmt.Errorf("parse workflow_call.inputs: %w", err)
}
case "secrets":
if err := decodeWorkflowCallMapping(val, spec.Secrets); err != nil {
return nil, fmt.Errorf("parse workflow_call.secrets: %w", err)
}
case "outputs":
if err := decodeWorkflowCallMapping(val, spec.Outputs); err != nil {
return nil, fmt.Errorf("parse workflow_call.outputs: %w", err)
}
}
}
for name, in := range spec.Inputs {
if in.Type == "" {
return nil, fmt.Errorf("workflow_call input %q is missing required field \"type\"", name)
}
switch in.Type {
case InputTypeString, InputTypeBoolean, InputTypeNumber:
default:
return nil, fmt.Errorf("workflow_call input %q has unsupported type %q", name, in.Type)
}
}
return spec, nil
}
// findWorkflowCallNode walks the "on:" node and returns the value mapping (or nil) for "workflow_call".
// "ok" is true when the workflow declares workflow_call (even with an empty body).
func findWorkflowCallNode(on *yaml.Node) (val *yaml.Node, ok bool) {
if on == nil || on.Kind == 0 {
return nil, false
}
switch on.Kind {
case yaml.ScalarNode:
return nil, on.Value == "workflow_call"
case yaml.SequenceNode:
for _, item := range on.Content {
if item.Kind == yaml.ScalarNode && item.Value == "workflow_call" {
return nil, true
}
}
return nil, false
case yaml.MappingNode:
for i := 0; i+1 < len(on.Content); i += 2 {
k := on.Content[i]
v := on.Content[i+1]
if k.Value != "workflow_call" {
continue
}
if v.Kind == yaml.MappingNode {
return v, true
}
return nil, true
}
}
return nil, false
}
func decodeWorkflowCallMapping[T any](node *yaml.Node, dst map[string]T) error {
if node == nil || node.Kind != yaml.MappingNode {
return nil
}
for i := 0; i+1 < len(node.Content); i += 2 {
name := node.Content[i].Value
var v T
if err := node.Content[i+1].Decode(&v); err != nil {
return fmt.Errorf("%q: %w", name, err)
}
dst[name] = v
}
return nil
}
// EvaluateCallerWith evaluates the caller-side expressions in `job.With` against the provided contexts
func EvaluateCallerWith(
jobID string,
job *Job,
gitCtx map[string]any,
results map[string]*JobResult,
vars map[string]string,
inputs map[string]any,
) (map[string]any, error) {
actJob := &model.Job{Strategy: &model.Strategy{
FailFastString: job.Strategy.FailFastString,
MaxParallelString: job.Strategy.MaxParallelString,
RawMatrix: job.Strategy.RawMatrix,
}}
var matrix map[string]any
matrixes, err := actJob.GetMatrixes()
if err != nil {
return nil, fmt.Errorf("get caller %q matrix: %w", jobID, err)
}
if len(matrixes) > 0 {
matrix = matrixes[0]
}
evaluator := NewExpressionEvaluator(NewInterpeter(jobID, actJob, matrix, toGitContext(gitCtx), results, vars, inputs))
out := make(map[string]any, len(job.With))
for k, raw := range job.With {
var evaluated any
switch v := raw.(type) {
case string:
node := yaml.Node{}
if err := node.Encode(v); err != nil {
return nil, fmt.Errorf("encode caller %q with[%q]: %w", jobID, k, err)
}
if err := evaluator.EvaluateYamlNode(&node); err != nil {
return nil, fmt.Errorf("evaluate caller %q with[%q]: %w", jobID, k, err)
}
if err := node.Decode(&evaluated); err != nil {
return nil, fmt.Errorf("decode caller %q with[%q]: %w", jobID, k, err)
}
default:
evaluated = v
}
out[k] = evaluated
}
return out, nil
}
// MatchCallerInputsAgainstSpec checks the caller's already-evaluated `with:` values against the callee's declared `on.workflow_call.inputs` schema
func MatchCallerInputsAgainstSpec(spec *WorkflowCallSpec, evaluated map[string]any) (map[string]any, error) {
resolved := make(map[string]any, len(spec.Inputs))
// fill defaults first
for name, in := range spec.Inputs {
if in.Default.IsZero() {
continue
}
var defaultVal any
if err := in.Default.Decode(&defaultVal); err != nil {
return nil, fmt.Errorf("decode workflow_call input %q default: %w", name, err)
}
v, err := parseWorkflowCallInput(name, in.Type, defaultVal)
if err != nil {
return nil, err
}
resolved[name] = v
}
for k, raw := range evaluated {
inputSpec, ok := spec.Inputs[k]
if !ok {
// ignore unknown "with:" keys
continue
}
converted, err := parseWorkflowCallInput(k, inputSpec.Type, raw)
if err != nil {
return nil, err
}
resolved[k] = converted
}
for name, in := range spec.Inputs {
if !in.Required {
continue
}
// resolved[name] is set when caller provided it OR when spec has a non-zero default - both satisfy "required".
if _, ok := resolved[name]; ok {
continue
}
return nil, fmt.Errorf("workflow_call input %q is required", name)
}
return resolved, nil
}
func parseWorkflowCallInput(name string, typ InputType, v any) (any, error) {
switch typ {
case InputTypeString:
return toString(v), nil
case InputTypeBoolean:
// strict type matching: a boolean input only accepts a native bool, not a "true"/"false" string
if b, ok := v.(bool); ok {
return b, nil
}
return false, fmt.Errorf("workflow_call input %q expects boolean", name)
case InputTypeNumber:
// strict type matching: a number input rejects "123"/"3.14" strings.
if _, isString := v.(string); isString {
return 0.0, fmt.Errorf("workflow_call input %q expects number", name)
}
return util.ToFloat64(v)
default:
return nil, fmt.Errorf("workflow_call input %q has unsupported type %q", name, typ)
}
}
// SecretsInherit is the literal keyword used in a caller's `secrets: inherit` directive
const SecretsInherit = "inherit"
// callerSecretValueRegexp matches the `${{ secrets.NAME }}` form expected for each value in a caller's `secrets:` mapping.
var callerSecretValueRegexp = regexp.MustCompile(`^\s*\$\{\{\s*secrets\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}\s*$`)
// ParseCallerSecrets decodes a caller's "secrets:" YAML node into one of two forms:
// - inherit == true: the caller wrote `secrets: inherit`; mapping is nil
// - inherit == false, mapping == {alias: source_name}: explicit mapping. Each value must be of the form `${{ secrets.NAME }}`.
//
// Both alias and source name are upper-cased: secret names are case-insensitive (matching GitHub),
// and Gitea stores secrets upper-cased, so this keeps lookups and schema validation consistent.
func ParseCallerSecrets(node yaml.Node) (inherit bool, mapping map[string]string, err error) {
if node.IsZero() {
return false, nil, nil
}
if node.Kind == yaml.ScalarNode && strings.TrimSpace(node.Value) == SecretsInherit {
return true, nil, nil
}
if node.Kind != yaml.MappingNode {
return false, nil, errors.New("invalid secrets: section, expected mapping or 'inherit'")
}
out := make(map[string]string, len(node.Content)/2)
for i := 0; i+1 < len(node.Content); i += 2 {
k := node.Content[i]
v := node.Content[i+1]
var sv string
if err := v.Decode(&sv); err != nil {
return false, nil, fmt.Errorf("decode secret %q: %w", k.Value, err)
}
matches := callerSecretValueRegexp.FindStringSubmatch(sv)
if len(matches) != 2 {
return false, nil, fmt.Errorf("caller secret %q value must be of the form ${{ secrets.NAME }}", k.Value)
}
out[strings.ToUpper(k.Value)] = strings.ToUpper(matches[1])
}
return false, out, nil
}
// ValidateCallerSecrets checks a caller's parsed explicit-mapping `secrets:` against the called workflow's declared `on.workflow_call.secrets` schema.
func ValidateCallerSecrets(spec *WorkflowCallSpec, mapping map[string]string) error {
if spec == nil {
return errors.New("ValidateCallerSecrets: nil workflow_call spec")
}
// Secret names are case-insensitive, so compare declared names and caller aliases upper-cased.
declaredNames := make(container.Set[string], len(spec.Secrets))
for name := range spec.Secrets {
declaredNames.Add(strings.ToUpper(name))
}
provided := make(container.Set[string], len(mapping))
for alias := range mapping {
up := strings.ToUpper(alias)
provided.Add(up)
if !declaredNames.Contains(up) {
return fmt.Errorf("caller secret %q is not declared in the called workflow's on.workflow_call.secrets", alias)
}
}
for name, sec := range spec.Secrets {
if sec.Required && !provided.Contains(strings.ToUpper(name)) {
return fmt.Errorf("required secret %q is not provided by the caller", name)
}
}
return nil
}
// EvaluateWorkflowCallOutputs evaluates a called workflow's "on.workflow_call.outputs.<name>.value" expressions against the provided contexts.
func EvaluateWorkflowCallOutputs(spec *WorkflowCallSpec, gitCtx *model.GithubContext, vars map[string]string, inputs map[string]any, jobOutputs JobOutputs) (map[string]string, error) {
if spec == nil || len(spec.Outputs) == 0 {
return map[string]string{}, nil
}
jobsCtx := make(map[string]*model.WorkflowCallResult, len(jobOutputs))
for jobID, outputs := range jobOutputs {
jobsCtx[jobID] = &model.WorkflowCallResult{Outputs: outputs}
}
// See `on.workflow_call.outputs.<output_id>.value` in https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#context-availability
env := &exprparser.EvaluationEnvironment{
Github: gitCtx,
Jobs: &jobsCtx,
Vars: vars,
Inputs: inputs,
}
interpreter := exprparser.NewInterpeter(env, exprparser.Config{})
out := make(map[string]string, len(spec.Outputs))
for name, o := range spec.Outputs {
v, err := evaluateWorkflowCallOutputValue(interpreter, o.Value)
if err != nil {
return nil, fmt.Errorf("workflow_call output %q: %w", name, err)
}
out[name] = v
}
return out, nil
}
func evaluateWorkflowCallOutputValue(interpreter exprparser.Interpreter, value string) (string, error) {
if !strings.Contains(value, "${{") || !strings.Contains(value, "}}") {
return value, nil
}
expr, err := rewriteSubExpression(value, true)
if err != nil {
return "", err
}
evaluated, err := interpreter.Evaluate(expr, exprparser.DefaultStatusCheckNone)
if err != nil {
return "", err
}
return toString(evaluated), nil
}
func toString(v any) string {
switch s := v.(type) {
case string:
return s
case nil:
return ""
default:
return fmt.Sprintf("%v", s)
}
}
@@ -0,0 +1,471 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package jobparser
import (
"maps"
"testing"
"gitea.com/gitea/runner/act/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.yaml.in/yaml/v4"
)
func TestParseWorkflowCallSpec(t *testing.T) {
t.Run("malformed YAML surfaces a parse error", func(t *testing.T) {
// Mismatched flow-sequence brackets — yaml.Unmarshal must reject this.
_, err := ParseWorkflowCallSpec([]byte(`name: bad
on: [workflow_call
jobs:
noop: { }
`))
require.Error(t, err)
})
t.Run("workflow without on.workflow_call is rejected", func(t *testing.T) {
notCallable := []byte(`name: ordinary
on: push
jobs:
noop:
runs-on: ubuntu-latest
steps:
- run: echo
`)
_, err := ParseWorkflowCallSpec(notCallable)
require.Error(t, err)
assert.Contains(t, err.Error(), "does not declare on.workflow_call")
})
t.Run("input missing the required type field is rejected", func(t *testing.T) {
content := callableWorkflow(t, `inputs:
x:
description: missing type
`)
_, err := ParseWorkflowCallSpec(content)
require.Error(t, err)
assert.Contains(t, err.Error(), `missing required field "type"`)
})
t.Run("inputs/secrets/outputs are decoded", func(t *testing.T) {
content := callableWorkflow(t, `inputs:
env:
type: string
required: true
secrets:
DEPLOY_KEY:
required: true
outputs:
sha:
value: ${{ jobs.build.outputs.commit }}
`)
spec, err := ParseWorkflowCallSpec(content)
require.NoError(t, err)
assert.Equal(t, InputTypeString, spec.Inputs["env"].Type)
assert.True(t, spec.Inputs["env"].Required)
assert.True(t, spec.Secrets["DEPLOY_KEY"].Required)
assert.Equal(t, "${{ jobs.build.outputs.commit }}", spec.Outputs["sha"].Value)
})
}
func TestEvaluateCallerWith(t *testing.T) {
t.Run("empty with: returns empty map", func(t *testing.T) {
out, err := EvaluateCallerWith("caller", &Job{}, nil, callerResults("caller", nil, nil), nil, nil)
require.NoError(t, err)
assert.Empty(t, out)
})
t.Run("non-string raw values pass through unchanged", func(t *testing.T) {
job := &Job{With: map[string]any{
"already_bool": true,
"already_int": 42,
"already_slice": []any{"a", "b"},
}}
out, err := EvaluateCallerWith("caller", job, nil, callerResults("caller", nil, nil), nil, nil)
require.NoError(t, err)
assert.Equal(t, true, out["already_bool"])
assert.Equal(t, 42, out["already_int"])
assert.Equal(t, []any{"a", "b"}, out["already_slice"])
})
t.Run("expressions resolve against vars/inputs/results", func(t *testing.T) {
job := &Job{With: map[string]any{
"env_name": "${{ vars.ENV }}",
"from_inputs": "${{ inputs.PARENT_VAR }}",
"from_needs": "${{ needs.upstream.outputs.commit }}",
}}
gitCtx := map[string]any{"event": map[string]any{}}
results := callerResults("caller", []string{"upstream"}, map[string]*JobResult{
"upstream": {Result: "success", Outputs: map[string]string{"commit": "abc123"}},
})
vars := map[string]string{"ENV": "staging"}
inputs := map[string]any{"PARENT_VAR": "from-parent"}
out, err := EvaluateCallerWith("caller", job, gitCtx, results, vars, inputs)
require.NoError(t, err)
assert.Equal(t, "staging", out["env_name"])
assert.Equal(t, "from-parent", out["from_inputs"])
assert.Equal(t, "abc123", out["from_needs"])
})
t.Run("matrix.X resolves to this caller row's matrix instance", func(t *testing.T) {
var rawMatrix yaml.Node
require.NoError(t, rawMatrix.Encode(map[string][]any{"target": {"staging"}}))
job := &Job{
With: map[string]any{"env": "${{ matrix.target }}"},
Strategy: Strategy{RawMatrix: rawMatrix},
}
out, err := EvaluateCallerWith("caller", job, nil, callerResults("caller", nil, nil), nil, nil)
require.NoError(t, err)
assert.Equal(t, "staging", out["env"])
})
}
func TestMatchCallerInputsAgainstSpec(t *testing.T) {
// mustParseSpec wraps ParseWorkflowCallSpec for test brevity.
mustParseSpec := func(t *testing.T, content []byte) *WorkflowCallSpec {
t.Helper()
spec, err := ParseWorkflowCallSpec(content)
require.NoError(t, err)
return spec
}
t.Run("default is filled when caller does not provide the input", func(t *testing.T) {
spec := mustParseSpec(t, callableWorkflow(t, `inputs:
greeting:
type: string
default: hi
`))
out, err := MatchCallerInputsAgainstSpec(spec, nil)
require.NoError(t, err)
assert.Equal(t, map[string]any{"greeting": "hi"}, out)
})
t.Run("caller-provided value wins over default", func(t *testing.T) {
spec := mustParseSpec(t, callableWorkflow(t, `inputs:
greeting:
type: string
default: hi
`))
out, err := MatchCallerInputsAgainstSpec(spec, map[string]any{"greeting": "hello"})
require.NoError(t, err)
assert.Equal(t, map[string]any{"greeting": "hello"}, out)
})
t.Run("required input must be provided", func(t *testing.T) {
spec := mustParseSpec(t, callableWorkflow(t, `inputs:
target:
type: string
required: true
`))
_, err := MatchCallerInputsAgainstSpec(spec, nil)
require.Error(t, err)
assert.Contains(t, err.Error(), `"target" is required`)
})
t.Run("required input is satisfied by a default value", func(t *testing.T) {
spec := mustParseSpec(t, callableWorkflow(t, `inputs:
target:
type: string
required: true
default: prod
`))
out, err := MatchCallerInputsAgainstSpec(spec, nil)
require.NoError(t, err)
assert.Equal(t, map[string]any{"target": "prod"}, out)
})
t.Run("boolean inputs accept native bool values and bool defaults", func(t *testing.T) {
spec := mustParseSpec(t, callableWorkflow(t, `inputs:
flag1:
type: boolean
flag2:
type: boolean
default: true
flag3:
type: boolean
`))
out, err := MatchCallerInputsAgainstSpec(spec, map[string]any{
"flag1": true,
"flag3": false,
})
require.NoError(t, err)
assert.Equal(t, true, out["flag1"])
assert.Equal(t, true, out["flag2"]) // from default
assert.Equal(t, false, out["flag3"])
})
t.Run("boolean input rejects strings", func(t *testing.T) {
spec := mustParseSpec(t, callableWorkflow(t, `inputs:
flag:
type: boolean
`))
_, err := MatchCallerInputsAgainstSpec(spec, map[string]any{"flag": "true"})
require.Error(t, err)
assert.Contains(t, err.Error(), "expects boolean")
})
t.Run("number inputs accept native numeric values and number defaults", func(t *testing.T) {
spec := mustParseSpec(t, callableWorkflow(t, `inputs:
count:
type: number
ratio:
type: number
default: 0.5
`))
out, err := MatchCallerInputsAgainstSpec(spec, map[string]any{"count": 42})
require.NoError(t, err)
assert.InDelta(t, 42.0, out["count"], 0)
assert.InDelta(t, 0.5, out["ratio"], 0)
})
t.Run("number input rejects strings", func(t *testing.T) {
spec := mustParseSpec(t, callableWorkflow(t, `inputs:
count:
type: number
`))
_, err := MatchCallerInputsAgainstSpec(spec, map[string]any{"count": "42"})
require.Error(t, err)
assert.Contains(t, err.Error(), "expects number")
})
t.Run("unknown caller-with key is silently dropped", func(t *testing.T) {
spec := mustParseSpec(t, callableWorkflow(t, `inputs:
known:
type: string
default: ok
`))
out, err := MatchCallerInputsAgainstSpec(spec, map[string]any{
"known": "yes",
"unknown": "ignored",
})
require.NoError(t, err)
assert.Equal(t, map[string]any{"known": "yes"}, out)
})
}
func TestParseCallerSecrets(t *testing.T) {
// secretYAMLNode unmarshals raw YAML text into a yaml.Node so tests can hand it to ParseCallerSecrets.
secretYAMLNode := func(t *testing.T, s string) yaml.Node {
t.Helper()
var node yaml.Node
require.NoError(t, yaml.Unmarshal([]byte(s), &node))
// yaml.Unmarshal wraps content in a DocumentNode; the meaningful node is the first child.
if node.Kind == yaml.DocumentNode && len(node.Content) > 0 {
return *node.Content[0]
}
return node
}
t.Run("zero node returns no inherit, no mapping", func(t *testing.T) {
inherit, mapping, err := ParseCallerSecrets(yaml.Node{})
require.NoError(t, err)
assert.False(t, inherit)
assert.Nil(t, mapping)
})
t.Run("\"inherit\" scalar sets inherit=true", func(t *testing.T) {
inherit, mapping, err := ParseCallerSecrets(secretYAMLNode(t, `inherit`))
require.NoError(t, err)
assert.True(t, inherit)
assert.Nil(t, mapping)
})
t.Run("non-inherit scalar is rejected", func(t *testing.T) {
_, _, err := ParseCallerSecrets(secretYAMLNode(t, `something-else`))
require.Error(t, err)
assert.Contains(t, err.Error(), "expected mapping or 'inherit'")
})
t.Run("mapping of secrets-style references is parsed", func(t *testing.T) {
inherit, mapping, err := ParseCallerSecrets(secretYAMLNode(t, `
DEPLOY_KEY: ${{ secrets.GITEA_DEPLOY_KEY }}
DB_PASS: ${{ secrets.PROD_DB_PASS }}
`))
require.NoError(t, err)
assert.False(t, inherit)
assert.Equal(t, map[string]string{
"DEPLOY_KEY": "GITEA_DEPLOY_KEY",
"DB_PASS": "PROD_DB_PASS",
}, mapping)
})
t.Run("alias and source names are upper-cased", func(t *testing.T) {
inherit, mapping, err := ParseCallerSecrets(secretYAMLNode(t, `
deploy_key: ${{ secrets.gitea_deploy_key }}
`))
require.NoError(t, err)
assert.False(t, inherit)
assert.Equal(t, map[string]string{"DEPLOY_KEY": "GITEA_DEPLOY_KEY"}, mapping)
})
t.Run("mapping value not in ${{ secrets.NAME }} form is rejected", func(t *testing.T) {
// plain string
_, _, err := ParseCallerSecrets(secretYAMLNode(t, `KEY: not-an-expression`))
require.Error(t, err)
assert.Contains(t, err.Error(), `must be of the form ${{ secrets.NAME }}`)
// expression but referencing the wrong context (vars instead of secrets)
_, _, err = ParseCallerSecrets(secretYAMLNode(t, `KEY: ${{ vars.NAME }}`))
require.Error(t, err)
assert.Contains(t, err.Error(), `must be of the form ${{ secrets.NAME }}`)
})
}
func TestValidateCallerSecrets(t *testing.T) {
specWith := func(secrets map[string]SecretSpec) *WorkflowCallSpec {
return &WorkflowCallSpec{Secrets: secrets}
}
t.Run("explicit mapping with all required + only declared aliases is accepted", func(t *testing.T) {
spec := specWith(map[string]SecretSpec{
"DEPLOY_KEY": {Required: true},
"OPTIONAL": {},
})
mapping := map[string]string{
"DEPLOY_KEY": "PROD_DEPLOY_KEY",
"OPTIONAL": "SOMETHING_ELSE",
}
require.NoError(t, ValidateCallerSecrets(spec, mapping))
})
t.Run("alias not in callee schema is rejected", func(t *testing.T) {
spec := specWith(map[string]SecretSpec{"DEPLOY_KEY": {}})
mapping := map[string]string{
"DEPLOY_KEY": "PROD_DEPLOY_KEY",
"EXTRA": "SOMETHING_NOT_DECLARED",
}
err := ValidateCallerSecrets(spec, mapping)
require.Error(t, err)
assert.Contains(t, err.Error(), `caller secret "EXTRA"`)
assert.Contains(t, err.Error(), `not declared`)
})
t.Run("missing required secret is rejected", func(t *testing.T) {
spec := specWith(map[string]SecretSpec{
"MUST_HAVE": {Required: true},
"OPTIONAL": {},
})
mapping := map[string]string{"OPTIONAL": "X"}
err := ValidateCallerSecrets(spec, mapping)
require.Error(t, err)
assert.Contains(t, err.Error(), `required secret "MUST_HAVE"`)
assert.Contains(t, err.Error(), `not provided`)
})
t.Run("callee with no secrets schema accepts an empty mapping", func(t *testing.T) {
spec := specWith(map[string]SecretSpec{})
require.NoError(t, ValidateCallerSecrets(spec, nil))
require.NoError(t, ValidateCallerSecrets(spec, map[string]string{}))
})
t.Run("callee with no secrets schema rejects a non-empty mapping", func(t *testing.T) {
spec := specWith(map[string]SecretSpec{})
err := ValidateCallerSecrets(spec, map[string]string{"X": "Y"})
require.Error(t, err)
assert.Contains(t, err.Error(), `caller secret "X"`)
})
t.Run("name matching is case-insensitive", func(t *testing.T) {
// declared name and caller alias differ only in case; both should match.
spec := specWith(map[string]SecretSpec{"deploy_key": {Required: true}})
mapping := map[string]string{"DEPLOY_KEY": "PROD_DEPLOY_KEY"}
require.NoError(t, ValidateCallerSecrets(spec, mapping))
})
t.Run("nil spec is rejected", func(t *testing.T) {
err := ValidateCallerSecrets(nil, map[string]string{"X": "Y"})
require.Error(t, err)
assert.Contains(t, err.Error(), "nil workflow_call spec")
})
}
func TestEvaluateWorkflowCallOutputs(t *testing.T) {
t.Run("nil spec returns empty map", func(t *testing.T) {
out, err := EvaluateWorkflowCallOutputs(nil, &model.GithubContext{}, nil, nil, nil)
require.NoError(t, err)
assert.Empty(t, out)
})
t.Run("spec with no outputs returns empty map", func(t *testing.T) {
spec := &WorkflowCallSpec{Outputs: map[string]OutputSpec{}}
out, err := EvaluateWorkflowCallOutputs(spec, &model.GithubContext{}, nil, nil, nil)
require.NoError(t, err)
assert.Empty(t, out)
})
t.Run("plain string value passes through unchanged", func(t *testing.T) {
spec := &WorkflowCallSpec{Outputs: map[string]OutputSpec{
"name": {Value: "static-value"},
}}
out, err := EvaluateWorkflowCallOutputs(spec, &model.GithubContext{}, nil, nil, nil)
require.NoError(t, err)
assert.Equal(t, map[string]string{"name": "static-value"}, out)
})
t.Run("output references jobs.<id>.outputs.<name>", func(t *testing.T) {
spec := &WorkflowCallSpec{Outputs: map[string]OutputSpec{
"sha": {Value: "${{ jobs.build.outputs.commit }}"},
}}
jobOutputs := JobOutputs{
"build": {"commit": "deadbeef"},
}
out, err := EvaluateWorkflowCallOutputs(spec, &model.GithubContext{}, nil, nil, jobOutputs)
require.NoError(t, err)
assert.Equal(t, "deadbeef", out["sha"])
})
t.Run("output references inputs.<name>", func(t *testing.T) {
spec := &WorkflowCallSpec{Outputs: map[string]OutputSpec{
"target": {Value: "${{ inputs.env_name }}"},
}}
inputs := map[string]any{"env_name": "staging"}
out, err := EvaluateWorkflowCallOutputs(spec, &model.GithubContext{}, nil, inputs, nil)
require.NoError(t, err)
assert.Equal(t, "staging", out["target"])
})
t.Run("multiple outputs are all evaluated", func(t *testing.T) {
spec := &WorkflowCallSpec{Outputs: map[string]OutputSpec{
"static": {Value: "static-value"},
"dynamic": {Value: "${{ vars.SUFFIX }}"},
}}
vars := map[string]string{"SUFFIX": "abc"}
out, err := EvaluateWorkflowCallOutputs(spec, &model.GithubContext{}, vars, nil, nil)
require.NoError(t, err)
assert.Equal(t, "static-value", out["static"])
assert.Equal(t, "abc", out["dynamic"])
})
t.Run("expression referencing an undefined symbol surfaces an error", func(t *testing.T) {
spec := &WorkflowCallSpec{Outputs: map[string]OutputSpec{
"bad": {Value: "${{ this.is.not.valid() }}"},
}}
_, err := EvaluateWorkflowCallOutputs(spec, &model.GithubContext{}, nil, nil, nil)
require.Error(t, err)
assert.Contains(t, err.Error(), `output "bad"`)
})
}
// callableWorkflow returns a minimal valid called-workflow YAML with on.workflow_call.
func callableWorkflow(t *testing.T, body string) []byte {
t.Helper()
return []byte(`name: callable
on:
workflow_call:
` + body + `
jobs:
noop:
runs-on: ubuntu-latest
steps:
- run: "echo"
`)
}
// callerResults returns the minimum results map shape that NewInterpeter expects
func callerResults(callerJobID string, callerNeeds []string, deps map[string]*JobResult) map[string]*JobResult {
out := make(map[string]*JobResult, len(deps)+1)
maps.Copy(out, deps)
out[callerJobID] = &JobResult{Needs: callerNeeds}
return out
}
+226
View File
@@ -0,0 +1,226 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
"bufio"
"context"
"fmt"
"io"
"os"
"strings"
"time"
runnerv1 "gitea.dev/actions-proto-go/runner/v1"
"gitea.dev/models/dbfs"
"gitea.dev/modules/log"
"gitea.dev/modules/storage"
"gitea.dev/modules/zstd"
"google.golang.org/protobuf/types/known/timestamppb"
)
const (
MaxLineSize = 64 * 1024
DBFSPrefix = "actions_log/"
timeFormat = "2006-01-02T15:04:05.0000000Z07:00"
defaultBufSize = MaxLineSize
)
// WriteLogs appends logs to DBFS file for temporary storage.
// It doesn't respect the file format in the filename like ".zst", since it's difficult to reopen a closed compressed file and append new content.
// Why doesn't it store logs in object storage directly? Because it's not efficient to append content to object storage.
func WriteLogs(ctx context.Context, filename string, offset int64, rows []*runnerv1.LogRow) ([]int, error) {
flag, openFileFor := os.O_WRONLY, "write-only"
if offset == 0 {
// Only allow to create file if offset is 0 (the first write), see #25560.
// Otherwise, it might result in content holes if the file has been deleted after transferred (actions.TransferLogs).
flag, openFileFor = os.O_WRONLY|os.O_CREATE, "write-create"
}
name := DBFSPrefix + filename
f, err := dbfs.OpenFile(ctx, name, flag)
if err != nil {
return nil, fmt.Errorf("dbfs.OpenFile %q for %s: %w", name, openFileFor, err)
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
return nil, fmt.Errorf("dbfs.Stat %q: %w", name, err)
}
if stat.Size() < offset {
// If the size is less than offset, refuse to write, or it could result in content holes.
// However, if the size is greater than offset, we can still write to overwrite the content.
return nil, fmt.Errorf("size of %q is less than offset", name)
}
if _, err := f.Seek(offset, io.SeekStart); err != nil {
return nil, fmt.Errorf("dbfs.Seek %q: %w", name, err)
}
writer := bufio.NewWriterSize(f, defaultBufSize)
ns := make([]int, 0, len(rows))
for _, row := range rows {
n, err := writer.WriteString(FormatLog(row.Time.AsTime(), row.Content) + "\n")
if err != nil {
return nil, err
}
ns = append(ns, n)
}
if err := writer.Flush(); err != nil {
return nil, err
}
return ns, nil
}
func ReadLogs(ctx context.Context, inStorage bool, filename string, offset, limit int64) ([]*runnerv1.LogRow, error) {
f, err := OpenLogs(ctx, inStorage, filename)
if err != nil {
return nil, err
}
defer f.Close()
if _, err := f.Seek(offset, io.SeekStart); err != nil {
return nil, fmt.Errorf("file seek: %w", err)
}
scanner := bufio.NewScanner(f)
maxLineSize := len(timeFormat) + MaxLineSize + 1
scanner.Buffer(make([]byte, maxLineSize), maxLineSize)
var rows []*runnerv1.LogRow
for scanner.Scan() && (int64(len(rows)) < limit || limit < 0) {
t, c, err := ParseLog(scanner.Text())
if err != nil {
return nil, fmt.Errorf("parse log %q: %w", scanner.Text(), err)
}
rows = append(rows, &runnerv1.LogRow{
Time: timestamppb.New(t),
Content: c,
})
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("ReadLogs scan: %w", err)
}
return rows, nil
}
const (
// logZstdBlockSize is the block size for zstd compression.
// 128KB leads the compression ratio to be close to the regular zstd compression.
// And it means each read from the underlying object storage will be at least 128KB*(compression ratio).
// The compression ratio is about 30% for text files, so the actual read size is about 38KB, which should be acceptable.
logZstdBlockSize = 128 * 1024 // 128KB
)
// TransferLogs transfers logs from DBFS to object storage.
// It happens when the file is complete and no more logs will be appended.
// It respects the file format in the filename like ".zst", and compresses the content if needed.
// The task log file must be marked as "log_in_storage=true" after the transfer.
func TransferLogs(ctx context.Context, filename string) (func(), error) {
name := DBFSPrefix + filename
remove := func() {
if err := dbfs.Remove(ctx, name); err != nil {
log.Warn("dbfs.Remove %q: %v", name, err)
}
}
f, err := dbfs.Open(ctx, name)
if err != nil {
return nil, fmt.Errorf("dbfs.Open %q: %w", name, err)
}
defer f.Close()
var reader io.Reader = f
if strings.HasSuffix(filename, ".zst") {
r, w := io.Pipe()
reader = r
zstdWriter, err := zstd.NewSeekableWriter(w, logZstdBlockSize)
if err != nil {
return nil, fmt.Errorf("zstd NewSeekableWriter: %w", err)
}
go func() {
defer func() {
_ = w.CloseWithError(zstdWriter.Close())
}()
if _, err := io.Copy(zstdWriter, f); err != nil {
_ = w.CloseWithError(err)
return
}
}()
}
if _, err := storage.Actions.Save(filename, reader, -1); err != nil {
return nil, fmt.Errorf("storage save %q: %w", filename, err)
}
return remove, nil
}
func RemoveLogs(ctx context.Context, inStorage bool, filename string) error {
if !inStorage {
name := DBFSPrefix + filename
err := dbfs.Remove(ctx, name)
if err != nil {
return fmt.Errorf("dbfs.Remove %q: %w", name, err)
}
return nil
}
err := storage.Actions.Delete(filename)
if err != nil {
return fmt.Errorf("storage delete %q: %w", filename, err)
}
return nil
}
func OpenLogs(ctx context.Context, inStorage bool, filename string) (io.ReadSeekCloser, error) {
if !inStorage {
name := DBFSPrefix + filename
f, err := dbfs.Open(ctx, name)
if err != nil {
return nil, fmt.Errorf("dbfs.Open %q: %w", name, err)
}
return f, nil
}
f, err := storage.Actions.Open(filename)
if err != nil {
return nil, fmt.Errorf("storage open %q: %w", filename, err)
}
var reader io.ReadSeekCloser = f
if strings.HasSuffix(filename, ".zst") {
r, err := zstd.NewSeekableReader(f)
if err != nil {
return nil, fmt.Errorf("zstd NewSeekableReader: %w", err)
}
reader = r
}
return reader, nil
}
func FormatLog(timestamp time.Time, content string) string {
// Content shouldn't contain new line, it will break log indexes, other control chars are safe.
content = strings.ReplaceAll(content, "\n", `\n`)
if len(content) > MaxLineSize {
content = content[:MaxLineSize]
}
return fmt.Sprintf("%s %s", timestamp.UTC().Format(timeFormat), content)
}
func ParseLog(in string) (time.Time, string, error) {
index := strings.IndexRune(in, ' ')
if index < 0 {
return time.Time{}, "", fmt.Errorf("invalid log: %q", in)
}
timestamp, err := time.Parse(timeFormat, in[:index])
if err != nil {
return time.Time{}, "", err
}
return timestamp, in[index+1:], nil
}
+123
View File
@@ -0,0 +1,123 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
actions_model "gitea.dev/models/actions"
)
const (
preStepName = "Set up job"
postStepName = "Complete job"
)
// FullSteps returns steps with "Set up job" and "Complete job"
func FullSteps(task *actions_model.ActionTask) []*actions_model.ActionTaskStep {
if len(task.Steps) == 0 {
return fullStepsOfEmptySteps(task)
}
// firstStep is the first step that has run or running, not include preStep.
// For example,
// 1. preStep(Success) -> step1(Success) -> step2(Running) -> step3(Waiting) -> postStep(Waiting): firstStep is step1.
// 2. preStep(Success) -> step1(Skipped) -> step2(Success) -> postStep(Success): firstStep is step2.
// 3. preStep(Success) -> step1(Running) -> step2(Waiting) -> postStep(Waiting): firstStep is step1.
// 4. preStep(Success) -> step1(Skipped) -> step2(Skipped) -> postStep(Skipped): firstStep is nil.
// 5. preStep(Success) -> step1(Cancelled) -> step2(Cancelled) -> postStep(Cancelled): firstStep is nil.
var firstStep *actions_model.ActionTaskStep
// lastHasRunStep is the last step that has run.
// For example,
// 1. preStep(Success) -> step1(Success) -> step2(Running) -> step3(Waiting) -> postStep(Waiting): lastHasRunStep is step1.
// 2. preStep(Success) -> step1(Success) -> step2(Success) -> step3(Success) -> postStep(Success): lastHasRunStep is step3.
// 3. preStep(Success) -> step1(Success) -> step2(Failure) -> step3 -> postStep(Waiting): lastHasRunStep is step2.
// So its Stopped is the Started of postStep when there are no more steps to run.
var lastHasRunStep *actions_model.ActionTaskStep
var logIndex int64
for _, step := range task.Steps {
if firstStep == nil && (step.Status.HasRun() || step.Status.IsRunning()) {
firstStep = step
}
if step.Status.HasRun() {
lastHasRunStep = step
}
logIndex += step.LogLength
}
preStep := &actions_model.ActionTaskStep{
Name: preStepName,
LogLength: task.LogLength,
Started: task.Started,
Status: actions_model.StatusRunning,
}
// No step has run or is running, so preStep is equal to the task
if firstStep == nil {
preStep.Stopped = task.Stopped
preStep.Status = task.Status
} else {
preStep.LogLength = firstStep.LogIndex
preStep.Stopped = firstStep.Started
preStep.Status = actions_model.StatusSuccess
}
logIndex += preStep.LogLength
if lastHasRunStep == nil {
lastHasRunStep = preStep
}
postStep := &actions_model.ActionTaskStep{
Name: postStepName,
Status: actions_model.StatusWaiting,
}
// If the lastHasRunStep is the last step, or it has failed, postStep has started.
if lastHasRunStep.Status.IsFailure() || lastHasRunStep == task.Steps[len(task.Steps)-1] {
postStep.LogIndex = logIndex
postStep.LogLength = task.LogLength - postStep.LogIndex
postStep.Started = lastHasRunStep.Stopped
postStep.Status = actions_model.StatusRunning
}
if task.Status.IsDone() {
postStep.Status = task.Status
postStep.Stopped = task.Stopped
}
ret := make([]*actions_model.ActionTaskStep, 0, len(task.Steps)+2)
ret = append(ret, preStep)
ret = append(ret, task.Steps...)
ret = append(ret, postStep)
return ret
}
func fullStepsOfEmptySteps(task *actions_model.ActionTask) []*actions_model.ActionTaskStep {
preStep := &actions_model.ActionTaskStep{
Name: preStepName,
LogLength: task.LogLength,
Started: task.Started,
Stopped: task.Stopped,
Status: actions_model.StatusRunning,
}
postStep := &actions_model.ActionTaskStep{
Name: postStepName,
LogIndex: task.LogLength,
Started: task.Stopped,
Stopped: task.Stopped,
Status: actions_model.StatusWaiting,
}
if task.Status.IsDone() {
preStep.Status = task.Status
if preStep.Status.IsSuccess() {
postStep.Status = actions_model.StatusSuccess
} else {
postStep.Status = actions_model.StatusCancelled
}
}
return []*actions_model.ActionTaskStep{
preStep,
postStep,
}
}
+165
View File
@@ -0,0 +1,165 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
"testing"
actions_model "gitea.dev/models/actions"
"github.com/stretchr/testify/assert"
)
func TestFullSteps(t *testing.T) {
tests := []struct {
name string
task *actions_model.ActionTask
want []*actions_model.ActionTaskStep
}{
{
name: "regular",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
},
Status: actions_model.StatusSuccess,
Started: 10000,
Stopped: 10100,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
{Name: postStepName, Status: actions_model.StatusSuccess, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 10100},
},
},
{
name: "failed step",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 20, Started: 10010, Stopped: 10020},
{Status: actions_model.StatusFailure, LogIndex: 30, LogLength: 60, Started: 10020, Stopped: 10090},
{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
},
Status: actions_model.StatusFailure,
Started: 10000,
Stopped: 10100,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 20, Started: 10010, Stopped: 10020},
{Status: actions_model.StatusFailure, LogIndex: 30, LogLength: 60, Started: 10020, Stopped: 10090},
{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
{Name: postStepName, Status: actions_model.StatusFailure, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 10100},
},
},
{
name: "first step is running",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{
{Status: actions_model.StatusRunning, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 0},
},
Status: actions_model.StatusRunning,
Started: 10000,
Stopped: 10100,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
{Status: actions_model.StatusRunning, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 0},
{Name: postStepName, Status: actions_model.StatusWaiting, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
},
},
{
name: "first step has canceled",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{
{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
},
Status: actions_model.StatusFailure,
Started: 10000,
Stopped: 10100,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusFailure, LogIndex: 0, LogLength: 100, Started: 10000, Stopped: 10100},
{Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
{Name: postStepName, Status: actions_model.StatusFailure, LogIndex: 100, LogLength: 0, Started: 10100, Stopped: 10100},
},
},
{
name: "empty steps",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{},
Status: actions_model.StatusSuccess,
Started: 10000,
Stopped: 10100,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 100, Started: 10000, Stopped: 10100},
{Name: postStepName, Status: actions_model.StatusSuccess, LogIndex: 100, LogLength: 0, Started: 10100, Stopped: 10100},
},
},
{
name: "all steps finished but task is running",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
},
Status: actions_model.StatusRunning,
Started: 10000,
Stopped: 0,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
{Name: postStepName, Status: actions_model.StatusRunning, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 0},
},
},
{
name: "skipped task",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{
{Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
},
Status: actions_model.StatusSkipped,
Started: 0,
Stopped: 0,
LogLength: 0,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
{Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
{Name: postStepName, Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
},
},
{
name: "first step is skipped",
task: &actions_model.ActionTask{
Steps: []*actions_model.ActionTaskStep{
{Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
},
Status: actions_model.StatusSuccess,
Started: 10000,
Stopped: 10100,
LogLength: 100,
},
want: []*actions_model.ActionTaskStep{
{Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010},
{Status: actions_model.StatusSkipped, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0},
{Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090},
{Name: postStepName, Status: actions_model.StatusSuccess, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 10100},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, FullSteps(tt.task), "FullSteps(%v)", tt.task)
})
}
}
@@ -0,0 +1,65 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package workflowpattern
import (
"strings"
"gitea.dev/modules/glob"
)
type WorkflowPattern struct {
negative bool
glob glob.Glob
}
func CompilePatterns(patterns ...string) ([]*WorkflowPattern, error) {
ret := make([]*WorkflowPattern, 0, len(patterns))
for _, pattern := range patterns {
cp, err := glob.CompileWorkflow(pattern)
if err != nil {
return nil, err
}
ret = append(ret, &WorkflowPattern{glob: cp, negative: strings.HasPrefix(pattern, "!")})
}
return ret, nil
}
// Skip returns true if the workflow should be skipped per paths/branches semantics.
func Skip(sequence []*WorkflowPattern, input []string) bool {
allSkipped := true
for _, file := range input {
shouldSkip := true
for _, item := range sequence {
if item.negative {
// "!README.md" doesn't match "README.md", so "README.md" should be skipped
// "!README.md" matches "help.md" but it shouldn't affect "skip or not", because "help.md" might have been skipped by other rules like "docs/*.md"
if !item.glob.Match(file) {
shouldSkip = true
}
} else if item.glob.Match(file) {
// if "*.md" matches "help.md" so it shouldn't be skipped
shouldSkip = false
}
}
allSkipped = allSkipped && shouldSkip
}
return len(sequence) > 0 && allSkipped
}
// Filter returns true if the workflow should be skipped per paths-ignore/branches-ignore semantics.
func Filter(sequence []*WorkflowPattern, input []string) bool {
for _, file := range input {
anyMatched := false
for _, item := range sequence {
if anyMatched = item.glob.Match(file); anyMatched {
break
}
}
if !anyMatched {
return false
}
}
return len(sequence) != 0
}
@@ -0,0 +1,416 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package workflowpattern
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestMatchPattern(t *testing.T) {
kases := []struct {
inputs []string
patterns []string
skipResult bool
filterResult bool
}{
{
patterns: []string{"*"},
inputs: []string{"path/with/slash"},
skipResult: true,
filterResult: false,
},
{
patterns: []string{"path/a", "path/b", "path/c"},
inputs: []string{"meta", "path/b", "otherfile"},
skipResult: false,
filterResult: false,
},
{
patterns: []string{"path/a", "path/b", "path/c"},
inputs: []string{"path/b"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"path/a", "path/b", "path/c"},
inputs: []string{"path/c", "path/b"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"path/a", "path/b", "path/c"},
inputs: []string{"path/c", "path/b", "path/a"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"path/a", "path/b", "path/c"},
inputs: []string{"path/c", "path/b", "path/d", "path/a"},
skipResult: false,
filterResult: false,
},
{
patterns: []string{},
inputs: []string{},
skipResult: false,
filterResult: false,
},
{
patterns: []string{"\\!file"},
inputs: []string{"!file"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"escape\\\\backslash"},
inputs: []string{"escape\\backslash"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{".yml"},
inputs: []string{"fyml"},
skipResult: true,
filterResult: false,
},
// https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#patterns-to-match-branches-and-tags
{
patterns: []string{"feature/*"},
inputs: []string{"feature/my-branch"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"feature/*"},
inputs: []string{"feature/your-branch"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"feature/**"},
inputs: []string{"feature/beta-a/my-branch"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"feature/**"},
inputs: []string{"feature/beta-a/my-branch"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"feature/**"},
inputs: []string{"feature/mona/the/octocat"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"main", "releases/mona-the-octocat"},
inputs: []string{"main"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"main", "releases/mona-the-octocat"},
inputs: []string{"releases/mona-the-octocat"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*"},
inputs: []string{"main"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*"},
inputs: []string{"releases"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**"},
inputs: []string{"all/the/branches"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**"},
inputs: []string{"every/tag"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*feature"},
inputs: []string{"mona-feature"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*feature"},
inputs: []string{"feature"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*feature"},
inputs: []string{"ver-10-feature"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"v2*"},
inputs: []string{"v2"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"v2*"},
inputs: []string{"v2.0"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"v2*"},
inputs: []string{"v2.9"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"v[12].[0-9]+.[0-9]+"},
inputs: []string{"v1.10.1"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"v[12].[0-9]+.[0-9]+"},
inputs: []string{"v2.0.0"},
skipResult: false,
filterResult: true,
},
// https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#patterns-to-match-file-paths
{
patterns: []string{"*"},
inputs: []string{"README.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*"},
inputs: []string{"server.rb"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*.jsx?"},
inputs: []string{"page.js"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*.jsx?"},
inputs: []string{"page.jsx"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**"},
inputs: []string{"all/the/files.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*.js"},
inputs: []string{"app.js"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*.js"},
inputs: []string{"index.js"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**.js"},
inputs: []string{"index.js"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**.js"},
inputs: []string{"js/index.js"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**.js"},
inputs: []string{"src/js/app.js"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"docs/*"},
inputs: []string{"docs/README.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"docs/*"},
inputs: []string{"docs/file.txt"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"docs/**"},
inputs: []string{"docs/README.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"docs/**"},
inputs: []string{"docs/mona/octocat.txt"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"docs/**/*.md"},
inputs: []string{"docs/README.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"docs/**/*.md"},
inputs: []string{"docs/mona/hello-world.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"docs/**/*.md"},
inputs: []string{"docs/a/markdown/file.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/docs/**"},
inputs: []string{"docs/hello.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/docs/**"},
inputs: []string{"dir/docs/my-file.txt"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/docs/**"},
inputs: []string{"space/docs/plan/space.doc"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/README.md"},
inputs: []string{"README.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/README.md"},
inputs: []string{"js/README.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/*src/**"},
inputs: []string{"a/src/app.js"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/*src/**"},
inputs: []string{"my-src/code/js/app.js"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/*-post.md"},
inputs: []string{"my-post.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/*-post.md"},
inputs: []string{"path/their-post.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/migrate-*.sql"},
inputs: []string{"migrate-10909.sql"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/migrate-*.sql"},
inputs: []string{"db/migrate-v1.0.sql"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"**/migrate-*.sql"},
inputs: []string{"db/sept/migrate-v1.sql"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*.md", "!README.md"},
inputs: []string{"hello.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*.md", "!README.md"},
inputs: []string{"README.md"},
skipResult: true,
filterResult: true,
},
{
patterns: []string{"*.md", "!README.md"},
inputs: []string{"docs/hello.md"},
skipResult: true,
filterResult: true,
},
{
patterns: []string{"*.md", "!README.md", "README*"},
inputs: []string{"hello.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*.md", "!README.md", "README*"},
inputs: []string{"README.md"},
skipResult: false,
filterResult: true,
},
{
patterns: []string{"*.md", "!README.md", "README*"},
inputs: []string{"README.doc"},
skipResult: false,
filterResult: true,
},
}
for _, kase := range kases {
msg := fmt.Sprintf("patterns=%s, input=%s", strings.Join(kase.patterns, ","), strings.Join(kase.inputs, ","))
patterns, err := CompilePatterns(kase.patterns...)
assert.NoError(t, err, "compile error: %s", msg)
assert.Equal(t, kase.skipResult, Skip(patterns, kase.inputs), "unexpected skip result: %s", msg)
assert.Equal(t, kase.filterResult, Filter(patterns, kase.inputs), "unexpected filter result: %s", msg)
}
}
+774
View File
@@ -0,0 +1,774 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
"bytes"
"slices"
"strings"
"gitea.dev/modules/actions/jobparser"
"gitea.dev/modules/actions/workflowpattern"
"gitea.dev/modules/git"
"gitea.dev/modules/glob"
"gitea.dev/modules/log"
"gitea.dev/modules/setting"
api "gitea.dev/modules/structs"
"gitea.dev/modules/util"
webhook_module "gitea.dev/modules/webhook"
"gitea.com/gitea/runner/act/model"
"go.yaml.in/yaml/v4"
)
type DetectedWorkflow struct {
EntryName string
TriggerEvent *jobparser.Event
Content []byte
}
func init() {
model.OnDecodeNodeError = func(node yaml.Node, out any, err error) {
// Log the error instead of panic or fatal.
// It will be a big job to refactor act/pkg/model to return decode error,
// so we just log the error and return empty value, and improve it later.
log.Error("Failed to decode node %v into %T: %v", node, out, err)
}
}
func IsWorkflow(path string) bool {
if (!strings.HasSuffix(path, ".yaml")) && (!strings.HasSuffix(path, ".yml")) {
return false
}
for _, workflowDir := range setting.Actions.WorkflowDirs {
if strings.HasPrefix(path, workflowDir+"/") {
return true
}
}
return false
}
func ListWorkflows(commit *git.Commit) (string, git.Entries, error) {
var tree *git.Tree
var err error
var workflowDir string
for _, workflowDir = range setting.Actions.WorkflowDirs {
tree, err = commit.SubTree(workflowDir)
if err == nil {
break
}
if !git.IsErrNotExist(err) {
return "", nil, err
}
}
if tree == nil {
return "", nil, nil
}
entries, err := tree.ListEntriesRecursiveFast()
if err != nil {
return "", nil, err
}
ret := make(git.Entries, 0, len(entries))
for _, entry := range entries {
if strings.HasSuffix(entry.Name(), ".yml") || strings.HasSuffix(entry.Name(), ".yaml") {
ret = append(ret, entry)
}
}
return workflowDir, ret, nil
}
func GetContentFromEntry(entry *git.TreeEntry) ([]byte, error) {
f, err := entry.Blob().DataAsync()
if err != nil {
return nil, err
}
content, err := util.ReadWithLimit(f, 1024*1024)
_ = f.Close()
if err != nil {
return nil, err
}
return content, nil
}
func GetEventsFromContent(content []byte) ([]*jobparser.Event, error) {
workflow, err := model.ReadWorkflow(bytes.NewReader(content))
if err != nil {
return nil, err
}
events, err := jobparser.ParseRawOn(&workflow.RawOn)
if err != nil {
return nil, err
}
if err := ValidateWorkflowContent(content); err != nil {
return nil, err
}
return events, nil
}
// ValidateWorkflowContent catches structural errors (e.g. blank lines in run: | blocks)
// that model.ReadWorkflow alone does not detect.
func ValidateWorkflowContent(content []byte) error {
_, err := jobparser.Parse(content)
return err
}
func DetectWorkflows(
gitRepo *git.Repository,
commit *git.Commit,
triggedEvent webhook_module.HookEventType,
payload api.Payloader,
detectSchedule bool,
) ([]*DetectedWorkflow, []*DetectedWorkflow, error) {
_, entries, err := ListWorkflows(commit)
if err != nil {
return nil, nil, err
}
workflows := make([]*DetectedWorkflow, 0, len(entries))
schedules := make([]*DetectedWorkflow, 0, len(entries))
for _, entry := range entries {
content, err := GetContentFromEntry(entry)
if err != nil {
return nil, nil, err
}
// one workflow may have multiple events
events, err := GetEventsFromContent(content)
if err != nil {
log.Warn("ignore invalid workflow %q: %v", entry.Name(), err)
continue
}
for _, evt := range events {
log.Trace("detect workflow %q for event %#v matching %q", entry.Name(), evt, triggedEvent)
if evt.IsSchedule() {
if detectSchedule {
dwf := &DetectedWorkflow{
EntryName: entry.Name(),
TriggerEvent: evt,
Content: content,
}
schedules = append(schedules, dwf)
}
} else if detectMatched(gitRepo, commit, triggedEvent, payload, evt) {
dwf := &DetectedWorkflow{
EntryName: entry.Name(),
TriggerEvent: evt,
Content: content,
}
workflows = append(workflows, dwf)
}
}
}
return workflows, schedules, nil
}
func DetectScheduledWorkflows(gitRepo *git.Repository, commit *git.Commit) ([]*DetectedWorkflow, error) {
_, entries, err := ListWorkflows(commit)
if err != nil {
return nil, err
}
wfs := make([]*DetectedWorkflow, 0, len(entries))
for _, entry := range entries {
content, err := GetContentFromEntry(entry)
if err != nil {
return nil, err
}
// one workflow may have multiple events
events, err := GetEventsFromContent(content)
if err != nil {
log.Warn("ignore invalid workflow %q: %v", entry.Name(), err)
continue
}
for _, evt := range events {
if evt.IsSchedule() {
log.Trace("detect scheduled workflow: %q", entry.Name())
dwf := &DetectedWorkflow{
EntryName: entry.Name(),
TriggerEvent: evt,
Content: content,
}
wfs = append(wfs, dwf)
}
}
}
return wfs, nil
}
func detectMatched(gitRepo *git.Repository, commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader, evt *jobparser.Event) bool {
if !canGithubEventMatch(evt.Name, triggedEvent) {
return false
}
switch triggedEvent {
case // events with no activity types
webhook_module.HookEventCreate,
webhook_module.HookEventDelete,
webhook_module.HookEventFork,
webhook_module.HookEventWiki,
webhook_module.HookEventSchedule:
if len(evt.Acts()) != 0 {
log.Warn("Ignore unsupported %s event arguments %v", triggedEvent, evt.Acts())
}
// no special filter parameters for these events, just return true if name matched
return true
case // push
webhook_module.HookEventPush:
return matchPushEvent(commit, payload.(*api.PushPayload), evt)
case // issues
webhook_module.HookEventIssues,
webhook_module.HookEventIssueAssign,
webhook_module.HookEventIssueLabel,
webhook_module.HookEventIssueMilestone:
return matchIssuesEvent(payload.(*api.IssuePayload), evt)
case // issue_comment
webhook_module.HookEventIssueComment,
// `pull_request_comment` is same as `issue_comment`
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_comment-use-issue_comment
webhook_module.HookEventPullRequestComment:
return matchIssueCommentEvent(payload.(*api.IssueCommentPayload), evt)
case // pull_request
webhook_module.HookEventPullRequest,
webhook_module.HookEventPullRequestSync,
webhook_module.HookEventPullRequestAssign,
webhook_module.HookEventPullRequestLabel,
webhook_module.HookEventPullRequestReviewRequest,
webhook_module.HookEventPullRequestMilestone:
return matchPullRequestEvent(gitRepo, commit, payload.(*api.PullRequestPayload), evt)
case // pull_request_review
webhook_module.HookEventPullRequestReviewApproved,
webhook_module.HookEventPullRequestReviewRejected:
return matchPullRequestReviewEvent(payload.(*api.PullRequestPayload), evt)
case // pull_request_review_comment
webhook_module.HookEventPullRequestReviewComment:
return matchPullRequestReviewCommentEvent(payload.(*api.PullRequestPayload), evt)
case // release
webhook_module.HookEventRelease:
return matchReleaseEvent(payload.(*api.ReleasePayload), evt)
case // registry_package
webhook_module.HookEventPackage:
return matchPackageEvent(payload.(*api.PackagePayload), evt)
case // workflow_run
webhook_module.HookEventWorkflowRun:
return matchWorkflowRunEvent(payload.(*api.WorkflowRunPayload), evt)
default:
log.Warn("unsupported event %q", triggedEvent)
return false
}
}
func matchPushEvent(commit *git.Commit, pushPayload *api.PushPayload, evt *jobparser.Event) bool {
// with no special filter parameters
if len(evt.Acts()) == 0 {
return true
}
matchTimes := 0
hasBranchFilter := false
hasTagFilter := false
refName := git.RefName(pushPayload.Ref)
// all acts conditions should be satisfied
for cond, vals := range evt.Acts() {
switch cond {
case "branches":
hasBranchFilter = true
if !refName.IsBranch() {
break
}
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Skip(patterns, []string{refName.BranchName()}) {
matchTimes++
}
case "branches-ignore":
hasBranchFilter = true
if !refName.IsBranch() {
break
}
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Filter(patterns, []string{refName.BranchName()}) {
matchTimes++
}
case "tags":
hasTagFilter = true
if !refName.IsTag() {
break
}
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Skip(patterns, []string{refName.TagName()}) {
matchTimes++
}
case "tags-ignore":
hasTagFilter = true
if !refName.IsTag() {
break
}
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Filter(patterns, []string{refName.TagName()}) {
matchTimes++
}
case "paths":
if refName.IsTag() {
matchTimes++
break
}
filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before)
if err != nil {
log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err)
} else {
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Skip(patterns, filesChanged) {
matchTimes++
}
}
case "paths-ignore":
if refName.IsTag() {
matchTimes++
break
}
filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before)
if err != nil {
log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err)
} else {
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Filter(patterns, filesChanged) {
matchTimes++
}
}
default:
log.Warn("push event unsupported condition %q", cond)
}
}
// if both branch and tag filter are defined in the workflow only one needs to match
if hasBranchFilter && hasTagFilter {
matchTimes++
}
return matchTimes == len(evt.Acts())
}
func matchIssuesEvent(issuePayload *api.IssuePayload, evt *jobparser.Event) bool {
// with no special filter parameters
if len(evt.Acts()) == 0 {
return true
}
matchTimes := 0
// all acts conditions should be satisfied
for cond, vals := range evt.Acts() {
switch cond {
case "types":
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#issues
// Actions with the same name:
// opened, edited, closed, reopened, assigned, unassigned, milestoned, demilestoned
// Actions need to be converted:
// label_updated -> labeled (when adding) or unlabeled (when removing)
// label_cleared -> unlabeled
// Unsupported activity types:
// deleted, transferred, pinned, unpinned, locked, unlocked
actions := []string{}
switch issuePayload.Action {
case api.HookIssueLabelUpdated:
if len(issuePayload.Changes.AddedLabels) > 0 {
actions = append(actions, "labeled")
}
if len(issuePayload.Changes.RemovedLabels) > 0 {
actions = append(actions, "unlabeled")
}
case api.HookIssueLabelCleared:
actions = append(actions, "unlabeled")
default:
actions = append(actions, string(issuePayload.Action))
}
for _, val := range vals {
if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) {
matchTimes++
break
}
}
default:
log.Warn("issue event unsupported condition %q", cond)
}
}
return matchTimes == len(evt.Acts())
}
func matchPullRequestEvent(gitRepo *git.Repository, commit *git.Commit, prPayload *api.PullRequestPayload, evt *jobparser.Event) bool {
acts := evt.Acts()
activityTypeMatched := false
matchTimes := 0
if vals, ok := acts["types"]; !ok {
// defaultly, only pull request `opened`, `reopened` and `synchronized` will trigger workflow
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
activityTypeMatched = prPayload.Action == api.HookIssueSynchronized || prPayload.Action == api.HookIssueOpened || prPayload.Action == api.HookIssueReOpened
} else {
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
// Actions with the same name:
// opened, edited, closed, reopened, assigned, unassigned, review_requested, review_request_removed, milestoned, demilestoned
// Actions need to be converted:
// synchronized -> synchronize
// label_updated -> labeled
// label_cleared -> unlabeled
// Unsupported activity types:
// converted_to_draft, ready_for_review, locked, unlocked, auto_merge_enabled, auto_merge_disabled, enqueued, dequeued
action := prPayload.Action
switch action {
case api.HookIssueSynchronized:
action = "synchronize"
case api.HookIssueLabelUpdated:
action = "labeled"
case api.HookIssueLabelCleared:
action = "unlabeled"
}
log.Trace("matching pull_request %s with %v", action, vals)
for _, val := range vals {
if glob.MustCompile(val, '/').Match(string(action)) {
activityTypeMatched = true
matchTimes++
break
}
}
}
var (
headCommit = commit
err error
)
if evt.Name == GithubEventPullRequestTarget && (len(acts["paths"]) > 0 || len(acts["paths-ignore"]) > 0) {
headCommit, err = gitRepo.GetCommit(prPayload.PullRequest.Head.Sha)
if err != nil {
log.Error("GetCommit [ref: %s]: %v", prPayload.PullRequest.Head.Sha, err)
return false
}
}
// all acts conditions should be satisfied
for cond, vals := range acts {
switch cond {
case "types":
// types have been checked
continue
case "branches":
refName := git.RefName(prPayload.PullRequest.Base.Ref)
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Skip(patterns, []string{refName.ShortName()}) {
matchTimes++
}
case "branches-ignore":
refName := git.RefName(prPayload.PullRequest.Base.Ref)
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Filter(patterns, []string{refName.ShortName()}) {
matchTimes++
}
case "paths":
filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.MergeBase)
if err != nil {
log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", headCommit.ID.String(), err)
} else {
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Skip(patterns, filesChanged) {
matchTimes++
}
}
case "paths-ignore":
filesChanged, err := headCommit.GetFilesChangedSinceCommit(prPayload.PullRequest.MergeBase)
if err != nil {
log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", headCommit.ID.String(), err)
} else {
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Filter(patterns, filesChanged) {
matchTimes++
}
}
default:
log.Warn("pull request event unsupported condition %q", cond)
}
}
return activityTypeMatched && matchTimes == len(evt.Acts())
}
func matchIssueCommentEvent(issueCommentPayload *api.IssueCommentPayload, evt *jobparser.Event) bool {
// with no special filter parameters
if len(evt.Acts()) == 0 {
return true
}
matchTimes := 0
// all acts conditions should be satisfied
for cond, vals := range evt.Acts() {
switch cond {
case "types":
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#issue_comment
// Actions with the same name:
// created, edited, deleted
// Actions need to be converted:
// NONE
// Unsupported activity types:
// NONE
for _, val := range vals {
if glob.MustCompile(val, '/').Match(string(issueCommentPayload.Action)) {
matchTimes++
break
}
}
default:
log.Warn("issue comment event unsupported condition %q", cond)
}
}
return matchTimes == len(evt.Acts())
}
func matchPullRequestReviewEvent(prPayload *api.PullRequestPayload, evt *jobparser.Event) bool {
// with no special filter parameters
if len(evt.Acts()) == 0 {
return true
}
matchTimes := 0
// all acts conditions should be satisfied
for cond, vals := range evt.Acts() {
switch cond {
case "types":
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_review
// Activity types with the same name:
// NONE
// Activity types need to be converted:
// reviewed -> submitted
// reviewed -> edited
// Unsupported activity types:
// dismissed
actions := make([]string, 0)
if prPayload.Action == api.HookIssueReviewed {
// the `reviewed` HookIssueAction can match the two activity types: `submitted` and `edited`
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_review
actions = append(actions, "submitted", "edited")
}
for _, val := range vals {
if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) {
matchTimes++
break
}
}
default:
log.Warn("pull request review event unsupported condition %q", cond)
}
}
return matchTimes == len(evt.Acts())
}
func matchPullRequestReviewCommentEvent(prPayload *api.PullRequestPayload, evt *jobparser.Event) bool {
// with no special filter parameters
if len(evt.Acts()) == 0 {
return true
}
matchTimes := 0
// all acts conditions should be satisfied
for cond, vals := range evt.Acts() {
switch cond {
case "types":
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_review_comment
// Activity types with the same name:
// NONE
// Activity types need to be converted:
// reviewed -> created
// reviewed -> edited
// Unsupported activity types:
// deleted
actions := make([]string, 0)
if prPayload.Action == api.HookIssueReviewed {
// the `reviewed` HookIssueAction can match the two activity types: `created` and `edited`
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_review_comment
actions = append(actions, "created", "edited")
}
for _, val := range vals {
if slices.ContainsFunc(actions, glob.MustCompile(val, '/').Match) {
matchTimes++
break
}
}
default:
log.Warn("pull request review comment event unsupported condition %q", cond)
}
}
return matchTimes == len(evt.Acts())
}
func matchReleaseEvent(payload *api.ReleasePayload, evt *jobparser.Event) bool {
// with no special filter parameters
if len(evt.Acts()) == 0 {
return true
}
matchTimes := 0
// all acts conditions should be satisfied
for cond, vals := range evt.Acts() {
switch cond {
case "types":
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#release
// Activity types with the same name:
// published
// Activity types need to be converted:
// updated -> edited
// Unsupported activity types:
// unpublished, created, deleted, prereleased, released
action := payload.Action
switch action {
case api.HookReleaseUpdated:
action = "edited"
}
for _, val := range vals {
if glob.MustCompile(val, '/').Match(string(action)) {
matchTimes++
break
}
}
default:
log.Warn("release event unsupported condition %q", cond)
}
}
return matchTimes == len(evt.Acts())
}
func matchPackageEvent(payload *api.PackagePayload, evt *jobparser.Event) bool {
// with no special filter parameters
if len(evt.Acts()) == 0 {
return true
}
matchTimes := 0
// all acts conditions should be satisfied
for cond, vals := range evt.Acts() {
switch cond {
case "types":
// See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#registry_package
// Activity types with the same name:
// NONE
// Activity types need to be converted:
// created -> published
// Unsupported activity types:
// updated
action := payload.Action
switch action {
case api.HookPackageCreated:
action = "published"
}
for _, val := range vals {
if glob.MustCompile(val, '/').Match(string(action)) {
matchTimes++
break
}
}
default:
log.Warn("package event unsupported condition %q", cond)
}
}
return matchTimes == len(evt.Acts())
}
func matchWorkflowRunEvent(payload *api.WorkflowRunPayload, evt *jobparser.Event) bool {
// with no special filter parameters
if len(evt.Acts()) == 0 {
return true
}
matchTimes := 0
// all acts conditions should be satisfied
for cond, vals := range evt.Acts() {
switch cond {
case "types":
action := payload.Action
for _, val := range vals {
if glob.MustCompile(val, '/').Match(action) {
matchTimes++
break
}
}
case "workflows":
workflow := payload.Workflow
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Skip(patterns, []string{workflow.Name}) {
matchTimes++
}
case "branches":
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Skip(patterns, []string{payload.WorkflowRun.HeadBranch}) {
matchTimes++
}
case "branches-ignore":
patterns, err := workflowpattern.CompilePatterns(vals...)
if err != nil {
break
}
if !workflowpattern.Filter(patterns, []string{payload.WorkflowRun.HeadBranch}) {
matchTimes++
}
default:
log.Warn("workflow run event unsupported condition %q", cond)
}
}
return matchTimes == len(evt.Acts())
}
+418
View File
@@ -0,0 +1,418 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package actions
import (
"testing"
"gitea.dev/modules/git"
"gitea.dev/modules/setting"
api "gitea.dev/modules/structs"
"gitea.dev/modules/test"
webhook_module "gitea.dev/modules/webhook"
"github.com/stretchr/testify/assert"
)
func fullWorkflowContent(part string) []byte {
return []byte(`
name: test
` + part + `
jobs:
test:
runs-on: ubuntu-latest
steps:
- run: echo hello
`)
}
func TestIsWorkflow(t *testing.T) {
defer test.MockVariableValue(&setting.Actions.WorkflowDirs)()
tests := []struct {
name string
dirs []string
path string
expected bool
}{
{
name: "default with yml extension",
dirs: []string{".gitea/workflows", ".github/workflows"},
path: ".gitea/workflows/test.yml",
expected: true,
},
{
name: "default with yaml extension",
dirs: []string{".gitea/workflows", ".github/workflows"},
path: ".github/workflows/test.yaml",
expected: true,
},
{
name: "only gitea configured, github path rejected",
dirs: []string{".gitea/workflows"},
path: ".github/workflows/test.yml",
expected: false,
},
{
name: "only github configured, gitea path rejected",
dirs: []string{".github/workflows"},
path: ".gitea/workflows/test.yml",
expected: false,
},
{
name: "custom workflow dir",
dirs: []string{".custom/workflows"},
path: ".custom/workflows/deploy.yml",
expected: true,
},
{
name: "non-workflow file",
dirs: []string{".gitea/workflows", ".github/workflows"},
path: ".gitea/workflows/readme.md",
expected: false,
},
{
name: "directory boundary",
dirs: []string{".gitea/workflows"},
path: ".gitea/workflows2/test.yml",
expected: false,
},
{
name: "unrelated path",
dirs: []string{".gitea/workflows", ".github/workflows"},
path: "src/main.go",
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
setting.Actions.WorkflowDirs = tt.dirs
assert.Equal(t, tt.expected, IsWorkflow(tt.path))
})
}
}
func TestDetectMatched(t *testing.T) {
testCases := []struct {
desc string
commit *git.Commit
triggedEvent webhook_module.HookEventType
payload api.Payloader
yamlOn string
expected bool
}{
{
desc: "HookEventCreate(create) matches GithubEventCreate(create)",
triggedEvent: webhook_module.HookEventCreate,
payload: nil,
yamlOn: "on: create",
expected: true,
},
{
desc: "HookEventIssues(issues) `opened` action matches GithubEventIssues(issues)",
triggedEvent: webhook_module.HookEventIssues,
payload: &api.IssuePayload{Action: api.HookIssueOpened},
yamlOn: "on: issues",
expected: true,
},
{
desc: "HookEventIssues(issues) `milestoned` action matches GithubEventIssues(issues)",
triggedEvent: webhook_module.HookEventIssues,
payload: &api.IssuePayload{Action: api.HookIssueMilestoned},
yamlOn: "on: issues",
expected: true,
},
{
desc: "HookEventPullRequestSync(pull_request_sync) matches GithubEventPullRequest(pull_request)",
triggedEvent: webhook_module.HookEventPullRequestSync,
payload: &api.PullRequestPayload{Action: api.HookIssueSynchronized},
yamlOn: "on: pull_request",
expected: true,
},
{
desc: "HookEventPullRequest(pull_request) `label_updated` action doesn't match GithubEventPullRequest(pull_request) with no activity type",
triggedEvent: webhook_module.HookEventPullRequest,
payload: &api.PullRequestPayload{Action: api.HookIssueLabelUpdated},
yamlOn: "on: pull_request",
expected: false,
},
{
desc: "HookEventPullRequest(pull_request) `closed` action doesn't match GithubEventPullRequest(pull_request) with no activity type",
triggedEvent: webhook_module.HookEventPullRequest,
payload: &api.PullRequestPayload{Action: api.HookIssueClosed},
yamlOn: "on: pull_request",
expected: false,
},
{
desc: "HookEventPullRequest(pull_request) `closed` action doesn't match GithubEventPullRequest(pull_request) with branches",
triggedEvent: webhook_module.HookEventPullRequest,
payload: &api.PullRequestPayload{
Action: api.HookIssueClosed,
PullRequest: &api.PullRequest{
Base: &api.PRBranchInfo{},
},
},
yamlOn: "on:\n pull_request:\n branches: [main]",
expected: false,
},
{
desc: "HookEventPullRequest(pull_request) `label_updated` action matches GithubEventPullRequest(pull_request) with `label` activity type",
triggedEvent: webhook_module.HookEventPullRequest,
payload: &api.PullRequestPayload{Action: api.HookIssueLabelUpdated},
yamlOn: "on:\n pull_request:\n types: [labeled]",
expected: true,
},
{
desc: "HookEventPullRequestReviewComment(pull_request_review_comment) matches GithubEventPullRequestReviewComment(pull_request_review_comment)",
triggedEvent: webhook_module.HookEventPullRequestReviewComment,
payload: &api.PullRequestPayload{Action: api.HookIssueReviewed},
yamlOn: "on:\n pull_request_review_comment:\n types: [created]",
expected: true,
},
{
desc: "HookEventPullRequestReviewRejected(pull_request_review_rejected) doesn't match GithubEventPullRequestReview(pull_request_review) with `dismissed` activity type (we don't support `dismissed` at present)",
triggedEvent: webhook_module.HookEventPullRequestReviewRejected,
payload: &api.PullRequestPayload{Action: api.HookIssueReviewed},
yamlOn: "on:\n pull_request_review:\n types: [dismissed]",
expected: false,
},
{
desc: "HookEventRelease(release) `published` action matches GithubEventRelease(release) with `published` activity type",
triggedEvent: webhook_module.HookEventRelease,
payload: &api.ReleasePayload{Action: api.HookReleasePublished},
yamlOn: "on:\n release:\n types: [published]",
expected: true,
},
{
desc: "HookEventPackage(package) `created` action doesn't match GithubEventRegistryPackage(registry_package) with `updated` activity type",
triggedEvent: webhook_module.HookEventPackage,
payload: &api.PackagePayload{Action: api.HookPackageCreated},
yamlOn: "on:\n registry_package:\n types: [updated]",
expected: false,
},
{
desc: "HookEventWiki(wiki) matches GithubEventGollum(gollum)",
triggedEvent: webhook_module.HookEventWiki,
payload: nil,
yamlOn: "on: gollum",
expected: true,
},
{
desc: "HookEventSchedule(schedule) matches GithubEventSchedule(schedule)",
triggedEvent: webhook_module.HookEventSchedule,
payload: nil,
yamlOn: "on: schedule",
expected: true,
},
{
desc: "push to tag matches workflow with paths condition (should skip paths check)",
triggedEvent: webhook_module.HookEventPush,
payload: &api.PushPayload{
Ref: "refs/tags/v1.0.0",
Before: "0000000",
Commits: []*api.PayloadCommit{
{
ID: "abcdef123456",
Added: []string{"src/main.go"},
Message: "Release v1.0.0",
},
},
},
commit: nil,
yamlOn: "on:\n push:\n paths:\n - src/**",
expected: true,
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
evts, err := GetEventsFromContent(fullWorkflowContent(tc.yamlOn))
assert.NoError(t, err)
assert.Len(t, evts, 1)
assert.Equal(t, tc.expected, detectMatched(nil, tc.commit, tc.triggedEvent, tc.payload, evts[0]))
})
}
}
func TestMatchIssuesEvent(t *testing.T) {
testCases := []struct {
desc string
payload *api.IssuePayload
yamlOn string
expected bool
eventType string
}{
{
desc: "Label deletion should trigger unlabeled event",
payload: &api.IssuePayload{
Action: api.HookIssueLabelUpdated,
Issue: &api.Issue{
Labels: []*api.Label{},
},
Changes: &api.ChangesPayload{
RemovedLabels: []*api.Label{
{ID: 123, Name: "deleted-label"},
},
},
},
yamlOn: "on:\n issues:\n types: [unlabeled]",
expected: true,
eventType: "unlabeled",
},
{
desc: "Label deletion with existing labels should trigger unlabeled event",
payload: &api.IssuePayload{
Action: api.HookIssueLabelUpdated,
Issue: &api.Issue{
Labels: []*api.Label{
{ID: 456, Name: "existing-label"},
},
},
Changes: &api.ChangesPayload{
AddedLabels: nil,
RemovedLabels: []*api.Label{
{ID: 123, Name: "deleted-label"},
},
},
},
yamlOn: "on:\n issues:\n types: [unlabeled]",
expected: true,
eventType: "unlabeled",
},
{
desc: "Label addition should trigger labeled event",
payload: &api.IssuePayload{
Action: api.HookIssueLabelUpdated,
Issue: &api.Issue{
Labels: []*api.Label{
{ID: 123, Name: "new-label"},
},
},
Changes: &api.ChangesPayload{
AddedLabels: []*api.Label{
{ID: 123, Name: "new-label"},
},
RemovedLabels: []*api.Label{}, // Empty array, no labels removed
},
},
yamlOn: "on:\n issues:\n types: [labeled]",
expected: true,
eventType: "labeled",
},
{
desc: "Label clear should trigger unlabeled event",
payload: &api.IssuePayload{
Action: api.HookIssueLabelCleared,
Issue: &api.Issue{
Labels: []*api.Label{},
},
},
yamlOn: "on:\n issues:\n types: [unlabeled]",
expected: true,
eventType: "unlabeled",
},
{
desc: "Both adding and removing labels should trigger labeled event",
payload: &api.IssuePayload{
Action: api.HookIssueLabelUpdated,
Issue: &api.Issue{
Labels: []*api.Label{
{ID: 789, Name: "new-label"},
},
},
Changes: &api.ChangesPayload{
AddedLabels: []*api.Label{
{ID: 789, Name: "new-label"},
},
RemovedLabels: []*api.Label{
{ID: 123, Name: "deleted-label"},
},
},
},
yamlOn: "on:\n issues:\n types: [labeled]",
expected: true,
eventType: "labeled",
},
{
desc: "Both adding and removing labels should trigger unlabeled event",
payload: &api.IssuePayload{
Action: api.HookIssueLabelUpdated,
Issue: &api.Issue{
Labels: []*api.Label{
{ID: 789, Name: "new-label"},
},
},
Changes: &api.ChangesPayload{
AddedLabels: []*api.Label{
{ID: 789, Name: "new-label"},
},
RemovedLabels: []*api.Label{
{ID: 123, Name: "deleted-label"},
},
},
},
yamlOn: "on:\n issues:\n types: [unlabeled]",
expected: true,
eventType: "unlabeled",
},
{
desc: "Both adding and removing labels should trigger both events",
payload: &api.IssuePayload{
Action: api.HookIssueLabelUpdated,
Issue: &api.Issue{
Labels: []*api.Label{
{ID: 789, Name: "new-label"},
},
},
Changes: &api.ChangesPayload{
AddedLabels: []*api.Label{
{ID: 789, Name: "new-label"},
},
RemovedLabels: []*api.Label{
{ID: 123, Name: "deleted-label"},
},
},
},
yamlOn: "on:\n issues:\n types: [labeled, unlabeled]",
expected: true,
eventType: "multiple",
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
evts, err := GetEventsFromContent(fullWorkflowContent(tc.yamlOn))
assert.NoError(t, err)
assert.Len(t, evts, 1)
// Test if the event matches as expected
assert.Equal(t, tc.expected, matchIssuesEvent(tc.payload, evts[0]))
// For extra validation, check that action mapping works correctly
if tc.eventType == "multiple" {
// Skip direct action mapping validation for multiple events case
// as one action can map to multiple event types
return
}
// Determine expected action for single event case
var expectedAction string
switch tc.payload.Action {
case api.HookIssueLabelUpdated:
if tc.eventType == "labeled" {
expectedAction = "labeled"
} else if tc.eventType == "unlabeled" {
expectedAction = "unlabeled"
}
case api.HookIssueLabelCleared:
expectedAction = "unlabeled"
default:
expectedAction = string(tc.payload.Action)
}
assert.Equal(t, expectedAction, tc.eventType, "Event type should match expected")
})
}
}
+28
View File
@@ -0,0 +1,28 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package analyze
import (
"path"
"github.com/go-enry/go-enry/v2"
)
// GetCodeLanguage detects code language based on file name and content
// It can be slow when the content is used for detection
func GetCodeLanguage(filename string, content []byte) string {
if language, ok := enry.GetLanguageByExtension(filename); ok {
return language
}
if language, ok := enry.GetLanguageByFilename(filename); ok {
return language
}
if len(content) == 0 {
return enry.OtherLanguage
}
return enry.GetLanguage(path.Base(filename), content)
}
+27
View File
@@ -0,0 +1,27 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package analyze
import (
"path/filepath"
"strings"
"github.com/go-enry/go-enry/v2/data"
)
// IsGenerated returns whether or not path is a generated path.
func IsGenerated(path string) bool {
ext := strings.ToLower(filepath.Ext(path))
if _, ok := data.GeneratedCodeExtensions[ext]; ok {
return true
}
for _, m := range data.GeneratedCodeNameMatchers {
if m(path) {
return true
}
}
return false
}
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package analyze
import (
"path"
"strings"
"github.com/go-enry/go-enry/v2"
)
// IsVendor returns whether the path is a vendor path.
// It uses go-enry's IsVendor function but overrides its detection for certain
// special cases that shouldn't be marked as vendored in the diff view.
func IsVendor(treePath string) bool {
if !enry.IsVendor(treePath) {
return false
}
// Override detection for single files
basename := path.Base(treePath)
switch basename {
case ".gitignore", ".gitattributes", ".gitmodules":
return false
}
if strings.HasPrefix(treePath, ".github/") || strings.HasPrefix(treePath, ".gitea/") {
return false
}
return true
}
+53
View File
@@ -0,0 +1,53 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package analyze
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestIsVendor(t *testing.T) {
tests := []struct {
path string
want bool
}{
// Original go-enry test cases
{"cache/", true},
{"random/cache/", true},
{"cache", false},
{"dependencies/", true},
{"Dependencies/", true},
{"dependency/", false},
{"dist/", true},
{"dist", false},
{"random/dist/", true},
{"random/dist", false},
{"deps/", true},
{"configure", true},
{"a/configure", true},
{"config.guess", true},
{"config.guess/", false},
{".vscode/", true},
{"doc/_build/", true},
{"a/docs/_build/", true},
{"a/dasdocs/_build-vsdoc.js", true},
{"a/dasdocs/_build-vsdoc.j", false},
// Override: Git/GitHub/Gitea-related paths should NOT be detected as vendored
{".gitignore", false},
{".gitattributes", false},
{".gitmodules", false},
{"src/.gitignore", false},
{".github/workflows/ci.yml", false},
{".gitea/workflows/ci.yml", false},
}
for _, tt := range tests {
t.Run(tt.path, func(t *testing.T) {
got := IsVendor(tt.path)
assert.Equal(t, tt.want, got)
})
}
}
+375
View File
@@ -0,0 +1,375 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package assetfs
import (
"bytes"
"compress/gzip"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"
"gitea.dev/modules/json"
"gitea.dev/modules/util"
)
type EmbeddedFile interface {
io.ReadSeeker
fs.ReadDirFile
ReadDir(n int) ([]fs.DirEntry, error)
}
type EmbeddedFileInfo interface {
fs.FileInfo
fs.DirEntry
GetGzipContent() ([]byte, bool)
}
type decompressor interface {
io.Reader
Close() error
Reset(io.Reader) error
}
type embeddedFileInfo struct {
fs *embeddedFS
fullName string
data []byte
BaseName string `json:"n"`
OriginSize int64 `json:"s,omitempty"`
DataBegin int64 `json:"b,omitempty"`
DataLen int64 `json:"l,omitempty"`
Children []*embeddedFileInfo `json:"c,omitempty"`
}
func (fi *embeddedFileInfo) GetGzipContent() ([]byte, bool) {
// when generating the bindata, if the compressed data equals or is larger than the original data, we store the original data
if fi.DataLen == fi.OriginSize {
return nil, false
}
return fi.data, true
}
type EmbeddedFileBase struct {
info *embeddedFileInfo
dataReader io.ReadSeeker
seekPos int64
}
func (f *EmbeddedFileBase) ReadDir(n int) ([]fs.DirEntry, error) {
// this method is used to satisfy the "func (f ioFile) ReadDir(...)" in httpfs
l, err := f.info.fs.ReadDir(f.info.fullName)
if err != nil {
return nil, err
}
if n < 0 || n > len(l) {
return l, nil
}
return l[:n], nil
}
type EmbeddedOriginFile struct {
EmbeddedFileBase
}
type EmbeddedCompressedFile struct {
EmbeddedFileBase
decompressor decompressor
decompressorPos int64
}
type embeddedFS struct {
meta func() *EmbeddedMeta
files map[string]*embeddedFileInfo
filesMu sync.RWMutex
data []byte
}
type EmbeddedMeta struct {
Root *embeddedFileInfo
}
func NewEmbeddedFS(data []byte) fs.ReadDirFS {
efs := &embeddedFS{data: data, files: make(map[string]*embeddedFileInfo)}
efs.meta = sync.OnceValue(func() *EmbeddedMeta {
var meta EmbeddedMeta
p := bytes.LastIndexByte(data, '\n')
if p < 0 {
return &meta
}
if err := json.Unmarshal(data[p+1:], &meta); err != nil {
panic("embedded file is not valid")
}
return &meta
})
return efs
}
var _ fs.ReadDirFS = (*embeddedFS)(nil)
func (e *embeddedFS) ReadDir(name string) (l []fs.DirEntry, err error) {
fi, err := e.getFileInfo(name)
if err != nil {
return nil, err
}
if !fi.IsDir() {
return nil, fs.ErrNotExist
}
l = make([]fs.DirEntry, len(fi.Children))
for i, child := range fi.Children {
l[i], err = e.getFileInfo(name + "/" + child.BaseName)
if err != nil {
return nil, err
}
}
return l, nil
}
func (e *embeddedFS) getFileInfo(fullName string) (*embeddedFileInfo, error) {
// no need to do heavy "path.Clean()" because we don't want to support "foo/../bar" or absolute paths
fullName = strings.TrimPrefix(fullName, "./")
if fullName == "" {
fullName = "."
}
e.filesMu.RLock()
fi := e.files[fullName]
e.filesMu.RUnlock()
if fi != nil {
return fi, nil
}
fields := strings.Split(fullName, "/")
fi = e.meta().Root
if fullName != "." {
found := true
for _, field := range fields {
for _, child := range fi.Children {
if found = child.BaseName == field; found {
fi = child
break
}
}
if !found {
return nil, fs.ErrNotExist
}
}
}
e.filesMu.Lock()
defer e.filesMu.Unlock()
if fi != nil {
fi.fs = e
fi.fullName = fullName
fi.data = e.data[fi.DataBegin : fi.DataBegin+fi.DataLen]
e.files[fullName] = fi // do not cache nil, otherwise keeping accessing random non-existing file will cause OOM
return fi, nil
}
return nil, fs.ErrNotExist
}
func (e *embeddedFS) Open(name string) (fs.File, error) {
info, err := e.getFileInfo(name)
if err != nil {
return nil, err
}
base := EmbeddedFileBase{info: info}
base.dataReader = bytes.NewReader(base.info.data)
if info.DataLen != info.OriginSize {
decomp, err := gzip.NewReader(base.dataReader)
if err != nil {
return nil, err
}
return &EmbeddedCompressedFile{EmbeddedFileBase: base, decompressor: decomp}, nil
}
return &EmbeddedOriginFile{base}, nil
}
var (
_ EmbeddedFileInfo = (*embeddedFileInfo)(nil)
_ EmbeddedFile = (*EmbeddedOriginFile)(nil)
_ EmbeddedFile = (*EmbeddedCompressedFile)(nil)
)
func (f *EmbeddedOriginFile) Read(p []byte) (n int, err error) {
return f.dataReader.Read(p)
}
func (f *EmbeddedCompressedFile) Read(p []byte) (n int, err error) {
if f.decompressorPos > f.seekPos {
if err = f.decompressor.Reset(bytes.NewReader(f.info.data)); err != nil {
return 0, err
}
f.decompressorPos = 0
}
if f.decompressorPos < f.seekPos {
if _, err = io.CopyN(io.Discard, f.decompressor, f.seekPos-f.decompressorPos); err != nil {
return 0, err
}
f.decompressorPos = f.seekPos
}
n, err = f.decompressor.Read(p)
f.decompressorPos += int64(n)
f.seekPos = f.decompressorPos
return n, err
}
func (f *EmbeddedFileBase) Seek(offset int64, whence int) (int64, error) {
switch whence {
case io.SeekStart:
f.seekPos = offset
case io.SeekCurrent:
f.seekPos += offset
case io.SeekEnd:
f.seekPos = f.info.OriginSize + offset
}
return f.seekPos, nil
}
func (f *EmbeddedFileBase) Stat() (fs.FileInfo, error) {
return f.info, nil
}
func (f *EmbeddedOriginFile) Close() error {
return nil
}
func (f *EmbeddedCompressedFile) Close() error {
return f.decompressor.Close()
}
func (fi *embeddedFileInfo) Name() string {
return fi.BaseName
}
func (fi *embeddedFileInfo) Size() int64 {
return fi.OriginSize
}
func (fi *embeddedFileInfo) Mode() fs.FileMode {
return util.Iif(fi.IsDir(), fs.ModeDir|0o555, 0o444)
}
func (fi *embeddedFileInfo) ModTime() time.Time {
return GetExecutableModTime()
}
func (fi *embeddedFileInfo) IsDir() bool {
return fi.Children != nil
}
func (fi *embeddedFileInfo) Sys() any {
return nil
}
func (fi *embeddedFileInfo) Type() fs.FileMode {
return util.Iif(fi.IsDir(), fs.ModeDir, 0)
}
func (fi *embeddedFileInfo) Info() (fs.FileInfo, error) {
return fi, nil
}
// GetExecutableModTime returns the modification time of the executable file.
// In bindata, we can't use the ModTime of the files because we need to make the build reproducible
var GetExecutableModTime = sync.OnceValue(func() (modTime time.Time) {
exePath, err := os.Executable()
if err != nil {
return modTime
}
exePath, err = filepath.Abs(exePath)
if err != nil {
return modTime
}
exePath, err = filepath.EvalSymlinks(exePath)
if err != nil {
return modTime
}
st, err := os.Stat(exePath)
if err != nil {
return modTime
}
return st.ModTime()
})
func GenerateEmbedBindata(fsRootPath, outputFile string) error {
output, err := os.OpenFile(outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
if err != nil {
return err
}
defer output.Close()
meta := &EmbeddedMeta{}
meta.Root = &embeddedFileInfo{}
var outputOffset int64
var embedFiles func(parent *embeddedFileInfo, fsPath, embedPath string) error
embedFiles = func(parent *embeddedFileInfo, fsPath, embedPath string) error {
dirEntries, err := os.ReadDir(fsPath)
if err != nil {
return err
}
for _, dirEntry := range dirEntries {
if err != nil {
return err
}
if dirEntry.IsDir() {
child := &embeddedFileInfo{
BaseName: dirEntry.Name(),
Children: []*embeddedFileInfo{}, // non-nil means it's a directory
}
parent.Children = append(parent.Children, child)
if err = embedFiles(child, filepath.Join(fsPath, dirEntry.Name()), path.Join(embedPath, dirEntry.Name())); err != nil {
return err
}
} else {
data, err := os.ReadFile(filepath.Join(fsPath, dirEntry.Name()))
if err != nil {
return err
}
var compressed bytes.Buffer
gz, _ := gzip.NewWriterLevel(&compressed, gzip.BestCompression)
if _, err = gz.Write(data); err != nil {
return err
}
if err = gz.Close(); err != nil {
return err
}
// only use the compressed data if it is smaller than the original data
outputBytes := util.Iif(len(compressed.Bytes()) < len(data), compressed.Bytes(), data)
child := &embeddedFileInfo{
BaseName: dirEntry.Name(),
OriginSize: int64(len(data)),
DataBegin: outputOffset,
DataLen: int64(len(outputBytes)),
}
if _, err = output.Write(outputBytes); err != nil {
return err
}
outputOffset += child.DataLen
parent.Children = append(parent.Children, child)
}
}
return nil
}
if err = embedFiles(meta.Root, fsRootPath, ""); err != nil {
return err
}
jsonBuf, err := json.Marshal(meta)
if err != nil {
return err
}
_, _ = output.Write([]byte{'\n'})
_, err = output.Write(bytes.TrimSpace(jsonBuf))
return err
}
+98
View File
@@ -0,0 +1,98 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package assetfs
import (
"bytes"
"io/fs"
"net/http"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestEmbed(t *testing.T) {
tmpDir := t.TempDir()
tmpDataDir := tmpDir + "/data"
_ = os.MkdirAll(tmpDataDir+"/foo/bar", 0o755)
_ = os.WriteFile(tmpDataDir+"/a.txt", []byte("a"), 0o644)
_ = os.WriteFile(tmpDataDir+"/foo/bar/b.txt", bytes.Repeat([]byte("a"), 1000), 0o644)
_ = os.WriteFile(tmpDataDir+"/foo/c.txt", []byte("c"), 0o644)
require.NoError(t, GenerateEmbedBindata(tmpDataDir, tmpDir+"/out.dat"))
data, err := os.ReadFile(tmpDir + "/out.dat")
require.NoError(t, err)
efs := NewEmbeddedFS(data)
// test a non-existing file
_, err = fs.ReadFile(efs, "not exist")
assert.ErrorIs(t, err, fs.ErrNotExist)
// test a normal file (no compression)
content, err := fs.ReadFile(efs, "a.txt")
require.NoError(t, err)
assert.Equal(t, "a", string(content))
fi, err := fs.Stat(efs, "a.txt")
require.NoError(t, err)
_, ok := fi.(EmbeddedFileInfo).GetGzipContent()
assert.False(t, ok)
// test a compressed file
content, err = fs.ReadFile(efs, "foo/bar/b.txt")
require.NoError(t, err)
assert.Equal(t, bytes.Repeat([]byte("a"), 1000), content)
fi, err = fs.Stat(efs, "foo/bar/b.txt")
require.NoError(t, err)
assert.False(t, fi.Mode().IsDir())
assert.True(t, fi.Mode().IsRegular())
gzipContent, ok := fi.(EmbeddedFileInfo).GetGzipContent()
assert.True(t, ok)
assert.Greater(t, len(gzipContent), 1)
assert.Less(t, len(gzipContent), 1000)
// test list root directory
entries, err := fs.ReadDir(efs, ".")
require.NoError(t, err)
assert.Len(t, entries, 2)
assert.Equal(t, "a.txt", entries[0].Name())
assert.False(t, entries[0].IsDir())
// test list subdirectory
entries, err = fs.ReadDir(efs, "foo")
require.NoError(t, err)
require.Len(t, entries, 2)
assert.Equal(t, "bar", entries[0].Name())
assert.True(t, entries[0].IsDir())
assert.Equal(t, "c.txt", entries[1].Name())
assert.False(t, entries[1].IsDir())
// test directory mode
fi, err = fs.Stat(efs, "foo")
require.NoError(t, err)
assert.True(t, fi.IsDir())
assert.True(t, fi.Mode().IsDir())
assert.False(t, fi.Mode().IsRegular())
// test httpfs
hfs := http.FS(efs)
hf, err := hfs.Open("foo/bar/b.txt")
require.NoError(t, err)
hi, err := hf.Stat()
require.NoError(t, err)
fiEmbedded, ok := hi.(EmbeddedFileInfo)
require.True(t, ok)
gzipContent, ok = fiEmbedded.GetGzipContent()
assert.True(t, ok)
assert.Greater(t, len(gzipContent), 1)
assert.Less(t, len(gzipContent), 1000)
// test httpfs directory listing
hf, err = hfs.Open("foo")
require.NoError(t, err)
dirs, err := hf.Readdir(1)
require.NoError(t, err)
assert.Len(t, dirs, 1)
}
+273
View File
@@ -0,0 +1,273 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package assetfs
import (
"context"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"time"
"gitea.dev/modules/container"
"gitea.dev/modules/log"
"gitea.dev/modules/process"
"gitea.dev/modules/util"
"github.com/fsnotify/fsnotify"
)
// Layer represents a layer in a layered asset file-system. It has a name and works like http.FileSystem
type Layer struct {
name string
fs fs.FS
localPath string
}
func (l *Layer) Name() string {
return l.name
}
// Open opens the named file. The caller is responsible for closing the file.
func (l *Layer) Open(name string) (fs.File, error) {
return l.fs.Open(name)
}
// Local returns a new Layer with the given name, it serves files from the given local path.
func Local(name, base string, sub ...string) *Layer {
// TODO: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before
// Ideally, the caller should guarantee the base is absolute, guessing a relative path based on the current working directory is unreliable.
base, err := filepath.Abs(base)
if err != nil {
// This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths.
panic(fmt.Sprintf("Unable to get absolute path for %q: %v", base, err))
}
root := util.FilePathJoinAbs(base, sub...)
return &Layer{name: name, fs: os.DirFS(root), localPath: root}
}
// Bindata returns a new Layer with the given name, it serves files from the given bindata asset.
func Bindata(name string, fs fs.FS) *Layer {
return &Layer{name: name, fs: fs}
}
// LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers.
// The first layer is the top layer, and it will be used first.
// If the file is not found in the top layer, it will be searched in the next layer.
type LayeredFS struct {
layers []*Layer
}
var _ fs.ReadDirFS = (*LayeredFS)(nil)
// Layered returns a new LayeredFS with the given layers. The first layer is the top layer.
func Layered(layers ...*Layer) *LayeredFS {
return &LayeredFS{layers: layers}
}
// Open opens the named file. The caller is responsible for closing the file.
func (l *LayeredFS) Open(name string) (fs.File, error) {
for _, layer := range l.layers {
f, err := layer.Open(name)
if err == nil || !os.IsNotExist(err) {
return f, err
}
}
return nil, fs.ErrNotExist
}
// ReadFile reads the named file.
func (l *LayeredFS) ReadFile(elems ...string) ([]byte, error) {
bs, _, err := l.ReadLayeredFile(elems...)
return bs, err
}
func (l *LayeredFS) ReadDir(name string) (files []fs.DirEntry, _ error) {
filesMap := map[string]fs.DirEntry{}
for _, layer := range l.layers {
entries, err := readDirOptional(layer, name)
if err != nil {
return nil, err
}
for _, entry := range entries {
entryName := entry.Name()
if _, exist := filesMap[entryName]; !exist && shouldInclude(entry) {
filesMap[entryName] = entry
}
}
}
for _, file := range filesMap {
files = append(files, file)
}
slices.SortFunc(files, func(a, b fs.DirEntry) int { return strings.Compare(a.Name(), b.Name()) })
return files, nil
}
// ReadLayeredFile reads the named file, and returns the layer name.
func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) {
name := util.PathJoinRel(elems...)
for _, layer := range l.layers {
bs, err := fs.ReadFile(layer, name)
if os.IsNotExist(err) {
continue
} else if err != nil {
return nil, layer.name, err
}
return bs, layer.name, err
}
return nil, "", fs.ErrNotExist
}
func shouldInclude(dirEntry fs.DirEntry, fileMode ...bool) bool {
if util.IsCommonHiddenFileName(dirEntry.Name()) {
return false
}
if len(fileMode) == 0 {
return true
} else if len(fileMode) == 1 {
return fileMode[0] == !dirEntry.IsDir()
}
panic("too many arguments for fileMode in shouldInclude")
}
func readDirOptional(layer *Layer, name string) (entries []fs.DirEntry, err error) {
if entries, err = fs.ReadDir(layer, name); os.IsNotExist(err) {
return nil, nil
}
return entries, err
}
// ListFiles lists files/directories in the given directory. The fileMode controls the returned files.
// * omitted: all files and directories will be returned.
// * true: only files will be returned.
// * false: only directories will be returned.
// The returned files are sorted by name.
func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) {
fileSet := make(container.Set[string])
for _, layer := range l.layers {
entries, err := readDirOptional(layer, name)
if err != nil {
return nil, err
}
for _, entry := range entries {
if shouldInclude(entry, fileMode...) {
fileSet.Add(entry.Name())
}
}
}
files := fileSet.Values()
sort.Strings(files)
return files, nil
}
// ListAllFiles returns files/directories in the given directory, including subdirectories, recursively.
// The fileMode controls the returned files:
// * omitted: all files and directories will be returned.
// * true: only files will be returned.
// * false: only directories will be returned.
// The returned files are sorted by name.
func (l *LayeredFS) ListAllFiles(name string, fileMode ...bool) ([]string, error) {
return listAllFiles(l.layers, name, fileMode...)
}
func listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, error) {
fileSet := make(container.Set[string])
var list func(dir string) error
list = func(dir string) error {
for _, layer := range layers {
entries, err := readDirOptional(layer, dir)
if err != nil {
return err
}
for _, entry := range entries {
path := util.PathJoinRelX(dir, entry.Name())
if shouldInclude(entry, fileMode...) {
fileSet.Add(path)
}
if entry.IsDir() {
if err = list(path); err != nil {
return err
}
}
}
}
return nil
}
if err := list(name); err != nil {
return nil, err
}
files := fileSet.Values()
sort.Strings(files)
return files, nil
}
// WatchLocalChanges watches local changes in the file-system. It's used to help to reload assets when the local file-system changes.
func (l *LayeredFS) WatchLocalChanges(ctx context.Context, callback func()) {
ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Asset Local FileSystem Watcher", process.SystemProcessType, true)
defer finished()
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Error("Unable to create watcher for asset local file-system: %v", err)
return
}
defer watcher.Close()
for _, layer := range l.layers {
if layer.localPath == "" {
continue
}
layerDirs, err := listAllFiles([]*Layer{layer}, ".", false)
if err != nil {
log.Error("Unable to list directories for asset local file-system %q: %v", layer.localPath, err)
continue
}
layerDirs = append(layerDirs, ".")
for _, dir := range layerDirs {
if err = watcher.Add(util.FilePathJoinAbs(layer.localPath, dir)); err != nil && !os.IsNotExist(err) {
log.Error("Unable to watch directory %s: %v", dir, err)
}
}
}
debounce := util.Debounce(100 * time.Millisecond)
for {
select {
case <-ctx.Done():
return
case event, ok := <-watcher.Events:
if !ok {
return
}
log.Trace("Watched asset local file-system had event: %v", event)
debounce(callback)
case err, ok := <-watcher.Errors:
if !ok {
return
}
log.Error("Watched asset local file-system had error: %v", err)
}
}
}
// GetFileLayerName returns the name of the first-seen layer that contains the given file.
func (l *LayeredFS) GetFileLayerName(elems ...string) string {
name := util.PathJoinRel(elems...)
for _, layer := range l.layers {
f, err := layer.Open(name)
if os.IsNotExist(err) {
continue
} else if err != nil {
return ""
}
_ = f.Close()
return layer.name
}
return ""
}
+109
View File
@@ -0,0 +1,109 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package assetfs
import (
"io"
"io/fs"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLayered(t *testing.T) {
dir := filepath.Join(t.TempDir(), "assetfs-layers")
dir1 := filepath.Join(dir, "l1")
dir2 := filepath.Join(dir, "l2")
mkdir := func(elems ...string) {
assert.NoError(t, os.MkdirAll(filepath.Join(elems...), 0o755))
}
write := func(content string, elems ...string) {
assert.NoError(t, os.WriteFile(filepath.Join(elems...), []byte(content), 0o644))
}
// d1 & f1: only in "l1"; d2 & f2: only in "l2"
// da & fa: in both "l1" and "l2"
mkdir(dir1, "d1")
mkdir(dir1, "da")
mkdir(dir1, "da/sub1")
mkdir(dir2, "d2")
mkdir(dir2, "da")
mkdir(dir2, "da/sub2")
write("dummy", dir1, ".DS_Store")
write("f1", dir1, "f1")
write("fa-1", dir1, "fa")
write("d1-f", dir1, "d1/f")
write("da-f-1", dir1, "da/f")
write("f2", dir2, "f2")
write("fa-2", dir2, "fa")
write("d2-f", dir2, "d2/f")
write("da-f-2", dir2, "da/f")
assets := Layered(Local("l1", dir1), Local("l2", dir2))
f, err := assets.Open("f1")
assert.NoError(t, err)
bs, err := io.ReadAll(f)
assert.NoError(t, err)
assert.Equal(t, "f1", string(bs))
_ = f.Close()
assertRead := func(expected string, expectedErr error, elems ...string) {
bs, err := assets.ReadFile(elems...)
if err != nil {
assert.ErrorIs(t, err, expectedErr)
} else {
assert.NoError(t, err)
assert.Equal(t, expected, string(bs))
}
}
assertRead("f1", nil, "f1")
assertRead("f2", nil, "f2")
assertRead("fa-1", nil, "fa")
assertRead("d1-f", nil, "d1/f")
assertRead("d2-f", nil, "d2/f")
assertRead("da-f-1", nil, "da/f")
assertRead("", fs.ErrNotExist, "no-such")
files, err := assets.ListFiles(".", true)
assert.NoError(t, err)
assert.Equal(t, []string{"f1", "f2", "fa"}, files)
files, err = assets.ListFiles(".", false)
assert.NoError(t, err)
assert.Equal(t, []string{"d1", "d2", "da"}, files)
files, err = assets.ListFiles(".")
assert.NoError(t, err)
assert.Equal(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files)
files, err = assets.ListAllFiles(".", true)
assert.NoError(t, err)
assert.Equal(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files)
files, err = assets.ListAllFiles(".", false)
assert.NoError(t, err)
assert.Equal(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files)
files, err = assets.ListAllFiles(".")
assert.NoError(t, err)
assert.Equal(t, []string{
"d1", "d1/f",
"d2", "d2/f",
"da", "da/f", "da/sub1", "da/sub2",
"f1", "f2", "fa",
}, files)
assert.Empty(t, assets.GetFileLayerName("no-such"))
assert.Equal(t, "l1", assets.GetFileLayerName("f1"))
assert.Equal(t, "l2", assets.GetFileLayerName("f2"))
}
+22
View File
@@ -0,0 +1,22 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package auth
import (
"gitea.dev/modules/json"
"gitea.dev/modules/log"
)
func UnmarshalGroupTeamMapping(raw string) (map[string]map[string][]string, error) {
groupTeamMapping := make(map[string]map[string][]string)
if raw == "" {
return groupTeamMapping, nil
}
err := json.Unmarshal([]byte(raw), &groupTeamMapping)
if err != nil {
log.Error("Failed to unmarshal group team mapping: %v", err)
return nil, err
}
return groupTeamMapping, nil
}
+47
View File
@@ -0,0 +1,47 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package httpauth
import (
"encoding/base64"
"strings"
"gitea.dev/modules/util"
)
type BasicAuth struct {
Username, Password string
}
type BearerToken struct {
Token string
}
type ParsedAuthorizationHeader struct {
BasicAuth *BasicAuth
BearerToken *BearerToken
}
func ParseAuthorizationHeader(header string) (ret ParsedAuthorizationHeader, _ bool) {
parts := strings.Fields(header)
if len(parts) != 2 {
return ret, false
}
if util.AsciiEqualFold(parts[0], "basic") {
s, err := base64.StdEncoding.DecodeString(parts[1])
if err != nil {
return ret, false
}
u, p, ok := strings.Cut(string(s), ":")
if !ok {
return ret, false
}
ret.BasicAuth = &BasicAuth{Username: u, Password: p}
return ret, true
} else if util.AsciiEqualFold(parts[0], "token") || util.AsciiEqualFold(parts[0], "bearer") {
ret.BearerToken = &BearerToken{Token: parts[1]}
return ret, true
}
return ret, false
}
+43
View File
@@ -0,0 +1,43 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package httpauth
import (
"encoding/base64"
"testing"
"github.com/stretchr/testify/assert"
)
func TestParseAuthorizationHeader(t *testing.T) {
type parsed = ParsedAuthorizationHeader
type basic = BasicAuth
type bearer = BearerToken
cases := []struct {
headerValue string
expected parsed
ok bool
}{
{"", parsed{}, false},
{"?", parsed{}, false},
{"foo", parsed{}, false},
{"any value", parsed{}, false},
{"Basic ?", parsed{}, false},
{"Basic " + base64.StdEncoding.EncodeToString([]byte("foo")), parsed{}, false},
{"Basic " + base64.StdEncoding.EncodeToString([]byte("foo:bar")), parsed{BasicAuth: &basic{"foo", "bar"}}, true},
{"basic " + base64.StdEncoding.EncodeToString([]byte("foo:bar")), parsed{BasicAuth: &basic{"foo", "bar"}}, true},
{"token value", parsed{BearerToken: &bearer{"value"}}, true},
{"Token value", parsed{BearerToken: &bearer{"value"}}, true},
{"bearer value", parsed{BearerToken: &bearer{"value"}}, true},
{"Bearer value", parsed{BearerToken: &bearer{"value"}}, true},
{"Bearer wrong value", parsed{}, false},
}
for _, c := range cases {
ret, ok := ParseAuthorizationHeader(c.headerValue)
assert.Equal(t, c.ok, ok, "header %q", c.headerValue)
assert.Equal(t, c.expected, ret, "header %q", c.headerValue)
}
}
+57
View File
@@ -0,0 +1,57 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package openid
import (
"sync"
"time"
"github.com/yohcop/openid-go"
)
type timedDiscoveredInfo struct {
info openid.DiscoveredInfo
time time.Time
}
type timedDiscoveryCache struct {
cache map[string]timedDiscoveredInfo
ttl time.Duration
mutex *sync.Mutex
}
func newTimedDiscoveryCache(ttl time.Duration) *timedDiscoveryCache {
return &timedDiscoveryCache{cache: map[string]timedDiscoveredInfo{}, ttl: ttl, mutex: &sync.Mutex{}}
}
func (s *timedDiscoveryCache) Put(id string, info openid.DiscoveredInfo) {
s.mutex.Lock()
defer s.mutex.Unlock()
s.cache[id] = timedDiscoveredInfo{info: info, time: time.Now()}
}
// Delete timed-out cache entries
func (s *timedDiscoveryCache) cleanTimedOut() {
now := time.Now()
for k, e := range s.cache {
diff := now.Sub(e.time)
if diff > s.ttl {
delete(s.cache, k)
}
}
}
func (s *timedDiscoveryCache) Get(id string) openid.DiscoveredInfo {
s.mutex.Lock()
defer s.mutex.Unlock()
// Delete old cached while we are at it.
s.cleanTimedOut()
if info, has := s.cache[id]; has {
return info.info
}
return nil
}
@@ -0,0 +1,49 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package openid
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type testDiscoveredInfo struct{}
func (s *testDiscoveredInfo) ClaimedID() string {
return "claimedID"
}
func (s *testDiscoveredInfo) OpEndpoint() string {
return "opEndpoint"
}
func (s *testDiscoveredInfo) OpLocalID() string {
return "opLocalID"
}
func TestTimedDiscoveryCache(t *testing.T) {
ttl := 50 * time.Millisecond
dc := newTimedDiscoveryCache(ttl)
// Put some initial values
dc.Put("foo", &testDiscoveredInfo{}) // openid.opEndpoint: "a", openid.opLocalID: "b", openid.claimedID: "c"})
// Make sure we can retrieve them
di := dc.Get("foo")
require.NotNil(t, di)
assert.Equal(t, "opEndpoint", di.OpEndpoint())
assert.Equal(t, "opLocalID", di.OpLocalID())
assert.Equal(t, "claimedID", di.ClaimedID())
// Attempt to get a non-existent value
assert.Nil(t, dc.Get("bar"))
// Sleep for a while and try to retrieve again
time.Sleep(ttl * 3 / 2)
assert.Nil(t, dc.Get("foo"))
}
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package openid
import (
"time"
"github.com/yohcop/openid-go"
)
// For the demo, we use in-memory infinite storage nonce and discovery
// cache. In your app, do not use this as it will eat up memory and
// never
// free it. Use your own implementation, on a better database system.
// If you have multiple servers for example, you may need to share at
// least
// the nonceStore between them.
var (
nonceStore = openid.NewSimpleNonceStore()
discoveryCache = newTimedDiscoveryCache(24 * time.Hour)
)
// Verify handles response from OpenID provider
func Verify(fullURL string) (id string, err error) {
return openid.Verify(fullURL, discoveryCache, nonceStore)
}
// Normalize normalizes an OpenID URI
func Normalize(url string) (id string, err error) {
return openid.Normalize(url)
}
// RedirectURL redirects browser
func RedirectURL(id, callbackURL, realm string) (string, error) {
return openid.RedirectURL(id, callbackURL, realm)
}
+44
View File
@@ -0,0 +1,44 @@
// Copyright 2014 The Gogs Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build pam
package pam
import (
"errors"
"github.com/msteinert/pam/v2"
)
// Supported is true when built with PAM
var Supported = true
// Auth pam auth service
func Auth(serviceName, userName, passwd string) (string, error) {
t, err := pam.StartFunc(serviceName, userName, func(s pam.Style, msg string) (string, error) {
switch s {
case pam.PromptEchoOff:
return passwd, nil
case pam.PromptEchoOn, pam.ErrorMsg, pam.TextInfo:
return "", nil
}
return "", errors.New("Unrecognized PAM message style")
})
if err != nil {
return "", err
}
defer t.End()
if err = t.Authenticate(0); err != nil {
return "", err
}
if err = t.AcctMgmt(0); err != nil {
return "", err
}
// PAM login names might suffer transformations in the PAM stack.
// We should take whatever the PAM stack returns for it.
return t.GetItem(pam.User)
}
+22
View File
@@ -0,0 +1,22 @@
// Copyright 2014 The Gogs Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !pam
package pam
import (
"errors"
)
// Supported is false when built without PAM
var Supported = false
// Auth not supported lack of pam tag
func Auth(serviceName, userName, passwd string) (string, error) {
// bypass the lint on callers: SA4023: this comparison is always true (staticcheck)
if !Supported {
return "", errors.New("PAM not supported")
}
return "", nil
}
+19
View File
@@ -0,0 +1,19 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build pam
package pam
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestPamAuth(t *testing.T) {
result, err := Auth("gitea", "user1", "false-pwd")
assert.Error(t, err)
assert.EqualError(t, err, "Authentication failure")
assert.Empty(t, result)
}
+74
View File
@@ -0,0 +1,74 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hash
import (
"encoding/hex"
"strings"
"gitea.dev/modules/log"
"golang.org/x/crypto/argon2"
)
func init() {
MustRegister("argon2", NewArgon2Hasher)
}
// Argon2Hasher implements PasswordHasher
// and uses the Argon2 key derivation function, hybrant variant
type Argon2Hasher struct {
time uint32
memory uint32
threads uint8
keyLen uint32
}
// HashWithSaltBytes a provided password and salt
func (hasher *Argon2Hasher) HashWithSaltBytes(password string, salt []byte) string {
if hasher == nil {
return ""
}
return hex.EncodeToString(argon2.IDKey([]byte(password), salt, hasher.time, hasher.memory, hasher.threads, hasher.keyLen))
}
// NewArgon2Hasher is a factory method to create an Argon2Hasher
// The provided config should be either empty or of the form:
// "<time>$<memory>$<threads>$<keyLen>", where <x> is the string representation
// of an integer
func NewArgon2Hasher(config string) *Argon2Hasher {
// This default configuration uses the following parameters:
// time=2, memory=64*1024, threads=8, keyLen=50.
// It will make two passes through the memory, using 64MiB in total.
// This matches the original configuration for `argon2` prior to storing hash parameters
// in the database.
// THESE VALUES MUST NOT BE CHANGED OR BACKWARDS COMPATIBILITY WILL BREAK
hasher := &Argon2Hasher{
time: 2,
memory: 1 << 16,
threads: 8,
keyLen: 50,
}
if config == "" {
return hasher
}
vals := strings.SplitN(config, "$", 4)
if len(vals) != 4 {
log.Error("invalid argon2 hash spec %s", config)
return nil
}
var err error
hasher.time, err = parseUintParam[uint32](vals[0], "time", "argon2", config, nil)
hasher.memory, err = parseUintParam[uint32](vals[1], "memory", "argon2", config, err)
hasher.threads, err = parseUintParam[uint8](vals[2], "threads", "argon2", config, err)
hasher.keyLen, err = parseUintParam[uint32](vals[3], "keyLen", "argon2", config, err)
if err != nil {
return nil
}
return hasher
}
+54
View File
@@ -0,0 +1,54 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hash
import (
"golang.org/x/crypto/bcrypt"
)
func init() {
MustRegister("bcrypt", NewBcryptHasher)
}
// BcryptHasher implements PasswordHasher
// and uses the bcrypt password hash function.
type BcryptHasher struct {
cost int
}
// HashWithSaltBytes a provided password and salt
func (hasher *BcryptHasher) HashWithSaltBytes(password string, salt []byte) string {
if hasher == nil {
return ""
}
hashedPassword, _ := bcrypt.GenerateFromPassword([]byte(password), hasher.cost)
return string(hashedPassword)
}
func (hasher *BcryptHasher) VerifyPassword(password, hashedPassword, salt string) bool {
return bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte(password)) == nil
}
// NewBcryptHasher is a factory method to create an BcryptHasher
// The provided config should be either empty or the string representation of the "<cost>"
// as an integer
func NewBcryptHasher(config string) *BcryptHasher {
// This matches the original configuration for `bcrypt` prior to storing hash parameters
// in the database.
// THESE VALUES MUST NOT BE CHANGED OR BACKWARDS COMPATIBILITY WILL BREAK
hasher := &BcryptHasher{
cost: 10, // cost=10. i.e. 2^10 rounds of key expansion.
}
if config == "" {
return hasher
}
var err error
hasher.cost, err = parseIntParam(config, "cost", "bcrypt", config, nil)
if err != nil {
return nil
}
return hasher
}
+30
View File
@@ -0,0 +1,30 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hash
import (
"strconv"
"gitea.dev/modules/log"
"gitea.dev/modules/util"
)
func parseIntParam(value, param, algorithmName, config string, previousErr error) (int, error) {
parsed, err := strconv.Atoi(value)
if err != nil {
log.Error("invalid integer for %s representation in %s hash spec %s", param, algorithmName, config)
return 0, err
}
return parsed, previousErr // <- Keep the previous error as this function should still return an error once everything has been checked if any call failed
}
func parseUintParam[T uint32 | uint8](value, param, algorithmName, config string, previousErr error) (ret T, _ error) {
_, isUint32 := any(ret).(uint32)
parsed, err := strconv.ParseUint(value, 10, util.Iif(isUint32, 32, 8))
if err != nil {
log.Error("invalid integer for %s representation in %s hash spec %s", param, algorithmName, config)
return 0, err
}
return T(parsed), previousErr // <- Keep the previous error as this function should still return an error once everything has been checked if any call failed
}
+33
View File
@@ -0,0 +1,33 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hash
import (
"encoding/hex"
)
// DummyHasher implements PasswordHasher and is a dummy hasher that simply
// puts the password in place with its salt
// This SHOULD NOT be used in production and is provided to make the integration
// tests faster only
type DummyHasher struct{}
// HashWithSaltBytes a provided password and salt
func (hasher *DummyHasher) HashWithSaltBytes(password string, salt []byte) string {
if hasher == nil {
return ""
}
if len(salt) == 10 {
return string(salt) + ":" + password
}
return hex.EncodeToString(salt) + ":" + password
}
// NewDummyHasher is a factory method to create a DummyHasher
// Any provided configuration is ignored
func NewDummyHasher(_ string) *DummyHasher {
return &DummyHasher{}
}
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hash
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestDummyHasher(t *testing.T) {
dummy := &PasswordHashAlgorithm{
PasswordSaltHasher: NewDummyHasher(""),
Specification: "dummy",
}
password, salt := "password", "ZogKvWdyEx"
hash, err := dummy.Hash(password, salt)
assert.NoError(t, err)
assert.Equal(t, hash, salt+":"+password)
assert.True(t, dummy.VerifyPassword(password, hash, salt))
}
+189
View File
@@ -0,0 +1,189 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hash
import (
"crypto/subtle"
"encoding/hex"
"fmt"
"strings"
"sync/atomic"
"gitea.dev/modules/log"
)
// This package takes care of hashing passwords, verifying passwords, defining
// available password algorithms, defining recommended password algorithms and
// choosing the default password algorithm.
// PasswordSaltHasher will hash a provided password with the provided saltBytes
type PasswordSaltHasher interface {
HashWithSaltBytes(password string, saltBytes []byte) string
}
// PasswordHasher will hash a provided password with the salt
type PasswordHasher interface {
Hash(password, salt string) (string, error)
}
// PasswordVerifier will ensure that a providedPassword matches the hashPassword when hashed with the salt
type PasswordVerifier interface {
VerifyPassword(providedPassword, hashedPassword, salt string) bool
}
// PasswordHashAlgorithms are named PasswordSaltHashers with a default verifier and hash function
type PasswordHashAlgorithm struct {
PasswordSaltHasher
Specification string // The specification that is used to create the internal PasswordSaltHasher
}
// Hash the provided password with the salt and return the hash
func (algorithm *PasswordHashAlgorithm) Hash(password, salt string) (string, error) {
var saltBytes []byte
// There are two formats for the salt value:
// * The new format is a (32+)-byte hex-encoded string
// * The old format was a 10-byte binary format
// We have to tolerate both here.
if len(salt) == 10 {
saltBytes = []byte(salt)
} else {
var err error
saltBytes, err = hex.DecodeString(salt)
if err != nil {
return "", err
}
}
return algorithm.HashWithSaltBytes(password, saltBytes), nil
}
// Verify the provided password matches the hashPassword when hashed with the salt
func (algorithm *PasswordHashAlgorithm) VerifyPassword(providedPassword, hashedPassword, salt string) bool {
// Some PasswordSaltHashers have their own specialised compare function that takes into
// account the stored parameters within the hash. e.g. bcrypt
if verifier, ok := algorithm.PasswordSaltHasher.(PasswordVerifier); ok {
return verifier.VerifyPassword(providedPassword, hashedPassword, salt)
}
// Compute the hash of the password.
providedPasswordHash, err := algorithm.Hash(providedPassword, salt)
if err != nil {
log.Error("passwordhash: %v.Hash(): %v", algorithm.Specification, err)
return false
}
// Compare it against the hashed password in constant-time.
return subtle.ConstantTimeCompare([]byte(hashedPassword), []byte(providedPasswordHash)) == 1
}
var (
lastNonDefaultAlgorithm atomic.Value
availableHasherFactories = map[string]func(string) PasswordSaltHasher{}
)
// MustRegister registers a PasswordSaltHasher with the availableHasherFactories
// Caution: This is not thread safe.
func MustRegister[T PasswordSaltHasher](name string, newFn func(config string) T) {
if err := Register(name, newFn); err != nil {
panic(err)
}
}
// Register registers a PasswordSaltHasher with the availableHasherFactories
// Caution: This is not thread safe.
func Register[T PasswordSaltHasher](name string, newFn func(config string) T) error {
if _, has := availableHasherFactories[name]; has {
return fmt.Errorf("duplicate registration of password salt hasher: %s", name)
}
availableHasherFactories[name] = func(config string) PasswordSaltHasher {
n := newFn(config)
return n
}
return nil
}
// In early versions of gitea the password hash algorithm field of a user could be
// empty. At that point the default was `pbkdf2` without configuration values
//
// Please note this is not the same as the DefaultAlgorithm which is used
// to determine what an empty PASSWORD_HASH_ALGO setting in the app.ini means.
// These are not the same even if they have the same apparent value and they mean different things.
//
// DO NOT COALESCE THESE VALUES
const defaultEmptyHashAlgorithmSpecification = "pbkdf2"
// Parse will convert the provided algorithm specification in to a PasswordHashAlgorithm
// If the provided specification matches the DefaultHashAlgorithm Specification it will be
// used.
// In addition the last non-default hasher will be cached to help reduce the load from
// parsing specifications.
//
// NOTE: No de-aliasing is done in this function, thus any specification which does not
// contain a configuration will use the default values for that hasher. These are not
// necessarily the same values as those obtained by dealiasing. This allows for
// seamless backwards compatibility with the original configuration.
//
// To further labour this point, running `Parse("pbkdf2")` does not obtain the
// same algorithm as setting `PASSWORD_HASH_ALGO=pbkdf2` in app.ini, nor is it intended to.
// A user that has `password_hash_algo='pbkdf2'` in the db means get the original, unconfigured algorithm
// Users will be migrated automatically as they log-in to have the complete specification stored
// in their `password_hash_algo` fields by other code.
func Parse(algorithmSpec string) *PasswordHashAlgorithm {
if algorithmSpec == "" {
algorithmSpec = defaultEmptyHashAlgorithmSpecification
}
if DefaultHashAlgorithm != nil && algorithmSpec == DefaultHashAlgorithm.Specification {
return DefaultHashAlgorithm
}
ptr := lastNonDefaultAlgorithm.Load()
if ptr != nil {
hashAlgorithm, ok := ptr.(*PasswordHashAlgorithm)
if ok && hashAlgorithm.Specification == algorithmSpec {
return hashAlgorithm
}
}
// Now convert the provided specification in to a hasherType +/- some configuration parameters
vals := strings.SplitN(algorithmSpec, "$", 2)
var hasherType string
var config string
if len(vals) == 0 {
// This should not happen as algorithmSpec should not be empty
// due to it being assigned to defaultEmptyHashAlgorithmSpecification above
// but we should be absolutely cautious here
return nil
}
hasherType = vals[0]
if len(vals) > 1 {
config = vals[1]
}
newFn, has := availableHasherFactories[hasherType]
if !has {
// unknown hasher type
return nil
}
ph := newFn(config)
if ph == nil {
// The provided configuration is likely invalid - it will have been logged already
// but we cannot hash safely
return nil
}
hashAlgorithm := &PasswordHashAlgorithm{
PasswordSaltHasher: ph,
Specification: algorithmSpec,
}
lastNonDefaultAlgorithm.Store(hashAlgorithm)
return hashAlgorithm
}
+190
View File
@@ -0,0 +1,190 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hash
import (
"encoding/hex"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
type testSaltHasher string
func (t testSaltHasher) HashWithSaltBytes(password string, salt []byte) string {
return password + "$" + string(salt) + "$" + string(t)
}
func Test_registerHasher(t *testing.T) {
MustRegister("Test_registerHasher", func(config string) testSaltHasher {
return testSaltHasher(config)
})
assert.Panics(t, func() {
MustRegister("Test_registerHasher", func(config string) testSaltHasher {
return testSaltHasher(config)
})
})
assert.Error(t, Register("Test_registerHasher", func(config string) testSaltHasher {
return testSaltHasher(config)
}))
assert.Equal(t, "password$salt$",
Parse("Test_registerHasher").PasswordSaltHasher.HashWithSaltBytes("password", []byte("salt")))
assert.Equal(t, "password$salt$config",
Parse("Test_registerHasher$config").PasswordSaltHasher.HashWithSaltBytes("password", []byte("salt")))
delete(availableHasherFactories, "Test_registerHasher")
}
func TestParse(t *testing.T) {
hashAlgorithmsToTest := []string{}
for plainHashAlgorithmNames := range availableHasherFactories {
hashAlgorithmsToTest = append(hashAlgorithmsToTest, plainHashAlgorithmNames)
}
for _, aliased := range aliasAlgorithmNames {
if strings.Contains(aliased, "$") {
hashAlgorithmsToTest = append(hashAlgorithmsToTest, aliased)
}
}
for _, algorithmName := range hashAlgorithmsToTest {
t.Run(algorithmName, func(t *testing.T) {
algo := Parse(algorithmName)
assert.NotNil(t, algo, "Algorithm %s resulted in an empty algorithm", algorithmName)
})
}
}
func TestHashing(t *testing.T) {
hashAlgorithmsToTest := []string{}
for plainHashAlgorithmNames := range availableHasherFactories {
hashAlgorithmsToTest = append(hashAlgorithmsToTest, plainHashAlgorithmNames)
}
for _, aliased := range aliasAlgorithmNames {
if strings.Contains(aliased, "$") {
hashAlgorithmsToTest = append(hashAlgorithmsToTest, aliased)
}
}
runTests := func(password, salt string, shouldPass bool) {
for _, algorithmName := range hashAlgorithmsToTest {
t.Run(algorithmName, func(t *testing.T) {
output, err := Parse(algorithmName).Hash(password, salt)
if shouldPass {
assert.NoError(t, err)
assert.NotEmpty(t, output, "output for %s was empty", algorithmName)
} else {
assert.Error(t, err)
}
assert.Equal(t, Parse(algorithmName).VerifyPassword(password, output, salt), shouldPass)
})
}
}
// Test with new salt format.
runTests(strings.Repeat("a", 16), hex.EncodeToString([]byte{0x01, 0x02, 0x03}), true)
// Test with legacy salt format.
runTests(strings.Repeat("a", 16), strings.Repeat("b", 10), true)
// Test with invalid salt.
runTests(strings.Repeat("a", 16), "a", false)
}
// vectors were generated using the current codebase.
var vectors = []struct {
algorithms []string
password string
salt string
output string
shouldfail bool
}{
{
algorithms: []string{"bcrypt", "bcrypt$10"},
password: "abcdef",
salt: strings.Repeat("a", 10),
output: "$2a$10$fjtm8BsQ2crym01/piJroenO3oSVUBhSLKaGdTYJ4tG0ePVCrU0G2",
shouldfail: false,
},
{
algorithms: []string{"scrypt", "scrypt$65536$16$2$50"},
password: "abcdef",
salt: strings.Repeat("a", 10),
output: "3b571d0c07c62d42b7bad3dbf18fb0cd67d4d8cd4ad4c6928e1090e5b2a4a84437c6fd2627d897c0e7e65025ca62b67a0002",
shouldfail: false,
},
{
algorithms: []string{"argon2", "argon2$2$65536$8$50"},
password: "abcdef",
salt: strings.Repeat("a", 10),
output: "551f089f570f989975b6f7c6a8ff3cf89bc486dd7bbe87ed4d80ad4362f8ee599ec8dda78dac196301b98456402bcda775dc",
shouldfail: false,
},
{
algorithms: []string{"pbkdf2", "pbkdf2$10000$50"},
password: "abcdef",
salt: strings.Repeat("a", 10),
output: "ab48d5471b7e6ed42d10001db88c852ff7303c788e49da5c3c7b63d5adf96360303724b74b679223a3dea8a242d10abb1913",
shouldfail: false,
},
{
algorithms: []string{"bcrypt", "bcrypt$10"},
password: "abcdef",
salt: hex.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04}),
output: "$2a$10$qhgm32w9ZpqLygugWJsLjey8xRGcaq9iXAfmCeNBXxddgyoaOC3Gq",
shouldfail: false,
},
{
algorithms: []string{"scrypt", "scrypt$65536$16$2$50"},
password: "abcdef",
salt: hex.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04}),
output: "25fe5f66b43fa4eb7b6717905317cd2223cf841092dc8e0a1e8c75720ad4846cb5d9387303e14bc3c69faa3b1c51ef4b7de1",
shouldfail: false,
},
{
algorithms: []string{"argon2", "argon2$2$65536$8$50"},
password: "abcdef",
salt: hex.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04}),
output: "9c287db63a91d18bb1414b703216da4fc431387c1ae7c8acdb280222f11f0929831055dbfd5126a3b48566692e83ec750d2a",
shouldfail: false,
},
{
algorithms: []string{"pbkdf2", "pbkdf2$10000$50"},
password: "abcdef",
salt: hex.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04}),
output: "45d6cdc843d65cf0eda7b90ab41435762a282f7df013477a1c5b212ba81dbdca2edf1ecc4b5cb05956bb9e0c37ab29315d78",
shouldfail: false,
},
{
algorithms: []string{"pbkdf2$320000$50"},
password: "abcdef",
salt: hex.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04}),
output: "84e233114499e8721da80e85568e5b7b5900b3e49a30845fcda9d1e1756da4547d70f8740ac2b4a5d82f88cebcd27f21bfe2",
shouldfail: false,
},
{
algorithms: []string{"pbkdf2", "pbkdf2$10000$50"},
password: "abcdef",
salt: "",
output: "",
shouldfail: true,
},
}
// Ensure that the current code will correctly verify against the test vectors.
func TestVectors(t *testing.T) {
for i, vector := range vectors {
for _, algorithm := range vector.algorithms {
t.Run(strconv.Itoa(i)+": "+algorithm, func(t *testing.T) {
pa := Parse(algorithm)
assert.Equal(t, !vector.shouldfail, pa.VerifyPassword(vector.password, vector.output, vector.salt))
})
}
}
}
+67
View File
@@ -0,0 +1,67 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hash
import (
"crypto/sha256"
"encoding/hex"
"strings"
"gitea.dev/modules/log"
"golang.org/x/crypto/pbkdf2"
)
func init() {
MustRegister("pbkdf2", NewPBKDF2Hasher)
}
// PBKDF2Hasher implements PasswordHasher
// and uses the PBKDF2 key derivation function.
type PBKDF2Hasher struct {
iter, keyLen int
}
// HashWithSaltBytes a provided password and salt
func (hasher *PBKDF2Hasher) HashWithSaltBytes(password string, salt []byte) string {
if hasher == nil {
return ""
}
return hex.EncodeToString(pbkdf2.Key([]byte(password), salt, hasher.iter, hasher.keyLen, sha256.New))
}
// NewPBKDF2Hasher is a factory method to create an PBKDF2Hasher
// config should be either empty or of the form:
// "<iter>$<keyLen>", where <x> is the string representation
// of an integer
func NewPBKDF2Hasher(config string) *PBKDF2Hasher {
// This default configuration uses the following parameters:
// iter=10000, keyLen=50.
// This matches the original configuration for `pbkdf2` prior to storing parameters
// in the database.
// THESE VALUES MUST NOT BE CHANGED OR BACKWARDS COMPATIBILITY WILL BREAK
hasher := &PBKDF2Hasher{
iter: 10_000,
keyLen: 50,
}
if config == "" {
return hasher
}
vals := strings.SplitN(config, "$", 2)
if len(vals) != 2 {
log.Error("invalid pbkdf2 hash spec %s", config)
return nil
}
var err error
hasher.iter, err = parseIntParam(vals[0], "iter", "pbkdf2", config, nil)
hasher.keyLen, err = parseIntParam(vals[1], "keyLen", "pbkdf2", config, err)
if err != nil {
return nil
}
return hasher
}
+67
View File
@@ -0,0 +1,67 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hash
import (
"encoding/hex"
"strings"
"gitea.dev/modules/log"
"golang.org/x/crypto/scrypt"
)
func init() {
MustRegister("scrypt", NewScryptHasher)
}
// ScryptHasher implements PasswordHasher
// and uses the scrypt key derivation function.
type ScryptHasher struct {
n, r, p, keyLen int
}
// HashWithSaltBytes a provided password and salt
func (hasher *ScryptHasher) HashWithSaltBytes(password string, salt []byte) string {
if hasher == nil {
return ""
}
hashedPassword, _ := scrypt.Key([]byte(password), salt, hasher.n, hasher.r, hasher.p, hasher.keyLen)
return hex.EncodeToString(hashedPassword)
}
// NewScryptHasher is a factory method to create an ScryptHasher
// The provided config should be either empty or of the form:
// "<n>$<r>$<p>$<keyLen>", where <x> is the string representation
// of an integer
func NewScryptHasher(config string) *ScryptHasher {
// This matches the original configuration for `scrypt` prior to storing hash parameters
// in the database.
// THESE VALUES MUST NOT BE CHANGED OR BACKWARDS COMPATIBILITY WILL BREAK
hasher := &ScryptHasher{
n: 1 << 16,
r: 16,
p: 2, // 2 passes through memory - this default config will use 128MiB in total.
keyLen: 50,
}
if config == "" {
return hasher
}
vals := strings.SplitN(config, "$", 4)
if len(vals) != 4 {
log.Error("invalid scrypt hash spec %s", config)
return nil
}
var err error
hasher.n, err = parseIntParam(vals[0], "n", "scrypt", config, nil)
hasher.r, err = parseIntParam(vals[1], "r", "scrypt", config, err)
hasher.p, err = parseIntParam(vals[2], "p", "scrypt", config, err)
hasher.keyLen, err = parseIntParam(vals[3], "keyLen", "scrypt", config, err)
if err != nil {
return nil
}
return hasher
}
+76
View File
@@ -0,0 +1,76 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hash
// DefaultHashAlgorithmName represents the default value of PASSWORD_HASH_ALGO
// configured in app.ini.
//
// It is NOT the same and does NOT map to the defaultEmptyHashAlgorithmSpecification.
//
// It will be dealiased as per aliasAlgorithmNames whereas
// defaultEmptyHashAlgorithmSpecification does not undergo dealiasing.
const DefaultHashAlgorithmName = "pbkdf2"
var DefaultHashAlgorithm *PasswordHashAlgorithm
// aliasAlgorithmNames provides a mapping between the value of PASSWORD_HASH_ALGO
// configured in the app.ini and the parameters used within the hashers internally.
//
// If it is necessary to change the default parameters for any hasher in future you
// should change these values and not those in argon2.go etc.
var aliasAlgorithmNames = map[string]string{
"argon2": "argon2$2$65536$8$50",
"bcrypt": "bcrypt$10",
"scrypt": "scrypt$65536$16$2$50",
"pbkdf2": "pbkdf2_v2", // pbkdf2 should default to pbkdf2_v2
"pbkdf2_v1": "pbkdf2$10000$50",
// The latest PBKDF2 password algorithm is used as the default since it doesn't
// use a lot of memory and is safer to use on less powerful devices.
"pbkdf2_v2": "pbkdf2$50000$50",
// The pbkdf2_hi password algorithm is offered as a stronger alternative to the
// slightly improved pbkdf2_v2 algorithm
"pbkdf2_hi": "pbkdf2$320000$50",
}
var RecommendedHashAlgorithms = []string{
"pbkdf2",
"argon2",
"bcrypt",
"scrypt",
"pbkdf2_hi",
}
// hashAlgorithmToSpec converts an algorithm name or a specification to a full algorithm specification
func hashAlgorithmToSpec(algorithmName string) string {
if algorithmName == "" {
algorithmName = DefaultHashAlgorithmName
}
alias, has := aliasAlgorithmNames[algorithmName]
for has {
algorithmName = alias
alias, has = aliasAlgorithmNames[algorithmName]
}
return algorithmName
}
// SetDefaultPasswordHashAlgorithm will take a provided algorithmName and de-alias it to
// a complete algorithm specification.
func SetDefaultPasswordHashAlgorithm(algorithmName string) (string, *PasswordHashAlgorithm) {
algoSpec := hashAlgorithmToSpec(algorithmName)
// now we get a full specification, e.g. pbkdf2$50000$50 rather than pbdkf2
DefaultHashAlgorithm = Parse(algoSpec)
return algoSpec, DefaultHashAlgorithm
}
// ConfigHashAlgorithm will try to find a "recommended algorithm name" defined by RecommendedHashAlgorithms for config
// This function is not fast and is only used for the installation page
func ConfigHashAlgorithm(algorithm string) string {
algorithm = hashAlgorithmToSpec(algorithm)
for _, recommAlgo := range RecommendedHashAlgorithms {
if algorithm == hashAlgorithmToSpec(recommAlgo) {
return recommAlgo
}
}
return algorithm
}
@@ -0,0 +1,38 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package hash
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestCheckSettingPasswordHashAlgorithm(t *testing.T) {
t.Run("pbkdf2 is pbkdf2_v2", func(t *testing.T) {
pbkdf2v2Config, pbkdf2v2Algo := SetDefaultPasswordHashAlgorithm("pbkdf2_v2")
pbkdf2Config, pbkdf2Algo := SetDefaultPasswordHashAlgorithm("pbkdf2")
assert.Equal(t, pbkdf2v2Config, pbkdf2Config)
assert.Equal(t, pbkdf2v2Algo.Specification, pbkdf2Algo.Specification)
})
for a, b := range aliasAlgorithmNames {
t.Run(a+"="+b, func(t *testing.T) {
aConfig, aAlgo := SetDefaultPasswordHashAlgorithm(a)
bConfig, bAlgo := SetDefaultPasswordHashAlgorithm(b)
assert.Equal(t, bConfig, aConfig)
assert.Equal(t, aAlgo.Specification, bAlgo.Specification)
})
}
t.Run("pbkdf2_v2 is the default when default password hash algorithm is empty", func(t *testing.T) {
emptyConfig, emptyAlgo := SetDefaultPasswordHashAlgorithm("")
pbkdf2v2Config, pbkdf2v2Algo := SetDefaultPasswordHashAlgorithm("pbkdf2_v2")
assert.Equal(t, pbkdf2v2Config, emptyConfig)
assert.Equal(t, pbkdf2v2Algo.Specification, emptyAlgo.Specification)
})
}
+136
View File
@@ -0,0 +1,136 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package password
import (
"bytes"
"context"
"crypto/rand"
"errors"
"html/template"
"math/big"
"strings"
"sync"
"gitea.dev/modules/setting"
"gitea.dev/modules/translation"
)
var (
ErrComplexity = errors.New("password not complex enough")
ErrMinLength = errors.New("password not long enough")
)
// complexity contains information about a particular kind of password complexity
type complexity struct {
ValidChars string
TrNameOne string
}
var (
matchComplexityOnce sync.Once
validChars string
requiredList []complexity
charComplexities = map[string]complexity{
"lower": {
`abcdefghijklmnopqrstuvwxyz`,
"form.password_lowercase_one",
},
"upper": {
`ABCDEFGHIJKLMNOPQRSTUVWXYZ`,
"form.password_uppercase_one",
},
"digit": {
`0123456789`,
"form.password_digit_one",
},
"spec": {
` !"#$%&'()*+,-./:;<=>?@[\]^_{|}~` + "`",
"form.password_special_one",
},
}
)
// NewComplexity for preparation
func NewComplexity() {
matchComplexityOnce.Do(func() {
setupComplexity(setting.PasswordComplexity)
})
}
func setupComplexity(values []string) {
if len(values) != 1 || values[0] != "off" {
for _, val := range values {
if complexity, ok := charComplexities[val]; ok {
validChars += complexity.ValidChars
requiredList = append(requiredList, complexity)
}
}
if len(requiredList) == 0 {
// No valid character classes found; use all classes as default
for _, complexity := range charComplexities {
validChars += complexity.ValidChars
requiredList = append(requiredList, complexity)
}
}
}
if validChars == "" {
// No complexities to check; provide a sensible default for password generation
validChars = charComplexities["lower"].ValidChars + charComplexities["upper"].ValidChars + charComplexities["digit"].ValidChars
}
}
// IsComplexEnough return True if password meets complexity settings
func IsComplexEnough(pwd string) bool {
NewComplexity()
if len(validChars) > 0 {
for _, req := range requiredList {
if !strings.ContainsAny(req.ValidChars, pwd) {
return false
}
}
}
return true
}
// Generate a random password
func Generate(n int) (string, error) {
NewComplexity()
buffer := make([]byte, n)
maxInt := big.NewInt(int64(len(validChars)))
for {
for j := range n {
rnd, err := rand.Int(rand.Reader, maxInt)
if err != nil {
return "", err
}
buffer[j] = validChars[rnd.Int64()]
}
if err := IsPwned(context.Background(), string(buffer)); err != nil {
if errors.Is(err, ErrIsPwned) {
continue
}
return "", err
}
if IsComplexEnough(string(buffer)) && string(buffer[0]) != " " && string(buffer[n-1]) != " " {
return string(buffer), nil
}
}
}
// BuildComplexityError builds the error message when password complexity checks fail
func BuildComplexityError(locale translation.Locale) template.HTML {
var buffer bytes.Buffer
buffer.WriteString(locale.TrString("form.password_complexity"))
buffer.WriteString("<ul>")
for _, c := range requiredList {
buffer.WriteString("<li>")
buffer.WriteString(locale.TrString(c.TrNameOne))
buffer.WriteString("</li>")
}
buffer.WriteString("</ul>")
return template.HTML(buffer.String())
}
+76
View File
@@ -0,0 +1,76 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package password
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestComplexity_IsComplexEnough(t *testing.T) {
matchComplexityOnce.Do(func() {})
testlist := []struct {
complexity []string
truevalues []string
falsevalues []string
}{
{[]string{"off"}, []string{"1", "-", "a", "A", "ñ", "日本語"}, []string{}},
{[]string{"lower"}, []string{"abc", "abc!"}, []string{"ABC", "123", "=!$", ""}},
{[]string{"upper"}, []string{"ABC"}, []string{"abc", "123", "=!$", "abc!", ""}},
{[]string{"digit"}, []string{"123"}, []string{"abc", "ABC", "=!$", "abc!", ""}},
{[]string{"spec"}, []string{"=!$", "abc!"}, []string{"abc", "ABC", "123", ""}},
{[]string{"off"}, []string{"abc", "ABC", "123", "=!$", "abc!", ""}, nil},
{[]string{"lower", "spec"}, []string{"abc!"}, []string{"abc", "ABC", "123", "=!$", "abcABC123", ""}},
{[]string{"lower", "upper", "digit"}, []string{"abcABC123"}, []string{"abc", "ABC", "123", "=!$", "abc!", ""}},
{[]string{""}, []string{"abC=1", "abc!9D"}, []string{"ABC", "123", "=!$", ""}},
}
for _, test := range testlist {
testComplextity(test.complexity)
for _, val := range test.truevalues {
assert.True(t, IsComplexEnough(val))
}
for _, val := range test.falsevalues {
assert.False(t, IsComplexEnough(val))
}
}
// Remove settings for other tests
testComplextity([]string{"off"})
}
func TestComplexity_Generate(t *testing.T) {
matchComplexityOnce.Do(func() {})
const maxCount = 50
const pwdLen = 50
test := func(t *testing.T, modes []string) {
testComplextity(modes)
for range maxCount {
pwd, err := Generate(pwdLen)
assert.NoError(t, err)
assert.Len(t, pwd, pwdLen)
assert.True(t, IsComplexEnough(pwd), "Failed complexities with modes %+v for generated: %s", modes, pwd)
}
}
test(t, []string{"lower"})
test(t, []string{"upper"})
test(t, []string{"lower", "upper", "spec"})
test(t, []string{"off"})
test(t, []string{""})
// Remove settings for other tests
testComplextity([]string{"off"})
}
func testComplextity(values []string) {
// Cleanup previous values
validChars = ""
requiredList = make([]complexity, 0, len(values))
setupComplexity(values)
}
+52
View File
@@ -0,0 +1,52 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package password
import (
"context"
"errors"
"fmt"
"gitea.dev/modules/auth/password/pwn"
"gitea.dev/modules/setting"
)
var ErrIsPwned = errors.New("password has been pwned")
type ErrIsPwnedRequest struct {
err error
}
func IsErrIsPwnedRequest(err error) bool {
_, ok := err.(ErrIsPwnedRequest)
return ok
}
func (err ErrIsPwnedRequest) Error() string {
return fmt.Sprintf("using Have-I-Been-Pwned service failed: %v", err.err)
}
func (err ErrIsPwnedRequest) Unwrap() error {
return err.err
}
// IsPwned checks whether a password has been pwned
// If a password has not been pwned, no error is returned.
func IsPwned(ctx context.Context, password string) error {
if !setting.PasswordCheckPwn {
return nil
}
client := pwn.New(pwn.WithContext(ctx))
count, err := client.CheckPassword(password, true)
if err != nil {
return ErrIsPwnedRequest{err}
}
if count > 0 {
return ErrIsPwned
}
return nil
}
+118
View File
@@ -0,0 +1,118 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package pwn
import (
"context"
"crypto/sha1"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"gitea.dev/modules/setting"
)
const passwordURL = "https://api.pwnedpasswords.com/range/"
// ErrEmptyPassword is an empty password error
var ErrEmptyPassword = errors.New("password cannot be empty")
// Client is a HaveIBeenPwned client
type Client struct {
ctx context.Context
http *http.Client
}
// New returns a new HaveIBeenPwned Client
func New(options ...ClientOption) *Client {
client := &Client{
ctx: context.Background(),
http: http.DefaultClient,
}
for _, opt := range options {
opt(client)
}
return client
}
// ClientOption is a way to modify a new Client
type ClientOption func(*Client)
// WithHTTP will set the http.Client of a Client
func WithHTTP(httpClient *http.Client) func(pwnClient *Client) {
return func(pwnClient *Client) {
pwnClient.http = httpClient
}
}
// WithContext will set the context.Context of a Client
func WithContext(ctx context.Context) func(pwnClient *Client) {
return func(pwnClient *Client) {
pwnClient.ctx = ctx
}
}
func newRequest(ctx context.Context, method, url string, body io.ReadCloser) (*http.Request, error) {
req, err := http.NewRequestWithContext(ctx, method, url, body)
if err != nil {
return nil, err
}
req.Header.Add("User-Agent", "Gitea "+setting.AppVer)
return req, nil
}
// CheckPassword returns the number of times a password has been compromised
// Adding padding will make requests more secure, however is also slower
// because artificial responses will be added to the response
// For more information, see https://www.troyhunt.com/enhancing-pwned-passwords-privacy-with-padding/
func (c *Client) CheckPassword(pw string, padding bool) (int64, error) {
if pw == "" {
return -1, ErrEmptyPassword
}
sha := sha1.New()
sha.Write([]byte(pw))
enc := hex.EncodeToString(sha.Sum(nil))
prefix, suffix := enc[:5], enc[5:]
req, err := newRequest(c.ctx, http.MethodGet, fmt.Sprintf("%s%s", passwordURL, prefix), nil)
if err != nil {
return -1, nil
}
if padding {
req.Header.Add("Add-Padding", "true")
}
resp, err := c.http.Do(req)
if err != nil {
return -1, err
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return -1, err
}
defer resp.Body.Close()
for pair := range strings.SplitSeq(string(body), "\n") {
parts := strings.Split(pair, ":")
if len(parts) != 2 {
continue
}
if strings.EqualFold(suffix, parts[0]) {
count, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64)
if err != nil {
return -1, err
}
return count, nil
}
}
return 0, nil
}
+61
View File
@@ -0,0 +1,61 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package pwn
import (
"errors"
"io"
"net/http"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
type mockTransport struct{}
func (mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
if req.URL.Host != "api.pwnedpasswords.com" {
return nil, errors.New("unsupported host")
}
respMap := map[string]string{
"/range/5c1d8": "EAF2F254732680E8AC339B84F3266ECCBB5:1\r\nFC446EB88938834178CB9322C1EE273C2A7:2",
"/range/ba189": "FD4CB34F0378BCB15D23F6FFD28F0775C9E:3\r\nFDF342FCD8C3611DAE4D76E8A992A3E4169:4",
"/range/a1733": "C4CE0F1F0062B27B9E2F41AF0C08218017C:1\r\nFC446EB88938834178CB9322C1EE273C2A7:2\r\nFE81480327C992FE62065A827429DD1318B:0",
"/range/5617b": "FD4CB34F0378BCB15D23F6FFD28F0775C9E:3\r\nFDF342FCD8C3611DAE4D76E8A992A3E4169:4\r\nFE81480327C992FE62065A827429DD1318B:0",
"/range/79082": "FDF342FCD8C3611DAE4D76E8A992A3E4169:4\r\nFE81480327C992FE62065A827429DD1318B:0\r\nAFEF386F56EB0B4BE314E07696E5E6E6536:0",
}
if resp, ok := respMap[req.URL.Path]; ok {
return &http.Response{Request: req, Body: io.NopCloser(strings.NewReader(resp))}, nil
}
return nil, errors.New("unsupported path")
}
func TestPassword(t *testing.T) {
client := New(WithHTTP(&http.Client{Transport: mockTransport{}}))
count, err := client.CheckPassword("", false)
assert.ErrorIs(t, err, ErrEmptyPassword, "blank input should return ErrEmptyPassword")
assert.EqualValues(t, -1, count)
count, err = client.CheckPassword("pwned", false)
assert.NoError(t, err)
assert.EqualValues(t, 1, count)
count, err = client.CheckPassword("notpwned", false)
assert.NoError(t, err)
assert.EqualValues(t, 0, count)
count, err = client.CheckPassword("paddedpwned", true)
assert.NoError(t, err)
assert.EqualValues(t, 1, count)
count, err = client.CheckPassword("paddednotpwned", true)
assert.NoError(t, err)
assert.EqualValues(t, 0, count)
count, err = client.CheckPassword("paddednotpwnedzero", true)
assert.NoError(t, err)
assert.EqualValues(t, 0, count)
}
+80
View File
@@ -0,0 +1,80 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package webauthn
import (
"context"
"encoding/binary"
"encoding/gob"
"gitea.dev/models/auth"
user_model "gitea.dev/models/user"
"gitea.dev/modules/setting"
"gitea.dev/modules/util"
"github.com/go-webauthn/webauthn/protocol"
"github.com/go-webauthn/webauthn/webauthn"
)
// WebAuthn represents the global WebAuthn instance
var WebAuthn *webauthn.WebAuthn
// Init initializes the WebAuthn instance from the config.
func Init() {
gob.Register(&webauthn.SessionData{}) // TODO: CHI-SESSION-GOB-REGISTER.
appURL, _ := protocol.FullyQualifiedOrigin(setting.AppURL)
WebAuthn = &webauthn.WebAuthn{
Config: &webauthn.Config{
RPDisplayName: setting.AppName,
RPID: setting.Domain,
RPOrigins: []string{appURL},
AuthenticatorSelection: protocol.AuthenticatorSelection{
UserVerification: protocol.VerificationDiscouraged,
},
AttestationPreference: protocol.PreferDirectAttestation,
},
}
}
// user represents an implementation of webauthn.User based on User model
type user struct {
ctx context.Context
User *user_model.User
defaultAuthFlags protocol.AuthenticatorFlags
}
var _ webauthn.User = (*user)(nil)
func NewWebAuthnUser(ctx context.Context, u *user_model.User, defaultAuthFlags ...protocol.AuthenticatorFlags) webauthn.User {
return &user{ctx: ctx, User: u, defaultAuthFlags: util.OptionalArg(defaultAuthFlags)}
}
// WebAuthnID implements the webauthn.User interface
func (u *user) WebAuthnID() []byte {
id := make([]byte, 8)
binary.PutVarint(id, u.User.ID)
return id
}
// WebAuthnName implements the webauthn.User interface
func (u *user) WebAuthnName() string {
return util.IfZero(u.User.LoginName, u.User.Name)
}
// WebAuthnDisplayName implements the webauthn.User interface
func (u *user) WebAuthnDisplayName() string {
return u.User.DisplayName()
}
// WebAuthnCredentials implements the webauthn.User interface
func (u *user) WebAuthnCredentials() []webauthn.Credential {
dbCreds, err := auth.GetWebAuthnCredentialsByUID(u.ctx, u.User.ID)
if err != nil {
return nil
}
return dbCreds.ToCredentials(u.defaultAuthFlags)
}
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package webauthn
import (
"testing"
"gitea.dev/modules/setting"
"github.com/stretchr/testify/assert"
)
func TestInit(t *testing.T) {
setting.Domain = "domain"
setting.AppName = "AppName"
setting.AppURL = "https://domain/"
rpOrigin := []string{"https://domain"}
Init()
assert.Equal(t, setting.Domain, WebAuthn.Config.RPID)
assert.Equal(t, setting.AppName, WebAuthn.Config.RPDisplayName)
assert.Equal(t, rpOrigin, WebAuthn.Config.RPOrigins)
}
+135
View File
@@ -0,0 +1,135 @@
// Copyright 2014 The Gogs Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package avatar
import (
"bytes"
"errors"
"fmt"
"image"
"image/color"
"image/png"
"gitea.dev/modules/avatar/identicon"
"gitea.dev/modules/setting"
_ "golang.org/x/image/webp" // for processing webp images
_ "image/gif" // for processing gif images
_ "image/jpeg" // for processing jpeg images
"golang.org/x/image/draw"
)
// DefaultAvatarSize is the target CSS pixel size for avatar generation. It is
// multiplied by setting.Avatar.RenderedSizeFactor and the resulting size is the
// usual size of avatar image saved on server, unless the original file is smaller
// than the size after resizing.
const DefaultAvatarSize = 256
// RandomImageWithSize generates and returns a random avatar image unique to input data
// in custom size (height and width).
func RandomImageWithSize(size int, data []byte) image.Image {
// we use white as background, and use dark colors to draw blocks
imgMaker := identicon.New(size, color.White, identicon.DarkColors)
return imgMaker.Make(data)
}
// RandomImageDefaultSize generates and returns a random avatar image unique to input data
// in default size (height and width).
func RandomImageDefaultSize(data []byte) image.Image {
return RandomImageWithSize(DefaultAvatarSize*setting.Avatar.RenderedSizeFactor, data)
}
// processAvatarImage process the avatar image data, crop and resize it if necessary.
// the returned data could be the original image if no processing is needed.
func processAvatarImage(data []byte, maxOriginSize int64) ([]byte, error) {
imgCfg, imgType, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("image.DecodeConfig: %w", err)
}
// for safety, only accept known types explicitly
if imgType != "png" && imgType != "jpeg" && imgType != "gif" && imgType != "webp" {
return nil, errors.New("unsupported avatar image type")
}
// do not process image which is too large, it would consume too much memory
if imgCfg.Width > setting.Avatar.MaxWidth {
return nil, fmt.Errorf("image width is too large: %d > %d", imgCfg.Width, setting.Avatar.MaxWidth)
}
if imgCfg.Height > setting.Avatar.MaxHeight {
return nil, fmt.Errorf("image height is too large: %d > %d", imgCfg.Height, setting.Avatar.MaxHeight)
}
// If the origin is small enough, just use it, then APNG could be supported,
// otherwise, if the image is processed later, APNG loses animation.
// And one more thing, webp is not fully supported, for animated webp, image.DecodeConfig works but Decode fails.
// So for animated webp, if the uploaded file is smaller than maxOriginSize, it will be used, if it's larger, there will be an error.
if len(data) < int(maxOriginSize) {
return data, nil
}
img, _, err := image.Decode(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("image.Decode: %w", err)
}
// try to crop and resize the origin image if necessary
img = cropSquare(img)
targetSize := DefaultAvatarSize * setting.Avatar.RenderedSizeFactor
img = scale(img, targetSize, targetSize, draw.BiLinear)
// try to encode the cropped/resized image to png
bs := bytes.Buffer{}
if err = png.Encode(&bs, img); err != nil {
return nil, err
}
resized := bs.Bytes()
// usually the png compression is not good enough, use the original image (no cropping/resizing) if the origin is smaller
if len(data) <= len(resized) {
return data, nil
}
return resized, nil
}
// ProcessAvatarImage process the avatar image data, crop and resize it if necessary.
// the returned data could be the original image if no processing is needed.
func ProcessAvatarImage(data []byte) ([]byte, error) {
return processAvatarImage(data, setting.Avatar.MaxOriginSize)
}
// scale resizes the image to width x height using the given scaler.
func scale(src image.Image, width, height int, scale draw.Scaler) image.Image {
rect := image.Rect(0, 0, width, height)
dst := image.NewRGBA(rect)
scale.Scale(dst, rect, src, src.Bounds(), draw.Over, nil)
return dst
}
// cropSquare crops the largest square image from the center of the image.
// If the image is already square, it is returned unchanged.
func cropSquare(src image.Image) image.Image {
bounds := src.Bounds()
if bounds.Dx() == bounds.Dy() {
return src
}
var rect image.Rectangle
if bounds.Dx() > bounds.Dy() {
// width > height
size := bounds.Dy()
rect = image.Rect((bounds.Dx()-size)/2, 0, (bounds.Dx()+size)/2, size)
} else {
// width < height
size := bounds.Dx()
rect = image.Rect(0, (bounds.Dy()-size)/2, size, (bounds.Dy()+size)/2)
}
dst := image.NewRGBA(rect)
draw.Draw(dst, rect, src, rect.Min, draw.Src)
return dst
}
+138
View File
@@ -0,0 +1,138 @@
// Copyright 2016 The Gogs Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package avatar
import (
"bytes"
"image"
"image/png"
"os"
"testing"
"gitea.dev/modules/setting"
"github.com/stretchr/testify/assert"
)
func Test_ProcessAvatarPNG(t *testing.T) {
setting.Avatar.MaxWidth = 4096
setting.Avatar.MaxHeight = 4096
data, err := os.ReadFile("testdata/avatar.png")
assert.NoError(t, err)
_, err = processAvatarImage(data, 262144)
assert.NoError(t, err)
}
func Test_ProcessAvatarJPEG(t *testing.T) {
setting.Avatar.MaxWidth = 4096
setting.Avatar.MaxHeight = 4096
data, err := os.ReadFile("testdata/avatar.jpeg")
assert.NoError(t, err)
_, err = processAvatarImage(data, 262144)
assert.NoError(t, err)
}
func Test_ProcessAvatarInvalidData(t *testing.T) {
setting.Avatar.MaxWidth = 5
setting.Avatar.MaxHeight = 5
_, err := processAvatarImage([]byte{}, 12800)
assert.EqualError(t, err, "image.DecodeConfig: image: unknown format")
}
func Test_ProcessAvatarInvalidImageSize(t *testing.T) {
setting.Avatar.MaxWidth = 5
setting.Avatar.MaxHeight = 5
data, err := os.ReadFile("testdata/avatar.png")
assert.NoError(t, err)
_, err = processAvatarImage(data, 12800)
assert.EqualError(t, err, "image width is too large: 10 > 5")
}
func Test_ProcessAvatarImage(t *testing.T) {
setting.Avatar.MaxWidth = 4096
setting.Avatar.MaxHeight = 4096
scaledSize := DefaultAvatarSize * setting.Avatar.RenderedSizeFactor
newImgData := func(size int, optHeight ...int) []byte {
width := size
height := size
if len(optHeight) == 1 {
height = optHeight[0]
}
img := image.NewRGBA(image.Rect(0, 0, width, height))
bs := bytes.Buffer{}
err := png.Encode(&bs, img)
assert.NoError(t, err)
return bs.Bytes()
}
// if origin image canvas is too large, crop and resize it
origin := newImgData(500, 600)
result, err := processAvatarImage(origin, 0)
assert.NoError(t, err)
assert.NotEqual(t, origin, result)
decoded, err := png.Decode(bytes.NewReader(result))
assert.NoError(t, err)
assert.Equal(t, scaledSize, decoded.Bounds().Max.X)
assert.Equal(t, scaledSize, decoded.Bounds().Max.Y)
// if origin image is smaller than the default size, use the origin image
origin = newImgData(1)
result, err = processAvatarImage(origin, 0)
assert.NoError(t, err)
assert.Equal(t, origin, result)
// use the origin image if the origin is smaller
origin = newImgData(scaledSize + 100)
result, err = processAvatarImage(origin, 0)
assert.NoError(t, err)
assert.Less(t, len(result), len(origin))
// still use the origin image if the origin doesn't exceed the max-origin-size
origin = newImgData(scaledSize + 100)
result, err = processAvatarImage(origin, 262144)
assert.NoError(t, err)
assert.Equal(t, origin, result)
// allow to use known image format (eg: webp) if it is small enough
origin, err = os.ReadFile("testdata/animated.webp")
assert.NoError(t, err)
result, err = processAvatarImage(origin, 262144)
assert.NoError(t, err)
assert.Equal(t, origin, result)
// do not support unknown image formats, eg: SVG may contain embedded JS
origin = []byte("<svg></svg>")
_, err = processAvatarImage(origin, 262144)
assert.ErrorContains(t, err, "image: unknown format")
// make sure the canvas size limit works
setting.Avatar.MaxWidth = 5
setting.Avatar.MaxHeight = 5
origin = newImgData(10)
_, err = processAvatarImage(origin, 262144)
assert.ErrorContains(t, err, "image width is too large: 10 > 5")
}
func BenchmarkRandomImage(b *testing.B) {
b.Run("size-48", func(b *testing.B) {
for b.Loop() {
// BenchmarkRandomImage/size-48-12 49549 22899 ns/op
RandomImageWithSize(48, []byte("test-content"))
}
})
b.Run("size-96", func(b *testing.B) {
for b.Loop() {
// BenchmarkRandomImage/size-96-12 13816 88187 ns/op
RandomImageWithSize(96, []byte("test-content"))
}
})
}
+28
View File
@@ -0,0 +1,28 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package avatar
import (
"crypto/sha256"
"encoding/hex"
"strconv"
)
// HashAvatar will generate a unique string, which ensures that when there's a
// different unique ID while the data is the same, it will generate a different
// output. It will generate the output according to:
// HEX(HASH(uniqueID || - || data))
// The hash being used is SHA256.
// The sole purpose of the unique ID is to generate a distinct hash Such that
// two unique IDs with the same data will have a different hash output.
// The "-" byte is important to ensure that data cannot be modified such that
// the first byte is a number, which could lead to a "collision" with the hash
// of another unique ID.
func HashAvatar(uniqueID int64, data []byte) string {
h := sha256.New()
h.Write([]byte(strconv.FormatInt(uniqueID, 10)))
h.Write([]byte{'-'})
h.Write(data)
return hex.EncodeToString(h.Sum(nil))
}
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package avatar_test
import (
"bytes"
"image"
"image/png"
"testing"
"gitea.dev/modules/avatar"
"github.com/stretchr/testify/assert"
)
func Test_HashAvatar(t *testing.T) {
myImage := image.NewRGBA(image.Rect(0, 0, 32, 32))
var buff bytes.Buffer
png.Encode(&buff, myImage)
assert.Equal(t, "9ddb5bac41d57e72aa876321d0c09d71090c05f94bc625303801be2f3240d2cb", avatar.HashAvatar(1, buff.Bytes()))
assert.Equal(t, "9a5d44e5d637b9582a976676e8f3de1dccd877c2fe3e66ca3fab1629f2f47609", avatar.HashAvatar(8, buff.Bytes()))
assert.Equal(t, "ed7399158672088770de6f5211ce15528ebd675e92fc4fc060c025f4b2794ccb", avatar.HashAvatar(1024, buff.Bytes()))
assert.Equal(t, "161178642c7d59eb25a61dddced5e6b66eae1c70880d5f148b1b497b767e72d9", avatar.HashAvatar(1024, []byte{}))
}
+717
View File
@@ -0,0 +1,717 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
// Copied and modified from https://github.com/issue9/identicon/ (MIT License)
package identicon
import "image"
var (
// the blocks can appear in center, these blocks can be more beautiful
centerBlocks = []blockFunc{b0, b1, b2, b3, b19, b26, b27}
// all blocks
blocks = []blockFunc{b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15, b16, b17, b18, b19, b20, b21, b22, b23, b24, b25, b26, b27}
)
type blockFunc func(img *image.Paletted, x, y, size, angle int)
// draw a polygon by points, and the polygon is rotated by angle.
func drawBlock(img *image.Paletted, x, y, size, angle int, points []int) {
if angle != 0 {
m := size / 2
rotate(points, m, m, angle)
}
for i := range size {
for j := range size {
if pointInPolygon(i, j, points) {
img.SetColorIndex(x+i, y+j, 1)
}
}
}
}
// blank
//
// --------
// | |
// | |
// | |
// --------
func b0(img *image.Paletted, x, y, size, angle int) {}
// full-filled
//
// --------
// |######|
// |######|
// |######|
// --------
func b1(img *image.Paletted, x, y, size, angle int) {
for i := x; i < x+size; i++ {
for j := y; j < y+size; j++ {
img.SetColorIndex(i, j, 1)
}
}
}
// a small block
//
// ----------
// | |
// | #### |
// | #### |
// | |
// ----------
func b2(img *image.Paletted, x, y, size, angle int) {
l := size / 4
x += l
y += l
for i := x; i < x+2*l; i++ {
for j := y; j < y+2*l; j++ {
img.SetColorIndex(i, j, 1)
}
}
}
// diamond
//
// ---------
// | # |
// | ### |
// | ##### |
// |#######|
// | ##### |
// | ### |
// | # |
// ---------
func b3(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, 0, []int{
m, 0,
size, m,
m, size,
0, m,
m, 0,
})
}
// b4
//
// -------
// |#####|
// |#### |
// |### |
// |## |
// |# |
// |------
func b4(img *image.Paletted, x, y, size, angle int) {
drawBlock(img, x, y, size, angle, []int{
0, 0,
size, 0,
0, size,
0, 0,
})
}
// b5
//
// ---------
// | # |
// | ### |
// | ##### |
// |#######|
func b5(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
m, 0,
size, size,
0, size,
m, 0,
})
}
// b6
//
// --------
// |### |
// |### |
// |### |
// --------
func b6(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
0, 0,
m, 0,
m, size,
0, size,
0, 0,
})
}
// b7 italic cone
//
// ---------
// | # |
// | ## |
// | #####|
// | ####|
// |--------
func b7(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
0, 0,
size, m,
size, size,
m, size,
0, 0,
})
}
// b8 three small triangles
//
// -----------
// | # |
// | ### |
// | ##### |
// | # # |
// | ### ### |
// |#########|
// -----------
func b8(img *image.Paletted, x, y, size, angle int) {
m := size / 2
mm := m / 2
// top
drawBlock(img, x, y, size, angle, []int{
m, 0,
3 * mm, m,
mm, m,
m, 0,
})
// bottom left
drawBlock(img, x, y, size, angle, []int{
mm, m,
m, size,
0, size,
mm, m,
})
// bottom right
drawBlock(img, x, y, size, angle, []int{
3 * mm, m,
size, size,
m, size,
3 * mm, m,
})
}
// b9 italic triangle
//
// ---------
// |# |
// | #### |
// | #####|
// | #### |
// | # |
// ---------
func b9(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
0, 0,
size, m,
m, size,
0, 0,
})
}
// b10
//
// ----------
// | ####|
// | ### |
// | ## |
// | # |
// |#### |
// |### |
// |## |
// |# |
// ----------
func b10(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
m, 0,
size, 0,
m, m,
m, 0,
})
drawBlock(img, x, y, size, angle, []int{
0, m,
m, m,
0, size,
0, m,
})
}
// b11
//
// ----------
// |#### |
// |#### |
// |#### |
// | |
// | |
// ----------
func b11(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
0, 0,
m, 0,
m, m,
0, m,
0, 0,
})
}
// b12
//
// -----------
// | |
// | |
// |#########|
// | ##### |
// | # |
// -----------
func b12(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
0, m,
size, m,
m, size,
0, m,
})
}
// b13
//
// -----------
// | |
// | |
// | # |
// | ##### |
// |#########|
// -----------
func b13(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
m, m,
size, size,
0, size,
m, m,
})
}
// b14
//
// ---------
// | # |
// | ### |
// |#### |
// | |
// | |
// ---------
func b14(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
m, 0,
m, m,
0, m,
m, 0,
})
}
// b15
//
// ----------
// |##### |
// |### |
// |# |
// | |
// | |
// ----------
func b15(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
0, 0,
m, 0,
0, m,
0, 0,
})
}
// b16
//
// ---------
// | # |
// | ##### |
// |#######|
// | # |
// | ##### |
// |#######|
// ---------
func b16(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
m, 0,
size, m,
0, m,
m, 0,
})
drawBlock(img, x, y, size, angle, []int{
m, m,
size, size,
0, size,
m, m,
})
}
// b17
//
// ----------
// |##### |
// |### |
// |# |
// | ##|
// | ##|
// ----------
func b17(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
0, 0,
m, 0,
0, m,
0, 0,
})
quarter := size / 4
drawBlock(img, x, y, size, angle, []int{
size - quarter, size - quarter,
size, size - quarter,
size, size,
size - quarter, size,
size - quarter, size - quarter,
})
}
// b18
//
// ----------
// |##### |
// |#### |
// |### |
// |## |
// |# |
// ----------
func b18(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
0, 0,
m, 0,
0, size,
0, 0,
})
}
// b19
//
// ----------
// |########|
// |### ###|
// |# #|
// |### ###|
// |########|
// ----------
func b19(img *image.Paletted, x, y, size, angle int) {
m := size / 2
drawBlock(img, x, y, size, angle, []int{
0, 0,
m, 0,
0, m,
0, 0,
})
drawBlock(img, x, y, size, angle, []int{
m, 0,
size, 0,
size, m,
m, 0,
})
drawBlock(img, x, y, size, angle, []int{
size, m,
size, size,
m, size,
size, m,
})
drawBlock(img, x, y, size, angle, []int{
0, m,
m, size,
0, size,
0, m,
})
}
// b20
//
// ----------
// | ## |
// |### |
// |## |
// |## |
// |# |
// ----------
func b20(img *image.Paletted, x, y, size, angle int) {
m := size / 2
q := size / 4
drawBlock(img, x, y, size, angle, []int{
q, 0,
0, size,
0, m,
q, 0,
})
}
// b21
//
// ----------
// | #### |
// |## #####|
// |## ##|
// |## |
// |# |
// ----------
func b21(img *image.Paletted, x, y, size, angle int) {
m := size / 2
q := size / 4
drawBlock(img, x, y, size, angle, []int{
q, 0,
0, size,
0, m,
q, 0,
})
drawBlock(img, x, y, size, angle, []int{
q, 0,
size, q,
size, m,
q, 0,
})
}
// b22
//
// ----------
// | #### |
// |## ### |
// |## ##|
// |## ##|
// |# #|
// ----------
func b22(img *image.Paletted, x, y, size, angle int) {
m := size / 2
q := size / 4
drawBlock(img, x, y, size, angle, []int{
q, 0,
0, size,
0, m,
q, 0,
})
drawBlock(img, x, y, size, angle, []int{
q, 0,
size, q,
size, size,
q, 0,
})
}
// b23
//
// ----------
// | #######|
// |### #|
// |## |
// |## |
// |# |
// ----------
func b23(img *image.Paletted, x, y, size, angle int) {
m := size / 2
q := size / 4
drawBlock(img, x, y, size, angle, []int{
q, 0,
0, size,
0, m,
q, 0,
})
drawBlock(img, x, y, size, angle, []int{
q, 0,
size, 0,
size, q,
q, 0,
})
}
// b24
//
// ----------
// | ## ###|
// |### ###|
// |## ## |
// |## ## |
// |# # |
// ----------
func b24(img *image.Paletted, x, y, size, angle int) {
m := size / 2
q := size / 4
drawBlock(img, x, y, size, angle, []int{
q, 0,
0, size,
0, m,
q, 0,
})
drawBlock(img, x, y, size, angle, []int{
m, 0,
size, 0,
m, size,
m, 0,
})
}
// b25
//
// ----------
// |# #|
// |## ###|
// |## ## |
// |###### |
// |#### |
// ----------
func b25(img *image.Paletted, x, y, size, angle int) {
m := size / 2
q := size / 4
drawBlock(img, x, y, size, angle, []int{
0, 0,
0, size,
q, size,
0, 0,
})
drawBlock(img, x, y, size, angle, []int{
0, m,
size, 0,
q, size,
0, m,
})
}
// b26
//
// ----------
// |# #|
// |### ###|
// | #### |
// |### ###|
// |# #|
// ----------
func b26(img *image.Paletted, x, y, size, angle int) {
m := size / 2
q := size / 4
drawBlock(img, x, y, size, angle, []int{
0, 0,
m, q,
q, m,
0, 0,
})
drawBlock(img, x, y, size, angle, []int{
size, 0,
m + q, m,
m, q,
size, 0,
})
drawBlock(img, x, y, size, angle, []int{
size, size,
m, m + q,
q + m, m,
size, size,
})
drawBlock(img, x, y, size, angle, []int{
0, size,
q, m,
m, q + m,
0, size,
})
}
// b27
//
// ----------
// |########|
// |## ###|
// |# #|
// |### ##|
// |########|
// ----------
func b27(img *image.Paletted, x, y, size, angle int) {
m := size / 2
q := size / 4
drawBlock(img, x, y, size, angle, []int{
0, 0,
size, 0,
0, q,
0, 0,
})
drawBlock(img, x, y, size, angle, []int{
q + m, 0,
size, 0,
size, size,
q + m, 0,
})
drawBlock(img, x, y, size, angle, []int{
size, q + m,
size, size,
0, size,
size, q + m,
})
drawBlock(img, x, y, size, angle, []int{
0, size,
0, 0,
q, size,
0, size,
})
}
+134
View File
@@ -0,0 +1,134 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package identicon
import "image/color"
// DarkColors are dark colors for avatar blocks, they come from image/color/palette.WebSafe, and light colors (0xff) are removed
var DarkColors = []color.Color{
color.RGBA{0x00, 0x00, 0x33, 0xff},
color.RGBA{0x00, 0x00, 0x66, 0xff},
color.RGBA{0x00, 0x00, 0x99, 0xff},
color.RGBA{0x00, 0x00, 0xcc, 0xff},
color.RGBA{0x00, 0x33, 0x00, 0xff},
color.RGBA{0x00, 0x33, 0x33, 0xff},
color.RGBA{0x00, 0x33, 0x66, 0xff},
color.RGBA{0x00, 0x33, 0x99, 0xff},
color.RGBA{0x00, 0x33, 0xcc, 0xff},
color.RGBA{0x00, 0x66, 0x00, 0xff},
color.RGBA{0x00, 0x66, 0x33, 0xff},
color.RGBA{0x00, 0x66, 0x66, 0xff},
color.RGBA{0x00, 0x66, 0x99, 0xff},
color.RGBA{0x00, 0x66, 0xcc, 0xff},
color.RGBA{0x00, 0x99, 0x00, 0xff},
color.RGBA{0x00, 0x99, 0x33, 0xff},
color.RGBA{0x00, 0x99, 0x66, 0xff},
color.RGBA{0x00, 0x99, 0x99, 0xff},
color.RGBA{0x00, 0x99, 0xcc, 0xff},
color.RGBA{0x00, 0xcc, 0x00, 0xff},
color.RGBA{0x00, 0xcc, 0x33, 0xff},
color.RGBA{0x00, 0xcc, 0x66, 0xff},
color.RGBA{0x00, 0xcc, 0x99, 0xff},
color.RGBA{0x00, 0xcc, 0xcc, 0xff},
color.RGBA{0x33, 0x00, 0x00, 0xff},
color.RGBA{0x33, 0x00, 0x33, 0xff},
color.RGBA{0x33, 0x00, 0x66, 0xff},
color.RGBA{0x33, 0x00, 0x99, 0xff},
color.RGBA{0x33, 0x00, 0xcc, 0xff},
color.RGBA{0x33, 0x33, 0x00, 0xff},
color.RGBA{0x33, 0x33, 0x33, 0xff},
color.RGBA{0x33, 0x33, 0x66, 0xff},
color.RGBA{0x33, 0x33, 0x99, 0xff},
color.RGBA{0x33, 0x33, 0xcc, 0xff},
color.RGBA{0x33, 0x66, 0x00, 0xff},
color.RGBA{0x33, 0x66, 0x33, 0xff},
color.RGBA{0x33, 0x66, 0x66, 0xff},
color.RGBA{0x33, 0x66, 0x99, 0xff},
color.RGBA{0x33, 0x66, 0xcc, 0xff},
color.RGBA{0x33, 0x99, 0x00, 0xff},
color.RGBA{0x33, 0x99, 0x33, 0xff},
color.RGBA{0x33, 0x99, 0x66, 0xff},
color.RGBA{0x33, 0x99, 0x99, 0xff},
color.RGBA{0x33, 0x99, 0xcc, 0xff},
color.RGBA{0x33, 0xcc, 0x00, 0xff},
color.RGBA{0x33, 0xcc, 0x33, 0xff},
color.RGBA{0x33, 0xcc, 0x66, 0xff},
color.RGBA{0x33, 0xcc, 0x99, 0xff},
color.RGBA{0x33, 0xcc, 0xcc, 0xff},
color.RGBA{0x66, 0x00, 0x00, 0xff},
color.RGBA{0x66, 0x00, 0x33, 0xff},
color.RGBA{0x66, 0x00, 0x66, 0xff},
color.RGBA{0x66, 0x00, 0x99, 0xff},
color.RGBA{0x66, 0x00, 0xcc, 0xff},
color.RGBA{0x66, 0x33, 0x00, 0xff},
color.RGBA{0x66, 0x33, 0x33, 0xff},
color.RGBA{0x66, 0x33, 0x66, 0xff},
color.RGBA{0x66, 0x33, 0x99, 0xff},
color.RGBA{0x66, 0x33, 0xcc, 0xff},
color.RGBA{0x66, 0x66, 0x00, 0xff},
color.RGBA{0x66, 0x66, 0x33, 0xff},
color.RGBA{0x66, 0x66, 0x66, 0xff},
color.RGBA{0x66, 0x66, 0x99, 0xff},
color.RGBA{0x66, 0x66, 0xcc, 0xff},
color.RGBA{0x66, 0x99, 0x00, 0xff},
color.RGBA{0x66, 0x99, 0x33, 0xff},
color.RGBA{0x66, 0x99, 0x66, 0xff},
color.RGBA{0x66, 0x99, 0x99, 0xff},
color.RGBA{0x66, 0x99, 0xcc, 0xff},
color.RGBA{0x66, 0xcc, 0x00, 0xff},
color.RGBA{0x66, 0xcc, 0x33, 0xff},
color.RGBA{0x66, 0xcc, 0x66, 0xff},
color.RGBA{0x66, 0xcc, 0x99, 0xff},
color.RGBA{0x66, 0xcc, 0xcc, 0xff},
color.RGBA{0x99, 0x00, 0x00, 0xff},
color.RGBA{0x99, 0x00, 0x33, 0xff},
color.RGBA{0x99, 0x00, 0x66, 0xff},
color.RGBA{0x99, 0x00, 0x99, 0xff},
color.RGBA{0x99, 0x00, 0xcc, 0xff},
color.RGBA{0x99, 0x33, 0x00, 0xff},
color.RGBA{0x99, 0x33, 0x33, 0xff},
color.RGBA{0x99, 0x33, 0x66, 0xff},
color.RGBA{0x99, 0x33, 0x99, 0xff},
color.RGBA{0x99, 0x33, 0xcc, 0xff},
color.RGBA{0x99, 0x66, 0x00, 0xff},
color.RGBA{0x99, 0x66, 0x33, 0xff},
color.RGBA{0x99, 0x66, 0x66, 0xff},
color.RGBA{0x99, 0x66, 0x99, 0xff},
color.RGBA{0x99, 0x66, 0xcc, 0xff},
color.RGBA{0x99, 0x99, 0x00, 0xff},
color.RGBA{0x99, 0x99, 0x33, 0xff},
color.RGBA{0x99, 0x99, 0x66, 0xff},
color.RGBA{0x99, 0x99, 0x99, 0xff},
color.RGBA{0x99, 0x99, 0xcc, 0xff},
color.RGBA{0x99, 0xcc, 0x00, 0xff},
color.RGBA{0x99, 0xcc, 0x33, 0xff},
color.RGBA{0x99, 0xcc, 0x66, 0xff},
color.RGBA{0x99, 0xcc, 0x99, 0xff},
color.RGBA{0x99, 0xcc, 0xcc, 0xff},
color.RGBA{0xcc, 0x00, 0x00, 0xff},
color.RGBA{0xcc, 0x00, 0x33, 0xff},
color.RGBA{0xcc, 0x00, 0x66, 0xff},
color.RGBA{0xcc, 0x00, 0x99, 0xff},
color.RGBA{0xcc, 0x00, 0xcc, 0xff},
color.RGBA{0xcc, 0x33, 0x00, 0xff},
color.RGBA{0xcc, 0x33, 0x33, 0xff},
color.RGBA{0xcc, 0x33, 0x66, 0xff},
color.RGBA{0xcc, 0x33, 0x99, 0xff},
color.RGBA{0xcc, 0x33, 0xcc, 0xff},
color.RGBA{0xcc, 0x66, 0x00, 0xff},
color.RGBA{0xcc, 0x66, 0x33, 0xff},
color.RGBA{0xcc, 0x66, 0x66, 0xff},
color.RGBA{0xcc, 0x66, 0x99, 0xff},
color.RGBA{0xcc, 0x66, 0xcc, 0xff},
color.RGBA{0xcc, 0x99, 0x00, 0xff},
color.RGBA{0xcc, 0x99, 0x33, 0xff},
color.RGBA{0xcc, 0x99, 0x66, 0xff},
color.RGBA{0xcc, 0x99, 0x99, 0xff},
color.RGBA{0xcc, 0x99, 0xcc, 0xff},
color.RGBA{0xcc, 0xcc, 0x00, 0xff},
color.RGBA{0xcc, 0xcc, 0x33, 0xff},
color.RGBA{0xcc, 0xcc, 0x66, 0xff},
color.RGBA{0xcc, 0xcc, 0x99, 0xff},
color.RGBA{0xcc, 0xcc, 0xcc, 0xff},
}
+134
View File
@@ -0,0 +1,134 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
// Copied and modified from https://github.com/issue9/identicon/ (MIT License)
// Generate pseudo-random avatars by IP, E-mail, etc.
package identicon
import (
"crypto/sha256"
"image"
"image/color"
)
const (
minImageSize = 16
maxImageSize = 2048
)
// Identicon is used to generate pseudo-random avatars
type Identicon struct {
foreColors []color.Color
backColor color.Color
size int
rect image.Rectangle
}
// New returns an Identicon struct.
// Only one foreground color will be picked randomly for one image.
func New(size int, backColor color.Color, foreColors []color.Color) *Identicon {
size = max(size, minImageSize)
size = min(size, maxImageSize)
return &Identicon{
foreColors: foreColors,
backColor: backColor,
size: size,
rect: image.Rect(0, 0, size, size),
}
}
// Make generates an avatar by data
func (i *Identicon) Make(data []byte) image.Image {
h := sha256.New()
h.Write(data)
sum := h.Sum(nil)
b1 := int(sum[0]+sum[1]+sum[2]) % len(blocks)
b2 := int(sum[3]+sum[4]+sum[5]) % len(blocks)
c := int(sum[6]+sum[7]+sum[8]) % len(centerBlocks)
b1Angle := int(sum[9]+sum[10]) % 4
b2Angle := int(sum[11]+sum[12]) % 4
foreColor := int(sum[11]+sum[12]+sum[15]) % len(i.foreColors)
return i.render(c, b1, b2, b1Angle, b2Angle, foreColor)
}
func (i *Identicon) render(c, b1, b2, b1Angle, b2Angle, foreColor int) image.Image {
p := image.NewPaletted(i.rect, []color.Color{i.backColor, i.foreColors[foreColor]})
drawBlocks(p, i.size, centerBlocks[c], blocks[b1], blocks[b2], b1Angle, b2Angle)
return p
}
/*
# Algorithm
Origin: An image is split into 9 areas
```
-------------
| 1 | 2 | 3 |
-------------
| 4 | 5 | 6 |
-------------
| 7 | 8 | 9 |
-------------
```
Area 1/3/9/7 use a 90-degree rotating pattern.
Area 1/3/9/7 use another 90-degree rotating pattern.
Area 5 uses a random pattern.
The Patched Fix: make the image left-right mirrored to get rid of something like "swastika"
*/
// draw blocks to the paletted
// c: the block drawer for the center block
// b1,b2: the block drawers for other blocks (around the center block)
// b1Angle,b2Angle: the angle for the rotation of b1/b2
func drawBlocks(p *image.Paletted, size int, c, b1, b2 blockFunc, b1Angle, b2Angle int) {
nextAngle := func(a int) int {
return (a + 1) % 4
}
padding := (size % 3) / 2 // in cased the size can not be aligned by 3 blocks.
blockSize := size / 3
twoBlockSize := 2 * blockSize
// center
c(p, blockSize+padding, blockSize+padding, blockSize, 0)
// left top (1)
b1(p, 0+padding, 0+padding, blockSize, b1Angle)
// center top (2)
b2(p, blockSize+padding, 0+padding, blockSize, b2Angle)
b1Angle = nextAngle(b1Angle)
b2Angle = nextAngle(b2Angle)
// right top (3)
// b1(p, twoBlockSize+padding, 0+padding, blockSize, b1Angle)
// right middle (6)
// b2(p, twoBlockSize+padding, blockSize+padding, blockSize, b2Angle)
b1Angle = nextAngle(b1Angle)
b2Angle = nextAngle(b2Angle)
// right bottom (9)
// b1(p, twoBlockSize+padding, twoBlockSize+padding, blockSize, b1Angle)
// center bottom (8)
b2(p, blockSize+padding, twoBlockSize+padding, blockSize, b2Angle)
b1Angle = nextAngle(b1Angle)
b2Angle = nextAngle(b2Angle)
// lef bottom (7)
b1(p, 0+padding, twoBlockSize+padding, blockSize, b1Angle)
// left middle (4)
b2(p, 0+padding, blockSize+padding, blockSize, b2Angle)
// then we make it left-right mirror, so we didn't draw 3/6/9 before
for x := 0; x < size/2; x++ {
for y := range size {
p.SetColorIndex(size-x, y, p.ColorIndexAt(x, y))
}
}
}
@@ -0,0 +1,40 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build test_avatar_identicon
package identicon
import (
"image/color"
"image/png"
"os"
"strconv"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGenerate(t *testing.T) {
dir, _ := os.Getwd()
dir = dir + "/testdata"
if st, err := os.Stat(dir); err != nil || !st.IsDir() {
t.Errorf("can not save generated images to %s", dir)
}
backColor := color.White
imgMaker, err := New(64, backColor, DarkColors)
assert.NoError(t, err)
for i := 0; i < 100; i++ {
s := strconv.Itoa(i)
img := imgMaker.Make([]byte(s))
f, err := os.Create(dir + "/" + s + ".png")
if !assert.NoError(t, err) {
continue
}
defer f.Close()
err = png.Encode(f, img)
assert.NoError(t, err)
}
}
+68
View File
@@ -0,0 +1,68 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
// Copied and modified from https://github.com/issue9/identicon/ (MIT License)
package identicon
var (
// cos(0),cos(90),cos(180),cos(270)
cos = []int{1, 0, -1, 0}
// sin(0),sin(90),sin(180),sin(270)
sin = []int{0, 1, 0, -1}
)
// rotate the points by center point (x,y)
// angle: [0,1,2,3] means [090180270] degree
func rotate(points []int, x, y, angle int) {
// the angle is only used internally, and it has been guaranteed to be 0/1/2/3, so we do not check it again
for i := 0; i < len(points); i += 2 {
px, py := points[i]-x, points[i+1]-y
points[i] = px*cos[angle] - py*sin[angle] + x
points[i+1] = px*sin[angle] + py*cos[angle] + y
}
}
// check whether the point is inside the polygon (defined by the points)
// the first and the last point must be the same
func pointInPolygon(x, y int, polygonPoints []int) bool {
if len(polygonPoints) < 8 { // a valid polygon must have more than 2 points
return false
}
// reference: nonzero winding rule, https://en.wikipedia.org/wiki/Nonzero-rule
// split the plane into two by the check point horizontally:
// y>0includes (x>0 && y==0)
// y<0includes (x<0 && y==0)
//
// then scan every point in the polygon.
//
// if current point and previous point are in different planes (eg: curY>0 && prevY<0),
// check the clock-direction from previous point to current point (use check point as origin).
// if the direction is clockwise, then r++, otherwise then r--
// finally, if 2==abs(r), then the check point is inside the polygon
r := 0
prevX, prevY := polygonPoints[0], polygonPoints[1]
prev := (prevY > y) || ((prevX > x) && (prevY == y))
for i := 2; i < len(polygonPoints); i += 2 {
currX, currY := polygonPoints[i], polygonPoints[i+1]
curr := (currY > y) || ((currX > x) && (currY == y))
if curr == prev {
prevX, prevY = currX, currY
continue
}
if mul := (prevX-x)*(currY-y) - (currX-x)*(prevY-y); mul >= 0 {
r++
} else { // mul < 0
r--
}
prevX, prevY = currX, currY
prev = curr
}
return r == 2 || r == -2
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 4.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 521 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 159 B

+129
View File
@@ -0,0 +1,129 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package badge
import (
"strings"
"sync"
"unicode"
actions_model "gitea.dev/models/actions"
)
// The Badge layout: |offset|label|message|
// We use 10x scale to calculate more precisely
// Then scale down to normal size in tmpl file
type Text struct {
text string
width int
x int
}
func (t Text) Text() string {
return t.text
}
func (t Text) Width() int {
return t.width
}
func (t Text) X() int {
return t.x
}
func (t Text) TextLength() int {
return int(float64(t.width-defaultOffset) * 10)
}
type Badge struct {
IDPrefix string
FontFamily string
Color string
FontSize int
Label Text
Message Text
}
func (b Badge) Width() int {
return b.Label.width + b.Message.width
}
// Style follows https://shields.io/badges
const (
StyleFlat = "flat"
StyleFlatSquare = "flat-square"
)
const (
defaultOffset = 10
defaultFontSize = 11
DefaultColor = "#9f9f9f" // Grey
DefaultFontFamily = "DejaVu Sans,Verdana,Geneva,sans-serif"
DefaultStyle = StyleFlat
)
var GlobalVars = sync.OnceValue(func() (ret struct {
StatusColorMap map[actions_model.Status]string
DejaVuGlyphWidthData map[rune]uint8
AllStyles []string
},
) {
ret.StatusColorMap = map[actions_model.Status]string{
actions_model.StatusSuccess: "#4c1", // Green
actions_model.StatusSkipped: "#dfb317", // Yellow
actions_model.StatusUnknown: "#97ca00", // Light Green
actions_model.StatusFailure: "#e05d44", // Red
actions_model.StatusCancelled: "#fe7d37", // Orange
actions_model.StatusWaiting: "#dfb317", // Yellow
actions_model.StatusRunning: "#dfb317", // Yellow
actions_model.StatusBlocked: "#dfb317", // Yellow
}
ret.DejaVuGlyphWidthData = dejaVuGlyphWidthDataFunc()
ret.AllStyles = []string{StyleFlat, StyleFlatSquare}
return ret
})
// GenerateBadge generates badge with given template
func GenerateBadge(label, message, color string) Badge {
lw := calculateTextWidth(label) + defaultOffset
mw := calculateTextWidth(message) + defaultOffset
lx := lw * 5
mx := lw*10 + mw*5 - 10
return Badge{
FontFamily: DefaultFontFamily,
Label: Text{
text: label,
width: lw,
x: lx,
},
Message: Text{
text: message,
width: mw,
x: mx,
},
FontSize: defaultFontSize * 10,
Color: color,
}
}
func calculateTextWidth(text string) int {
width := 0
widthData := GlobalVars().DejaVuGlyphWidthData
for _, char := range strings.TrimSpace(text) {
charWidth, ok := widthData[char]
if !ok {
// use the width of 'm' in case of missing glyph width data for a printable character
if unicode.IsPrint(char) {
charWidth = widthData['m']
} else {
charWidth = 0
}
}
width += int(charWidth)
}
return width
}
+206
View File
@@ -0,0 +1,206 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package badge
// DejaVuGlyphWidthData is generated by `sfnt.Face.GlyphAdvance(nil, <rune>, 11, font.HintingNone)` with DejaVu Sans
// v2.37 (https://github.com/dejavu-fonts/dejavu-fonts/releases/download/version_2_37/dejavu-sans-ttf-2.37.zip).
//
// Fonts defined in "DefaultFontFamily" all have similar widths (including "DejaVu Sans"),
// and these widths are fixed and don't seem to change.
//
// A devtest page "/devtest/badge-actions-svg" could be used to check the rendered images.
func dejaVuGlyphWidthDataFunc() map[rune]uint8 {
return map[rune]uint8{
32: 3,
33: 4,
34: 5,
35: 9,
36: 7,
37: 10,
38: 9,
39: 3,
40: 4,
41: 4,
42: 6,
43: 9,
44: 3,
45: 4,
46: 3,
47: 4,
48: 7,
49: 7,
50: 7,
51: 7,
52: 7,
53: 7,
54: 7,
55: 7,
56: 7,
57: 7,
58: 4,
59: 4,
60: 9,
61: 9,
62: 9,
63: 6,
64: 11,
65: 8,
66: 8,
67: 8,
68: 8,
69: 7,
70: 6,
71: 9,
72: 8,
73: 3,
74: 3,
75: 7,
76: 6,
77: 9,
78: 8,
79: 9,
80: 7,
81: 9,
82: 8,
83: 7,
84: 7,
85: 8,
86: 8,
87: 11,
88: 8,
89: 7,
90: 8,
91: 4,
92: 4,
93: 4,
94: 9,
95: 6,
96: 6,
97: 7,
98: 7,
99: 6,
100: 7,
101: 7,
102: 4,
103: 7,
104: 7,
105: 3,
106: 3,
107: 6,
108: 3,
109: 11,
110: 7,
111: 7,
112: 7,
113: 7,
114: 5,
115: 6,
116: 4,
117: 7,
118: 7,
119: 9,
120: 7,
121: 7,
122: 6,
123: 7,
124: 4,
125: 7,
126: 9,
161: 4,
162: 7,
163: 7,
164: 7,
165: 7,
166: 4,
167: 6,
168: 6,
169: 11,
170: 5,
171: 7,
172: 9,
174: 11,
175: 6,
176: 6,
177: 9,
178: 4,
179: 4,
180: 6,
181: 7,
182: 7,
183: 3,
184: 6,
185: 4,
186: 5,
187: 7,
188: 11,
189: 11,
190: 11,
191: 6,
192: 8,
193: 8,
194: 8,
195: 8,
196: 8,
197: 8,
198: 11,
199: 8,
200: 7,
201: 7,
202: 7,
203: 7,
204: 3,
205: 3,
206: 3,
207: 3,
208: 9,
209: 8,
210: 9,
211: 9,
212: 9,
213: 9,
214: 9,
215: 9,
216: 9,
217: 8,
218: 8,
219: 8,
220: 8,
221: 7,
222: 7,
223: 7,
224: 7,
225: 7,
226: 7,
227: 7,
228: 7,
229: 7,
230: 11,
231: 6,
232: 7,
233: 7,
234: 7,
235: 7,
236: 3,
237: 3,
238: 3,
239: 3,
240: 7,
241: 7,
242: 7,
243: 7,
244: 7,
245: 7,
246: 7,
247: 9,
248: 7,
249: 7,
250: 7,
251: 7,
252: 7,
253: 7,
254: 7,
255: 7,
}
}
+70
View File
@@ -0,0 +1,70 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package base
import (
"unicode/utf8"
"golang.org/x/text/collate"
"golang.org/x/text/language"
)
func naturalSortGetRune(str string, pos int) (r rune, size int, has bool) {
if pos >= len(str) {
return 0, 0, false
}
r, size = utf8.DecodeRuneInString(str[pos:])
if r == utf8.RuneError {
r, size = rune(str[pos]), 1 // if invalid input, treat it as a single byte ascii
}
return r, size, true
}
func naturalSortAdvance(str string, pos int) (end int, isNumber bool) {
end = pos
for {
r, size, has := naturalSortGetRune(str, end)
if !has {
break
}
isCurRuneNum := '0' <= r && r <= '9'
if end == pos {
isNumber = isCurRuneNum
end += size
} else if isCurRuneNum == isNumber {
end += size
} else {
break
}
}
return end, isNumber
}
// NaturalSortCompare compares two strings so that they could be sorted in natural order
func NaturalSortCompare(s1, s2 string) int {
// There is a bug in Golang's collate package: https://github.com/golang/go/issues/67997
// text/collate: CompareString(collate.Numeric) returns wrong result for "0.0" vs "1.0" #67997
// So we need to handle the number parts by ourselves
c := collate.New(language.English, collate.Numeric)
pos1, pos2 := 0, 0
for pos1 < len(s1) && pos2 < len(s2) {
end1, isNum1 := naturalSortAdvance(s1, pos1)
end2, isNum2 := naturalSortAdvance(s2, pos2)
part1, part2 := s1[pos1:end1], s2[pos2:end2]
if isNum1 && isNum2 {
if part1 != part2 {
if len(part1) != len(part2) {
return len(part1) - len(part2)
}
return c.CompareString(part1, part2)
}
} else {
if cmp := c.CompareString(part1, part2); cmp != 0 {
return cmp
}
}
pos1, pos2 = end1, end2
}
return len(s1) - len(s2)
}
+43
View File
@@ -0,0 +1,43 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package base
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNaturalSortLess(t *testing.T) {
testLess := func(s1, s2 string) {
assert.Negative(t, NaturalSortCompare(s1, s2), "s1<s2 should be true: s1=%q, s2=%q", s1, s2)
}
testEqual := func(s1, s2 string) {
assert.Zero(t, NaturalSortCompare(s1, s2), "s1<s2 should be false: s1=%q, s2=%q", s1, s2)
}
testEqual("", "")
testLess("", "a")
testLess("", "1")
testLess("v1.2", "v1.2.0")
testLess("v1.2.0", "v1.10.0")
testLess("v1.20.0", "v1.29.0")
testEqual("v1.20.0", "v1.20.0")
testLess("a", "A")
testLess("a", "B")
testLess("A", "b")
testLess("A", "ab")
testLess("abc", "bcd")
testLess("a-1-a", "a-1-b")
testLess("2", "12")
testLess("cafe", "café")
testLess("café", "caff")
testLess("A-2", "A-11")
testLess("0.txt", "1.txt")
}
+124
View File
@@ -0,0 +1,124 @@
// Copyright 2014 The Gogs Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package base
import (
"crypto/hmac"
"crypto/sha1"
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"fmt"
"hash"
"strconv"
"time"
"gitea.dev/modules/setting"
"gitea.dev/modules/util"
"github.com/dustin/go-humanize"
)
// EncodeSha256 string to sha256 hex value.
func EncodeSha256(str string) string {
h := sha256.New()
_, _ = h.Write([]byte(str))
return hex.EncodeToString(h.Sum(nil))
}
// ShortSha is basically just truncating.
// It is DEPRECATED and will be removed in the future.
func ShortSha(sha1 string) string {
return util.TruncateRunes(sha1, 10)
}
// VerifyTimeLimitCode verify time limit code
func VerifyTimeLimitCode(now time.Time, data string, minutes int, code string) bool {
if len(code) <= 18 {
return false
}
startTimeStr := code[:12]
aliveTimeStr := code[12:18]
aliveTime, _ := strconv.Atoi(aliveTimeStr) // no need to check err, if anything wrong, the following code check will fail soon
// check code
retCode := CreateTimeLimitCode(data, aliveTime, startTimeStr, nil)
if subtle.ConstantTimeCompare([]byte(retCode), []byte(code)) != 1 {
return false
}
// check time is expired or not: startTime <= now && now < startTime + minutes
startTime, _ := time.ParseInLocation("200601021504", startTimeStr, time.Local)
return (startTime.Before(now) || startTime.Equal(now)) && now.Before(startTime.Add(time.Minute*time.Duration(minutes)))
}
// TimeLimitCodeLength default value for time limit code
const TimeLimitCodeLength = 12 + 6 + 40
// CreateTimeLimitCode create a time-limited code.
// Format: 12 length date time string + 6 minutes string (not used) + 40 hash string, some other code depends on this fixed length
// If h is nil, then use the default hmac hash.
func CreateTimeLimitCode[T time.Time | string](data string, minutes int, startTimeGeneric T, h hash.Hash) string {
const format = "200601021504"
var start time.Time
var startTimeAny any = startTimeGeneric
if t, ok := startTimeAny.(time.Time); ok {
start = t
} else {
var err error
start, err = time.ParseInLocation(format, startTimeAny.(string), time.Local)
if err != nil {
return "" // return an invalid code because the "parse" failed
}
}
startStr := start.Format(format)
end := start.Add(time.Minute * time.Duration(minutes))
if h == nil {
h = hmac.New(sha1.New, setting.GetGeneralTokenSigningSecret())
}
_, _ = fmt.Fprintf(h, "%s%s%s%s%d", data, hex.EncodeToString(setting.GetGeneralTokenSigningSecret()), startStr, end.Format(format), minutes)
encoded := hex.EncodeToString(h.Sum(nil))
code := fmt.Sprintf("%s%06d%s", startStr, minutes, encoded)
if len(code) != TimeLimitCodeLength {
panic("there is a hard requirement for the length of time-limited code") // it shouldn't happen
}
return code
}
// FileSize calculates the file size and generate user-friendly string.
func FileSize(s int64) string {
return humanize.IBytes(uint64(s))
}
// StringsToInt64s converts a slice of string to a slice of int64.
func StringsToInt64s(strs []string) ([]int64, error) {
if strs == nil {
return nil, nil
}
ints := make([]int64, 0, len(strs))
for _, s := range strs {
if s == "" {
continue
}
n, err := strconv.ParseInt(s, 10, 64)
if err != nil {
return nil, err
}
ints = append(ints, n)
}
return ints, nil
}
// Int64sToStrings converts a slice of int64 to a slice of string.
func Int64sToStrings(ints []int64) []string {
strs := make([]string, len(ints))
for i := range ints {
strs[i] = strconv.FormatInt(ints[i], 10)
}
return strs
}
+119
View File
@@ -0,0 +1,119 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package base
import (
"crypto/sha1"
"fmt"
"testing"
"time"
"gitea.dev/modules/setting"
"gitea.dev/modules/test"
"github.com/stretchr/testify/assert"
)
func TestEncodeSha256(t *testing.T) {
assert.Equal(t,
"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
EncodeSha256("foobar"),
)
}
func TestShortSha(t *testing.T) {
assert.Equal(t, "veryverylo", ShortSha("veryverylong"))
}
func TestVerifyTimeLimitCode(t *testing.T) {
defer test.MockVariableValue(&setting.InstallLock, true)()
initGeneralSecret := func(secret string) {
setting.CfgProvider, _ = setting.NewConfigProviderFromData(fmt.Sprintf(`
[security]
INTERNAL_TOKEN = dummy
INSTALL_LOCK = true
[oauth2]
JWT_SECRET = %s
`, secret))
setting.LoadCommonSettings()
}
initGeneralSecret("KZb_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko")
now := time.Now()
t.Run("TestGenericParameter", func(t *testing.T) {
time2000 := time.Date(2000, 1, 2, 3, 4, 5, 0, time.Local)
assert.Equal(t, "2000010203040000026fa5221b2731b7cf80b1b506f5e39e38c115fee5", CreateTimeLimitCode("test-sha1", 2, time2000, sha1.New()))
assert.Equal(t, "2000010203040000026fa5221b2731b7cf80b1b506f5e39e38c115fee5", CreateTimeLimitCode("test-sha1", 2, "200001020304", sha1.New()))
assert.Equal(t, "2000010203040000024842227a2f87041ff82025199c0187410a9297bf", CreateTimeLimitCode("test-hmac", 2, time2000, nil))
assert.Equal(t, "2000010203040000024842227a2f87041ff82025199c0187410a9297bf", CreateTimeLimitCode("test-hmac", 2, "200001020304", nil))
})
t.Run("TestInvalidCode", func(t *testing.T) {
assert.False(t, VerifyTimeLimitCode(now, "data", 2, ""))
assert.False(t, VerifyTimeLimitCode(now, "data", 2, "invalid code"))
})
t.Run("TestCreateAndVerify", func(t *testing.T) {
code := CreateTimeLimitCode("data", 2, now, nil)
assert.False(t, VerifyTimeLimitCode(now.Add(-time.Minute), "data", 2, code)) // not started yet
assert.True(t, VerifyTimeLimitCode(now, "data", 2, code))
assert.True(t, VerifyTimeLimitCode(now.Add(time.Minute), "data", 2, code))
assert.False(t, VerifyTimeLimitCode(now.Add(time.Minute), "DATA", 2, code)) // invalid data
assert.False(t, VerifyTimeLimitCode(now.Add(2*time.Minute), "data", 2, code)) // expired
})
t.Run("TestDifferentSecret", func(t *testing.T) {
// use another secret to ensure the code is invalid for different secret
verifyDataCode := func(c string) bool {
return VerifyTimeLimitCode(now, "data", 2, c)
}
code := CreateTimeLimitCode("data", 2, now, nil)
assert.True(t, verifyDataCode(code))
initGeneralSecret("000_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko")
assert.False(t, verifyDataCode(code))
})
}
func TestFileSize(t *testing.T) {
var size int64 = 512
assert.Equal(t, "512 B", FileSize(size))
size *= 1024
assert.Equal(t, "512 KiB", FileSize(size))
size *= 1024
assert.Equal(t, "512 MiB", FileSize(size))
size *= 1024
assert.Equal(t, "512 GiB", FileSize(size))
size *= 1024
assert.Equal(t, "512 TiB", FileSize(size))
size *= 1024
assert.Equal(t, "512 PiB", FileSize(size))
size *= 4
assert.Equal(t, "2.0 EiB", FileSize(size))
}
func TestStringsToInt64s(t *testing.T) {
testSuccess := func(input []string, expected []int64) {
result, err := StringsToInt64s(input)
assert.NoError(t, err)
assert.Equal(t, expected, result)
}
testSuccess(nil, nil)
testSuccess([]string{}, []int64{})
testSuccess([]string{""}, []int64{})
testSuccess([]string{"-1234"}, []int64{-1234})
testSuccess([]string{"1", "4", "16", "64", "256"}, []int64{1, 4, 16, 64, 256})
ints, err := StringsToInt64s([]string{"-1", "a"})
assert.Empty(t, ints)
assert.Error(t, err)
}
func TestInt64sToStrings(t *testing.T) {
assert.Equal(t, []string{}, Int64sToStrings([]int64{}))
assert.Equal(t,
[]string{"1", "4", "16", "64", "256"},
Int64sToStrings([]int64{1, 4, 16, 64, 256}),
)
}
+119
View File
@@ -0,0 +1,119 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"encoding/hex"
"errors"
"fmt"
"strconv"
"time"
"gitea.dev/modules/setting"
_ "gitea.com/go-chi/cache/memcache" //nolint:depguard // memcache plugin for cache, it is required for config "ADAPTER=memcache"
)
var defaultCache StringCache
// Init start cache service
func Init() error {
if defaultCache == nil {
c, err := NewStringCache(setting.CacheService.Cache)
if err != nil {
return err
}
for range 10 {
if err = c.Ping(); err == nil {
break
}
time.Sleep(time.Second)
}
if err != nil {
return err
}
defaultCache = c
}
return nil
}
const (
testCacheKey = "DefaultCache.TestKey"
// SlowCacheThreshold marks cache tests as slow
// set to 30ms per discussion: https://github.com/go-gitea/gitea/issues/33190
// TODO: Replace with metrics histogram
SlowCacheThreshold = 30 * time.Millisecond
)
// Test performs delete, put and get operations on a predefined key
// returns
func Test() (time.Duration, error) {
if defaultCache == nil {
return 0, errors.New("default cache not initialized")
}
testData := hex.EncodeToString(make([]byte, 500))
start := time.Now()
if err := defaultCache.Delete(testCacheKey); err != nil {
return 0, fmt.Errorf("expect cache to delete data based on key if exist but got: %w", err)
}
if err := defaultCache.Put(testCacheKey, testData, 10); err != nil {
return 0, fmt.Errorf("expect cache to store data but got: %w", err)
}
testVal, hit := defaultCache.Get(testCacheKey)
if !hit {
return 0, errors.New("expect cache hit but got none")
}
if testVal != testData {
return 0, errors.New("expect cache to return same value as stored but got other")
}
return time.Since(start), nil
}
// GetCache returns the currently configured cache
func GetCache() StringCache {
return defaultCache
}
// GetString returns the key value from cache with callback when no key exists in cache
func GetString(key string, getFunc func() (string, error)) (string, error) {
if defaultCache == nil || setting.CacheService.TTL == 0 {
return getFunc()
}
cached, exist := defaultCache.Get(key)
if !exist {
value, err := getFunc()
if err != nil {
return value, err
}
return value, defaultCache.Put(key, value, setting.CacheService.TTLSeconds())
}
return cached, nil
}
// GetInt64 returns key value from cache with callback when no key exists in cache
func GetInt64(key string, getFunc func() (int64, error)) (int64, error) {
s, err := GetString(key, func() (string, error) {
v, err := getFunc()
return strconv.FormatInt(v, 10), err
})
if err != nil {
return 0, err
}
if s == "" {
return 0, nil
}
return strconv.ParseInt(s, 10, 64)
}
// Remove key from cache
func Remove(key string) {
if defaultCache == nil {
return
}
_ = defaultCache.Delete(key)
}
+162
View File
@@ -0,0 +1,162 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"fmt"
"strconv"
"time"
"gitea.dev/modules/graceful"
"gitea.dev/modules/nosql"
"gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here
"github.com/redis/go-redis/v9"
)
// RedisCacher represents a redis cache adapter implementation.
type RedisCacher struct {
c redis.UniversalClient
prefix string
hsetName string
occupyMode bool
}
// toStr convert string/int/int64 interface to string. it's only used by the RedisCacher.Put internally
func toStr(v any) string {
if v == nil {
return ""
}
switch v := v.(type) {
case string:
return v
case []byte:
return string(v)
case int:
return strconv.FormatInt(int64(v), 10)
case int64:
return strconv.FormatInt(v, 10)
default:
return fmt.Sprint(v) // as what the old com.ToStr does in most cases
}
}
// Put puts value (string type) into cache with key and expire time.
// If expired is 0, it lives forever.
func (c *RedisCacher) Put(key string, val any, expire int64) error {
// this function is not well-designed, it only puts string values into cache
key = c.prefix + key
if expire == 0 {
if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), 0).Err(); err != nil {
return err
}
} else {
dur := time.Duration(expire) * time.Second
if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), dur).Err(); err != nil {
return err
}
}
if c.occupyMode {
return nil
}
return c.c.HSet(graceful.GetManager().HammerContext(), c.hsetName, key, "0").Err()
}
// Get gets cached value by given key.
func (c *RedisCacher) Get(key string) any {
val, err := c.c.Get(graceful.GetManager().HammerContext(), c.prefix+key).Result()
if err != nil {
return nil
}
return val
}
// Delete deletes cached value by given key.
func (c *RedisCacher) Delete(key string) error {
key = c.prefix + key
if err := c.c.Del(graceful.GetManager().HammerContext(), key).Err(); err != nil {
return err
}
if c.occupyMode {
return nil
}
return c.c.HDel(graceful.GetManager().HammerContext(), c.hsetName, key).Err()
}
// Incr increases cached int-type value by given key as a counter.
func (c *RedisCacher) Incr(key string) error {
if !c.IsExist(key) {
return fmt.Errorf("key '%s' not exist", key)
}
return c.c.Incr(graceful.GetManager().HammerContext(), c.prefix+key).Err()
}
// Decr decreases cached int-type value by given key as a counter.
func (c *RedisCacher) Decr(key string) error {
if !c.IsExist(key) {
return fmt.Errorf("key '%s' not exist", key)
}
return c.c.Decr(graceful.GetManager().HammerContext(), c.prefix+key).Err()
}
// IsExist returns true if cached value exists.
func (c *RedisCacher) IsExist(key string) bool {
if c.c.Exists(graceful.GetManager().HammerContext(), c.prefix+key).Val() == 1 {
return true
}
if !c.occupyMode {
c.c.HDel(graceful.GetManager().HammerContext(), c.hsetName, c.prefix+key)
}
return false
}
// Flush deletes all cached data.
func (c *RedisCacher) Flush() error {
if c.occupyMode {
return c.c.FlushDB(graceful.GetManager().HammerContext()).Err()
}
keys, err := c.c.HKeys(graceful.GetManager().HammerContext(), c.hsetName).Result()
if err != nil {
return err
}
if err = c.c.Del(graceful.GetManager().HammerContext(), keys...).Err(); err != nil {
return err
}
return c.c.Del(graceful.GetManager().HammerContext(), c.hsetName).Err()
}
// StartAndGC starts GC routine based on config string settings.
// AdapterConfig: network=tcp,addr=:6379,password=macaron,db=0,pool_size=100,idle_timeout=180,hset_name=MacaronCache,prefix=cache:
func (c *RedisCacher) StartAndGC(opts cache.Options) error {
c.hsetName = "MacaronCache"
c.occupyMode = opts.OccupyMode
uri := nosql.ToRedisURI(opts.AdapterConfig)
c.c = nosql.GetManager().GetRedisClient(uri.String())
for k, v := range uri.Query() {
switch k {
case "hset_name":
c.hsetName = v[0]
case "prefix":
c.prefix = v[0]
}
}
return c.c.Ping(graceful.GetManager().HammerContext()).Err()
}
// Ping tests if the cache is alive.
func (c *RedisCacher) Ping() error {
return c.c.Ping(graceful.GetManager().HammerContext()).Err()
}
func init() {
cache.Register("redis", &RedisCacher{})
}
+126
View File
@@ -0,0 +1,126 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"errors"
"testing"
"time"
"gitea.dev/modules/setting"
"github.com/stretchr/testify/assert"
)
func createTestCache() {
defaultCache, _ = NewStringCache(setting.Cache{
Adapter: "memory",
TTL: time.Minute,
})
setting.CacheService.TTL = 24 * time.Hour
}
func TestNewContext(t *testing.T) {
assert.NoError(t, Init())
setting.CacheService.Cache = setting.Cache{Adapter: "redis", Conn: "some random string"}
con, err := NewStringCache(setting.Cache{
Adapter: "rand",
Conn: "false conf",
Interval: 100,
})
assert.Error(t, err)
assert.Nil(t, con)
}
func TestTest(t *testing.T) {
defaultCache = nil
_, err := Test()
assert.Error(t, err)
createTestCache()
elapsed, err := Test()
assert.NoError(t, err)
// mem cache should take from 300ns up to 1ms on modern hardware ...
assert.Positive(t, elapsed)
assert.Less(t, elapsed, SlowCacheThreshold)
}
func TestGetCache(t *testing.T) {
createTestCache()
assert.NotNil(t, GetCache())
}
func TestGetString(t *testing.T) {
createTestCache()
data, err := GetString("key", func() (string, error) {
return "", errors.New("some error")
})
assert.Error(t, err)
assert.Empty(t, data)
data, err = GetString("key", func() (string, error) {
return "", nil
})
assert.NoError(t, err)
assert.Empty(t, data)
data, err = GetString("key", func() (string, error) {
return "some data", nil
})
assert.NoError(t, err)
assert.Empty(t, data)
Remove("key")
data, err = GetString("key", func() (string, error) {
return "some data", nil
})
assert.NoError(t, err)
assert.Equal(t, "some data", data)
data, err = GetString("key", func() (string, error) {
return "", errors.New("some error")
})
assert.NoError(t, err)
assert.Equal(t, "some data", data)
Remove("key")
}
func TestGetInt64(t *testing.T) {
createTestCache()
data, err := GetInt64("key", func() (int64, error) {
return 0, errors.New("some error")
})
assert.Error(t, err)
assert.EqualValues(t, 0, data)
data, err = GetInt64("key", func() (int64, error) {
return 0, nil
})
assert.NoError(t, err)
assert.EqualValues(t, 0, data)
data, err = GetInt64("key", func() (int64, error) {
return 100, nil
})
assert.NoError(t, err)
assert.EqualValues(t, 0, data)
Remove("key")
data, err = GetInt64("key", func() (int64, error) {
return 100, nil
})
assert.NoError(t, err)
assert.EqualValues(t, 100, data)
data, err = GetInt64("key", func() (int64, error) {
return 0, errors.New("some error")
})
assert.NoError(t, err)
assert.EqualValues(t, 100, data)
Remove("key")
}
+208
View File
@@ -0,0 +1,208 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"strconv"
"sync"
"time"
"gitea.dev/modules/json"
mc "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here
lru "github.com/hashicorp/golang-lru/v2"
)
// TwoQueueCache represents a LRU 2Q cache adapter implementation
type TwoQueueCache struct {
lock sync.Mutex
cache *lru.TwoQueueCache[string, any]
interval int
}
// TwoQueueCacheConfig describes the configuration for TwoQueueCache
type TwoQueueCacheConfig struct {
Size int `ini:"SIZE" json:"size"`
RecentRatio float64 `ini:"RECENT_RATIO" json:"recent_ratio"`
GhostRatio float64 `ini:"GHOST_RATIO" json:"ghost_ratio"`
}
// MemoryItem represents a memory cache item.
type MemoryItem struct {
Val any
Created int64
Timeout int64
}
func (item *MemoryItem) hasExpired() bool {
return item.Timeout > 0 &&
(time.Now().Unix()-item.Created) >= item.Timeout
}
var _ mc.Cache = &TwoQueueCache{}
// Put puts value into cache with key and expire time.
func (c *TwoQueueCache) Put(key string, val any, timeout int64) error {
item := &MemoryItem{
Val: val,
Created: time.Now().Unix(),
Timeout: timeout,
}
c.lock.Lock()
defer c.lock.Unlock()
c.cache.Add(key, item)
return nil
}
// Get gets cached value by given key.
func (c *TwoQueueCache) Get(key string) any {
c.lock.Lock()
defer c.lock.Unlock()
cached, ok := c.cache.Get(key)
if !ok {
return nil
}
item, ok := cached.(*MemoryItem)
if !ok || item.hasExpired() {
c.cache.Remove(key)
return nil
}
return item.Val
}
// Delete deletes cached value by given key.
func (c *TwoQueueCache) Delete(key string) error {
c.lock.Lock()
defer c.lock.Unlock()
c.cache.Remove(key)
return nil
}
// Incr increases cached int-type value by given key as a counter.
func (c *TwoQueueCache) Incr(key string) error {
c.lock.Lock()
defer c.lock.Unlock()
cached, ok := c.cache.Get(key)
if !ok {
return nil
}
item, ok := cached.(*MemoryItem)
if !ok || item.hasExpired() {
c.cache.Remove(key)
return nil
}
var err error
item.Val, err = mc.Incr(item.Val)
return err
}
// Decr decreases cached int-type value by given key as a counter.
func (c *TwoQueueCache) Decr(key string) error {
c.lock.Lock()
defer c.lock.Unlock()
cached, ok := c.cache.Get(key)
if !ok {
return nil
}
item, ok := cached.(*MemoryItem)
if !ok || item.hasExpired() {
c.cache.Remove(key)
return nil
}
var err error
item.Val, err = mc.Decr(item.Val)
return err
}
// IsExist returns true if cached value exists.
func (c *TwoQueueCache) IsExist(key string) bool {
c.lock.Lock()
defer c.lock.Unlock()
cached, ok := c.cache.Peek(key)
if !ok {
return false
}
item, ok := cached.(*MemoryItem)
if !ok || item.hasExpired() {
c.cache.Remove(key)
return false
}
return true
}
// Flush deletes all cached data.
func (c *TwoQueueCache) Flush() error {
c.lock.Lock()
defer c.lock.Unlock()
c.cache.Purge()
return nil
}
func (c *TwoQueueCache) checkAndInvalidate(key string) {
c.lock.Lock()
defer c.lock.Unlock()
cached, ok := c.cache.Peek(key)
if !ok {
return
}
item, ok := cached.(*MemoryItem)
if !ok || item.hasExpired() {
c.cache.Remove(key)
}
}
func (c *TwoQueueCache) startGC() {
if c.interval < 0 {
return
}
for _, key := range c.cache.Keys() {
c.checkAndInvalidate(key)
}
time.AfterFunc(time.Duration(c.interval)*time.Second, c.startGC)
}
// StartAndGC starts GC routine based on config string settings.
func (c *TwoQueueCache) StartAndGC(opts mc.Options) error {
var err error
size := 50000
if opts.AdapterConfig != "" {
size, err = strconv.Atoi(opts.AdapterConfig)
}
if err != nil {
if !json.Valid([]byte(opts.AdapterConfig)) {
return err
}
cfg := &TwoQueueCacheConfig{
Size: 50000,
RecentRatio: lru.Default2QRecentRatio,
GhostRatio: lru.Default2QGhostEntries,
}
_ = json.Unmarshal([]byte(opts.AdapterConfig), cfg)
c.cache, err = lru.New2QParams[string, any](cfg.Size, cfg.RecentRatio, cfg.GhostRatio)
} else {
c.cache, err = lru.New2Q[string, any](size)
}
c.interval = opts.Interval
if c.interval > 0 {
go c.startGC()
}
return err
}
// Ping tests if the cache is alive.
func (c *TwoQueueCache) Ping() error {
return mc.GenericPing(c)
}
func init() {
mc.Register("twoqueue", &TwoQueueCache{})
}

Some files were not shown because too many files have changed in this diff Show More