peanut/new-api
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

初始提交: Gitea 项目代码

Browse Source
This commit is contained in:
root
2026-05-30 22:47:36 +08:00
commit f288f76350
6116 changed files with 776822 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/repository/archiver/archiver.go
+367
View File
@@ -0,0 +1,367 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package archiver
import (
"context"
"errors"
"fmt"
"io"
"os"
"strings"
"time"
"gitea.dev/models/db"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/git"
"gitea.dev/modules/git/gitcmd"
"gitea.dev/modules/gitrepo"
"gitea.dev/modules/graceful"
"gitea.dev/modules/httplib"
"gitea.dev/modules/log"
"gitea.dev/modules/process"
"gitea.dev/modules/queue"
"gitea.dev/modules/setting"
"gitea.dev/modules/storage"
"gitea.dev/modules/util"
gitea_context "gitea.dev/services/context"
)
// ArchiveRequest defines the parameters of an archive request, which notably
// includes the specific repository being archived as well as the commit, the
// name by which it was requested, and the kind of archive being requested.
// This is entirely opaque to external entities, though, and mostly used as a
// handle elsewhere.
type ArchiveRequest struct {
Repo *repo_model.Repository
Type repo_model.ArchiveType
CommitID string
Paths []string
archiveRefShortName string // the ref short name to download the archive, for example: "master", "v1.0.0", "commit id"
}
// NewRequest creates an archival request, based on the URI. The
// resulting ArchiveRequest is suitable for being passed to Await()
// if it's determined that the request still needs to be satisfied.
func NewRequest(repo *repo_model.Repository, gitRepo *git.Repository, archiveRefExt string, paths []string) (*ArchiveRequest, error) {
// here the archiveRefShortName is not a clear ref, it could be a tag, branch or commit id
archiveRefShortName, archiveType := repo_model.SplitArchiveNameType(archiveRefExt)
if archiveType == repo_model.ArchiveUnknown {
return nil, util.NewInvalidArgumentErrorf("unknown format: %s", archiveRefExt)
}
if archiveType == repo_model.ArchiveBundle && len(paths) != 0 {
return nil, util.NewInvalidArgumentErrorf("cannot specify paths when requesting a bundle")
}
// Get corresponding commit.
commitID, err := gitRepo.ConvertToGitID(archiveRefShortName)
if err != nil {
return nil, util.NewNotExistErrorf("unrecognized repository reference: %s", archiveRefShortName)
}
r := &ArchiveRequest{Repo: repo, archiveRefShortName: archiveRefShortName, Type: archiveType, Paths: paths}
r.CommitID = commitID.String()
return r, nil
}
// GetArchiveName returns the name of the caller, based on the ref used by the
// caller to create this request.
func (aReq *ArchiveRequest) GetArchiveName() string {
return strings.ReplaceAll(aReq.archiveRefShortName, "/", "-") + "." + aReq.Type.String()
}
// Await awaits the completion of an ArchiveRequest. If the archive has
// already been prepared the method returns immediately. Otherwise, an archiver
// process will be started and its completion awaited. On success the returned
// RepoArchiver may be used to download the archive. Note that even if the
// context is cancelled/times out a started archiver will still continue to run
// in the background.
func (aReq *ArchiveRequest) Await(ctx context.Context) (*repo_model.RepoArchiver, error) {
archiver, err := repo_model.GetRepoArchiver(ctx, aReq.Repo.ID, aReq.Type, aReq.CommitID)
if err != nil {
return nil, fmt.Errorf("models.GetRepoArchiver: %w", err)
}
if archiver != nil && archiver.Status == repo_model.ArchiverReady {
// Archive already generated, we're done.
return archiver, nil
}
if err := StartArchive(aReq); err != nil {
return nil, fmt.Errorf("archiver.StartArchive: %w", err)
}
poll := time.NewTicker(time.Second * 1)
defer poll.Stop()
for {
select {
case <-graceful.GetManager().HammerContext().Done():
// System stopped.
return nil, graceful.GetManager().HammerContext().Err()
case <-ctx.Done():
return nil, ctx.Err()
case <-poll.C:
archiver, err = repo_model.GetRepoArchiver(ctx, aReq.Repo.ID, aReq.Type, aReq.CommitID)
if err != nil {
return nil, fmt.Errorf("repo_model.GetRepoArchiver: %w", err)
}
if archiver != nil && archiver.Status == repo_model.ArchiverReady {
return archiver, nil
}
}
}
}
// Stream satisfies the ArchiveRequest being passed in. Processing
// will occur directly in this routine.
func (aReq *ArchiveRequest) Stream(ctx context.Context, w io.Writer) error {
if aReq.Type == repo_model.ArchiveBundle {
return gitrepo.CreateBundle(
ctx,
aReq.Repo,
aReq.CommitID,
w,
)
}
return gitrepo.CreateArchive(
ctx,
aReq.Repo,
aReq.Type.String(),
w,
setting.Repository.PrefixArchiveFiles,
aReq.CommitID,
aReq.Paths,
)
}
// doArchive satisfies the ArchiveRequest being passed in. Processing
// will occur in a separate goroutine, as this phase may take a while to
// complete. If the archive already exists, doArchive will not do
// anything. In all cases, the caller should be examining the *ArchiveRequest
// being returned for completion, as it may be different than the one they passed
// in.
func doArchive(ctx context.Context, r *ArchiveRequest) (*repo_model.RepoArchiver, error) {
ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("ArchiveRequest[%s]: %s", r.Repo.FullName(), r.GetArchiveName()))
defer finished()
archiver, err := repo_model.GetRepoArchiver(ctx, r.Repo.ID, r.Type, r.CommitID)
if err != nil {
return nil, err
}
if archiver != nil {
// FIXME: If another process are generating it, we think it's not ready and just return
// Or we should wait until the archive generated.
if archiver.Status == repo_model.ArchiverGenerating {
return nil, nil //nolint:nilnil // return nil because the archive is still being generated
}
} else {
archiver = &repo_model.RepoArchiver{
RepoID: r.Repo.ID,
Type: r.Type,
CommitID: r.CommitID,
Status: repo_model.ArchiverGenerating,
}
if err := db.Insert(ctx, archiver); err != nil {
return nil, err
}
}
rPath := archiver.RelativePath()
_, err = storage.RepoArchives.Stat(rPath)
if err == nil {
if archiver.Status == repo_model.ArchiverGenerating {
archiver.Status = repo_model.ArchiverReady
if err = repo_model.UpdateRepoArchiverStatus(ctx, archiver); err != nil {
return nil, err
}
}
return archiver, nil
}
if !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("unable to stat archive: %w", err)
}
rd, w := io.Pipe()
defer func() {
_ = w.Close()
_ = rd.Close()
}()
done := make(chan error, 1) // Ensure that there is some capacity which will ensure that the goroutine below can always finish
go func(done chan error, w *io.PipeWriter, archiveReq *ArchiveRequest) {
defer func() {
if r := recover(); r != nil {
done <- fmt.Errorf("%v", r)
}
}()
err := archiveReq.Stream(ctx, w)
_ = w.CloseWithError(err)
done <- err
}(done, w, r)
// TODO: add lfs data to zip
// TODO: add submodule data to zip
if _, err := storage.RepoArchives.Save(rPath, rd, -1); err != nil {
return nil, fmt.Errorf("unable to write archive: %w", err)
}
err = <-done
if err != nil {
return nil, err
}
if archiver.Status == repo_model.ArchiverGenerating {
archiver.Status = repo_model.ArchiverReady
if err = repo_model.UpdateRepoArchiverStatus(ctx, archiver); err != nil {
return nil, err
}
}
return archiver, nil
}
var archiverQueue *queue.WorkerPoolQueue[*ArchiveRequest]
// Init initializes archiver
func Init(ctx context.Context) error {
handler := func(items ...*ArchiveRequest) []*ArchiveRequest {
for _, archiveReq := range items {
log.Trace("ArchiverData Process: %#v", archiveReq)
if archiver, err := doArchive(ctx, archiveReq); err != nil {
log.Error("Archive %v failed: %v", archiveReq, err)
} else {
log.Trace("ArchiverData Success: %#v", archiver)
}
}
return nil
}
archiverQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo-archive", handler)
if archiverQueue == nil {
return errors.New("unable to create repo-archive queue")
}
go graceful.GetManager().RunWithCancel(archiverQueue)
return nil
}
// StartArchive push the archive request to the queue
func StartArchive(request *ArchiveRequest) error {
has, err := archiverQueue.Has(request)
if err != nil {
return err
}
if has {
return nil
}
return archiverQueue.Push(request)
}
func deleteOldRepoArchiver(ctx context.Context, archiver *repo_model.RepoArchiver) error {
if _, err := db.DeleteByID[repo_model.RepoArchiver](ctx, archiver.ID); err != nil {
return err
}
p := archiver.RelativePath()
if err := storage.RepoArchives.Delete(p); err != nil {
log.Error("delete repo archive file failed: %v", err)
}
return nil
}
// DeleteOldRepositoryArchives deletes old repository archives.
func DeleteOldRepositoryArchives(ctx context.Context, olderThan time.Duration) error {
log.Trace("Doing: ArchiveCleanup")
for {
archivers, err := db.Find[repo_model.RepoArchiver](ctx, repo_model.FindRepoArchiversOption{
ListOptions: db.ListOptions{
PageSize: 100,
Page: 1,
},
OlderThan: olderThan,
})
if err != nil {
log.Trace("Error: ArchiveClean: %v", err)
return err
}
for _, archiver := range archivers {
if err := deleteOldRepoArchiver(ctx, archiver); err != nil {
return err
}
}
if len(archivers) < 100 {
break
}
}
log.Trace("Finished: ArchiveCleanup")
return nil
}
// DeleteRepositoryArchives deletes all repositories' archives.
func DeleteRepositoryArchives(ctx context.Context) error {
if err := repo_model.DeleteAllRepoArchives(ctx); err != nil {
return err
}
return storage.Clean(storage.RepoArchives)
}
func ServeRepoArchive(ctx *gitea_context.Base, archiveReq *ArchiveRequest) error {
// Add nix format link header so tarballs lock correctly:
// https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md
ctx.Resp.Header().Add("Link", fmt.Sprintf(`<%s/archive/%s.%s?rev=%s>; rel="immutable"`,
archiveReq.Repo.APIURL(),
archiveReq.CommitID,
archiveReq.Type.String(),
archiveReq.CommitID,
))
downloadName := archiveReq.Repo.Name + "-" + archiveReq.GetArchiveName()
if setting.Repository.StreamArchives || len(archiveReq.Paths) > 0 {
// the header must be set before starting streaming even an error would occur,
// because errors may happen in git command and such cases aren't in our control.
httplib.ServeSetHeaders(ctx.Resp, httplib.ServeHeaderOptions{Filename: downloadName})
if err := archiveReq.Stream(ctx, ctx.Resp); err != nil && !ctx.Written() {
if gitcmd.StderrHasPrefix(err, "fatal: pathspec") {
return util.NewInvalidArgumentErrorf("path doesn't exist or is invalid")
}
return fmt.Errorf("archive repo %s: failed to stream: %w", archiveReq.Repo.FullName(), err)
}
return nil
}
archiver, err := archiveReq.Await(ctx)
if err != nil {
return fmt.Errorf("archive repo %s: failed to await: %w", archiveReq.Repo.FullName(), err)
}
rPath := archiver.RelativePath()
if setting.RepoArchive.Storage.ServeDirect() {
// If we have a signed url (S3, object storage), redirect to this directly.
u, err := storage.RepoArchives.ServeDirectURL(rPath, downloadName, ctx.Req.Method, nil)
if u != nil && err == nil {
ctx.Redirect(u.String())
return nil
}
}
fr, err := storage.RepoArchives.Open(rPath)
if err != nil {
return fmt.Errorf("archive repo %s: failed to open archive file: %w", archiveReq.Repo.FullName(), err)
}
defer fr.Close()
ctx.ServeContent(fr, gitea_context.ServeHeaderOptions{
Filename: downloadName,
LastModified: archiver.CreatedUnix.AsLocalTime(),
})
return nil
}
services/repository/archiver/archiver_test.go
+138
View File
@@ -0,0 +1,138 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package archiver
import (
"testing"
"time"
"gitea.dev/models/unittest"
"gitea.dev/modules/util"
"gitea.dev/services/contexttest"
_ "gitea.dev/models/actions"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMain(m *testing.M) {
unittest.MainTest(m)
}
func TestArchive_Basic(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
ctx, _ := contexttest.MockContext(t, "user27/repo49")
firstCommit, secondCommit := "51f84af23134", "aacbdfe9e1c4"
contexttest.LoadRepo(t, ctx, 49)
contexttest.LoadGitRepo(t, ctx)
defer ctx.Repo.GitRepo.Close()
bogusReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip", nil)
assert.NoError(t, err)
assert.NotNil(t, bogusReq)
assert.Equal(t, firstCommit+".zip", bogusReq.GetArchiveName())
// Check a series of bogus requests.
// Step 1, valid commit with a bad extension.
bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".unknown", nil)
assert.Error(t, err)
assert.Nil(t, bogusReq)
// Step 2, missing commit.
bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "dbffff.zip", nil)
assert.Error(t, err)
assert.Nil(t, bogusReq)
// Step 3, doesn't look like branch/tag/commit.
bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "db.zip", nil)
assert.Error(t, err)
assert.Nil(t, bogusReq)
bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "master.zip", nil)
assert.NoError(t, err)
assert.NotNil(t, bogusReq)
assert.Equal(t, "master.zip", bogusReq.GetArchiveName())
bogusReq, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, "test/archive.zip", nil)
assert.NoError(t, err)
assert.NotNil(t, bogusReq)
assert.Equal(t, "test-archive.zip", bogusReq.GetArchiveName())
// Now two valid requests, firstCommit with valid extensions.
zipReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip", nil)
assert.NoError(t, err)
assert.NotNil(t, zipReq)
tgzReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".tar.gz", nil)
assert.NoError(t, err)
assert.NotNil(t, tgzReq)
secondReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, secondCommit+".bundle", nil)
assert.NoError(t, err)
assert.NotNil(t, secondReq)
inFlight := make([]*ArchiveRequest, 3)
inFlight[0] = zipReq
inFlight[1] = tgzReq
inFlight[2] = secondReq
doArchive(t.Context(), zipReq)
doArchive(t.Context(), tgzReq)
doArchive(t.Context(), secondReq)
// Make sure sending an unprocessed request through doesn't affect the queue
// count.
doArchive(t.Context(), zipReq)
// Sleep two seconds to make sure the queue doesn't change.
time.Sleep(2 * time.Second)
zipReq2, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip", nil)
assert.NoError(t, err)
// This zipReq should match what's sitting in the queue, as we haven't
// let it release yet. From the consumer's point of view, this looks like
// a long-running archive task.
assert.Equal(t, zipReq, zipReq2)
// We still have the other three stalled at completion, waiting to remove
// from archiveInProgress. Try to submit this new one before its
// predecessor has cleared out of the queue.
doArchive(t.Context(), zipReq2)
// Now we'll submit a request and TimedWaitForCompletion twice, before and
// after we release it. We should trigger both the timeout and non-timeout
// cases.
timedReq, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, secondCommit+".tar.gz", nil)
assert.NoError(t, err)
assert.NotNil(t, timedReq)
doArchive(t.Context(), timedReq)
zipReq2, err = NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".zip", nil)
assert.NoError(t, err)
// Now, we're guaranteed to have released the original zipReq from the queue.
// Ensure that we don't get handed back the released entry somehow, but they
// should remain functionally equivalent in all fields. The exception here
// is zipReq.cchan, which will be non-nil because it's a completed request.
// It's fine to go ahead and set it to nil now.
assert.Equal(t, zipReq, zipReq2)
assert.NotSame(t, zipReq, zipReq2)
// Same commit, different compression formats should have different names.
// Ideally, the extension would match what we originally requested.
assert.NotEqual(t, zipReq.GetArchiveName(), tgzReq.GetArchiveName())
assert.NotEqual(t, zipReq.GetArchiveName(), secondReq.GetArchiveName())
t.Run("BadPath", func(t *testing.T) {
badRequest, err := NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, firstCommit+".tar.gz", []string{"not-a-path"})
require.NoError(t, err)
err = ServeRepoArchive(ctx.Base, badRequest)
require.Error(t, err)
assert.ErrorIs(t, err, util.ErrInvalidArgument)
assert.ErrorContains(t, err, "path doesn't exist or is invalid")
})
}