初始提交: Gitea 项目代码
This commit is contained in:
root
@@ -0,0 +1,263 @@
|
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package dump
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.dev/modules/log"
|
||||
"gitea.dev/modules/setting"
|
||||
"gitea.dev/modules/timeutil"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
)
|
||||
|
||||
var SupportedOutputTypes = []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"}
|
||||
|
||||
// PrepareFileNameAndType prepares the output file name and type, if the type is not supported, it returns an empty "outType"
|
||||
func PrepareFileNameAndType(argFile, argType string) (outFileName, outType string) {
|
||||
if argFile == "" && argType == "" {
|
||||
outType = SupportedOutputTypes[0]
|
||||
outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType)
|
||||
} else if argFile == "" {
|
||||
outType = argType
|
||||
outFileName = fmt.Sprintf("gitea-dump-%d.%s", timeutil.TimeStampNow(), outType)
|
||||
} else if argType == "" {
|
||||
if filepath.Ext(outFileName) == "" {
|
||||
outType = SupportedOutputTypes[0]
|
||||
outFileName = argFile
|
||||
} else {
|
||||
for _, t := range SupportedOutputTypes {
|
||||
if strings.HasSuffix(argFile, "."+t) {
|
||||
outFileName = argFile
|
||||
outType = t
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
outFileName, outType = argFile, argType
|
||||
}
|
||||
if !slices.Contains(SupportedOutputTypes, outType) {
|
||||
return "", ""
|
||||
}
|
||||
return outFileName, outType
|
||||
}
|
||||
|
||||
func IsSubdir(upper, lower string) (bool, error) {
|
||||
if relPath, err := filepath.Rel(upper, lower); err != nil {
|
||||
return false, err
|
||||
} else if relPath == "." || !strings.HasPrefix(relPath, ".") {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
type Dumper struct {
|
||||
Verbose bool
|
||||
|
||||
jobs chan archives.ArchiveAsyncJob
|
||||
errArchiveAsync chan error
|
||||
errArchiveJob chan error
|
||||
|
||||
globalExcludeAbsPaths []string
|
||||
}
|
||||
|
||||
func NewDumper(ctx context.Context, format string, output io.Writer) (*Dumper, error) {
|
||||
d := &Dumper{
|
||||
jobs: make(chan archives.ArchiveAsyncJob, 1),
|
||||
errArchiveAsync: make(chan error, 1),
|
||||
errArchiveJob: make(chan error, 1),
|
||||
}
|
||||
|
||||
// TODO: in the future, we could completely drop the "mholt/archives" dependency.
|
||||
// Then we only need to support "zip" and ".tar.gz" natively, and let users provide custom command line tools
|
||||
// like "zstd" or "xz" with compression-level arguments.
|
||||
var comp archives.ArchiverAsync
|
||||
switch format {
|
||||
case "zip":
|
||||
comp = archives.Zip{Compression: zip.Deflate}
|
||||
case "tar":
|
||||
comp = archives.Tar{}
|
||||
case "tar.sz":
|
||||
comp = archives.CompressedArchive{Compression: archives.Sz{}, Archival: archives.Tar{}}
|
||||
case "tar.gz":
|
||||
comp = archives.CompressedArchive{Compression: archives.Gz{}, Archival: archives.Tar{}}
|
||||
case "tar.xz":
|
||||
comp = archives.CompressedArchive{Compression: archives.Xz{}, Archival: archives.Tar{}}
|
||||
case "tar.bz2":
|
||||
comp = archives.CompressedArchive{Compression: archives.Bz2{}, Archival: archives.Tar{}}
|
||||
case "tar.br":
|
||||
comp = archives.CompressedArchive{Compression: archives.Brotli{}, Archival: archives.Tar{}}
|
||||
case "tar.lz4":
|
||||
comp = archives.CompressedArchive{Compression: archives.Lz4{}, Archival: archives.Tar{}}
|
||||
case "tar.zst":
|
||||
comp = archives.CompressedArchive{Compression: archives.Zstd{}, Archival: archives.Tar{}}
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported format: %s", format)
|
||||
}
|
||||
go func() {
|
||||
d.errArchiveAsync <- comp.ArchiveAsync(ctx, output, d.jobs)
|
||||
close(d.errArchiveAsync)
|
||||
}()
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (dumper *Dumper) runArchiveJob(job archives.ArchiveAsyncJob) error {
|
||||
dumper.jobs <- job
|
||||
select {
|
||||
case err := <-dumper.errArchiveAsync:
|
||||
if err == nil {
|
||||
return errors.New("archiver has been closed")
|
||||
}
|
||||
return err
|
||||
case err := <-dumper.errArchiveJob:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// AddFileByPath adds a file by its filesystem path
|
||||
func (dumper *Dumper) AddFileByPath(filePath, absPath string) error {
|
||||
if dumper.Verbose {
|
||||
log.Info("Adding local file %s", filePath)
|
||||
}
|
||||
|
||||
fileInfo, err := os.Stat(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
archiveFileInfo := archives.FileInfo{
|
||||
FileInfo: fileInfo,
|
||||
NameInArchive: filePath,
|
||||
Open: func() (fs.File, error) { return os.Open(absPath) },
|
||||
}
|
||||
|
||||
return dumper.runArchiveJob(archives.ArchiveAsyncJob{
|
||||
File: archiveFileInfo,
|
||||
Result: dumper.errArchiveJob,
|
||||
})
|
||||
}
|
||||
|
||||
type readerFile struct {
|
||||
r io.Reader
|
||||
info os.FileInfo
|
||||
}
|
||||
|
||||
var _ fs.File = (*readerFile)(nil)
|
||||
|
||||
func (f *readerFile) Stat() (fs.FileInfo, error) { return f.info, nil }
|
||||
func (f *readerFile) Read(bytes []byte) (int, error) { return f.r.Read(bytes) }
|
||||
func (f *readerFile) Close() error { return nil }
|
||||
|
||||
// AddFileByReader adds a file's contents from a Reader
|
||||
func (dumper *Dumper) AddFileByReader(r io.Reader, info os.FileInfo, customName string) error {
|
||||
if dumper.Verbose {
|
||||
log.Info("Adding storage file %s", customName)
|
||||
}
|
||||
|
||||
fileInfo := archives.FileInfo{
|
||||
FileInfo: info,
|
||||
NameInArchive: customName,
|
||||
Open: func() (fs.File, error) { return &readerFile{r, info}, nil },
|
||||
}
|
||||
return dumper.runArchiveJob(archives.ArchiveAsyncJob{
|
||||
File: fileInfo,
|
||||
Result: dumper.errArchiveJob,
|
||||
})
|
||||
}
|
||||
|
||||
func (dumper *Dumper) Close() error {
|
||||
close(dumper.jobs)
|
||||
return <-dumper.errArchiveAsync
|
||||
}
|
||||
|
||||
func (dumper *Dumper) normalizeFilePath(absPath string) string {
|
||||
absPath = filepath.Clean(absPath)
|
||||
if setting.IsWindows {
|
||||
absPath = strings.ToLower(absPath)
|
||||
}
|
||||
return absPath
|
||||
}
|
||||
|
||||
func (dumper *Dumper) GlobalExcludeAbsPath(absPaths ...string) {
|
||||
for _, absPath := range absPaths {
|
||||
dumper.globalExcludeAbsPaths = append(dumper.globalExcludeAbsPaths, dumper.normalizeFilePath(absPath))
|
||||
}
|
||||
}
|
||||
|
||||
func (dumper *Dumper) shouldExclude(absPath string, excludes []string) bool {
|
||||
norm := dumper.normalizeFilePath(absPath)
|
||||
return slices.Contains(dumper.globalExcludeAbsPaths, norm) || slices.Contains(excludes, norm)
|
||||
}
|
||||
|
||||
func (dumper *Dumper) AddRecursiveExclude(insidePath, absPath string, excludes []string) error {
|
||||
excludes = slices.Clone(excludes)
|
||||
for i := range excludes {
|
||||
excludes[i] = dumper.normalizeFilePath(excludes[i])
|
||||
}
|
||||
return dumper.addFileOrDir(insidePath, absPath, excludes)
|
||||
}
|
||||
|
||||
func (dumper *Dumper) addFileOrDir(insidePath, absPath string, excludes []string) error {
|
||||
absPath, err := filepath.Abs(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir, err := os.Open(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dir.Close()
|
||||
|
||||
files, err := dir.Readdir(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, file := range files {
|
||||
currentAbsPath := filepath.Join(absPath, file.Name())
|
||||
if dumper.shouldExclude(currentAbsPath, excludes) {
|
||||
continue
|
||||
}
|
||||
|
||||
currentInsidePath := path.Join(insidePath, file.Name())
|
||||
if file.IsDir() {
|
||||
if err := dumper.AddFileByPath(currentInsidePath, currentAbsPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = dumper.addFileOrDir(currentInsidePath, currentAbsPath, excludes); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// only copy regular files and symlink regular files, skip non-regular files like socket/pipe/...
|
||||
shouldAdd := file.Mode().IsRegular()
|
||||
if !shouldAdd && file.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
target, err := filepath.EvalSymlinks(currentAbsPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
targetStat, err := os.Stat(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
shouldAdd = targetStat.Mode().IsRegular()
|
||||
}
|
||||
if shouldAdd {
|
||||
if err = dumper.AddFileByPath(currentInsidePath, currentAbsPath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package dump
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dev/modules/timeutil"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPrepareFileNameAndType(t *testing.T) {
|
||||
defer timeutil.MockSet(time.Unix(1234, 0))()
|
||||
test := func(argFile, argType, expFile, expType string) {
|
||||
outFile, outType := PrepareFileNameAndType(argFile, argType)
|
||||
assert.Equal(t,
|
||||
fmt.Sprintf("outFile=%s, outType=%s", expFile, expType),
|
||||
fmt.Sprintf("outFile=%s, outType=%s", outFile, outType),
|
||||
"argFile=%s, argType=%s", argFile, argType,
|
||||
)
|
||||
}
|
||||
|
||||
test("", "", "gitea-dump-1234.zip", "zip")
|
||||
test("", "tar.gz", "gitea-dump-1234.tar.gz", "tar.gz")
|
||||
test("", "no-such", "", "")
|
||||
|
||||
test("-", "", "-", "zip")
|
||||
test("-", "tar.gz", "-", "tar.gz")
|
||||
test("-", "no-such", "", "")
|
||||
|
||||
test("a", "", "a", "zip")
|
||||
test("a", "tar.gz", "a", "tar.gz")
|
||||
test("a", "no-such", "", "")
|
||||
|
||||
test("a.zip", "", "a.zip", "zip")
|
||||
test("a.zip", "tar.gz", "a.zip", "tar.gz")
|
||||
test("a.zip", "no-such", "", "")
|
||||
|
||||
test("a.tar.gz", "", "a.tar.gz", "zip")
|
||||
test("a.tar.gz", "tar.gz", "a.tar.gz", "tar.gz")
|
||||
test("a.tar.gz", "no-such", "", "")
|
||||
}
|
||||
|
||||
func TestIsSubDir(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/sub"), 0o755)
|
||||
|
||||
isSub, err := IsSubdir(filepath.Join(tmpDir, "include"), filepath.Join(tmpDir, "include"))
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, isSub)
|
||||
|
||||
isSub, err = IsSubdir(filepath.Join(tmpDir, "include"), filepath.Join(tmpDir, "include/sub"))
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, isSub)
|
||||
|
||||
isSub, err = IsSubdir(filepath.Join(tmpDir, "include/sub"), filepath.Join(tmpDir, "include"))
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, isSub)
|
||||
}
|
||||
|
||||
func TestDumperIntegration(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
dumper, err := NewDumper(t.Context(), "zip", &buf)
|
||||
require.NoError(t, err)
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(tmpDir, "test.txt"), nil, 0o644)
|
||||
f, _ := os.Open(filepath.Join(tmpDir, "test.txt"))
|
||||
|
||||
fi, _ := f.Stat()
|
||||
err = dumper.AddFileByReader(f, fi, "test.txt")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = dumper.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Positive(t, buf.Len())
|
||||
}
|
||||
|
||||
func TestDumper(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude1"), 0o755)
|
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude2"), 0o755)
|
||||
_ = os.MkdirAll(filepath.Join(tmpDir, "include/sub"), 0o755)
|
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/a"), nil, 0o644)
|
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/sub/b"), nil, 0o644)
|
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/exclude1/a-1"), nil, 0o644)
|
||||
_ = os.WriteFile(filepath.Join(tmpDir, "include/exclude2/a-2"), nil, 0o644)
|
||||
|
||||
sortStrings := func(s []string) []string {
|
||||
sort.Strings(s)
|
||||
return s
|
||||
}
|
||||
|
||||
t.Run("IncludesWithExcludes", func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
dumper, err := NewDumper(t.Context(), "tar", &buf)
|
||||
require.NoError(t, err)
|
||||
dumper.GlobalExcludeAbsPath(filepath.Join(tmpDir, "include/exclude1"))
|
||||
err = dumper.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), []string{filepath.Join(tmpDir, "include/exclude2")})
|
||||
require.NoError(t, err)
|
||||
err = dumper.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
files := extractTarFileNames(t, &buf)
|
||||
expected := []string{"include/a", "include/sub", "include/sub/b"}
|
||||
assert.Equal(t, sortStrings(expected), sortStrings(files))
|
||||
})
|
||||
|
||||
t.Run("IncludesAll", func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
dumper, err := NewDumper(t.Context(), "tar", &buf)
|
||||
require.NoError(t, err)
|
||||
err = dumper.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), nil)
|
||||
require.NoError(t, err)
|
||||
err = dumper.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
files := extractTarFileNames(t, &buf)
|
||||
expected := []string{
|
||||
"include/exclude2", "include/exclude2/a-2",
|
||||
"include/a", "include/sub", "include/sub/b",
|
||||
"include/exclude1", "include/exclude1/a-1",
|
||||
}
|
||||
assert.Equal(t, sortStrings(expected), sortStrings(files))
|
||||
})
|
||||
}
|
||||
|
||||
func extractTarFileNames(t *testing.T, buf *bytes.Buffer) (fileNames []string) {
|
||||
tr := tar.NewReader(buf)
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
require.NoError(t, err, "Error reading tar archive")
|
||||
fileNames = append(fileNames, hdr.Name)
|
||||
}
|
||||
return fileNames
|
||||
}
|
||||
Reference in New Issue
Block a user