初始提交: Gitea 项目代码
This commit is contained in:
root
@@ -0,0 +1,434 @@
|
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.dev/models/db"
|
||||
packages_model "gitea.dev/models/packages"
|
||||
container_model "gitea.dev/models/packages/container"
|
||||
user_model "gitea.dev/models/user"
|
||||
"gitea.dev/modules/globallock"
|
||||
"gitea.dev/modules/json"
|
||||
"gitea.dev/modules/log"
|
||||
packages_module "gitea.dev/modules/packages"
|
||||
container_module "gitea.dev/modules/packages/container"
|
||||
"gitea.dev/modules/util"
|
||||
notify_service "gitea.dev/services/notify"
|
||||
packages_service "gitea.dev/services/packages"
|
||||
container_service "gitea.dev/services/packages/container"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
oci "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
)
|
||||
|
||||
// manifestCreationInfo describes a manifest to create
|
||||
type manifestCreationInfo struct {
|
||||
MediaType string
|
||||
Owner *user_model.User
|
||||
Creator *user_model.User
|
||||
Image string
|
||||
Reference string
|
||||
IsTagged bool
|
||||
Properties map[string]string
|
||||
}
|
||||
|
||||
func processManifest(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) {
|
||||
var index oci.Index
|
||||
if err := json.NewDecoder(buf).Decode(&index); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if index.SchemaVersion != 2 {
|
||||
return "", errUnsupported.WithMessage("Schema version is not supported")
|
||||
}
|
||||
if _, err := buf.Seek(0, io.SeekStart); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if !container_module.IsMediaTypeValid(mci.MediaType) {
|
||||
mci.MediaType = index.MediaType
|
||||
if !container_module.IsMediaTypeValid(mci.MediaType) {
|
||||
return "", errManifestInvalid.WithMessage("MediaType not recognized")
|
||||
}
|
||||
}
|
||||
|
||||
// .../container/manifest.go:453:createManifestBlob() [E] Error inserting package blob: Error 1062 (23000): Duplicate entry '..........' for key 'package_blob.UQE_package_blob_md5'
|
||||
releaser, err := globallock.Lock(ctx, containerGlobalLockKey(mci.Owner.ID, mci.Image, "manifest"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer releaser()
|
||||
|
||||
if container_module.IsMediaTypeImageManifest(mci.MediaType) {
|
||||
return processOciImageManifest(ctx, mci, buf)
|
||||
} else if container_module.IsMediaTypeImageIndex(mci.MediaType) {
|
||||
return processOciImageIndex(ctx, mci, buf)
|
||||
}
|
||||
return "", errManifestInvalid
|
||||
}
|
||||
|
||||
type processManifestTxRet struct {
|
||||
pv *packages_model.PackageVersion
|
||||
pb *packages_model.PackageBlob
|
||||
created bool
|
||||
digest string
|
||||
}
|
||||
|
||||
func handleCreateManifestResult(ctx context.Context, err error, mci *manifestCreationInfo, contentStore *packages_module.ContentStore, txRet *processManifestTxRet) (string, error) {
|
||||
if err != nil {
|
||||
if txRet.created && txRet.pb != nil {
|
||||
if err := contentStore.Delete(packages_module.BlobHash256Key(txRet.pb.HashSHA256)); err != nil {
|
||||
log.Error("Error deleting package blob from content store: %v", err)
|
||||
}
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
pd, err := packages_model.GetPackageDescriptor(ctx, txRet.pv)
|
||||
if err != nil {
|
||||
log.Error("Error getting package descriptor: %v", err) // ignore this error
|
||||
} else {
|
||||
notify_service.PackageCreate(ctx, mci.Creator, pd)
|
||||
}
|
||||
return txRet.digest, nil
|
||||
}
|
||||
|
||||
func processOciImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (manifestDigest string, errRet error) {
|
||||
manifest, configDescriptor, metadata, err := container_service.ParseManifestMetadata(ctx, buf, mci.Owner.ID, mci.Image)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err = buf.Seek(0, io.SeekStart); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
contentStore := packages_module.NewContentStore()
|
||||
var txRet processManifestTxRet
|
||||
err = db.WithTx(ctx, func(ctx context.Context) (err error) {
|
||||
blobReferences := make([]*blobReference, 0, 1+len(manifest.Layers))
|
||||
blobReferences = append(blobReferences, &blobReference{
|
||||
Digest: manifest.Config.Digest,
|
||||
MediaType: manifest.Config.MediaType,
|
||||
File: configDescriptor,
|
||||
ExpectedSize: manifest.Config.Size,
|
||||
})
|
||||
|
||||
for _, layer := range manifest.Layers {
|
||||
pfd, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
|
||||
OwnerID: mci.Owner.ID,
|
||||
Image: mci.Image,
|
||||
Digest: string(layer.Digest),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
blobReferences = append(blobReferences, &blobReference{
|
||||
Digest: layer.Digest,
|
||||
MediaType: layer.MediaType,
|
||||
File: pfd,
|
||||
ExpectedSize: layer.Size,
|
||||
})
|
||||
}
|
||||
|
||||
pv, err := createPackageAndVersion(ctx, mci, metadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uploadVersion, err := packages_model.GetInternalVersionByNameAndVersion(ctx, mci.Owner.ID, packages_model.TypeContainer, mci.Image, container_module.UploadVersion)
|
||||
if err != nil && !errors.Is(err, packages_model.ErrPackageNotExist) {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, ref := range blobReferences {
|
||||
if _, err = createFileFromBlobReference(ctx, pv, uploadVersion, ref); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
txRet.pv = pv
|
||||
txRet.pb, txRet.created, txRet.digest, err = createManifestBlob(ctx, contentStore, mci, pv, buf)
|
||||
return err
|
||||
})
|
||||
|
||||
return handleCreateManifestResult(ctx, err, mci, contentStore, &txRet)
|
||||
}
|
||||
|
||||
func processOciImageIndex(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (manifestDigest string, errRet error) {
|
||||
var index oci.Index
|
||||
if err := json.NewDecoder(buf).Decode(&index); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := buf.Seek(0, io.SeekStart); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
contentStore := packages_module.NewContentStore()
|
||||
var txRet processManifestTxRet
|
||||
err := db.WithTx(ctx, func(ctx context.Context) (err error) {
|
||||
metadata := &container_module.Metadata{
|
||||
Type: container_module.TypeOCI,
|
||||
Manifests: make([]*container_module.Manifest, 0, len(index.Manifests)),
|
||||
}
|
||||
|
||||
for _, manifest := range index.Manifests {
|
||||
if !container_module.IsMediaTypeImageManifest(manifest.MediaType) {
|
||||
return errManifestInvalid
|
||||
}
|
||||
|
||||
platform := container_module.DefaultPlatform
|
||||
if manifest.Platform != nil {
|
||||
platform = fmt.Sprintf("%s/%s", manifest.Platform.OS, manifest.Platform.Architecture)
|
||||
if manifest.Platform.Variant != "" {
|
||||
platform = fmt.Sprintf("%s/%s", platform, manifest.Platform.Variant)
|
||||
}
|
||||
}
|
||||
|
||||
pfd, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
|
||||
OwnerID: mci.Owner.ID,
|
||||
Image: mci.Image,
|
||||
Digest: string(manifest.Digest),
|
||||
IsManifest: true,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, container_model.ErrContainerBlobNotExist) {
|
||||
return errManifestBlobUnknown
|
||||
}
|
||||
return fmt.Errorf("GetContainerBlob: %w", err)
|
||||
}
|
||||
|
||||
size, err := packages_model.CalculateFileSize(ctx, &packages_model.PackageFileSearchOptions{
|
||||
VersionID: pfd.File.VersionID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("CalculateFileSize: %w", err)
|
||||
}
|
||||
|
||||
metadata.Manifests = append(metadata.Manifests, &container_module.Manifest{
|
||||
Platform: platform,
|
||||
Digest: string(manifest.Digest),
|
||||
Size: size,
|
||||
})
|
||||
}
|
||||
|
||||
pv, err := createPackageAndVersion(ctx, mci, metadata)
|
||||
if err != nil {
|
||||
return fmt.Errorf("createPackageAndVersion: %w", err)
|
||||
}
|
||||
|
||||
txRet.pv = pv
|
||||
txRet.pb, txRet.created, txRet.digest, err = createManifestBlob(ctx, contentStore, mci, pv, buf)
|
||||
return err
|
||||
})
|
||||
|
||||
return handleCreateManifestResult(ctx, err, mci, contentStore, &txRet)
|
||||
}
|
||||
|
||||
func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, metadata *container_module.Metadata) (*packages_model.PackageVersion, error) {
|
||||
created := true
|
||||
p := &packages_model.Package{
|
||||
OwnerID: mci.Owner.ID,
|
||||
Type: packages_model.TypeContainer,
|
||||
Name: strings.ToLower(mci.Image),
|
||||
LowerName: strings.ToLower(mci.Image),
|
||||
}
|
||||
var err error
|
||||
if p, err = packages_model.TryInsertPackage(ctx, p); err != nil {
|
||||
if !errors.Is(err, packages_model.ErrDuplicatePackage) {
|
||||
log.Error("Error inserting package: %v", err)
|
||||
return nil, fmt.Errorf("TryInsertPackage: %w", err)
|
||||
}
|
||||
created = false
|
||||
}
|
||||
|
||||
if created {
|
||||
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, strings.ToLower(mci.Owner.LowerName+"/"+mci.Image)); err != nil {
|
||||
log.Error("Error setting package property: %v", err)
|
||||
return nil, fmt.Errorf("InsertProperty(PropertyRepository): %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
metadata.IsTagged = mci.IsTagged
|
||||
|
||||
metadataJSON, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("json.Marshal(metadata): %w", err)
|
||||
}
|
||||
|
||||
// "docker buildx imagetools create" multi-arch operations:
|
||||
// {"type":"oci","is_tagged":false,"platform":"unknown/unknown"}
|
||||
// {"type":"oci","is_tagged":false,"platform":"linux/amd64","layer_creation":["ADD file:9233f6f2237d79659a9521f7e390df217cec49f1a8aa3a12147bbca1956acdb9 in /","CMD [\"/bin/sh\"]"]}
|
||||
// {"type":"oci","is_tagged":false,"platform":"unknown/unknown"}
|
||||
// {"type":"oci","is_tagged":false,"platform":"linux/arm64","layer_creation":["ADD file:df53811312284306901fdaaff0a357a4bf40d631e662fe9ce6d342442e494b6c in /","CMD [\"/bin/sh\"]"]}
|
||||
// {"type":"oci","is_tagged":true,"manifests":[{"platform":"linux/amd64","digest":"sha256:72bb73e706c0dec424d00a1febb21deaf1175a70ead009ad8b159729cfcf5769","size":2819478},{"platform":"linux/arm64","digest":"sha256:9e1426dd084a3221663b85ca1ee99d140c50b153917a5c5604c1f9b78229fd24","size":2716499},{"platform":"unknown/unknown","digest":"sha256:b93f03d0ae11b988243e1b2cd8d29accf5b9670547b7bd8c7d96abecc7283e6e","size":1798},{"platform":"unknown/unknown","digest":"sha256:f034b182ba66366c63a5d195c6dfcd3333c027409c0ac98e55ade36aaa3b2963","size":1798}]}
|
||||
|
||||
_pv := &packages_model.PackageVersion{
|
||||
PackageID: p.ID,
|
||||
CreatorID: mci.Creator.ID,
|
||||
Version: strings.ToLower(mci.Reference),
|
||||
LowerVersion: strings.ToLower(mci.Reference),
|
||||
MetadataJSON: string(metadataJSON),
|
||||
}
|
||||
pv, err := packages_model.GetOrInsertVersion(ctx, _pv)
|
||||
if err != nil {
|
||||
if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) {
|
||||
log.Error("Error GetOrInsertVersion (first try) package: %v", err)
|
||||
return nil, fmt.Errorf("GetOrInsertVersion: first try: %w", err)
|
||||
}
|
||||
if err = packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
|
||||
return nil, fmt.Errorf("DeletePackageVersionAndReferences: %w", err)
|
||||
}
|
||||
// keep download count on overwriting
|
||||
_pv.DownloadCount = pv.DownloadCount
|
||||
pv, err = packages_model.GetOrInsertVersion(ctx, _pv)
|
||||
if err != nil {
|
||||
if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) {
|
||||
log.Error("Error GetOrInsertVersion (second try) package: %v", err)
|
||||
return nil, fmt.Errorf("GetOrInsertVersion: second try: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := packages_service.CheckCountQuotaExceeded(ctx, mci.Creator, mci.Owner); err != nil {
|
||||
return nil, fmt.Errorf("CheckCountQuotaExceeded: %w", err)
|
||||
}
|
||||
|
||||
if mci.IsTagged {
|
||||
if err = packages_model.InsertOrUpdateProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged, ""); err != nil {
|
||||
return nil, fmt.Errorf("InsertOrUpdateProperty(ManifestTagged): %w", err)
|
||||
}
|
||||
} else {
|
||||
if err = packages_model.DeletePropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged); err != nil {
|
||||
return nil, fmt.Errorf("DeletePropertiesByName(ManifestTagged): %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err = packages_model.DeletePropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference); err != nil {
|
||||
return nil, fmt.Errorf("DeletePropertiesByName(ManifestReference): %w", err)
|
||||
}
|
||||
for _, manifest := range metadata.Manifests {
|
||||
if _, err = packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, manifest.Digest); err != nil {
|
||||
return nil, fmt.Errorf("InsertProperty(ManifestReference): %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return pv, nil
|
||||
}
|
||||
|
||||
type blobReference struct {
|
||||
Digest digest.Digest
|
||||
MediaType string
|
||||
Name string
|
||||
File *packages_model.PackageFileDescriptor
|
||||
ExpectedSize int64
|
||||
IsLead bool
|
||||
}
|
||||
|
||||
func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *packages_model.PackageVersion, ref *blobReference) (*packages_model.PackageFile, error) {
|
||||
if ref.File.Blob.Size != ref.ExpectedSize {
|
||||
return nil, errSizeInvalid
|
||||
}
|
||||
|
||||
if ref.Name == "" {
|
||||
ref.Name = strings.ToLower("sha256_" + ref.File.Blob.HashSHA256)
|
||||
}
|
||||
|
||||
pf := &packages_model.PackageFile{
|
||||
VersionID: pv.ID,
|
||||
BlobID: ref.File.Blob.ID,
|
||||
Name: ref.Name,
|
||||
LowerName: ref.Name,
|
||||
CompositeKey: string(ref.Digest),
|
||||
IsLead: ref.IsLead,
|
||||
}
|
||||
var err error
|
||||
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
|
||||
if errors.Is(err, packages_model.ErrDuplicatePackageFile) {
|
||||
// Skip this blob because the manifest contains the same filesystem layer multiple times.
|
||||
return pf, nil
|
||||
}
|
||||
log.Error("Error inserting package file: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
props := map[string]string{
|
||||
container_module.PropertyMediaType: ref.MediaType,
|
||||
container_module.PropertyDigest: string(ref.Digest),
|
||||
}
|
||||
for name, value := range props {
|
||||
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil {
|
||||
log.Error("Error setting package file property: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the ref file (old file) from the blob upload version
|
||||
if uploadVersion != nil && ref.File.File != nil && uploadVersion.ID == ref.File.File.VersionID {
|
||||
if err := packages_service.DeletePackageFile(ctx, ref.File.File); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return pf, nil
|
||||
}
|
||||
|
||||
func createManifestBlob(ctx context.Context, contentStore *packages_module.ContentStore, mci *manifestCreationInfo, pv *packages_model.PackageVersion, buf *packages_module.HashedBuffer) (_ *packages_model.PackageBlob, created bool, manifestDigest string, _ error) {
|
||||
pb, exists, err := packages_model.GetOrInsertBlob(ctx, packages_service.NewPackageBlob(buf))
|
||||
if err != nil {
|
||||
log.Error("Error inserting package blob: %v", err)
|
||||
return nil, false, "", err
|
||||
}
|
||||
// FIXME: Workaround to be removed in v1.20
|
||||
// https://github.com/go-gitea/gitea/issues/19586
|
||||
if exists {
|
||||
err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
|
||||
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
|
||||
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
|
||||
exists = false
|
||||
}
|
||||
}
|
||||
if !exists {
|
||||
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil {
|
||||
log.Error("Error saving package blob in content store: %v", err)
|
||||
return nil, false, "", err
|
||||
}
|
||||
}
|
||||
|
||||
manifestDigest = digestFromHashSummer(buf)
|
||||
pf, err := createFileFromBlobReference(ctx, pv, nil, &blobReference{
|
||||
Digest: digest.Digest(manifestDigest),
|
||||
MediaType: mci.MediaType,
|
||||
Name: container_module.ManifestFilename,
|
||||
File: &packages_model.PackageFileDescriptor{Blob: pb},
|
||||
ExpectedSize: pb.Size,
|
||||
IsLead: true,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, "", err
|
||||
}
|
||||
|
||||
oldManifestFiles, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
|
||||
OwnerID: mci.Owner.ID,
|
||||
PackageType: packages_model.TypeContainer,
|
||||
VersionID: pv.ID,
|
||||
Query: container_module.ManifestFilename,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, "", err
|
||||
}
|
||||
for _, oldManifestFile := range oldManifestFiles {
|
||||
if oldManifestFile.ID != pf.ID && oldManifestFile.IsLead {
|
||||
err = packages_model.UpdateFile(ctx, &packages_model.PackageFile{ID: oldManifestFile.ID, IsLead: false}, []string{"is_lead"})
|
||||
if err != nil {
|
||||
return nil, false, "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
return pb, !exists, manifestDigest, err
|
||||
}
|
||||
Reference in New Issue
Block a user