diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index be416ac26..13763db98 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -32,6 +32,13 @@ func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) { return s.store.Open(KeyToRelativePath(key)) } +// FIXME: Workaround to be removed in v1.20 +// https://github.com/go-gitea/gitea/issues/19586 +func (s *ContentStore) Has(key BlobHash256Key) error { + _, err := s.store.Stat(KeyToRelativePath(key)) + return err +} + // Save stores a package blob func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error { _, err := s.store.Save(KeyToRelativePath(key), r, size) diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go index df6b7aed9..912fdf643 100644 --- a/routers/api/packages/container/blob.go +++ b/routers/api/packages/container/blob.go @@ -7,8 +7,11 @@ package container import ( "context" "encoding/hex" + "errors" "fmt" + "os" "strings" + "sync" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" @@ -16,9 +19,12 @@ import ( "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" container_module "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" ) +var uploadVersionMutex sync.Mutex + // saveAsPackageBlob creates a package blob from an upload // The uploaded blob gets stored in a special upload version to link them to the package/image func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_service.PackageInfo) (*packages_model.PackageBlob, error) { @@ -27,6 +33,9 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic exists := false contentStore := packages_module.NewContentStore() + var uploadVersion *packages_model.PackageVersion + + uploadVersionMutex.Lock() err := db.WithTx(db.DefaultContext, func(ctx context.Context) error { created := true @@ -67,12 +76,31 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic return err } } + uploadVersion = pv + + return nil + }) + uploadVersionMutex.Unlock() + if err != nil { + return nil, err + } + + err = db.WithTx(db.DefaultContext, func(ctx context.Context) error { pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb) if err != nil { log.Error("Error inserting package blob: %v", err) return err } + // FIXME: Workaround to be removed in v1.20 + // https://github.com/go-gitea/gitea/issues/19586 + if exists { + err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256)) + if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) { + log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256) + exists = false + } + } if !exists { if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil { log.Error("Error saving package blob in content store: %v", err) @@ -83,7 +111,7 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256)) pf := &packages_model.PackageFile{ - VersionID: pv.ID, + VersionID: uploadVersion.ID, BlobID: pb.ID, Name: filename, LowerName: filename, diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go index 5bc64e1b2..4ea4ebd53 100644 --- a/routers/api/packages/container/container.go +++ b/routers/api/packages/container/container.go @@ -5,11 +5,13 @@ package container import ( + "code.gitea.io/gitea/modules/util" "errors" "fmt" "io" "net/http" "net/url" + "os" "regexp" "strconv" "strings" @@ -193,7 +195,7 @@ func InitiateUploadBlob(ctx *context.Context) { mount := ctx.FormTrim("mount") from := ctx.FormTrim("from") if mount != "" { - blob, _ := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + blob, _ := workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{ Image: from, Digest: mount, }) @@ -406,7 +408,7 @@ func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescri return nil, container_model.ErrContainerBlobNotExist } - return container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + return workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{ OwnerID: ctx.Package.Owner.ID, Image: ctx.Params("image"), Digest: digest, @@ -548,7 +550,7 @@ func getManifestFromContext(ctx *context.Context) (*packages_model.PackageFileDe return nil, container_model.ErrContainerBlobNotExist } - return container_model.GetContainerBlob(ctx, opts) + return workaroundGetContainerBlob(ctx, opts) } // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry @@ -688,3 +690,23 @@ func GetTagList(ctx *context.Context) { Tags: tags, }) } + +// FIXME: Workaround to be removed in v1.20 +// https://github.com/go-gitea/gitea/issues/19586 +func workaroundGetContainerBlob(ctx *context.Context, opts *container_model.BlobSearchOptions) (*packages_model.PackageFileDescriptor, error) { + blob, err := container_model.GetContainerBlob(ctx, opts) + if err != nil { + return nil, err + } + + err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(blob.Blob.HashSHA256)) + if err != nil { + if errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist) { + log.Debug("Package registry inconsistent: blob %s does not exist on file system", blob.Blob.HashSHA256) + return nil, container_model.ErrContainerBlobNotExist + } + return nil, err + } + + return blob, nil +} diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go index a48b1de3b..ede2c9527 100644 --- a/routers/api/packages/container/manifest.go +++ b/routers/api/packages/container/manifest.go @@ -5,9 +5,12 @@ package container import ( + "code.gitea.io/gitea/modules/util" "context" + "errors" "fmt" "io" + "os" "strings" "code.gitea.io/gitea/models/db" @@ -403,6 +406,16 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack log.Error("Error inserting package blob: %v", err) return nil, false, "", err } + // FIXME: Workaround to be removed in v1.20 + // https://github.com/go-gitea/gitea/issues/19586 + if exists { + err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(pb.HashSHA256)) + if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) { + log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256) + exists = false + } + } + if !exists { contentStore := packages_module.NewContentStore() if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil {