cerc-io/gitea
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backport of knacker fix attached to gitea pull 21862

Browse Source
This commit is contained in:
Michael Shaw 2022-11-22 12:11:18 -05:00
parent e77b76425e
commit 8c5c2a2ce9
4 changed files with 74 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
modules/packages/content_store.go
View File

@ -32,6 +32,13 @@ func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) {
return s.store.Open(KeyToRelativePath(key))
}
// FIXME: Workaround to be removed in v1.20
// https://github.com/go-gitea/gitea/issues/19586
func (s *ContentStore) Has(key BlobHash256Key) error {
_, err := s.store.Stat(KeyToRelativePath(key))
return err
}
// Save stores a package blob
func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error {
_, err := s.store.Save(KeyToRelativePath(key), r, size)

30
routers/api/packages/container/blob.go
View File

@ -7,8 +7,11 @@ package container
import (
"context"
"encoding/hex"
"errors"
"fmt"
"os"
"strings"
"sync"
"code.gitea.io/gitea/models/db"
packages_model "code.gitea.io/gitea/models/packages"
@ -16,9 +19,12 @@ import (
"code.gitea.io/gitea/modules/log"
packages_module "code.gitea.io/gitea/modules/packages"
container_module "code.gitea.io/gitea/modules/packages/container"
"code.gitea.io/gitea/modules/util"
packages_service "code.gitea.io/gitea/services/packages"
)
var uploadVersionMutex sync.Mutex
// saveAsPackageBlob creates a package blob from an upload
// The uploaded blob gets stored in a special upload version to link them to the package/image
func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_service.PackageInfo) (*packages_model.PackageBlob, error) {
@ -27,6 +33,9 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
exists := false
contentStore := packages_module.NewContentStore()
var uploadVersion *packages_model.PackageVersion
uploadVersionMutex.Lock()
err := db.WithTx(db.DefaultContext, func(ctx context.Context) error {
created := true
@ -67,12 +76,31 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
return err
}
}
uploadVersion = pv
return nil
})
uploadVersionMutex.Unlock()
if err != nil {
return nil, err
}
err = db.WithTx(db.DefaultContext, func(ctx context.Context) error {
pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb)
if err != nil {
log.Error("Error inserting package blob: %v", err)
return err
}
// FIXME: Workaround to be removed in v1.20
// https://github.com/go-gitea/gitea/issues/19586
if exists {
err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
exists = false
}
}
if !exists {
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
log.Error("Error saving package blob in content store: %v", err)
@ -83,7 +111,7 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256))
pf := &packages_model.PackageFile{
VersionID: pv.ID,
VersionID: uploadVersion.ID,
BlobID: pb.ID,
Name: filename,
LowerName: filename,

28
routers/api/packages/container/container.go
View File

@ -5,11 +5,13 @@
package container
import (
"code.gitea.io/gitea/modules/util"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"regexp"
"strconv"
"strings"
@ -193,7 +195,7 @@ func InitiateUploadBlob(ctx *context.Context) {
mount := ctx.FormTrim("mount")
from := ctx.FormTrim("from")
if mount != "" {
blob, _ := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
blob, _ := workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
Image: from,
Digest: mount,
})
@ -406,7 +408,7 @@ func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescri
return nil, container_model.ErrContainerBlobNotExist
}
return container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
return workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
OwnerID: ctx.Package.Owner.ID,
Image: ctx.Params("image"),
Digest: digest,
@ -548,7 +550,7 @@ func getManifestFromContext(ctx *context.Context) (*packages_model.PackageFileDe
return nil, container_model.ErrContainerBlobNotExist
}
return container_model.GetContainerBlob(ctx, opts)
return workaroundGetContainerBlob(ctx, opts)
}
// https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry
@ -688,3 +690,23 @@ func GetTagList(ctx *context.Context) {
Tags: tags,
})
}
// FIXME: Workaround to be removed in v1.20
// https://github.com/go-gitea/gitea/issues/19586
func workaroundGetContainerBlob(ctx *context.Context, opts *container_model.BlobSearchOptions) (*packages_model.PackageFileDescriptor, error) {
blob, err := container_model.GetContainerBlob(ctx, opts)
if err != nil {
return nil, err
}
err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(blob.Blob.HashSHA256))
if err != nil {
if errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist) {
log.Debug("Package registry inconsistent: blob %s does not exist on file system", blob.Blob.HashSHA256)
return nil, container_model.ErrContainerBlobNotExist
}
return nil, err
}
return blob, nil
}

13
routers/api/packages/container/manifest.go
View File

@ -5,9 +5,12 @@
package container
import (
"code.gitea.io/gitea/modules/util"
"context"
"errors"
"fmt"
"io"
"os"
"strings"
"code.gitea.io/gitea/models/db"
@ -403,6 +406,16 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack
log.Error("Error inserting package blob: %v", err)
return nil, false, "", err
}
// FIXME: Workaround to be removed in v1.20
// https://github.com/go-gitea/gitea/issues/19586
if exists {
err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(pb.HashSHA256))
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
exists = false
}
}
if !exists {
contentStore := packages_module.NewContentStore()
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil {