cerc-io/gitea
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HEAD check for container blobs deletes blob if content store cannot find

Browse Source 
blob nil on repeat

logging for blob insert

Upload blob recovery during push

console cleanup
This commit is contained in:
Michael Shaw 2022-11-16 15:35:49 -05:00
parent 3c07ed0911
commit 00f3f3aedf
3 changed files with 28 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
routers/api/packages/container/blob.go
View File

@ -23,7 +23,7 @@ import (
// The uploaded blob gets stored in a special upload version to link them to the package/image // The uploaded blob gets stored in a special upload version to link them to the package/image
func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_service.PackageInfo) (*packages_model.PackageBlob, error) { func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_service.PackageInfo) (*packages_model.PackageBlob, error) {
pb := packages_service.NewPackageBlob(hsr) pb := packages_service.NewPackageBlob(hsr)
fmt.Println("saveAsPackageBlob: ", pb.HashSHA256)
exists := false exists := false
contentStore := packages_module.NewContentStore() contentStore := packages_module.NewContentStore()
@ -45,7 +45,7 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
return err return err
} }
} }
fmt.Println("CREATED!!!")
if created { if created {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, strings.ToLower(pi.Owner.LowerName+"/"+pi.Name)); err != nil { if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, strings.ToLower(pi.Owner.LowerName+"/"+pi.Name)); err != nil {
log.Error("Error setting package property: %v", err) log.Error("Error setting package property: %v", err)
@ -73,6 +73,16 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
log.Error("Error inserting package blob: %v", err) log.Error("Error inserting package blob: %v", err)
return err return err
} }
if exists {
_, err := contentStore.Get(packages_module.BlobHash256Key(pb.HashSHA256))
if err != nil {
log.Info("BLOB MODEL and ERR in ContentStore. Resaving Blob to content store ", err.Error())
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
log.Error("Error saving package blob in content store during FIXITUP: %v", err)
return err
}
}
}
if !exists { if !exists {
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil { if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
log.Error("Error saving package blob in content store: %v", err) log.Error("Error saving package blob in content store: %v", err)

16
routers/api/packages/container/container.go
View File

@ -379,7 +379,23 @@ func HeadBlob(ctx *context.Context) {
} }
return return
} }
//check if actually in ContentStore
//if not, delete Blob and return 404
//content_store.Get()
cStore := packages_module.NewContentStore()
configReader, err := cStore.Get(packages_module.BlobHash256Key(blob.Blob.HashSHA256))
if err != nil {
log.Warn("ContentStore Get Error while HEAD for Blob: ", blob.Blob.HashSHA256, err.Error())
digest := ctx.Params("digest")
if err := deleteBlob(ctx.Package.Owner.ID, ctx.Params("image"), digest); err != nil {
log.Warn("Error while deleteBlob in HEAD: ", err.Error())
}
apiErrorDefined(ctx, errBlobUnknown)
return
}
defer configReader.Close()
setResponseHeaders(ctx.Resp, &containerHeaders{ setResponseHeaders(ctx.Resp, &containerHeaders{
ContentDigest: blob.Properties.GetByName(container_module.PropertyDigest), ContentDigest: blob.Properties.GetByName(container_module.PropertyDigest),
ContentLength: blob.Blob.Size, ContentLength: blob.Blob.Size,

1
services/packages/packages.go
View File

@ -224,7 +224,6 @@ func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.Packag
func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) { func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename) log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data)) pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data))
if err != nil { if err != nil {
log.Error("Error inserting package blob: %v", err) log.Error("Error inserting package blob: %v", err)