cerc-io/plugeth
18
0
Fork 1
Code Issues 4 Pull Requests Packages Projects Releases Wiki Activity

build/ci: handle split up listing (#21293)

Browse Source
This commit is contained in:
Martin Holst Swende 2020-07-04 20:10:48 +02:00 committed by GitHub
parent 490b380a04
commit fa01117498
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build/ci.go
View File

@ -1098,6 +1098,8 @@ func doPurge(cmdline []string) {
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
fmt.Printf("Found %d blobs\n", len(blobs))
// Iterate over the blobs, collect and sort all unstable builds // Iterate over the blobs, collect and sort all unstable builds
for i := 0; i < len(blobs); i++ { for i := 0; i < len(blobs); i++ {
if !strings.Contains(blobs[i].Name, "unstable") { if !strings.Contains(blobs[i].Name, "unstable") {
@ -1119,6 +1121,7 @@ func doPurge(cmdline []string) {
break break
} }
} }
fmt.Printf("Deleting %d blobs\n", len(blobs))
// Delete all marked as such and return // Delete all marked as such and return
if err := build.AzureBlobstoreDelete(auth, blobs); err != nil { if err := build.AzureBlobstoreDelete(auth, blobs); err != nil {
log.Fatal(err) log.Fatal(err)

22
internal/build/azure.go
View File

@ -71,26 +71,35 @@ func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig)
// AzureBlobstoreList lists all the files contained within an azure blobstore. // AzureBlobstoreList lists all the files contained within an azure blobstore.
func AzureBlobstoreList(config AzureBlobstoreConfig) ([]azblob.BlobItem, error) { func AzureBlobstoreList(config AzureBlobstoreConfig) ([]azblob.BlobItem, error) {
credential, err := azblob.NewSharedKeyCredential(config.Account, config.Token) credential := azblob.NewAnonymousCredential()
if len(config.Token) > 0 {
c, err := azblob.NewSharedKeyCredential(config.Account, config.Token)
if err != nil { if err != nil {
return nil, err return nil, err
} }
credential = c
}
pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{})
u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", config.Account)) u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", config.Account))
service := azblob.NewServiceURL(*u, pipeline) service := azblob.NewServiceURL(*u, pipeline)
var allBlobs []azblob.BlobItem
// List all the blobs from the container and return them // List all the blobs from the container and return them
container := service.NewContainerURL(config.Container) container := service.NewContainerURL(config.Container)
nextMarker := azblob.Marker{}
res, err := container.ListBlobsFlatSegment(context.Background(), azblob.Marker{}, azblob.ListBlobsSegmentOptions{ for nextMarker.NotDone() {
MaxResults: 1024 * 1024 * 1024, // Yes, fetch all of them res, err := container.ListBlobsFlatSegment(context.Background(), nextMarker, azblob.ListBlobsSegmentOptions{
MaxResults: 5000, // The server only gives max 5K items
}) })
if err != nil { if err != nil {
return nil, err return nil, err
} }
return res.Segment.BlobItems, nil allBlobs = append(allBlobs, res.Segment.BlobItems...)
nextMarker = res.NextMarker
}
return allBlobs, nil
} }
// AzureBlobstoreDelete iterates over a list of files to delete and removes them // AzureBlobstoreDelete iterates over a list of files to delete and removes them
@ -121,6 +130,7 @@ func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []azblob.BlobItem)
if _, err := blockblob.Delete(context.Background(), azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}); err != nil { if _, err := blockblob.Delete(context.Background(), azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}); err != nil {
return err return err
} }
fmt.Printf("deleted %s (%s)\n", blob.Name, blob.Properties.LastModified)
} }
return nil return nil
} }