Provide self-registering storage system (#12978)
* Provide self-registering storage system Signed-off-by: Andrew Thornton <art27@cantab.net> * More simplification Signed-off-by: Andrew Thornton <art27@cantab.net> * Remove old strings from setting Signed-off-by: Andrew Thornton <art27@cantab.net> * oops attachments not attachment Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
		
							parent
							
								
									ade9c8dc3c
								
							
						
					
					
						commit
						6b1266b6b3
					
				| @ -32,8 +32,8 @@ var CmdMigrateStorage = cli.Command{ | |||||||
| 		}, | 		}, | ||||||
| 		cli.StringFlag{ | 		cli.StringFlag{ | ||||||
| 			Name:  "storage, s", | 			Name:  "storage, s", | ||||||
| 			Value: setting.LocalStorageType, | 			Value: "", | ||||||
| 			Usage: "New storage type, local or minio", | 			Usage: "New storage type: local (default) or minio", | ||||||
| 		}, | 		}, | ||||||
| 		cli.StringFlag{ | 		cli.StringFlag{ | ||||||
| 			Name:  "path, p", | 			Name:  "path, p", | ||||||
| @ -107,6 +107,8 @@ func runMigrateStorage(ctx *cli.Context) error { | |||||||
| 		return err | 		return err | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
|  | 	goCtx := context.Background() | ||||||
|  | 
 | ||||||
| 	if err := storage.Init(); err != nil { | 	if err := storage.Init(); err != nil { | ||||||
| 		return err | 		return err | ||||||
| 	} | 	} | ||||||
| @ -114,24 +116,31 @@ func runMigrateStorage(ctx *cli.Context) error { | |||||||
| 	var dstStorage storage.ObjectStorage | 	var dstStorage storage.ObjectStorage | ||||||
| 	var err error | 	var err error | ||||||
| 	switch strings.ToLower(ctx.String("storage")) { | 	switch strings.ToLower(ctx.String("storage")) { | ||||||
| 	case setting.LocalStorageType: | 	case "": | ||||||
|  | 		fallthrough | ||||||
|  | 	case string(storage.LocalStorageType): | ||||||
| 		p := ctx.String("path") | 		p := ctx.String("path") | ||||||
| 		if p == "" { | 		if p == "" { | ||||||
| 			log.Fatal("Path must be given when storage is loal") | 			log.Fatal("Path must be given when storage is loal") | ||||||
| 			return nil | 			return nil | ||||||
| 		} | 		} | ||||||
| 		dstStorage, err = storage.NewLocalStorage(p) | 		dstStorage, err = storage.NewLocalStorage( | ||||||
| 	case setting.MinioStorageType: | 			goCtx, | ||||||
|  | 			storage.LocalStorageConfig{ | ||||||
|  | 				Path: p, | ||||||
|  | 			}) | ||||||
|  | 	case string(storage.MinioStorageType): | ||||||
| 		dstStorage, err = storage.NewMinioStorage( | 		dstStorage, err = storage.NewMinioStorage( | ||||||
| 			context.Background(), | 			goCtx, | ||||||
| 			ctx.String("minio-endpoint"), | 			storage.MinioStorageConfig{ | ||||||
| 			ctx.String("minio-access-key-id"), | 				Endpoint:        ctx.String("minio-endpoint"), | ||||||
| 			ctx.String("minio-secret-access-key"), | 				AccessKeyID:     ctx.String("minio-access-key-id"), | ||||||
| 			ctx.String("minio-bucket"), | 				SecretAccessKey: ctx.String("minio-secret-access-key"), | ||||||
| 			ctx.String("minio-location"), | 				Bucket:          ctx.String("minio-bucket"), | ||||||
| 			ctx.String("minio-base-path"), | 				Location:        ctx.String("minio-location"), | ||||||
| 			ctx.Bool("minio-use-ssl"), | 				BasePath:        ctx.String("minio-base-path"), | ||||||
| 		) | 				UseSSL:          ctx.Bool("minio-use-ssl"), | ||||||
|  | 			}) | ||||||
| 	default: | 	default: | ||||||
| 		return fmt.Errorf("Unsupported attachments storage type: %s", ctx.String("storage")) | 		return fmt.Errorf("Unsupported attachments storage type: %s", ctx.String("storage")) | ||||||
| 	} | 	} | ||||||
|  | |||||||
| @ -67,10 +67,8 @@ func MainTest(m *testing.M, pathToGiteaRoot string) { | |||||||
| 	if err != nil { | 	if err != nil { | ||||||
| 		fatalTestError("url.Parse: %v\n", err) | 		fatalTestError("url.Parse: %v\n", err) | ||||||
| 	} | 	} | ||||||
| 	setting.Attachment.Storage.Type = setting.LocalStorageType |  | ||||||
| 	setting.Attachment.Storage.Path = filepath.Join(setting.AppDataPath, "attachments") | 	setting.Attachment.Storage.Path = filepath.Join(setting.AppDataPath, "attachments") | ||||||
| 
 | 
 | ||||||
| 	setting.LFS.Storage.Type = setting.LocalStorageType |  | ||||||
| 	setting.LFS.Storage.Path = filepath.Join(setting.AppDataPath, "lfs") | 	setting.LFS.Storage.Path = filepath.Join(setting.AppDataPath, "lfs") | ||||||
| 	if err = storage.Init(); err != nil { | 	if err = storage.Init(); err != nil { | ||||||
| 		fatalTestError("storage.Init: %v\n", err) | 		fatalTestError("storage.Init: %v\n", err) | ||||||
|  | |||||||
| @ -4,12 +4,6 @@ | |||||||
| 
 | 
 | ||||||
| package setting | package setting | ||||||
| 
 | 
 | ||||||
| import ( |  | ||||||
| 	"path/filepath" |  | ||||||
| 
 |  | ||||||
| 	"code.gitea.io/gitea/modules/log" |  | ||||||
| ) |  | ||||||
| 
 |  | ||||||
| var ( | var ( | ||||||
| 	// Attachment settings
 | 	// Attachment settings
 | ||||||
| 	Attachment = struct { | 	Attachment = struct { | ||||||
| @ -20,7 +14,6 @@ var ( | |||||||
| 		Enabled      bool | 		Enabled      bool | ||||||
| 	}{ | 	}{ | ||||||
| 		Storage: Storage{ | 		Storage: Storage{ | ||||||
| 			Type:        LocalStorageType, |  | ||||||
| 			ServeDirect: false, | 			ServeDirect: false, | ||||||
| 		}, | 		}, | ||||||
| 		AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip", | 		AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip", | ||||||
| @ -32,37 +25,9 @@ var ( | |||||||
| 
 | 
 | ||||||
| func newAttachmentService() { | func newAttachmentService() { | ||||||
| 	sec := Cfg.Section("attachment") | 	sec := Cfg.Section("attachment") | ||||||
| 	Attachment.Storage.Type = sec.Key("STORAGE_TYPE").MustString("") | 	storageType := sec.Key("STORAGE_TYPE").MustString("") | ||||||
| 	if Attachment.Storage.Type == "" { |  | ||||||
| 		Attachment.Storage.Type = "default" |  | ||||||
| 	} |  | ||||||
| 
 | 
 | ||||||
| 	if Attachment.Storage.Type != LocalStorageType && Attachment.Storage.Type != MinioStorageType { | 	Attachment.Storage = getStorage("attachments", storageType, sec) | ||||||
| 		storage, ok := storages[Attachment.Storage.Type] |  | ||||||
| 		if !ok { |  | ||||||
| 			log.Fatal("Failed to get attachment storage type: %s", Attachment.Storage.Type) |  | ||||||
| 		} |  | ||||||
| 		Attachment.Storage = storage |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	// Override
 |  | ||||||
| 	Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(Attachment.ServeDirect) |  | ||||||
| 
 |  | ||||||
| 	switch Attachment.Storage.Type { |  | ||||||
| 	case LocalStorageType: |  | ||||||
| 		Attachment.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "attachments")) |  | ||||||
| 		if !filepath.IsAbs(Attachment.Path) { |  | ||||||
| 			Attachment.Path = filepath.Join(AppWorkPath, Attachment.Path) |  | ||||||
| 		} |  | ||||||
| 	case MinioStorageType: |  | ||||||
| 		Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString(Attachment.Minio.Endpoint) |  | ||||||
| 		Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString(Attachment.Minio.AccessKeyID) |  | ||||||
| 		Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString(Attachment.Minio.SecretAccessKey) |  | ||||||
| 		Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString(Attachment.Minio.Bucket) |  | ||||||
| 		Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString(Attachment.Minio.Location) |  | ||||||
| 		Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(Attachment.Minio.UseSSL) |  | ||||||
| 		Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/") |  | ||||||
| 	} |  | ||||||
| 
 | 
 | ||||||
| 	Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".docx,.gif,.gz,.jpeg,.jpg,.log,.pdf,.png,.pptx,.txt,.xlsx,.zip") | 	Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".docx,.gif,.gz,.jpeg,.jpg,.log,.pdf,.png,.pptx,.txt,.xlsx,.zip") | ||||||
| 	Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4) | 	Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4) | ||||||
|  | |||||||
| @ -37,40 +37,15 @@ func newLFSService() { | |||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	lfsSec := Cfg.Section("lfs") | 	lfsSec := Cfg.Section("lfs") | ||||||
| 	LFS.Storage.Type = lfsSec.Key("STORAGE_TYPE").MustString("") | 	storageType := lfsSec.Key("STORAGE_TYPE").MustString("") | ||||||
| 	if LFS.Storage.Type == "" { |  | ||||||
| 		LFS.Storage.Type = "default" |  | ||||||
| 	} |  | ||||||
| 
 | 
 | ||||||
| 	if LFS.Storage.Type != LocalStorageType && LFS.Storage.Type != MinioStorageType { | 	// Specifically default PATH to LFS_CONTENT_PATH
 | ||||||
| 		storage, ok := storages[LFS.Storage.Type] | 	lfsSec.Key("PATH").MustString( | ||||||
| 		if !ok { | 		sec.Key("LFS_CONTENT_PATH").String()) | ||||||
| 			log.Fatal("Failed to get lfs storage type: %s", LFS.Storage.Type) |  | ||||||
| 		} |  | ||||||
| 		LFS.Storage = storage |  | ||||||
| 	} |  | ||||||
| 
 | 
 | ||||||
| 	// Override
 | 	LFS.Storage = getStorage("lfs", storageType, lfsSec) | ||||||
| 	LFS.ServeDirect = lfsSec.Key("SERVE_DIRECT").MustBool(LFS.ServeDirect) |  | ||||||
| 	switch LFS.Storage.Type { |  | ||||||
| 	case LocalStorageType: |  | ||||||
| 		// keep compatible
 |  | ||||||
| 		LFS.Path = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs")) |  | ||||||
| 		LFS.Path = lfsSec.Key("PATH").MustString(LFS.Path) |  | ||||||
| 		if !filepath.IsAbs(LFS.Path) { |  | ||||||
| 			LFS.Path = filepath.Join(AppWorkPath, LFS.Path) |  | ||||||
| 		} |  | ||||||
| 
 |  | ||||||
| 	case MinioStorageType: |  | ||||||
| 		LFS.Minio.Endpoint = lfsSec.Key("MINIO_ENDPOINT").MustString(LFS.Minio.Endpoint) |  | ||||||
| 		LFS.Minio.AccessKeyID = lfsSec.Key("MINIO_ACCESS_KEY_ID").MustString(LFS.Minio.AccessKeyID) |  | ||||||
| 		LFS.Minio.SecretAccessKey = lfsSec.Key("MINIO_SECRET_ACCESS_KEY").MustString(LFS.Minio.SecretAccessKey) |  | ||||||
| 		LFS.Minio.Bucket = lfsSec.Key("MINIO_BUCKET").MustString(LFS.Minio.Bucket) |  | ||||||
| 		LFS.Minio.Location = lfsSec.Key("MINIO_LOCATION").MustString(LFS.Minio.Location) |  | ||||||
| 		LFS.Minio.UseSSL = lfsSec.Key("MINIO_USE_SSL").MustBool(LFS.Minio.UseSSL) |  | ||||||
| 		LFS.Minio.BasePath = lfsSec.Key("MINIO_BASE_PATH").MustString("lfs/") |  | ||||||
| 	} |  | ||||||
| 
 | 
 | ||||||
|  | 	// Rest of LFS service settings
 | ||||||
| 	if LFS.LocksPagingNum == 0 { | 	if LFS.LocksPagingNum == 0 { | ||||||
| 		LFS.LocksPagingNum = 50 | 		LFS.LocksPagingNum = 50 | ||||||
| 	} | 	} | ||||||
|  | |||||||
| @ -804,7 +804,6 @@ func NewContext() { | |||||||
| 		} | 		} | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	newStorageService() |  | ||||||
| 	newAttachmentService() | 	newAttachmentService() | ||||||
| 	newLFSService() | 	newLFSService() | ||||||
| 
 | 
 | ||||||
|  | |||||||
| @ -5,65 +5,77 @@ | |||||||
| package setting | package setting | ||||||
| 
 | 
 | ||||||
| import ( | import ( | ||||||
| 	"strings" | 	"path/filepath" | ||||||
|  | 	"reflect" | ||||||
| 
 | 
 | ||||||
| 	"code.gitea.io/gitea/modules/log" |  | ||||||
| 	ini "gopkg.in/ini.v1" | 	ini "gopkg.in/ini.v1" | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| // enumerate all storage types
 |  | ||||||
| const ( |  | ||||||
| 	LocalStorageType = "local" |  | ||||||
| 	MinioStorageType = "minio" |  | ||||||
| ) |  | ||||||
| 
 |  | ||||||
| // Storage represents configuration of storages
 | // Storage represents configuration of storages
 | ||||||
| type Storage struct { | type Storage struct { | ||||||
| 	Type        string | 	Type        string | ||||||
| 	Path        string | 	Path        string | ||||||
|  | 	Section     *ini.Section | ||||||
| 	ServeDirect bool | 	ServeDirect bool | ||||||
| 	Minio       struct { |  | ||||||
| 		Endpoint        string |  | ||||||
| 		AccessKeyID     string |  | ||||||
| 		SecretAccessKey string |  | ||||||
| 		UseSSL          bool |  | ||||||
| 		Bucket          string |  | ||||||
| 		Location        string |  | ||||||
| 		BasePath        string |  | ||||||
| 	} |  | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| var ( | // MapTo implements the Mappable interface
 | ||||||
| 	storages = make(map[string]Storage) | func (s *Storage) MapTo(v interface{}) error { | ||||||
| ) | 	pathValue := reflect.ValueOf(v).FieldByName("Path") | ||||||
| 
 | 	if pathValue.IsValid() && pathValue.Kind() == reflect.String { | ||||||
| func getStorage(sec *ini.Section) Storage { | 		pathValue.SetString(s.Path) | ||||||
| 	var storage Storage |  | ||||||
| 	storage.Type = sec.Key("STORAGE_TYPE").MustString(LocalStorageType) |  | ||||||
| 	storage.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false) |  | ||||||
| 	switch storage.Type { |  | ||||||
| 	case LocalStorageType: |  | ||||||
| 	case MinioStorageType: |  | ||||||
| 		storage.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000") |  | ||||||
| 		storage.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("") |  | ||||||
| 		storage.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("") |  | ||||||
| 		storage.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea") |  | ||||||
| 		storage.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1") |  | ||||||
| 		storage.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false) |  | ||||||
| 	} | 	} | ||||||
|  | 	if s.Section != nil { | ||||||
|  | 		return s.Section.MapTo(v) | ||||||
|  | 	} | ||||||
|  | 	return nil | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | func getStorage(name, typ string, overrides ...*ini.Section) Storage { | ||||||
|  | 	sectionName := "storage" | ||||||
|  | 	if len(name) > 0 { | ||||||
|  | 		sectionName = sectionName + "." + typ | ||||||
|  | 	} | ||||||
|  | 	sec := Cfg.Section(sectionName) | ||||||
|  | 
 | ||||||
|  | 	if len(overrides) == 0 { | ||||||
|  | 		overrides = []*ini.Section{ | ||||||
|  | 			Cfg.Section(sectionName + "." + name), | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	var storage Storage | ||||||
|  | 
 | ||||||
|  | 	storage.Type = sec.Key("STORAGE_TYPE").MustString("") | ||||||
|  | 	storage.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false) | ||||||
|  | 
 | ||||||
|  | 	// Global Defaults
 | ||||||
|  | 	sec.Key("MINIO_ENDPOINT").MustString("localhost:9000") | ||||||
|  | 	sec.Key("MINIO_ACCESS_KEY_ID").MustString("") | ||||||
|  | 	sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("") | ||||||
|  | 	sec.Key("MINIO_BUCKET").MustString("gitea") | ||||||
|  | 	sec.Key("MINIO_LOCATION").MustString("us-east-1") | ||||||
|  | 	sec.Key("MINIO_USE_SSL").MustBool(false) | ||||||
|  | 
 | ||||||
|  | 	storage.Section = sec | ||||||
|  | 
 | ||||||
|  | 	for _, override := range overrides { | ||||||
|  | 		for _, key := range storage.Section.Keys() { | ||||||
|  | 			if !override.HasKey(key.Name()) { | ||||||
|  | 				_, _ = override.NewKey(key.Name(), key.Value()) | ||||||
|  | 			} | ||||||
|  | 		} | ||||||
|  | 		storage.ServeDirect = override.Key("SERVE_DIRECT").MustBool(false) | ||||||
|  | 		storage.Section = override | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	// Specific defaults
 | ||||||
|  | 	storage.Path = storage.Section.Key("PATH").MustString(filepath.Join(AppDataPath, name)) | ||||||
|  | 	if !filepath.IsAbs(storage.Path) { | ||||||
|  | 		storage.Path = filepath.Join(AppWorkPath, storage.Path) | ||||||
|  | 		storage.Section.Key("PATH").SetValue(storage.Path) | ||||||
|  | 	} | ||||||
|  | 	storage.Section.Key("MINIO_BASE_PATH").MustString(name + "/") | ||||||
|  | 
 | ||||||
| 	return storage | 	return storage | ||||||
| } | } | ||||||
| 
 |  | ||||||
| func newStorageService() { |  | ||||||
| 	sec := Cfg.Section("storage") |  | ||||||
| 	storages["default"] = getStorage(sec) |  | ||||||
| 
 |  | ||||||
| 	for _, sec := range Cfg.Section("storage").ChildSections() { |  | ||||||
| 		name := strings.TrimPrefix(sec.Name(), "storage.") |  | ||||||
| 		if name == "default" || name == LocalStorageType || name == MinioStorageType { |  | ||||||
| 			log.Error("storage name %s is system reserved!", name) |  | ||||||
| 			continue |  | ||||||
| 		} |  | ||||||
| 		storages[name] = getStorage(sec) |  | ||||||
| 	} |  | ||||||
| } |  | ||||||
|  | |||||||
							
								
								
									
										65
									
								
								modules/storage/helper.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										65
									
								
								modules/storage/helper.go
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,65 @@ | |||||||
|  | // Copyright 2020 The Gitea Authors. All rights reserved.
 | ||||||
|  | // Use of this source code is governed by a MIT-style
 | ||||||
|  | // license that can be found in the LICENSE file.
 | ||||||
|  | 
 | ||||||
|  | package storage | ||||||
|  | 
 | ||||||
|  | import ( | ||||||
|  | 	"encoding/json" | ||||||
|  | 	"reflect" | ||||||
|  | ) | ||||||
|  | 
 | ||||||
|  | // Mappable represents an interface that can MapTo another interface
 | ||||||
|  | type Mappable interface { | ||||||
|  | 	MapTo(v interface{}) error | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // toConfig will attempt to convert a given configuration cfg into the provided exemplar type.
 | ||||||
|  | //
 | ||||||
|  | // It will tolerate the cfg being passed as a []byte or string of a json representation of the
 | ||||||
|  | // exemplar or the correct type of the exemplar itself
 | ||||||
|  | func toConfig(exemplar, cfg interface{}) (interface{}, error) { | ||||||
|  | 
 | ||||||
|  | 	// First of all check if we've got the same type as the exemplar - if so it's all fine.
 | ||||||
|  | 	if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) { | ||||||
|  | 		return cfg, nil | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	// Now if not - does it provide a MapTo function we can try?
 | ||||||
|  | 	if mappable, ok := cfg.(Mappable); ok { | ||||||
|  | 		newVal := reflect.New(reflect.TypeOf(exemplar)) | ||||||
|  | 		if err := mappable.MapTo(newVal.Interface()); err == nil { | ||||||
|  | 			return newVal.Elem().Interface(), nil | ||||||
|  | 		} | ||||||
|  | 		// MapTo has failed us ... let's try the json route ...
 | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	// OK we've been passed a byte array right?
 | ||||||
|  | 	configBytes, ok := cfg.([]byte) | ||||||
|  | 	if !ok { | ||||||
|  | 		// oh ... it's a string then?
 | ||||||
|  | 		var configStr string | ||||||
|  | 
 | ||||||
|  | 		configStr, ok = cfg.(string) | ||||||
|  | 		configBytes = []byte(configStr) | ||||||
|  | 	} | ||||||
|  | 	if !ok { | ||||||
|  | 		// hmm ... can we marshal it to json?
 | ||||||
|  | 		var err error | ||||||
|  | 
 | ||||||
|  | 		configBytes, err = json.Marshal(cfg) | ||||||
|  | 		ok = (err == nil) | ||||||
|  | 	} | ||||||
|  | 	if !ok { | ||||||
|  | 		// no ... we've tried hard enough at this point - throw an error!
 | ||||||
|  | 		return nil, ErrInvalidConfiguration{cfg: cfg} | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	// OK unmarshal the byte array into a new copy of the exemplar
 | ||||||
|  | 	newVal := reflect.New(reflect.TypeOf(exemplar)) | ||||||
|  | 	if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil { | ||||||
|  | 		// If we can't unmarshal it then return an error!
 | ||||||
|  | 		return nil, ErrInvalidConfiguration{cfg: cfg, err: err} | ||||||
|  | 	} | ||||||
|  | 	return newVal.Elem().Interface(), nil | ||||||
|  | } | ||||||
| @ -5,6 +5,7 @@ | |||||||
| package storage | package storage | ||||||
| 
 | 
 | ||||||
| import ( | import ( | ||||||
|  | 	"context" | ||||||
| 	"io" | 	"io" | ||||||
| 	"net/url" | 	"net/url" | ||||||
| 	"os" | 	"os" | ||||||
| @ -17,19 +18,35 @@ var ( | |||||||
| 	_ ObjectStorage = &LocalStorage{} | 	_ ObjectStorage = &LocalStorage{} | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
|  | // LocalStorageType is the type descriptor for local storage
 | ||||||
|  | const LocalStorageType Type = "local" | ||||||
|  | 
 | ||||||
|  | // LocalStorageConfig represents the configuration for a local storage
 | ||||||
|  | type LocalStorageConfig struct { | ||||||
|  | 	Path string `ini:"PATH"` | ||||||
|  | } | ||||||
|  | 
 | ||||||
| // LocalStorage represents a local files storage
 | // LocalStorage represents a local files storage
 | ||||||
| type LocalStorage struct { | type LocalStorage struct { | ||||||
|  | 	ctx context.Context | ||||||
| 	dir string | 	dir string | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // NewLocalStorage returns a local files
 | // NewLocalStorage returns a local files
 | ||||||
| func NewLocalStorage(bucket string) (*LocalStorage, error) { | func NewLocalStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) { | ||||||
| 	if err := os.MkdirAll(bucket, os.ModePerm); err != nil { | 	configInterface, err := toConfig(LocalStorageConfig{}, cfg) | ||||||
|  | 	if err != nil { | ||||||
|  | 		return nil, err | ||||||
|  | 	} | ||||||
|  | 	config := configInterface.(LocalStorageConfig) | ||||||
|  | 
 | ||||||
|  | 	if err := os.MkdirAll(config.Path, os.ModePerm); err != nil { | ||||||
| 		return nil, err | 		return nil, err | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	return &LocalStorage{ | 	return &LocalStorage{ | ||||||
| 		dir: bucket, | 		ctx: ctx, | ||||||
|  | 		dir: config.Path, | ||||||
| 	}, nil | 	}, nil | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| @ -80,6 +97,11 @@ func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) er | |||||||
| 		if err != nil { | 		if err != nil { | ||||||
| 			return err | 			return err | ||||||
| 		} | 		} | ||||||
|  | 		select { | ||||||
|  | 		case <-l.ctx.Done(): | ||||||
|  | 			return l.ctx.Err() | ||||||
|  | 		default: | ||||||
|  | 		} | ||||||
| 		if path == l.dir { | 		if path == l.dir { | ||||||
| 			return nil | 			return nil | ||||||
| 		} | 		} | ||||||
| @ -98,3 +120,7 @@ func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) er | |||||||
| 		return fn(relPath, obj) | 		return fn(relPath, obj) | ||||||
| 	}) | 	}) | ||||||
| } | } | ||||||
|  | 
 | ||||||
|  | func init() { | ||||||
|  | 	RegisterStorageType(LocalStorageType, NewLocalStorage) | ||||||
|  | } | ||||||
|  | |||||||
| @ -18,8 +18,9 @@ import ( | |||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| var ( | var ( | ||||||
| 	_            ObjectStorage = &MinioStorage{} | 	_ ObjectStorage = &MinioStorage{} | ||||||
| 	quoteEscaper               = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") | 
 | ||||||
|  | 	quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| type minioObject struct { | type minioObject struct { | ||||||
| @ -35,6 +36,20 @@ func (m *minioObject) Stat() (os.FileInfo, error) { | |||||||
| 	return &minioFileInfo{oi}, nil | 	return &minioFileInfo{oi}, nil | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | // MinioStorageType is the type descriptor for minio storage
 | ||||||
|  | const MinioStorageType Type = "minio" | ||||||
|  | 
 | ||||||
|  | // MinioStorageConfig represents the configuration for a minio storage
 | ||||||
|  | type MinioStorageConfig struct { | ||||||
|  | 	Endpoint        string `ini:"MINIO_ENDPOINT"` | ||||||
|  | 	AccessKeyID     string `ini:"MINIO_ACCESS_KEY_ID"` | ||||||
|  | 	SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"` | ||||||
|  | 	Bucket          string `ini:"MINIO_BUCKET"` | ||||||
|  | 	Location        string `ini:"MINIO_LOCATION"` | ||||||
|  | 	BasePath        string `ini:"MINIO_BASE_PATH"` | ||||||
|  | 	UseSSL          bool   `ini:"MINIO_USE_SSL"` | ||||||
|  | } | ||||||
|  | 
 | ||||||
| // MinioStorage returns a minio bucket storage
 | // MinioStorage returns a minio bucket storage
 | ||||||
| type MinioStorage struct { | type MinioStorage struct { | ||||||
| 	ctx      context.Context | 	ctx      context.Context | ||||||
| @ -44,20 +59,26 @@ type MinioStorage struct { | |||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // NewMinioStorage returns a minio storage
 | // NewMinioStorage returns a minio storage
 | ||||||
| func NewMinioStorage(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucket, location, basePath string, useSSL bool) (*MinioStorage, error) { | func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error) { | ||||||
| 	minioClient, err := minio.New(endpoint, &minio.Options{ | 	configInterface, err := toConfig(MinioStorageConfig{}, cfg) | ||||||
| 		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), | 	if err != nil { | ||||||
| 		Secure: useSSL, | 		return nil, err | ||||||
|  | 	} | ||||||
|  | 	config := configInterface.(MinioStorageConfig) | ||||||
|  | 
 | ||||||
|  | 	minioClient, err := minio.New(config.Endpoint, &minio.Options{ | ||||||
|  | 		Creds:  credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""), | ||||||
|  | 		Secure: config.UseSSL, | ||||||
| 	}) | 	}) | ||||||
| 	if err != nil { | 	if err != nil { | ||||||
| 		return nil, err | 		return nil, err | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	if err := minioClient.MakeBucket(ctx, bucket, minio.MakeBucketOptions{ | 	if err := minioClient.MakeBucket(ctx, config.Bucket, minio.MakeBucketOptions{ | ||||||
| 		Region: location, | 		Region: config.Location, | ||||||
| 	}); err != nil { | 	}); err != nil { | ||||||
| 		// Check to see if we already own this bucket (which happens if you run this twice)
 | 		// Check to see if we already own this bucket (which happens if you run this twice)
 | ||||||
| 		exists, errBucketExists := minioClient.BucketExists(ctx, bucket) | 		exists, errBucketExists := minioClient.BucketExists(ctx, config.Bucket) | ||||||
| 		if !exists || errBucketExists != nil { | 		if !exists || errBucketExists != nil { | ||||||
| 			return nil, err | 			return nil, err | ||||||
| 		} | 		} | ||||||
| @ -66,8 +87,8 @@ func NewMinioStorage(ctx context.Context, endpoint, accessKeyID, secretAccessKey | |||||||
| 	return &MinioStorage{ | 	return &MinioStorage{ | ||||||
| 		ctx:      ctx, | 		ctx:      ctx, | ||||||
| 		client:   minioClient, | 		client:   minioClient, | ||||||
| 		bucket:   bucket, | 		bucket:   config.Bucket, | ||||||
| 		basePath: basePath, | 		basePath: config.BasePath, | ||||||
| 	}, nil | 	}, nil | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| @ -183,3 +204,7 @@ func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) er | |||||||
| 	} | 	} | ||||||
| 	return nil | 	return nil | ||||||
| } | } | ||||||
|  | 
 | ||||||
|  | func init() { | ||||||
|  | 	RegisterStorageType(MinioStorageType, NewMinioStorage) | ||||||
|  | } | ||||||
|  | |||||||
| @ -22,6 +22,38 @@ var ( | |||||||
| 	ErrIterateObjectsNotSupported = errors.New("iterateObjects method not supported") | 	ErrIterateObjectsNotSupported = errors.New("iterateObjects method not supported") | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
|  | // ErrInvalidConfiguration is called when there is invalid configuration for a storage
 | ||||||
|  | type ErrInvalidConfiguration struct { | ||||||
|  | 	cfg interface{} | ||||||
|  | 	err error | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | func (err ErrInvalidConfiguration) Error() string { | ||||||
|  | 	if err.err != nil { | ||||||
|  | 		return fmt.Sprintf("Invalid Configuration Argument: %v: Error: %v", err.cfg, err.err) | ||||||
|  | 	} | ||||||
|  | 	return fmt.Sprintf("Invalid Configuration Argument: %v", err.cfg) | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // IsErrInvalidConfiguration checks if an error is an ErrInvalidConfiguration
 | ||||||
|  | func IsErrInvalidConfiguration(err error) bool { | ||||||
|  | 	_, ok := err.(ErrInvalidConfiguration) | ||||||
|  | 	return ok | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // Type is a type of Storage
 | ||||||
|  | type Type string | ||||||
|  | 
 | ||||||
|  | // NewStorageFunc is a function that creates a storage
 | ||||||
|  | type NewStorageFunc func(ctx context.Context, cfg interface{}) (ObjectStorage, error) | ||||||
|  | 
 | ||||||
|  | var storageMap = map[Type]NewStorageFunc{} | ||||||
|  | 
 | ||||||
|  | // RegisterStorageType registers a provided storage type with a function to create it
 | ||||||
|  | func RegisterStorageType(typ Type, fn func(ctx context.Context, cfg interface{}) (ObjectStorage, error)) { | ||||||
|  | 	storageMap[typ] = fn | ||||||
|  | } | ||||||
|  | 
 | ||||||
| // Object represents the object on the storage
 | // Object represents the object on the storage
 | ||||||
| type Object interface { | type Object interface { | ||||||
| 	io.ReadCloser | 	io.ReadCloser | ||||||
| @ -67,41 +99,25 @@ func Init() error { | |||||||
| 	return initLFS() | 	return initLFS() | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| func initStorage(storageCfg setting.Storage) (ObjectStorage, error) { | // NewStorage takes a storage type and some config and returns an ObjectStorage or an error
 | ||||||
| 	var err error | func NewStorage(typStr string, cfg interface{}) (ObjectStorage, error) { | ||||||
| 	var s ObjectStorage | 	if len(typStr) == 0 { | ||||||
| 	switch storageCfg.Type { | 		typStr = string(LocalStorageType) | ||||||
| 	case setting.LocalStorageType: | 	} | ||||||
| 		s, err = NewLocalStorage(storageCfg.Path) | 	fn, ok := storageMap[Type(typStr)] | ||||||
| 	case setting.MinioStorageType: | 	if !ok { | ||||||
| 		minio := storageCfg.Minio | 		return nil, fmt.Errorf("Unsupported storage type: %s", typStr) | ||||||
| 		s, err = NewMinioStorage( |  | ||||||
| 			context.Background(), |  | ||||||
| 			minio.Endpoint, |  | ||||||
| 			minio.AccessKeyID, |  | ||||||
| 			minio.SecretAccessKey, |  | ||||||
| 			minio.Bucket, |  | ||||||
| 			minio.Location, |  | ||||||
| 			minio.BasePath, |  | ||||||
| 			minio.UseSSL, |  | ||||||
| 		) |  | ||||||
| 	default: |  | ||||||
| 		return nil, fmt.Errorf("Unsupported attachment store type: %s", storageCfg.Type) |  | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	if err != nil { | 	return fn(context.Background(), cfg) | ||||||
| 		return nil, err |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	return s, nil |  | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| func initAttachments() (err error) { | func initAttachments() (err error) { | ||||||
| 	Attachments, err = initStorage(setting.Attachment.Storage) | 	Attachments, err = NewStorage(setting.Attachment.Storage.Type, setting.Attachment.Storage) | ||||||
| 	return | 	return | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| func initLFS() (err error) { | func initLFS() (err error) { | ||||||
| 	LFS, err = initStorage(setting.LFS.Storage) | 	LFS, err = NewStorage(setting.LFS.Storage.Type, setting.LFS.Storage) | ||||||
| 	return | 	return | ||||||
| } | } | ||||||
|  | |||||||
		Loading…
	
		Reference in New Issue
	
	Block a user