Merge pull request #1203 from percona/PBM-1618-minio-storage-type

PBM-1618: Minio as storage type for Custom-S3 storages

Author: boris-ilijic

cmd/pbm/status.go

@@ -540,10 +540,7 @@ func getStorageStat(
 	}
 
 	s.Type = cfg.Storage.Typ()
-
-	if cfg.Storage.Type == storage.S3 {
-		s.Region = cfg.Storage.S3.Region
-	}
+	s.Region = cfg.Storage.Region()
 	s.Path = cfg.Storage.Path()
 
 	bcps, err := pbm.GetAllBackups(ctx)

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/docker/go-connections v0.6.0
 	github.com/fsnotify/fsnotify v1.9.0
 	github.com/golang/snappy v1.0.0
+	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/googleapis/gax-go/v2 v2.15.0
 	github.com/klauspost/compress v1.18.0

packaging/conf/pbm-conf-reference.yml

@@ -10,7 +10,12 @@
 
 #storage:
 
-##  Remote backup storage type. Supported types: S3, GCS, shared filesystem, Azure
+##  Remote backup storage type. Supported types:
+##  - S3,
+##  - GCS,
+##  - Minio (for S3 compatible storage),
+##  - shared filesystem,
+##  - Azure
 
 
 #---------------------S3 Storage Configuration--------------------------
@@ -69,6 +74,53 @@
 #
 ## The maximum object size that will be stored on the storage
 #      maxObjSizeGB: 5018
+#
+
+#---------------------Minio Storage Configuration--------------------------
+#  type:
+#    minio:
+
+## Specify the location and name of the bucket that you have configured on the storage
+#      region:
+#      bucket:
+
+## The data directory to store backups in.
+## When undefined, backups are saved at the root of the bucket.
+#      prefix:
+
+## An optional custom URL to access the bucket. Useful for S3-compatible storage (e.g. MinIO)
+#      endpointUrl:
+
+#
+##  Use https
+#      secure: true
+
+## Allow PBM to upload data to storage with self-issued TLS certificates.
+## Use it with caution as it might leave a hole for man-in-the-middle attacks.
+#      insecureSkipTLSVerify: false
+
+## Enable path-style URL, by default it uses virtual-hosted style
+#      forcePathStyle: false
+
+## Access credentials
+#      credentials:
+#        access-key-id:
+#        secret-access-key:
+#        session-token:
+#        signature-ver: V4
+
+## The size of data chunks in bytes to be uploaded to the storage bucket in a single request.
+#      partSize: 10485760
+
+## Enable debug trace of HTTP communication
+#      debugTrace: true
+
+## Retry upload configuration options.
+#      retryer:
+#        numMaxRetries: 3
+#
+## The maximum object size that will be stored on the storage
+#      maxObjSizeGB: 5018
 
 #--------------------Google Cloud Storage Configuration-----------------
 

pbm/config/config.go

@@ -26,6 +26,7 @@ import (
 	"github.com/percona/percona-backup-mongodb/pbm/storage/azure"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/fs"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/gcs"
+	"github.com/percona/percona-backup-mongodb/pbm/storage/mio"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/oss"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/s3"
 	"github.com/percona/percona-backup-mongodb/pbm/topo"
@@ -142,6 +143,17 @@ func (c *Config) String() string {
 			c.Storage.S3.ServerSideEncryption.SseCustomerKey = "***"
 		}
 	}
+	if c.Storage.Minio != nil {
+		if c.Storage.Minio.Credentials.AccessKeyID != "" {
+			c.Storage.Minio.Credentials.AccessKeyID = "***"
+		}
+		if c.Storage.Minio.Credentials.SecretAccessKey != "" {
+			c.Storage.Minio.Credentials.SecretAccessKey = "***"
+		}
+		if c.Storage.Minio.Credentials.SessionToken != "" {
+			c.Storage.Minio.Credentials.SessionToken = "***"
+		}
+	}
 	if c.Storage.Azure != nil {
 		if c.Storage.Azure.Credentials.Key != "" {
 			c.Storage.Azure.Credentials.Key = "***"
@@ -236,6 +248,7 @@ func (cfg *PITRConf) Clone() *PITRConf {
 type StorageConf struct {
 	Type       storage.Type  `bson:"type" json:"type" yaml:"type"`
 	S3         *s3.Config    `bson:"s3,omitempty" json:"s3,omitempty" yaml:"s3,omitempty"`
+	Minio      *mio.Config   `bson:"minio,omitempty" json:"minio,omitempty" yaml:"minio,omitempty"`
 	GCS        *gcs.Config   `bson:"gcs,omitempty" json:"gcs,omitempty" yaml:"gcs,omitempty"`
 	Azure      *azure.Config `bson:"azure,omitempty" json:"azure,omitempty" yaml:"azure,omitempty"`
 	Filesystem *fs.Config    `bson:"filesystem,omitempty" json:"filesystem,omitempty" yaml:"filesystem,omitempty"`
@@ -256,6 +269,8 @@ func (s *StorageConf) Clone() *StorageConf {
 		rv.Filesystem = s.Filesystem.Clone()
 	case storage.S3:
 		rv.S3 = s.S3.Clone()
+	case storage.Minio:
+		rv.Minio = s.Minio.Clone()
 	case storage.Azure:
 		rv.Azure = s.Azure.Clone()
 	case storage.GCS:
@@ -276,6 +291,8 @@ func (s *StorageConf) Equal(other *StorageConf) bool {
 	switch s.Type {
 	case storage.S3:
 		return s.S3.Equal(other.S3)
+	case storage.Minio:
+		return s.Minio.Equal(other.Minio)
 	case storage.Azure:
 		return s.Azure.Equal(other.Azure)
 	case storage.GCS:
@@ -301,6 +318,8 @@ func (s *StorageConf) IsSameStorage(other *StorageConf) bool {
 	switch s.Type {
 	case storage.S3:
 		return s.S3.IsSameStorage(other.S3)
+	case storage.Minio:
+		return s.Minio.IsSameStorage(other.Minio)
 	case storage.Azure:
 		return s.Azure.IsSameStorage(other.Azure)
 	case storage.GCS:
@@ -320,6 +339,8 @@ func (s *StorageConf) Cast() error {
 		return s.Filesystem.Cast()
 	case storage.S3:
 		return s.S3.Cast()
+	case storage.Minio:
+		return s.Minio.Cast()
 	case storage.OSS:
 		return s.OSS.Cast()
 	case storage.GCS:
@@ -337,6 +358,8 @@ func (s *StorageConf) Typ() string {
 	switch s.Type {
 	case storage.S3:
 		return "S3"
+	case storage.Minio:
+		return "Minio"
 	case storage.Azure:
 		return "Azure"
 	case storage.GCS:
@@ -368,6 +391,19 @@ func (s *StorageConf) Path() string {
 		if s.S3.Prefix != "" {
 			path += "/" + s.S3.Prefix
 		}
+	case storage.Minio:
+		path = s.Minio.Endpoint
+		if path == "" {
+			path = "minio://" + s.Minio.Bucket
+		} else {
+			if !strings.Contains(path, "://") {
+				path = "minio://" + path
+			}
+			path += "/" + s.Minio.Bucket
+		}
+		if s.Minio.Prefix != "" {
+			path += "/" + s.Minio.Prefix
+		}
 	case storage.Azure:
 		epURL := s.Azure.EndpointURL
 		if epURL == "" {
@@ -389,6 +425,19 @@ func (s *StorageConf) Path() string {
 	return path
 }
 
+func (s *StorageConf) Region() string {
+	region := ""
+
+	switch s.Type {
+	case storage.S3:
+		region = s.S3.Region
+	case storage.Minio:
+		region = s.Minio.Region
+	}
+
+	return region
+}
+
 // RestoreConf is config options for the restore
 //
 //nolint:lll

pbm/config/config_test.go

@@ -3,9 +3,12 @@ package config
 import (
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
+
 	"github.com/percona/percona-backup-mongodb/pbm/storage/azure"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/fs"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/gcs"
+	"github.com/percona/percona-backup-mongodb/pbm/storage/mio"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/s3"
 )
 
@@ -151,6 +154,57 @@ func TestIsSameStorage(t *testing.T) {
 			t.Errorf("storage instances has different bucket: cfg=%+v, eq=%+v", cfg, neq)
 		}
 	})
+
+	t.Run("minio", func(t *testing.T) {
+		cfg := &mio.Config{
+			Region:   "eu",
+			Endpoint: "ep.com",
+			Bucket:   "b1",
+			Prefix:   "p1",
+			Credentials: mio.Credentials{
+				AccessKeyID:     "k1",
+				SecretAccessKey: "k2",
+				SessionToken:    "sess",
+			},
+			Secure:   true,
+			PartSize: 6 << 20,
+			Retryer:  &mio.Retryer{},
+		}
+		eq := &mio.Config{
+			Region:   "eu",
+			Endpoint: "ep.com",
+			Bucket:   "b1",
+			Prefix:   "p1",
+		}
+		if !cfg.IsSameStorage(eq) {
+			t.Errorf("config storage should identify the same instance: cfg=%+v, eq=%+v, diff=%s",
+				cfg, eq, cmp.Diff(*cfg, *eq))
+		}
+
+		neq := cfg.Clone()
+		neq.Region = "us"
+		if cfg.IsSameStorage(neq) {
+			t.Errorf("storage instances has different region: cfg=%+v, eq=%+v", cfg, neq)
+		}
+
+		neq = cfg.Clone()
+		neq.Endpoint = "ep2.com"
+		if cfg.IsSameStorage(neq) {
+			t.Errorf("storage instances has different EndpointURL: cfg=%+v, eq=%+v", cfg, neq)
+		}
+
+		neq = cfg.Clone()
+		neq.Bucket = "b2"
+		if cfg.IsSameStorage(neq) {
+			t.Errorf("storage instances has different bucket: cfg=%+v, eq=%+v", cfg, neq)
+		}
+
+		neq = cfg.Clone()
+		neq.Prefix = "p2"
+		if cfg.IsSameStorage(neq) {
+			t.Errorf("storage instances has different prefix: cfg=%+v, eq=%+v", cfg, neq)
+		}
+	})
 }
 
 func boolPtr(b bool) *bool {

pbm/log/discard.go

@@ -66,3 +66,4 @@ func (discardEventImpl) Info(msg string, args ...any)    {}
 func (discardEventImpl) Warning(msg string, args ...any) {}
 func (discardEventImpl) Error(msg string, args ...any)   {}
 func (discardEventImpl) Fatal(msg string, args ...any)   {}
+func (discardEventImpl) GetLogger() Logger               { return &discardLoggerImpl{} }

pbm/log/event.go

@@ -13,22 +13,26 @@ type eventImpl struct {
 	opid string
 }
 
-func (e *eventImpl) Debug(msg string, args ...interface{}) {
+func (e *eventImpl) Debug(msg string, args ...any) {
 	e.l.Debug(e.typ, e.obj, e.opid, e.ep, msg, args...)
 }
 
-func (e *eventImpl) Info(msg string, args ...interface{}) {
+func (e *eventImpl) Info(msg string, args ...any) {
 	e.l.Info(e.typ, e.obj, e.opid, e.ep, msg, args...)
 }
 
-func (e *eventImpl) Warning(msg string, args ...interface{}) {
+func (e *eventImpl) Warning(msg string, args ...any) {
 	e.l.Warning(e.typ, e.obj, e.opid, e.ep, msg, args...)
 }
 
-func (e *eventImpl) Error(msg string, args ...interface{}) {
+func (e *eventImpl) Error(msg string, args ...any) {
 	e.l.Error(e.typ, e.obj, e.opid, e.ep, msg, args...)
 }
 
-func (e *eventImpl) Fatal(msg string, args ...interface{}) {
+func (e *eventImpl) Fatal(msg string, args ...any) {
 	e.l.Fatal(e.typ, e.obj, e.opid, e.ep, msg, args...)
 }
+
+func (e *eventImpl) GetLogger() Logger {
+	return e.l
+}

pbm/log/log.go

@@ -45,6 +45,7 @@ type LogEvent interface {
 	Warning(msg string, args ...any)
 	Error(msg string, args ...any)
 	Fatal(msg string, args ...any)
+	GetLogger() Logger
 }
 
 type Buffer interface {

pbm/storage/gcs/gcs.go

@@ -13,6 +13,7 @@ import (
 )
 
 const (
+	gcsEndpointURL      = "storage.googleapis.com"
 	defaultMaxObjSizeGB = 5018 // 4.9 TB
 )
 

pbm/storage/gcs/hmac_client.go

@@ -36,7 +36,7 @@ func newHMACClient(opts *Config, l log.LogEvent) (*hmacClient, error) {
 		}
 	}
 
-	minioClient, err := minio.New("storage.googleapis.com", &minio.Options{
+	minioClient, err := minio.New(gcsEndpointURL, &minio.Options{
 		Creds: credentials.NewStaticV2(opts.Credentials.HMACAccessKey, opts.Credentials.HMACSecret, ""),
 	})
 	if err != nil {