bucket

package
v3.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 12, 2025 License: AGPL-3.0 Imports: 30 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// S3 is the value for the S3 storage backend.
	S3 = "s3"

	// GCS is the value for the GCS storage backend.
	GCS = "gcs"

	// Azure is the value for the Azure storage backend.
	Azure = "azure"

	// Swift is the value for the Openstack Swift storage backend.
	Swift = "swift"

	// Filesystem is the value for the filesystem storage backend.
	Filesystem = "filesystem"

	// Alibaba is the value for the Alibaba Cloud OSS storage backend
	Alibaba = "alibabacloud"

	// BOS is the value for the Baidu Cloud BOS storage backend
	BOS = "bos"
)

Variables

View Source 
var (
	SupportedBackends = []string{S3, GCS, Azure, Swift, Filesystem, Alibaba, BOS}

	ErrUnsupportedStorageBackend        = errors.New("unsupported storage backend")
	ErrInvalidCharactersInStoragePrefix = errors.New("storage prefix contains invalid characters, it may only contain digits and English alphabet letters")
)

Functions

func NewClient 

func NewClient(ctx context.Context, backend string, cfg Config, name string, logger log.Logger) (objstore.InstrumentedBucket, error)

NewClient creates a new bucket client based on the configured backend

func NewUserBucketClient 

func NewUserBucketClient(userID string, bucket objstore.Bucket, cfgProvider SSEConfigProvider) objstore.InstrumentedBucket

NewUserBucketClient returns a bucket client to use to access the storage on behalf of the provided user. The cfgProvider can be nil.

Types

type Config 

type Config struct {
	// Backends
	S3         s3.Config         `yaml:"s3"`
	GCS        gcs.Config        `yaml:"gcs"`
	Azure      azure.Config      `yaml:"azure"`
	Swift      swift.Config      `yaml:"swift"`
	Filesystem filesystem.Config `yaml:"filesystem"`
	Alibaba    oss.Config        `yaml:"alibaba"`
	BOS        bos.Config        `yaml:"bos"`

	StoragePrefix string `yaml:"storage_prefix"`

	// Used to inject additional backends into the config. Allows for this config to
	// be embedded in multiple contexts and support non-object storage based backends.
	ExtraBackends []string `yaml:"-"`

	// Not used internally, meant to allow callers to wrap Buckets
	// created using this config
	Middlewares []func(objstore.InstrumentedBucket) (objstore.InstrumentedBucket, error) `yaml:"-"`
}

Config holds configuration for accessing long-term storage.

func (*Config) RegisterFlags 

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the backend storage config.

func (*Config) RegisterFlagsWithPrefix 

func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

func (*Config) RegisterFlagsWithPrefixAndDefaultDirectory added in v3.3.0 

func (cfg *Config) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet)

func (*Config) SupportedBackends added in v3.4.0 

func (cfg *Config) SupportedBackends() []string

Returns the SupportedBackends for the package and any custom backends injected into the config.

func (*Config) Validate 

func (cfg *Config) Validate() error

type ConfigWithNamedStores added in v3.4.0 

type ConfigWithNamedStores struct {
	Config      `yaml:",inline"`
	NamedStores NamedStores `yaml:"named_stores"`
}

func (*ConfigWithNamedStores) Validate added in v3.4.0 

func (cfg *ConfigWithNamedStores) Validate() error

type NamedAzureStorageConfig added in v3.4.0 

type NamedAzureStorageConfig azure.Config

func (*NamedAzureStorageConfig) UnmarshalYAML added in v3.4.0 

func (cfg *NamedAzureStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedFilesystemStorageConfig added in v3.4.0 

type NamedFilesystemStorageConfig filesystem.Config

func (*NamedFilesystemStorageConfig) UnmarshalYAML added in v3.4.0 

func (cfg *NamedFilesystemStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedGCSStorageConfig added in v3.4.0 

type NamedGCSStorageConfig gcs.Config

func (*NamedGCSStorageConfig) UnmarshalYAML added in v3.4.0 

func (cfg *NamedGCSStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedS3StorageConfig added in v3.4.0 

type NamedS3StorageConfig s3.Config

func (*NamedS3StorageConfig) UnmarshalYAML added in v3.4.0 

func (cfg *NamedS3StorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (*NamedS3StorageConfig) Validate added in v3.4.0 

func (cfg *NamedS3StorageConfig) Validate() error

type NamedStores added in v3.4.0 

type NamedStores struct {
	Azure      map[string]NamedAzureStorageConfig      `yaml:"azure"`
	Filesystem map[string]NamedFilesystemStorageConfig `yaml:"filesystem"`
	GCS        map[string]NamedGCSStorageConfig        `yaml:"gcs"`
	S3         map[string]NamedS3StorageConfig         `yaml:"s3"`
	Swift      map[string]NamedSwiftStorageConfig      `yaml:"swift"`
	// contains filtered or unexported fields
}

NamedStores helps configure additional object stores from a given storage provider

func (*NamedStores) Exists added in v3.4.0 

func (ns *NamedStores) Exists(name string) bool

func (*NamedStores) LookupStoreType added in v3.4.0 

func (ns *NamedStores) LookupStoreType(name string) (string, bool)

func (*NamedStores) OverrideConfig added in v3.4.0 

func (ns *NamedStores) OverrideConfig(storeCfg *Config, namedStore string) error

OverrideConfig overrides the store config with the named store config

func (*NamedStores) Validate added in v3.4.0 

func (ns *NamedStores) Validate() error

type NamedSwiftStorageConfig added in v3.4.0 

type NamedSwiftStorageConfig swift.Config

func (*NamedSwiftStorageConfig) UnmarshalYAML added in v3.4.0 

func (cfg *NamedSwiftStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type ObjectClientAdapter added in v3.3.0 

type ObjectClientAdapter struct {
	// contains filtered or unexported fields
}

func NewObjectClient added in v3.4.0 

func NewObjectClient(ctx context.Context, backend string, cfg ConfigWithNamedStores, component string, hedgingCfg hedging.Config, disableRetries bool, logger log.Logger) (*ObjectClientAdapter, error)

func (*ObjectClientAdapter) DeleteObject added in v3.3.0 

func (o *ObjectClientAdapter) DeleteObject(ctx context.Context, objectKey string) error

DeleteObject deletes the specified object key from the configured bucket.

func (*ObjectClientAdapter) GetAttributes added in v3.3.0 

func (o *ObjectClientAdapter) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error)

GetAttributes returns the attributes of the specified object key from the configured bucket.

func (*ObjectClientAdapter) GetObject added in v3.3.0 

func (o *ObjectClientAdapter) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error)

GetObject returns a reader and the size for the specified object key from the configured bucket. size is set to -1 if it cannot be succefully determined, it is up to the caller to check this value before using it.

func (*ObjectClientAdapter) GetObjectRange added in v3.3.0 

func (o *ObjectClientAdapter) GetObjectRange(ctx context.Context, objectKey string, offset, length int64) (io.ReadCloser, error)

func (*ObjectClientAdapter) IsObjectNotFoundErr added in v3.3.0 

func (o *ObjectClientAdapter) IsObjectNotFoundErr(err error) bool

IsObjectNotFoundErr returns true if error means that object is not found. Relevant to GetObject and DeleteObject operations.

func (*ObjectClientAdapter) IsRetryableErr added in v3.3.0 

func (o *ObjectClientAdapter) IsRetryableErr(err error) bool

IsRetryableErr returns true if the request failed due to some retryable server-side scenario

func (*ObjectClientAdapter) List added in v3.3.0 

func (o *ObjectClientAdapter) List(ctx context.Context, prefix, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error)

List objects with given prefix.

func (*ObjectClientAdapter) ObjectExists added in v3.3.0 

func (o *ObjectClientAdapter) ObjectExists(ctx context.Context, objectKey string) (bool, error)

ObjectExists checks if a given objectKey exists in the bucket

func (*ObjectClientAdapter) PutObject added in v3.3.0 

func (o *ObjectClientAdapter) PutObject(ctx context.Context, objectKey string, object io.Reader) error

PutObject puts the specified bytes into the configured bucket at the provided key

func (*ObjectClientAdapter) Stop added in v3.3.0 

func (o *ObjectClientAdapter) Stop()

type PrefixedBucketClient 

type PrefixedBucketClient struct {
	// contains filtered or unexported fields
}

func NewPrefixedBucketClient 

func NewPrefixedBucketClient(bucket objstore.Bucket, prefix string) *PrefixedBucketClient

NewPrefixedBucketClient returns a new PrefixedBucketClient.

func (*PrefixedBucketClient) Attributes 

func (b *PrefixedBucketClient) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

Attributes returns attributes of the specified object.

func (*PrefixedBucketClient) Close 

func (b *PrefixedBucketClient) Close() error

Close implements io.Closer

func (*PrefixedBucketClient) Delete 

func (b *PrefixedBucketClient) Delete(ctx context.Context, name string) error

Delete removes the object with the given name.

func (*PrefixedBucketClient) Exists 

func (b *PrefixedBucketClient) Exists(ctx context.Context, name string) (bool, error)

Exists checks if the given object exists in the bucket.

func (*PrefixedBucketClient) Get 

func (b *PrefixedBucketClient) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get returns a reader for the given object name.

func (*PrefixedBucketClient) GetRange 

func (b *PrefixedBucketClient) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange returns a new range reader for the given object name and range.

func (*PrefixedBucketClient) IsAccessDeniedErr 

func (b *PrefixedBucketClient) IsAccessDeniedErr(err error) bool

IsAccessDeniedErr returns true if access to object is denied.

func (*PrefixedBucketClient) IsObjNotFoundErr 

func (b *PrefixedBucketClient) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations.

func (*PrefixedBucketClient) Iter 

func (b *PrefixedBucketClient) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full object name including the prefix of the inspected directory. The configured prefix will be stripped before supplied function is applied.

func (*PrefixedBucketClient) IterWithAttributes added in v3.4.0 

func (b *PrefixedBucketClient) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error

IterWithAttributes calls f for each entry in the given directory similar to Iter. In addition to Name, it also includes requested object attributes in the argument to f.

Attributes can be requested using IterOption. Not all IterOptions are supported by all providers, requesting for an unsupported option will fail with ErrOptionNotSupported.

func (*PrefixedBucketClient) Name 

func (b *PrefixedBucketClient) Name() string

Name returns the bucket name for the provider.

func (*PrefixedBucketClient) Provider added in v3.4.0 

func (b *PrefixedBucketClient) Provider() objstore.ObjProvider

Provider returns the provider of the bucket.

func (*PrefixedBucketClient) ReaderWithExpectedErrs 

func (b *PrefixedBucketClient) ReaderWithExpectedErrs(fn objstore.IsOpFailureExpectedFunc) objstore.BucketReader

WithExpectedErrs allows to specify a filter that marks certain errors as expected, so it will not increment thanos_objstore_bucket_operation_failures_total metric.

func (*PrefixedBucketClient) SupportedIterOptions added in v3.4.0 

func (b *PrefixedBucketClient) SupportedIterOptions() []objstore.IterOptionType

SupportedIterOptions returns a list of supported IterOptions by the underlying provider.

func (*PrefixedBucketClient) Upload 

func (b *PrefixedBucketClient) Upload(ctx context.Context, name string, r io.Reader) (err error)

Upload the contents of the reader as an object into the bucket.

func (*PrefixedBucketClient) WithExpectedErrs 

func (b *PrefixedBucketClient) WithExpectedErrs(fn objstore.IsOpFailureExpectedFunc) objstore.Bucket

ReaderWithExpectedErrs allows to specify a filter that marks certain errors as expected, so it will not increment thanos_objstore_bucket_operation_failures_total metric.

type SSEBucketClient 

type SSEBucketClient struct {
	// contains filtered or unexported fields
}

SSEBucketClient is a wrapper around a objstore.BucketReader that configures the object storage server-side encryption (SSE) for a given user.

func NewSSEBucketClient 

func NewSSEBucketClient(userID string, bucket objstore.Bucket, cfgProvider SSEConfigProvider) *SSEBucketClient

NewSSEBucketClient makes a new SSEBucketClient. The cfgProvider can be nil.

func (*SSEBucketClient) Attributes 

func (b *SSEBucketClient) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

Attributes implements objstore.Bucket.

func (*SSEBucketClient) Close 

func (b *SSEBucketClient) Close() error

Close implements objstore.Bucket.

func (*SSEBucketClient) Delete 

func (b *SSEBucketClient) Delete(ctx context.Context, name string) error

Delete implements objstore.Bucket.

func (*SSEBucketClient) Exists 

func (b *SSEBucketClient) Exists(ctx context.Context, name string) (bool, error)

Exists implements objstore.Bucket.

func (*SSEBucketClient) Get 

func (b *SSEBucketClient) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get implements objstore.Bucket.

func (*SSEBucketClient) GetRange 

func (b *SSEBucketClient) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange implements objstore.Bucket.

func (*SSEBucketClient) IsAccessDeniedErr 

func (b *SSEBucketClient) IsAccessDeniedErr(err error) bool

IsAccessDeniedErr returns true if access to object is denied.

func (*SSEBucketClient) IsObjNotFoundErr 

func (b *SSEBucketClient) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr implements objstore.Bucket.

func (*SSEBucketClient) Iter 

func (b *SSEBucketClient) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

Iter implements objstore.Bucket.

func (*SSEBucketClient) IterWithAttributes added in v3.4.0 

func (b *SSEBucketClient) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error

IterWithAttributes calls f for each entry in the given directory similar to Iter. In addition to Name, it also includes requested object attributes in the argument to f.

Attributes can be requested using IterOption. Not all IterOptions are supported by all providers, requesting for an unsupported option will fail with ErrOptionNotSupported.

func (*SSEBucketClient) Name 

func (b *SSEBucketClient) Name() string

Name implements objstore.Bucket.

func (*SSEBucketClient) Provider added in v3.4.0 

func (b *SSEBucketClient) Provider() objstore.ObjProvider

Provider returns the provider of the bucket.

func (*SSEBucketClient) ReaderWithExpectedErrs 

func (b *SSEBucketClient) ReaderWithExpectedErrs(fn objstore.IsOpFailureExpectedFunc) objstore.BucketReader

ReaderWithExpectedErrs implements objstore.Bucket.

func (*SSEBucketClient) SupportedIterOptions added in v3.4.0 

func (b *SSEBucketClient) SupportedIterOptions() []objstore.IterOptionType

SupportedIterOptions returns a list of supported IterOptions by the underlying provider.

func (*SSEBucketClient) Upload 

func (b *SSEBucketClient) Upload(ctx context.Context, name string, r io.Reader) error

Upload the contents of the reader as an object into the bucket.

func (*SSEBucketClient) WithExpectedErrs 

func (b *SSEBucketClient) WithExpectedErrs(fn objstore.IsOpFailureExpectedFunc) objstore.Bucket

WithExpectedErrs implements objstore.Bucket.

type SSEConfigProvider added in v3.3.0 

type SSEConfigProvider interface {
	// S3SSEType returns the per-tenant S3 SSE type.
	S3SSEType(userID string) string

	// S3SSEKMSKeyID returns the per-tenant S3 KMS-SSE key id or an empty string if not set.
	S3SSEKMSKeyID(userID string) string

	// S3SSEKMSEncryptionContext returns the per-tenant S3 KMS-SSE key id or an empty string if not set.
	S3SSEKMSEncryptionContext(userID string) string
}

SSEConfigProvider defines a per-tenant SSE config provider.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.
JackTT - Gopher 🇻🇳