diff options
Diffstat (limited to 'workhorse/internal/config')
-rw-r--r-- | workhorse/internal/config/config.go | 154 | ||||
-rw-r--r-- | workhorse/internal/config/config_test.go | 111 | ||||
-rw-r--r-- | workhorse/internal/config/url_openers.go | 51 | ||||
-rw-r--r-- | workhorse/internal/config/url_openers_test.go | 117 |
4 files changed, 433 insertions, 0 deletions
diff --git a/workhorse/internal/config/config.go b/workhorse/internal/config/config.go new file mode 100644 index 00000000000..84849c72744 --- /dev/null +++ b/workhorse/internal/config/config.go @@ -0,0 +1,154 @@ +package config + +import ( + "math" + "net/url" + "runtime" + "strings" + "time" + + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/BurntSushi/toml" + "gitlab.com/gitlab-org/labkit/log" + "gocloud.dev/blob" + "gocloud.dev/blob/azureblob" +) + +type TomlURL struct { + url.URL +} + +func (u *TomlURL) UnmarshalText(text []byte) error { + temp, err := url.Parse(string(text)) + u.URL = *temp + return err +} + +type TomlDuration struct { + time.Duration +} + +func (d *TomlDuration) UnmarshalTest(text []byte) error { + temp, err := time.ParseDuration(string(text)) + d.Duration = temp + return err +} + +type ObjectStorageCredentials struct { + Provider string + + S3Credentials S3Credentials `toml:"s3"` + AzureCredentials AzureCredentials `toml:"azurerm"` +} + +type ObjectStorageConfig struct { + URLMux *blob.URLMux `toml:"-"` +} + +type S3Credentials struct { + AwsAccessKeyID string `toml:"aws_access_key_id"` + AwsSecretAccessKey string `toml:"aws_secret_access_key"` +} + +type S3Config struct { + Region string `toml:"-"` + Bucket string `toml:"-"` + PathStyle bool `toml:"-"` + Endpoint string `toml:"-"` + UseIamProfile bool `toml:"-"` + ServerSideEncryption string `toml:"-"` // Server-side encryption mode (e.g. AES256, aws:kms) + SSEKMSKeyID string `toml:"-"` // Server-side encryption key-management service key ID (e.g. arn:aws:xxx) +} + +type GoCloudConfig struct { + URL string `toml:"-"` +} + +type AzureCredentials struct { + AccountName string `toml:"azure_storage_account_name"` + AccountKey string `toml:"azure_storage_access_key"` +} + +type RedisConfig struct { + URL TomlURL + Sentinel []TomlURL + SentinelMaster string + Password string + DB *int + ReadTimeout *TomlDuration + WriteTimeout *TomlDuration + KeepAlivePeriod *TomlDuration + MaxIdle *int + MaxActive *int +} + +type ImageResizerConfig struct { + MaxScalerProcs uint32 `toml:"max_scaler_procs"` + MaxFilesize uint64 `toml:"max_filesize"` +} + +type Config struct { + Redis *RedisConfig `toml:"redis"` + Backend *url.URL `toml:"-"` + CableBackend *url.URL `toml:"-"` + Version string `toml:"-"` + DocumentRoot string `toml:"-"` + DevelopmentMode bool `toml:"-"` + Socket string `toml:"-"` + CableSocket string `toml:"-"` + ProxyHeadersTimeout time.Duration `toml:"-"` + APILimit uint `toml:"-"` + APIQueueLimit uint `toml:"-"` + APIQueueTimeout time.Duration `toml:"-"` + APICILongPollingDuration time.Duration `toml:"-"` + ObjectStorageConfig ObjectStorageConfig `toml:"-"` + ObjectStorageCredentials ObjectStorageCredentials `toml:"object_storage"` + PropagateCorrelationID bool `toml:"-"` + ImageResizerConfig ImageResizerConfig `toml:"image_resizer"` + AltDocumentRoot string `toml:"alt_document_root"` +} + +var DefaultImageResizerConfig = ImageResizerConfig{ + MaxScalerProcs: uint32(math.Max(2, float64(runtime.NumCPU())/2)), + MaxFilesize: 250 * 1000, // 250kB, +} + +func LoadConfig(data string) (*Config, error) { + cfg := &Config{ImageResizerConfig: DefaultImageResizerConfig} + + if _, err := toml.Decode(data, cfg); err != nil { + return nil, err + } + + return cfg, nil +} + +func (c *Config) RegisterGoCloudURLOpeners() error { + c.ObjectStorageConfig.URLMux = new(blob.URLMux) + + creds := c.ObjectStorageCredentials + if strings.EqualFold(creds.Provider, "AzureRM") && creds.AzureCredentials.AccountName != "" && creds.AzureCredentials.AccountKey != "" { + accountName := azureblob.AccountName(creds.AzureCredentials.AccountName) + accountKey := azureblob.AccountKey(creds.AzureCredentials.AccountKey) + + credential, err := azureblob.NewCredential(accountName, accountKey) + if err != nil { + log.WithError(err).Error("error creating Azure credentials") + return err + } + + pipeline := azureblob.NewPipeline(credential, azblob.PipelineOptions{}) + + azureURLOpener := &azureURLOpener{ + &azureblob.URLOpener{ + AccountName: accountName, + Pipeline: pipeline, + Options: azureblob.Options{Credential: credential}, + }, + } + + c.ObjectStorageConfig.URLMux.RegisterBucket(azureblob.Scheme, azureURLOpener) + } + + return nil +} diff --git a/workhorse/internal/config/config_test.go b/workhorse/internal/config/config_test.go new file mode 100644 index 00000000000..102b29a0813 --- /dev/null +++ b/workhorse/internal/config/config_test.go @@ -0,0 +1,111 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +const azureConfig = ` +[object_storage] +provider = "AzureRM" + +[object_storage.azurerm] +azure_storage_account_name = "azuretester" +azure_storage_access_key = "deadbeef" +` + +func TestLoadEmptyConfig(t *testing.T) { + config := `` + + cfg, err := LoadConfig(config) + require.NoError(t, err) + + require.Empty(t, cfg.AltDocumentRoot) + require.Equal(t, cfg.ImageResizerConfig.MaxFilesize, uint64(250000)) + require.GreaterOrEqual(t, cfg.ImageResizerConfig.MaxScalerProcs, uint32(2)) + + require.Equal(t, ObjectStorageCredentials{}, cfg.ObjectStorageCredentials) + require.NoError(t, cfg.RegisterGoCloudURLOpeners()) +} + +func TestLoadObjectStorageConfig(t *testing.T) { + config := ` +[object_storage] +provider = "AWS" + +[object_storage.s3] +aws_access_key_id = "minio" +aws_secret_access_key = "gdk-minio" +` + + cfg, err := LoadConfig(config) + require.NoError(t, err) + + require.NotNil(t, cfg.ObjectStorageCredentials, "Expected object storage credentials") + + expected := ObjectStorageCredentials{ + Provider: "AWS", + S3Credentials: S3Credentials{ + AwsAccessKeyID: "minio", + AwsSecretAccessKey: "gdk-minio", + }, + } + + require.Equal(t, expected, cfg.ObjectStorageCredentials) +} + +func TestRegisterGoCloudURLOpeners(t *testing.T) { + cfg, err := LoadConfig(azureConfig) + require.NoError(t, err) + + require.NotNil(t, cfg.ObjectStorageCredentials, "Expected object storage credentials") + + expected := ObjectStorageCredentials{ + Provider: "AzureRM", + AzureCredentials: AzureCredentials{ + AccountName: "azuretester", + AccountKey: "deadbeef", + }, + } + + require.Equal(t, expected, cfg.ObjectStorageCredentials) + require.Nil(t, cfg.ObjectStorageConfig.URLMux) + + require.NoError(t, cfg.RegisterGoCloudURLOpeners()) + require.NotNil(t, cfg.ObjectStorageConfig.URLMux) + + require.True(t, cfg.ObjectStorageConfig.URLMux.ValidBucketScheme("azblob")) + require.Equal(t, []string{"azblob"}, cfg.ObjectStorageConfig.URLMux.BucketSchemes()) +} + +func TestLoadImageResizerConfig(t *testing.T) { + config := ` +[image_resizer] +max_scaler_procs = 200 +max_filesize = 350000 +` + + cfg, err := LoadConfig(config) + require.NoError(t, err) + + require.NotNil(t, cfg.ImageResizerConfig, "Expected image resizer config") + + expected := ImageResizerConfig{ + MaxScalerProcs: 200, + MaxFilesize: 350000, + } + + require.Equal(t, expected, cfg.ImageResizerConfig) +} + +func TestAltDocumentConfig(t *testing.T) { + config := ` +alt_document_root = "/path/to/documents" +` + + cfg, err := LoadConfig(config) + require.NoError(t, err) + + require.Equal(t, "/path/to/documents", cfg.AltDocumentRoot) +} diff --git a/workhorse/internal/config/url_openers.go b/workhorse/internal/config/url_openers.go new file mode 100644 index 00000000000..d3c96ee9eef --- /dev/null +++ b/workhorse/internal/config/url_openers.go @@ -0,0 +1,51 @@ +package config + +import ( + "context" + "fmt" + "net/url" + + "gocloud.dev/blob" + "gocloud.dev/blob/azureblob" +) + +// This code can be removed once https://github.com/google/go-cloud/pull/2851 is merged. + +// URLOpener opens Azure URLs like "azblob://mybucket". +// +// The URL host is used as the bucket name. +// +// The following query options are supported: +// - domain: The domain name used to access the Azure Blob storage (e.g. blob.core.windows.net) +type azureURLOpener struct { + *azureblob.URLOpener +} + +func (o *azureURLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket, error) { + opts := new(azureblob.Options) + *opts = o.Options + + err := setOptionsFromURLParams(u.Query(), opts) + if err != nil { + return nil, err + } + return azureblob.OpenBucket(ctx, o.Pipeline, o.AccountName, u.Host, opts) +} + +func setOptionsFromURLParams(q url.Values, opts *azureblob.Options) error { + for param, values := range q { + if len(values) > 1 { + return fmt.Errorf("multiple values of %v not allowed", param) + } + + value := values[0] + switch param { + case "domain": + opts.StorageDomain = azureblob.StorageDomain(value) + default: + return fmt.Errorf("unknown query parameter %q", param) + } + } + + return nil +} diff --git a/workhorse/internal/config/url_openers_test.go b/workhorse/internal/config/url_openers_test.go new file mode 100644 index 00000000000..6a851cacbb8 --- /dev/null +++ b/workhorse/internal/config/url_openers_test.go @@ -0,0 +1,117 @@ +package config + +import ( + "context" + "net/url" + "testing" + + "github.com/stretchr/testify/require" + "gocloud.dev/blob/azureblob" +) + +func TestURLOpeners(t *testing.T) { + cfg, err := LoadConfig(azureConfig) + require.NoError(t, err) + + require.NotNil(t, cfg.ObjectStorageCredentials, "Expected object storage credentials") + + require.NoError(t, cfg.RegisterGoCloudURLOpeners()) + require.NotNil(t, cfg.ObjectStorageConfig.URLMux) + + tests := []struct { + url string + valid bool + }{ + + { + url: "azblob://container/object", + valid: true, + }, + { + url: "azblob://container/object?domain=core.windows.net", + valid: true, + }, + { + url: "azblob://container/object?domain=core.windows.net&domain=test", + valid: false, + }, + { + url: "azblob://container/object?param=value", + valid: false, + }, + { + url: "s3://bucket/object", + valid: false, + }, + } + + for _, test := range tests { + t.Run(test.url, func(t *testing.T) { + ctx := context.Background() + url, err := url.Parse(test.url) + require.NoError(t, err) + + bucket, err := cfg.ObjectStorageConfig.URLMux.OpenBucketURL(ctx, url) + if bucket != nil { + defer bucket.Close() + } + + if test.valid { + require.NotNil(t, bucket) + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} + +func TestTestURLOpenersForParams(t *testing.T) { + tests := []struct { + name string + currOpts azureblob.Options + query url.Values + wantOpts azureblob.Options + wantErr bool + }{ + { + name: "InvalidParam", + query: url.Values{ + "foo": {"bar"}, + }, + wantErr: true, + }, + { + name: "StorageDomain", + query: url.Values{ + "domain": {"blob.core.usgovcloudapi.net"}, + }, + wantOpts: azureblob.Options{StorageDomain: "blob.core.usgovcloudapi.net"}, + }, + { + name: "duplicate StorageDomain", + query: url.Values{ + "domain": {"blob.core.usgovcloudapi.net", "blob.core.windows.net"}, + }, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + o := &azureURLOpener{ + URLOpener: &azureblob.URLOpener{ + Options: test.currOpts, + }, + } + err := setOptionsFromURLParams(test.query, &o.Options) + + if test.wantErr { + require.NotNil(t, err) + } else { + require.Nil(t, err) + require.Equal(t, test.wantOpts, o.Options) + } + }) + } +} |