diff options
Diffstat (limited to 'workhorse/internal/filestore/save_file_opts.go')
-rw-r--r-- | workhorse/internal/filestore/save_file_opts.go | 171 |
1 files changed, 171 insertions, 0 deletions
diff --git a/workhorse/internal/filestore/save_file_opts.go b/workhorse/internal/filestore/save_file_opts.go new file mode 100644 index 00000000000..1eb708c3f55 --- /dev/null +++ b/workhorse/internal/filestore/save_file_opts.go @@ -0,0 +1,171 @@ +package filestore + +import ( + "errors" + "strings" + "time" + + "gocloud.dev/blob" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/api" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/config" +) + +// DefaultObjectStoreTimeout is the timeout for ObjectStore upload operation +const DefaultObjectStoreTimeout = 4 * time.Hour + +type ObjectStorageConfig struct { + Provider string + + S3Credentials config.S3Credentials + S3Config config.S3Config + + // GoCloud mux that maps azureblob:// and future URLs (e.g. s3://, gcs://, etc.) to a handler + URLMux *blob.URLMux + + // Azure credentials are registered at startup in the GoCloud URLMux, so only the container name is needed + GoCloudConfig config.GoCloudConfig +} + +// SaveFileOpts represents all the options available for saving a file to object store +type SaveFileOpts struct { + // TempFilePrefix is the prefix used to create temporary local file + TempFilePrefix string + // LocalTempPath is the directory where to write a local copy of the file + LocalTempPath string + // RemoteID is the remote ObjectID provided by GitLab + RemoteID string + // RemoteURL is the final URL of the file + RemoteURL string + // PresignedPut is a presigned S3 PutObject compatible URL + PresignedPut string + // PresignedDelete is a presigned S3 DeleteObject compatible URL. + PresignedDelete string + // HTTP headers to be sent along with PUT request + PutHeaders map[string]string + // Whether to ignore Rails pre-signed URLs and have Workhorse directly access object storage provider + UseWorkhorseClient bool + // If UseWorkhorseClient is true, this is the temporary object name to store the file + RemoteTempObjectID string + // Workhorse object storage client (e.g. S3) parameters + ObjectStorageConfig ObjectStorageConfig + // Deadline it the S3 operation deadline, the upload will be aborted if not completed in time + Deadline time.Time + // The maximum accepted size in bytes of the upload + MaximumSize int64 + + //MultipartUpload parameters + // PartSize is the exact size of each uploaded part. Only the last one can be smaller + PartSize int64 + // PresignedParts contains the presigned URLs for each part + PresignedParts []string + // PresignedCompleteMultipart is a presigned URL for CompleteMulipartUpload + PresignedCompleteMultipart string + // PresignedAbortMultipart is a presigned URL for AbortMultipartUpload + PresignedAbortMultipart string +} + +// UseWorkhorseClientEnabled checks if the options require direct access to object storage +func (s *SaveFileOpts) UseWorkhorseClientEnabled() bool { + return s.UseWorkhorseClient && s.ObjectStorageConfig.IsValid() && s.RemoteTempObjectID != "" +} + +// IsLocal checks if the options require the writing of the file on disk +func (s *SaveFileOpts) IsLocal() bool { + return s.LocalTempPath != "" +} + +// IsMultipart checks if the options requires a Multipart upload +func (s *SaveFileOpts) IsMultipart() bool { + return s.PartSize > 0 +} + +// GetOpts converts GitLab api.Response to a proper SaveFileOpts +func GetOpts(apiResponse *api.Response) (*SaveFileOpts, error) { + timeout := time.Duration(apiResponse.RemoteObject.Timeout) * time.Second + if timeout == 0 { + timeout = DefaultObjectStoreTimeout + } + + opts := SaveFileOpts{ + LocalTempPath: apiResponse.TempPath, + RemoteID: apiResponse.RemoteObject.ID, + RemoteURL: apiResponse.RemoteObject.GetURL, + PresignedPut: apiResponse.RemoteObject.StoreURL, + PresignedDelete: apiResponse.RemoteObject.DeleteURL, + PutHeaders: apiResponse.RemoteObject.PutHeaders, + UseWorkhorseClient: apiResponse.RemoteObject.UseWorkhorseClient, + RemoteTempObjectID: apiResponse.RemoteObject.RemoteTempObjectID, + Deadline: time.Now().Add(timeout), + MaximumSize: apiResponse.MaximumSize, + } + + if opts.LocalTempPath != "" && opts.RemoteID != "" { + return nil, errors.New("API response has both TempPath and RemoteObject") + } + + if opts.LocalTempPath == "" && opts.RemoteID == "" { + return nil, errors.New("API response has neither TempPath nor RemoteObject") + } + + objectStorageParams := apiResponse.RemoteObject.ObjectStorage + if opts.UseWorkhorseClient && objectStorageParams != nil { + opts.ObjectStorageConfig.Provider = objectStorageParams.Provider + opts.ObjectStorageConfig.S3Config = objectStorageParams.S3Config + opts.ObjectStorageConfig.GoCloudConfig = objectStorageParams.GoCloudConfig + } + + // Backwards compatibility to ensure API servers that do not include the + // CustomPutHeaders flag will default to the original content type. + if !apiResponse.RemoteObject.CustomPutHeaders { + opts.PutHeaders = make(map[string]string) + opts.PutHeaders["Content-Type"] = "application/octet-stream" + } + + if multiParams := apiResponse.RemoteObject.MultipartUpload; multiParams != nil { + opts.PartSize = multiParams.PartSize + opts.PresignedCompleteMultipart = multiParams.CompleteURL + opts.PresignedAbortMultipart = multiParams.AbortURL + opts.PresignedParts = append([]string(nil), multiParams.PartURLs...) + } + + return &opts, nil +} + +func (c *ObjectStorageConfig) IsAWS() bool { + return strings.EqualFold(c.Provider, "AWS") || strings.EqualFold(c.Provider, "S3") +} + +func (c *ObjectStorageConfig) IsAzure() bool { + return strings.EqualFold(c.Provider, "AzureRM") +} + +func (c *ObjectStorageConfig) IsGoCloud() bool { + return c.GoCloudConfig.URL != "" +} + +func (c *ObjectStorageConfig) IsValid() bool { + if c.IsAWS() { + return c.S3Config.Bucket != "" && c.S3Config.Region != "" && c.s3CredentialsValid() + } else if c.IsGoCloud() { + // We could parse and validate the URL, but GoCloud providers + // such as AzureRM don't have a fallback to normal HTTP, so we + // always want to try the GoCloud path if there is a URL. + return true + } + + return false +} + +func (c *ObjectStorageConfig) s3CredentialsValid() bool { + // We need to be able to distinguish between two cases of AWS access: + // 1. AWS access via key and secret, but credentials not configured in Workhorse + // 2. IAM instance profiles used + if c.S3Config.UseIamProfile { + return true + } else if c.S3Credentials.AwsAccessKeyID != "" && c.S3Credentials.AwsSecretAccessKey != "" { + return true + } + + return false +} |