diff options
Diffstat (limited to 'workhorse/cmd')
-rw-r--r-- | workhorse/cmd/gitlab-resize-image/main.go | 37 | ||||
-rw-r--r-- | workhorse/cmd/gitlab-zip-cat/main.go | 96 | ||||
-rw-r--r-- | workhorse/cmd/gitlab-zip-metadata/limit/reader.go | 52 | ||||
-rw-r--r-- | workhorse/cmd/gitlab-zip-metadata/limit/reader_test.go | 90 | ||||
-rw-r--r-- | workhorse/cmd/gitlab-zip-metadata/main.go | 67 |
5 files changed, 342 insertions, 0 deletions
diff --git a/workhorse/cmd/gitlab-resize-image/main.go b/workhorse/cmd/gitlab-resize-image/main.go new file mode 100644 index 00000000000..662efd7306d --- /dev/null +++ b/workhorse/cmd/gitlab-resize-image/main.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "image" + "os" + "strconv" + + "github.com/disintegration/imaging" +) + +func main() { + if err := _main(); err != nil { + fmt.Fprintf(os.Stderr, "%s: fatal: %v\n", os.Args[0], err) + os.Exit(1) + } +} + +func _main() error { + widthParam := os.Getenv("GL_RESIZE_IMAGE_WIDTH") + requestedWidth, err := strconv.Atoi(widthParam) + if err != nil { + return fmt.Errorf("GL_RESIZE_IMAGE_WIDTH: %w", err) + } + + src, formatName, err := image.Decode(os.Stdin) + if err != nil { + return fmt.Errorf("decode: %w", err) + } + imagingFormat, err := imaging.FormatFromExtension(formatName) + if err != nil { + return fmt.Errorf("find imaging format: %w", err) + } + + image := imaging.Resize(src, requestedWidth, 0, imaging.Lanczos) + return imaging.Encode(os.Stdout, image, imagingFormat) +} diff --git a/workhorse/cmd/gitlab-zip-cat/main.go b/workhorse/cmd/gitlab-zip-cat/main.go new file mode 100644 index 00000000000..a0778a55ad2 --- /dev/null +++ b/workhorse/cmd/gitlab-zip-cat/main.go @@ -0,0 +1,96 @@ +package main + +import ( + "archive/zip" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + + "gitlab.com/gitlab-org/labkit/mask" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts" +) + +const progName = "gitlab-zip-cat" + +var Version = "unknown" + +var printVersion = flag.Bool("version", false, "Print version and exit") + +func main() { + flag.Parse() + + version := fmt.Sprintf("%s %s", progName, Version) + if *printVersion { + fmt.Println(version) + os.Exit(0) + } + + archivePath := os.Getenv("ARCHIVE_PATH") + encodedFileName := os.Getenv("ENCODED_FILE_NAME") + + if len(os.Args) != 1 || archivePath == "" || encodedFileName == "" { + fmt.Fprintf(os.Stderr, "Usage: %s\n", progName) + fmt.Fprintf(os.Stderr, "Env: ARCHIVE_PATH=https://path.to/archive.zip or /path/to/archive.zip\n") + fmt.Fprintf(os.Stderr, "Env: ENCODED_FILE_NAME=base64-encoded-file-name\n") + os.Exit(1) + } + + scrubbedArchivePath := mask.URL(archivePath) + + fileName, err := zipartifacts.DecodeFileEntry(encodedFileName) + if err != nil { + fatalError(fmt.Errorf("decode entry %q", encodedFileName), err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + archive, err := zipartifacts.OpenArchive(ctx, archivePath) + if err != nil { + fatalError(errors.New("open archive"), err) + } + + file := findFileInZip(fileName, archive) + if file == nil { + fatalError(fmt.Errorf("find %q in %q: not found", fileName, scrubbedArchivePath), zipartifacts.ErrorCode[zipartifacts.CodeEntryNotFound]) + } + // Start decompressing the file + reader, err := file.Open() + if err != nil { + fatalError(fmt.Errorf("open %q in %q", fileName, scrubbedArchivePath), err) + } + defer reader.Close() + + if _, err := fmt.Printf("%d\n", file.UncompressedSize64); err != nil { + fatalError(fmt.Errorf("write file size invalid"), err) + } + + if _, err := io.Copy(os.Stdout, reader); err != nil { + fatalError(fmt.Errorf("write %q from %q to stdout", fileName, scrubbedArchivePath), err) + } +} + +func findFileInZip(fileName string, archive *zip.Reader) *zip.File { + for _, file := range archive.File { + if file.Name == fileName { + return file + } + } + return nil +} + +func fatalError(contextErr error, statusErr error) { + code := zipartifacts.ExitCodeByError(statusErr) + + fmt.Fprintf(os.Stderr, "%s error: %v - %v, code: %d\n", progName, statusErr, contextErr, code) + + if code > 0 { + os.Exit(code) + } else { + os.Exit(1) + } +} diff --git a/workhorse/cmd/gitlab-zip-metadata/limit/reader.go b/workhorse/cmd/gitlab-zip-metadata/limit/reader.go new file mode 100644 index 00000000000..c1e9bd20993 --- /dev/null +++ b/workhorse/cmd/gitlab-zip-metadata/limit/reader.go @@ -0,0 +1,52 @@ +package limit + +import ( + "errors" + "io" + "sync/atomic" +) + +var ErrLimitExceeded = errors.New("reader limit exceeded") + +const megabyte = 1 << 20 + +// LimitedReaderAt supports running a callback in case of reaching a read limit +// (bytes), and allows using a smaller limit than a defined offset for a read. +type LimitedReaderAt struct { + read int64 + limit int64 + parent io.ReaderAt + limitFunc func(int64) +} + +func (r *LimitedReaderAt) ReadAt(p []byte, off int64) (int, error) { + if max := r.limit - r.read; int64(len(p)) > max { + p = p[0:max] + } + + n, err := r.parent.ReadAt(p, off) + atomic.AddInt64(&r.read, int64(n)) + + if r.read >= r.limit { + r.limitFunc(r.read) + + return n, ErrLimitExceeded + } + + return n, err +} + +func NewLimitedReaderAt(reader io.ReaderAt, limit int64, limitFunc func(int64)) io.ReaderAt { + return &LimitedReaderAt{parent: reader, limit: limit, limitFunc: limitFunc} +} + +// SizeToLimit tries to dermine an appropriate limit in bytes for an archive of +// a given size. If the size is less than 1 gigabyte we always limit a reader +// to 100 megabytes, otherwise the limit is 10% of a given size. +func SizeToLimit(size int64) int64 { + if size <= 1024*megabyte { + return 100 * megabyte + } + + return size / 10 +} diff --git a/workhorse/cmd/gitlab-zip-metadata/limit/reader_test.go b/workhorse/cmd/gitlab-zip-metadata/limit/reader_test.go new file mode 100644 index 00000000000..57ba8eeb214 --- /dev/null +++ b/workhorse/cmd/gitlab-zip-metadata/limit/reader_test.go @@ -0,0 +1,90 @@ +package limit + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReadAt(t *testing.T) { + t.Run("when limit has not been reached", func(t *testing.T) { + r := strings.NewReader("some string to read") + buf := make([]byte, 11) + + reader := NewLimitedReaderAt(r, 32, func(n int64) { + require.Zero(t, n) + }) + p, err := reader.ReadAt(buf, 0) + + require.NoError(t, err) + require.Equal(t, 11, p) + require.Equal(t, "some string", string(buf)) + }) + + t.Run("when read limit is exceeded", func(t *testing.T) { + r := strings.NewReader("some string to read") + buf := make([]byte, 11) + + reader := NewLimitedReaderAt(r, 9, func(n int64) { + require.Equal(t, 9, int(n)) + }) + p, err := reader.ReadAt(buf, 0) + + require.Error(t, err) + require.Equal(t, 9, p) + require.Equal(t, "some stri\x00\x00", string(buf)) + }) + + t.Run("when offset is higher than a limit", func(t *testing.T) { + r := strings.NewReader("some string to read") + buf := make([]byte, 4) + + reader := NewLimitedReaderAt(r, 5, func(n int64) { + require.Zero(t, n) + }) + + p, err := reader.ReadAt(buf, 15) + + require.NoError(t, err) + require.Equal(t, 4, p) + require.Equal(t, "read", string(buf)) + }) + + t.Run("when a read starts at the limit", func(t *testing.T) { + r := strings.NewReader("some string to read") + buf := make([]byte, 11) + + reader := NewLimitedReaderAt(r, 10, func(n int64) { + require.Equal(t, 10, int(n)) + }) + + reader.ReadAt(buf, 0) + p, err := reader.ReadAt(buf, 0) + + require.EqualError(t, err, ErrLimitExceeded.Error()) + require.Equal(t, 0, p) + require.Equal(t, "some strin\x00", string(buf)) + }) +} + +func TestSizeToLimit(t *testing.T) { + tests := []struct { + size int64 + limit int64 + name string + }{ + {size: 1, limit: 104857600, name: "1b to 100mb"}, + {size: 100, limit: 104857600, name: "100b to 100mb"}, + {size: 104857600, limit: 104857600, name: "100mb to 100mb"}, + {size: 1073741824, limit: 104857600, name: "1gb to 100mb"}, + {size: 10737418240, limit: 1073741824, name: "10gb to 1gb"}, + {size: 53687091200, limit: 5368709120, name: "50gb to 5gb"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.limit, SizeToLimit(test.size)) + }) + } +} diff --git a/workhorse/cmd/gitlab-zip-metadata/main.go b/workhorse/cmd/gitlab-zip-metadata/main.go new file mode 100644 index 00000000000..faa85ab5366 --- /dev/null +++ b/workhorse/cmd/gitlab-zip-metadata/main.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "os" + + "gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-metadata/limit" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts" +) + +const progName = "gitlab-zip-metadata" + +var Version = "unknown" + +var printVersion = flag.Bool("version", false, "Print version and exit") + +func main() { + flag.Parse() + + version := fmt.Sprintf("%s %s", progName, Version) + if *printVersion { + fmt.Println(version) + os.Exit(0) + } + + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP\n", progName) + os.Exit(1) + } + + readerFunc := func(reader io.ReaderAt, size int64) io.ReaderAt { + readLimit := limit.SizeToLimit(size) + + return limit.NewLimitedReaderAt(reader, readLimit, func(read int64) { + fmt.Fprintf(os.Stderr, "%s: zip archive limit exceeded after reading %d bytes\n", progName, read) + + fatalError(zipartifacts.ErrorCode[zipartifacts.CodeLimitsReached]) + }) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + archive, err := zipartifacts.OpenArchiveWithReaderFunc(ctx, os.Args[1], readerFunc) + if err != nil { + fatalError(err) + } + + if err := zipartifacts.GenerateZipMetadata(os.Stdout, archive); err != nil { + fatalError(err) + } +} + +func fatalError(err error) { + code := zipartifacts.ExitCodeByError(err) + + fmt.Fprintf(os.Stderr, "%s error: %v, code: %d\n", progName, err, code) + + if code > 0 { + os.Exit(code) + } else { + os.Exit(1) + } +} |