Add latest changes from gitlab-org/gitlab@13-7-stable-eev13.7.0-rc42

7 files changed, 1113 insertions, 0 deletions

diff --git a/workhorse/internal/artifacts/artifacts_store_test.go b/workhorse/internal/artifacts/artifacts_store_test.go
new file mode 100644
index 00000000000..bd56d9ea725
--- /dev/null
+++ b/workhorse/internal/artifacts/artifacts_store_test.go
@@ -0,0 +1,338 @@
+package artifacts
+
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/objectstore/test"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
+)
+
+func createTestZipArchive(t *testing.T) (data []byte, md5Hash string) {
+	var buffer bytes.Buffer
+	archive := zip.NewWriter(&buffer)
+	fileInArchive, err := archive.Create("test.file")
+	require.NoError(t, err)
+	fmt.Fprint(fileInArchive, "test")
+	archive.Close()
+	data = buffer.Bytes()
+
+	hasher := md5.New()
+	hasher.Write(data)
+	hexHash := hasher.Sum(nil)
+	md5Hash = hex.EncodeToString(hexHash)
+
+	return data, md5Hash
+}
+
+func createTestMultipartForm(t *testing.T, data []byte) (bytes.Buffer, string) {
+	var buffer bytes.Buffer
+	writer := multipart.NewWriter(&buffer)
+	file, err := writer.CreateFormFile("file", "my.file")
+	require.NoError(t, err)
+	file.Write(data)
+	writer.Close()
+	return buffer, writer.FormDataContentType()
+}
+
+func testUploadArtifactsFromTestZip(t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
+	archiveData, _ := createTestZipArchive(t)
+	contentBuffer, contentType := createTestMultipartForm(t, archiveData)
+
+	return testUploadArtifacts(t, contentType, ts.URL+Path, &contentBuffer)
+}
+
+func TestUploadHandlerSendingToExternalStorage(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	archiveData, md5 := createTestZipArchive(t)
+	archiveFile, err := ioutil.TempFile("", "artifact.zip")
+	require.NoError(t, err)
+	defer os.Remove(archiveFile.Name())
+	_, err = archiveFile.Write(archiveData)
+	require.NoError(t, err)
+	archiveFile.Close()
+
+	storeServerCalled := 0
+	storeServerMux := http.NewServeMux()
+	storeServerMux.HandleFunc("/url/put", func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "PUT", r.Method)
+
+		receivedData, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		require.Equal(t, archiveData, receivedData)
+
+		storeServerCalled++
+		w.Header().Set("ETag", md5)
+		w.WriteHeader(200)
+	})
+	storeServerMux.HandleFunc("/store-id", func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, archiveFile.Name())
+	})
+
+	responseProcessorCalled := 0
+	responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "store-id", r.FormValue("file.remote_id"))
+		require.NotEmpty(t, r.FormValue("file.remote_url"))
+		w.WriteHeader(200)
+		responseProcessorCalled++
+	}
+
+	storeServer := httptest.NewServer(storeServerMux)
+	defer storeServer.Close()
+
+	qs := fmt.Sprintf("?%s=%s", ArtifactFormatKey, ArtifactFormatZip)
+
+	tests := []struct {
+		name    string
+		preauth api.Response
+	}{
+		{
+			name: "ObjectStore Upload",
+			preauth: api.Response{
+				RemoteObject: api.RemoteObject{
+					StoreURL: storeServer.URL + "/url/put" + qs,
+					ID:       "store-id",
+					GetURL:   storeServer.URL + "/store-id",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			storeServerCalled = 0
+			responseProcessorCalled = 0
+
+			ts := testArtifactsUploadServer(t, test.preauth, responseProcessor)
+			defer ts.Close()
+
+			contentBuffer, contentType := createTestMultipartForm(t, archiveData)
+			response := testUploadArtifacts(t, contentType, ts.URL+Path+qs, &contentBuffer)
+			require.Equal(t, http.StatusOK, response.Code)
+			testhelper.RequireResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderPresent)
+			require.Equal(t, 1, storeServerCalled, "store should be called only once")
+			require.Equal(t, 1, responseProcessorCalled, "response processor should be called only once")
+		})
+	}
+}
+
+func TestUploadHandlerSendingToExternalStorageAndStorageServerUnreachable(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+		t.Fatal("it should not be called")
+	}
+
+	authResponse := api.Response{
+		TempPath: tempPath,
+		RemoteObject: api.RemoteObject{
+			StoreURL: "http://localhost:12323/invalid/url",
+			ID:       "store-id",
+		},
+	}
+
+	ts := testArtifactsUploadServer(t, authResponse, responseProcessor)
+	defer ts.Close()
+
+	response := testUploadArtifactsFromTestZip(t, ts)
+	require.Equal(t, http.StatusInternalServerError, response.Code)
+}
+
+func TestUploadHandlerSendingToExternalStorageAndInvalidURLIsUsed(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+		t.Fatal("it should not be called")
+	}
+
+	authResponse := api.Response{
+		TempPath: tempPath,
+		RemoteObject: api.RemoteObject{
+			StoreURL: "htt:////invalid-url",
+			ID:       "store-id",
+		},
+	}
+
+	ts := testArtifactsUploadServer(t, authResponse, responseProcessor)
+	defer ts.Close()
+
+	response := testUploadArtifactsFromTestZip(t, ts)
+	require.Equal(t, http.StatusInternalServerError, response.Code)
+}
+
+func TestUploadHandlerSendingToExternalStorageAndItReturnsAnError(t *testing.T) {
+	putCalledTimes := 0
+
+	storeServerMux := http.NewServeMux()
+	storeServerMux.HandleFunc("/url/put", func(w http.ResponseWriter, r *http.Request) {
+		putCalledTimes++
+		require.Equal(t, "PUT", r.Method)
+		w.WriteHeader(510)
+	})
+
+	responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+		t.Fatal("it should not be called")
+	}
+
+	storeServer := httptest.NewServer(storeServerMux)
+	defer storeServer.Close()
+
+	authResponse := api.Response{
+		RemoteObject: api.RemoteObject{
+			StoreURL: storeServer.URL + "/url/put",
+			ID:       "store-id",
+		},
+	}
+
+	ts := testArtifactsUploadServer(t, authResponse, responseProcessor)
+	defer ts.Close()
+
+	response := testUploadArtifactsFromTestZip(t, ts)
+	require.Equal(t, http.StatusInternalServerError, response.Code)
+	require.Equal(t, 1, putCalledTimes, "upload should be called only once")
+}
+
+func TestUploadHandlerSendingToExternalStorageAndSupportRequestTimeout(t *testing.T) {
+	putCalledTimes := 0
+
+	storeServerMux := http.NewServeMux()
+	storeServerMux.HandleFunc("/url/put", func(w http.ResponseWriter, r *http.Request) {
+		putCalledTimes++
+		require.Equal(t, "PUT", r.Method)
+		time.Sleep(10 * time.Second)
+		w.WriteHeader(510)
+	})
+
+	responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+		t.Fatal("it should not be called")
+	}
+
+	storeServer := httptest.NewServer(storeServerMux)
+	defer storeServer.Close()
+
+	authResponse := api.Response{
+		RemoteObject: api.RemoteObject{
+			StoreURL: storeServer.URL + "/url/put",
+			ID:       "store-id",
+			Timeout:  1,
+		},
+	}
+
+	ts := testArtifactsUploadServer(t, authResponse, responseProcessor)
+	defer ts.Close()
+
+	response := testUploadArtifactsFromTestZip(t, ts)
+	require.Equal(t, http.StatusInternalServerError, response.Code)
+	require.Equal(t, 1, putCalledTimes, "upload should be called only once")
+}
+
+func TestUploadHandlerMultipartUploadSizeLimit(t *testing.T) {
+	os, server := test.StartObjectStore()
+	defer server.Close()
+
+	err := os.InitiateMultipartUpload(test.ObjectPath)
+	require.NoError(t, err)
+
+	objectURL := server.URL + test.ObjectPath
+
+	uploadSize := 10
+	preauth := api.Response{
+		RemoteObject: api.RemoteObject{
+			ID: "store-id",
+			MultipartUpload: &api.MultipartUploadParams{
+				PartSize:    1,
+				PartURLs:    []string{objectURL + "?partNumber=1"},
+				AbortURL:    objectURL, // DELETE
+				CompleteURL: objectURL, // POST
+			},
+		},
+	}
+
+	responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+		t.Fatal("it should not be called")
+	}
+
+	ts := testArtifactsUploadServer(t, preauth, responseProcessor)
+	defer ts.Close()
+
+	contentBuffer, contentType := createTestMultipartForm(t, make([]byte, uploadSize))
+	response := testUploadArtifacts(t, contentType, ts.URL+Path, &contentBuffer)
+	require.Equal(t, http.StatusRequestEntityTooLarge, response.Code)
+
+	// Poll because AbortMultipartUpload is async
+	for i := 0; os.IsMultipartUpload(test.ObjectPath) && i < 100; i++ {
+		time.Sleep(10 * time.Millisecond)
+	}
+	require.False(t, os.IsMultipartUpload(test.ObjectPath), "MultipartUpload should not be in progress anymore")
+	require.Empty(t, os.GetObjectMD5(test.ObjectPath), "upload should have failed, so the object should not exists")
+}
+
+func TestUploadHandlerMultipartUploadMaximumSizeFromApi(t *testing.T) {
+	os, server := test.StartObjectStore()
+	defer server.Close()
+
+	err := os.InitiateMultipartUpload(test.ObjectPath)
+	require.NoError(t, err)
+
+	objectURL := server.URL + test.ObjectPath
+
+	uploadSize := int64(10)
+	maxSize := uploadSize - 1
+	preauth := api.Response{
+		MaximumSize: maxSize,
+		RemoteObject: api.RemoteObject{
+			ID: "store-id",
+			MultipartUpload: &api.MultipartUploadParams{
+				PartSize:    uploadSize,
+				PartURLs:    []string{objectURL + "?partNumber=1"},
+				AbortURL:    objectURL, // DELETE
+				CompleteURL: objectURL, // POST
+			},
+		},
+	}
+
+	responseProcessor := func(w http.ResponseWriter, r *http.Request) {
+		t.Fatal("it should not be called")
+	}
+
+	ts := testArtifactsUploadServer(t, preauth, responseProcessor)
+	defer ts.Close()
+
+	contentBuffer, contentType := createTestMultipartForm(t, make([]byte, uploadSize))
+	response := testUploadArtifacts(t, contentType, ts.URL+Path, &contentBuffer)
+	require.Equal(t, http.StatusRequestEntityTooLarge, response.Code)
+
+	testhelper.Retry(t, 5*time.Second, func() error {
+		if os.GetObjectMD5(test.ObjectPath) == "" {
+			return nil
+		}
+
+		return fmt.Errorf("file is still present")
+	})
+}
diff --git a/workhorse/internal/artifacts/artifacts_test.go b/workhorse/internal/artifacts/artifacts_test.go
new file mode 100644
index 00000000000..b9a42cc60c1
--- /dev/null
+++ b/workhorse/internal/artifacts/artifacts_test.go
@@ -0,0 +1,19 @@
+package artifacts
+
+import (
+	"os"
+	"testing"
+
+	"gitlab.com/gitlab-org/labkit/log"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
+)
+
+func TestMain(m *testing.M) {
+	if err := testhelper.BuildExecutables(); err != nil {
+		log.WithError(err).Fatal()
+	}
+
+	os.Exit(m.Run())
+
+}
diff --git a/workhorse/internal/artifacts/artifacts_upload.go b/workhorse/internal/artifacts/artifacts_upload.go
new file mode 100644
index 00000000000..3d4b8bf0931
--- /dev/null
+++ b/workhorse/internal/artifacts/artifacts_upload.go
@@ -0,0 +1,167 @@
+package artifacts
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"gitlab.com/gitlab-org/labkit/log"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/filestore"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/upload"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
+)
+
+// Sent by the runner: https://gitlab.com/gitlab-org/gitlab-runner/blob/c24da19ecce8808d9d2950896f70c94f5ea1cc2e/network/gitlab.go#L580
+const (
+	ArtifactFormatKey     = "artifact_format"
+	ArtifactFormatZip     = "zip"
+	ArtifactFormatDefault = ""
+)
+
+var zipSubcommandsErrorsCounter = promauto.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "gitlab_workhorse_zip_subcommand_errors_total",
+		Help: "Errors comming from subcommands used for processing ZIP archives",
+	}, []string{"error"})
+
+type artifactsUploadProcessor struct {
+	opts   *filestore.SaveFileOpts
+	format string
+
+	upload.SavedFileTracker
+}
+
+func (a *artifactsUploadProcessor) generateMetadataFromZip(ctx context.Context, file *filestore.FileHandler) (*filestore.FileHandler, error) {
+	metaReader, metaWriter := io.Pipe()
+	defer metaWriter.Close()
+
+	metaOpts := &filestore.SaveFileOpts{
+		LocalTempPath:  a.opts.LocalTempPath,
+		TempFilePrefix: "metadata.gz",
+	}
+	if metaOpts.LocalTempPath == "" {
+		metaOpts.LocalTempPath = os.TempDir()
+	}
+
+	fileName := file.LocalPath
+	if fileName == "" {
+		fileName = file.RemoteURL
+	}
+
+	zipMd := exec.CommandContext(ctx, "gitlab-zip-metadata", fileName)
+	zipMd.Stderr = log.ContextLogger(ctx).Writer()
+	zipMd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	zipMd.Stdout = metaWriter
+
+	if err := zipMd.Start(); err != nil {
+		return nil, err
+	}
+	defer helper.CleanUpProcessGroup(zipMd)
+
+	type saveResult struct {
+		error
+		*filestore.FileHandler
+	}
+	done := make(chan saveResult)
+	go func() {
+		var result saveResult
+		result.FileHandler, result.error = filestore.SaveFileFromReader(ctx, metaReader, -1, metaOpts)
+
+		done <- result
+	}()
+
+	if err := zipMd.Wait(); err != nil {
+		st, ok := helper.ExitStatus(err)
+
+		if !ok {
+			return nil, err
+		}
+
+		zipSubcommandsErrorsCounter.WithLabelValues(zipartifacts.ErrorLabelByCode(st)).Inc()
+
+		if st == zipartifacts.CodeNotZip {
+			return nil, nil
+		}
+
+		if st == zipartifacts.CodeLimitsReached {
+			return nil, zipartifacts.ErrBadMetadata
+		}
+	}
+
+	metaWriter.Close()
+	result := <-done
+	return result.FileHandler, result.error
+}
+
+func (a *artifactsUploadProcessor) ProcessFile(ctx context.Context, formName string, file *filestore.FileHandler, writer *multipart.Writer) error {
+	//  ProcessFile for artifacts requires file form-data field name to eq `file`
+
+	if formName != "file" {
+		return fmt.Errorf("invalid form field: %q", formName)
+	}
+	if a.Count() > 0 {
+		return fmt.Errorf("artifacts request contains more than one file")
+	}
+	a.Track(formName, file.LocalPath)
+
+	select {
+	case <-ctx.Done():
+		return fmt.Errorf("ProcessFile: context done")
+	default:
+	}
+
+	if !strings.EqualFold(a.format, ArtifactFormatZip) && a.format != ArtifactFormatDefault {
+		return nil
+	}
+
+	// TODO: can we rely on disk for shipping metadata? Not if we split workhorse and rails in 2 different PODs
+	metadata, err := a.generateMetadataFromZip(ctx, file)
+	if err != nil {
+		return err
+	}
+
+	if metadata != nil {
+		fields, err := metadata.GitLabFinalizeFields("metadata")
+		if err != nil {
+			return fmt.Errorf("finalize metadata field error: %v", err)
+		}
+
+		for k, v := range fields {
+			writer.WriteField(k, v)
+		}
+
+		a.Track("metadata", metadata.LocalPath)
+	}
+
+	return nil
+}
+
+func (a *artifactsUploadProcessor) Name() string {
+	return "artifacts"
+}
+
+func UploadArtifacts(myAPI *api.API, h http.Handler, p upload.Preparer) http.Handler {
+	return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
+		opts, _, err := p.Prepare(a)
+		if err != nil {
+			helper.Fail500(w, r, fmt.Errorf("UploadArtifacts: error preparing file storage options"))
+			return
+		}
+
+		format := r.URL.Query().Get(ArtifactFormatKey)
+
+		mg := &artifactsUploadProcessor{opts: opts, format: format, SavedFileTracker: upload.SavedFileTracker{Request: r}}
+		upload.HandleFileUploads(w, r, h, a, mg, opts)
+	}, "/authorize")
+}
diff --git a/workhorse/internal/artifacts/artifacts_upload_test.go b/workhorse/internal/artifacts/artifacts_upload_test.go
new file mode 100644
index 00000000000..c82ae791239
--- /dev/null
+++ b/workhorse/internal/artifacts/artifacts_upload_test.go
@@ -0,0 +1,322 @@
+package artifacts
+
+import (
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/dgrijalva/jwt-go"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/filestore"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/proxy"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/upload"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream/roundtripper"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
+
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	MetadataHeaderKey     = "Metadata-Status"
+	MetadataHeaderPresent = "present"
+	MetadataHeaderMissing = "missing"
+	Path                  = "/url/path"
+)
+
+func testArtifactsUploadServer(t *testing.T, authResponse api.Response, bodyProcessor func(w http.ResponseWriter, r *http.Request)) *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc(Path+"/authorize", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "POST" {
+			t.Fatal("Expected POST request")
+		}
+
+		w.Header().Set("Content-Type", api.ResponseContentType)
+
+		data, err := json.Marshal(&authResponse)
+		if err != nil {
+			t.Fatal("Expected to marshal")
+		}
+		w.Write(data)
+	})
+	mux.HandleFunc(Path, func(w http.ResponseWriter, r *http.Request) {
+		opts, err := filestore.GetOpts(&authResponse)
+		require.NoError(t, err)
+
+		if r.Method != "POST" {
+			t.Fatal("Expected POST request")
+		}
+		if opts.IsLocal() {
+			if r.FormValue("file.path") == "" {
+				t.Fatal("Expected file to be present")
+				return
+			}
+
+			_, err := ioutil.ReadFile(r.FormValue("file.path"))
+			if err != nil {
+				t.Fatal("Expected file to be readable")
+				return
+			}
+		} else {
+			if r.FormValue("file.remote_url") == "" {
+				t.Fatal("Expected file to be remote accessible")
+				return
+			}
+		}
+
+		if r.FormValue("metadata.path") != "" {
+			metadata, err := ioutil.ReadFile(r.FormValue("metadata.path"))
+			if err != nil {
+				t.Fatal("Expected metadata to be readable")
+				return
+			}
+			gz, err := gzip.NewReader(bytes.NewReader(metadata))
+			if err != nil {
+				t.Fatal("Expected metadata to be valid gzip")
+				return
+			}
+			defer gz.Close()
+			metadata, err = ioutil.ReadAll(gz)
+			if err != nil {
+				t.Fatal("Expected metadata to be valid")
+				return
+			}
+			if !bytes.HasPrefix(metadata, []byte(zipartifacts.MetadataHeaderPrefix+zipartifacts.MetadataHeader)) {
+				t.Fatal("Expected metadata to be of valid format")
+				return
+			}
+
+			w.Header().Set(MetadataHeaderKey, MetadataHeaderPresent)
+
+		} else {
+			w.Header().Set(MetadataHeaderKey, MetadataHeaderMissing)
+		}
+
+		w.WriteHeader(http.StatusOK)
+
+		if bodyProcessor != nil {
+			bodyProcessor(w, r)
+		}
+	})
+	return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
+}
+
+type testServer struct {
+	url        string
+	writer     *multipart.Writer
+	buffer     *bytes.Buffer
+	fileWriter io.Writer
+	cleanup    func()
+}
+
+func setupWithTmpPath(t *testing.T, filename string, includeFormat bool, format string, authResponse *api.Response, bodyProcessor func(w http.ResponseWriter, r *http.Request)) *testServer {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	require.NoError(t, err)
+
+	if authResponse == nil {
+		authResponse = &api.Response{TempPath: tempPath}
+	}
+
+	ts := testArtifactsUploadServer(t, *authResponse, bodyProcessor)
+
+	var buffer bytes.Buffer
+	writer := multipart.NewWriter(&buffer)
+	fileWriter, err := writer.CreateFormFile(filename, "my.file")
+	require.NotNil(t, fileWriter)
+	require.NoError(t, err)
+
+	cleanup := func() {
+		ts.Close()
+		require.NoError(t, os.RemoveAll(tempPath))
+		require.NoError(t, writer.Close())
+	}
+
+	qs := ""
+
+	if includeFormat {
+		qs = fmt.Sprintf("?%s=%s", ArtifactFormatKey, format)
+	}
+
+	return &testServer{url: ts.URL + Path + qs, writer: writer, buffer: &buffer, fileWriter: fileWriter, cleanup: cleanup}
+}
+
+func testUploadArtifacts(t *testing.T, contentType, url string, body io.Reader) *httptest.ResponseRecorder {
+	httpRequest, err := http.NewRequest("POST", url, body)
+	require.NoError(t, err)
+
+	httpRequest.Header.Set("Content-Type", contentType)
+	response := httptest.NewRecorder()
+	parsedURL := helper.URLMustParse(url)
+	roundTripper := roundtripper.NewTestBackendRoundTripper(parsedURL)
+	testhelper.ConfigureSecret()
+	apiClient := api.NewAPI(parsedURL, "123", roundTripper)
+	proxyClient := proxy.NewProxy(parsedURL, "123", roundTripper)
+	UploadArtifacts(apiClient, proxyClient, &upload.DefaultPreparer{}).ServeHTTP(response, httpRequest)
+	return response
+}
+
+func TestUploadHandlerAddingMetadata(t *testing.T) {
+	testCases := []struct {
+		desc          string
+		format        string
+		includeFormat bool
+	}{
+		{
+			desc:          "ZIP format",
+			format:        ArtifactFormatZip,
+			includeFormat: true,
+		},
+		{
+			desc:          "default format",
+			format:        ArtifactFormatDefault,
+			includeFormat: true,
+		},
+		{
+			desc:          "default format without artifact_format",
+			format:        ArtifactFormatDefault,
+			includeFormat: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			s := setupWithTmpPath(t, "file", tc.includeFormat, tc.format, nil,
+				func(w http.ResponseWriter, r *http.Request) {
+					token, err := jwt.ParseWithClaims(r.Header.Get(upload.RewrittenFieldsHeader), &upload.MultipartClaims{}, testhelper.ParseJWT)
+					require.NoError(t, err)
+
+					rewrittenFields := token.Claims.(*upload.MultipartClaims).RewrittenFields
+					require.Equal(t, 2, len(rewrittenFields))
+
+					require.Contains(t, rewrittenFields, "file")
+					require.Contains(t, rewrittenFields, "metadata")
+					require.Contains(t, r.PostForm, "file.gitlab-workhorse-upload")
+					require.Contains(t, r.PostForm, "metadata.gitlab-workhorse-upload")
+				},
+			)
+			defer s.cleanup()
+
+			archive := zip.NewWriter(s.fileWriter)
+			file, err := archive.Create("test.file")
+			require.NotNil(t, file)
+			require.NoError(t, err)
+
+			require.NoError(t, archive.Close())
+			require.NoError(t, s.writer.Close())
+
+			response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+			require.Equal(t, http.StatusOK, response.Code)
+			testhelper.RequireResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderPresent)
+		})
+	}
+}
+
+func TestUploadHandlerTarArtifact(t *testing.T) {
+	s := setupWithTmpPath(t, "file", true, "tar", nil,
+		func(w http.ResponseWriter, r *http.Request) {
+			token, err := jwt.ParseWithClaims(r.Header.Get(upload.RewrittenFieldsHeader), &upload.MultipartClaims{}, testhelper.ParseJWT)
+			require.NoError(t, err)
+
+			rewrittenFields := token.Claims.(*upload.MultipartClaims).RewrittenFields
+			require.Equal(t, 1, len(rewrittenFields))
+
+			require.Contains(t, rewrittenFields, "file")
+			require.Contains(t, r.PostForm, "file.gitlab-workhorse-upload")
+		},
+	)
+	defer s.cleanup()
+
+	file, err := os.Open("../../testdata/tarfile.tar")
+	require.NoError(t, err)
+
+	_, err = io.Copy(s.fileWriter, file)
+	require.NoError(t, err)
+	require.NoError(t, file.Close())
+	require.NoError(t, s.writer.Close())
+
+	response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+	require.Equal(t, http.StatusOK, response.Code)
+	testhelper.RequireResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderMissing)
+}
+
+func TestUploadHandlerForUnsupportedArchive(t *testing.T) {
+	s := setupWithTmpPath(t, "file", true, "other", nil, nil)
+	defer s.cleanup()
+	require.NoError(t, s.writer.Close())
+
+	response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+	require.Equal(t, http.StatusOK, response.Code)
+	testhelper.RequireResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderMissing)
+}
+
+func TestUploadHandlerForMultipleFiles(t *testing.T) {
+	s := setupWithTmpPath(t, "file", true, "", nil, nil)
+	defer s.cleanup()
+
+	file, err := s.writer.CreateFormFile("file", "my.file")
+	require.NotNil(t, file)
+	require.NoError(t, err)
+	require.NoError(t, s.writer.Close())
+
+	response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+	require.Equal(t, http.StatusInternalServerError, response.Code)
+}
+
+func TestUploadFormProcessing(t *testing.T) {
+	s := setupWithTmpPath(t, "metadata", true, "", nil, nil)
+	defer s.cleanup()
+	require.NoError(t, s.writer.Close())
+
+	response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+	require.Equal(t, http.StatusInternalServerError, response.Code)
+}
+
+func TestLsifFileProcessing(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	require.NoError(t, err)
+
+	s := setupWithTmpPath(t, "file", true, "zip", &api.Response{TempPath: tempPath, ProcessLsif: true}, nil)
+	defer s.cleanup()
+
+	file, err := os.Open("../../testdata/lsif/valid.lsif.zip")
+	require.NoError(t, err)
+
+	_, err = io.Copy(s.fileWriter, file)
+	require.NoError(t, err)
+	require.NoError(t, file.Close())
+	require.NoError(t, s.writer.Close())
+
+	response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+	require.Equal(t, http.StatusOK, response.Code)
+	testhelper.RequireResponseHeader(t, response, MetadataHeaderKey, MetadataHeaderPresent)
+}
+
+func TestInvalidLsifFileProcessing(t *testing.T) {
+	tempPath, err := ioutil.TempDir("", "uploads")
+	require.NoError(t, err)
+
+	s := setupWithTmpPath(t, "file", true, "zip", &api.Response{TempPath: tempPath, ProcessLsif: true}, nil)
+	defer s.cleanup()
+
+	file, err := os.Open("../../testdata/lsif/invalid.lsif.zip")
+	require.NoError(t, err)
+
+	_, err = io.Copy(s.fileWriter, file)
+	require.NoError(t, err)
+	require.NoError(t, file.Close())
+	require.NoError(t, s.writer.Close())
+
+	response := testUploadArtifacts(t, s.writer.FormDataContentType(), s.url, s.buffer)
+	require.Equal(t, http.StatusInternalServerError, response.Code)
+}
diff --git a/workhorse/internal/artifacts/entry.go b/workhorse/internal/artifacts/entry.go
new file mode 100644
index 00000000000..0c697d40020
--- /dev/null
+++ b/workhorse/internal/artifacts/entry.go
@@ -0,0 +1,123 @@
+package artifacts
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"gitlab.com/gitlab-org/labkit/log"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
+)
+
+type entry struct{ senddata.Prefix }
+type entryParams struct{ Archive, Entry string }
+
+var SendEntry = &entry{"artifacts-entry:"}
+
+// Artifacts downloader doesn't support ranges when downloading a single file
+func (e *entry) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
+	var params entryParams
+	if err := e.Unpack(&params, sendData); err != nil {
+		helper.Fail500(w, r, fmt.Errorf("SendEntry: unpack sendData: %v", err))
+		return
+	}
+
+	log.WithContextFields(r.Context(), log.Fields{
+		"entry":   params.Entry,
+		"archive": params.Archive,
+		"path":    r.URL.Path,
+	}).Print("SendEntry: sending")
+
+	if params.Archive == "" || params.Entry == "" {
+		helper.Fail500(w, r, fmt.Errorf("SendEntry: Archive or Entry is empty"))
+		return
+	}
+
+	err := unpackFileFromZip(r.Context(), params.Archive, params.Entry, w.Header(), w)
+
+	if os.IsNotExist(err) {
+		http.NotFound(w, r)
+	} else if err != nil {
+		helper.Fail500(w, r, fmt.Errorf("SendEntry: %v", err))
+	}
+}
+
+func detectFileContentType(fileName string) string {
+	contentType := mime.TypeByExtension(filepath.Ext(fileName))
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	return contentType
+}
+
+func unpackFileFromZip(ctx context.Context, archivePath, encodedFilename string, headers http.Header, output io.Writer) error {
+	fileName, err := zipartifacts.DecodeFileEntry(encodedFilename)
+	if err != nil {
+		return err
+	}
+
+	catFile := exec.Command("gitlab-zip-cat")
+	catFile.Env = append(os.Environ(),
+		"ARCHIVE_PATH="+archivePath,
+		"ENCODED_FILE_NAME="+encodedFilename,
+	)
+	catFile.Stderr = log.ContextLogger(ctx).Writer()
+	catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	stdout, err := catFile.StdoutPipe()
+	if err != nil {
+		return fmt.Errorf("create gitlab-zip-cat stdout pipe: %v", err)
+	}
+
+	if err := catFile.Start(); err != nil {
+		return fmt.Errorf("start %v: %v", catFile.Args, err)
+	}
+	defer helper.CleanUpProcessGroup(catFile)
+
+	basename := filepath.Base(fileName)
+	reader := bufio.NewReader(stdout)
+	contentLength, err := reader.ReadString('\n')
+	if err != nil {
+		if catFileErr := waitCatFile(catFile); catFileErr != nil {
+			return catFileErr
+		}
+		return fmt.Errorf("read content-length: %v", err)
+	}
+	contentLength = strings.TrimSuffix(contentLength, "\n")
+
+	// Write http headers about the file
+	headers.Set("Content-Length", contentLength)
+	headers.Set("Content-Type", detectFileContentType(fileName))
+	headers.Set("Content-Disposition", "attachment; filename=\""+escapeQuotes(basename)+"\"")
+	// Copy file body to client
+	if _, err := io.Copy(output, reader); err != nil {
+		return fmt.Errorf("copy stdout of %v: %v", catFile.Args, err)
+	}
+
+	return waitCatFile(catFile)
+}
+
+func waitCatFile(cmd *exec.Cmd) error {
+	err := cmd.Wait()
+	if err == nil {
+		return nil
+	}
+
+	st, ok := helper.ExitStatus(err)
+
+	if ok && (st == zipartifacts.CodeArchiveNotFound || st == zipartifacts.CodeEntryNotFound) {
+		return os.ErrNotExist
+	}
+	return fmt.Errorf("wait for %v to finish: %v", cmd.Args, err)
+
+}
diff --git a/workhorse/internal/artifacts/entry_test.go b/workhorse/internal/artifacts/entry_test.go
new file mode 100644
index 00000000000..6f1e9d360aa
--- /dev/null
+++ b/workhorse/internal/artifacts/entry_test.go
@@ -0,0 +1,134 @@
+package artifacts
+
+import (
+	"archive/zip"
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
+)
+
+func testEntryServer(t *testing.T, archive string, entry string) *httptest.ResponseRecorder {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "GET", r.Method)
+
+		encodedEntry := base64.StdEncoding.EncodeToString([]byte(entry))
+		jsonParams := fmt.Sprintf(`{"Archive":"%s","Entry":"%s"}`, archive, encodedEntry)
+		data := base64.URLEncoding.EncodeToString([]byte(jsonParams))
+
+		SendEntry.Inject(w, r, data)
+	})
+
+	httpRequest, err := http.NewRequest("GET", "/url/path", nil)
+	require.NoError(t, err)
+	response := httptest.NewRecorder()
+	mux.ServeHTTP(response, httpRequest)
+	return response
+}
+
+func TestDownloadingFromValidArchive(t *testing.T) {
+	tempFile, err := ioutil.TempFile("", "uploads")
+	require.NoError(t, err)
+	defer tempFile.Close()
+	defer os.Remove(tempFile.Name())
+
+	archive := zip.NewWriter(tempFile)
+	defer archive.Close()
+	fileInArchive, err := archive.Create("test.txt")
+	require.NoError(t, err)
+	fmt.Fprint(fileInArchive, "testtest")
+	archive.Close()
+
+	response := testEntryServer(t, tempFile.Name(), "test.txt")
+
+	require.Equal(t, 200, response.Code)
+
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"test.txt\"")
+
+	testhelper.RequireResponseBody(t, response, "testtest")
+}
+
+func TestDownloadingFromValidHTTPArchive(t *testing.T) {
+	tempDir, err := ioutil.TempDir("", "uploads")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDir)
+
+	f, err := os.Create(filepath.Join(tempDir, "archive.zip"))
+	require.NoError(t, err)
+	defer f.Close()
+
+	archive := zip.NewWriter(f)
+	defer archive.Close()
+	fileInArchive, err := archive.Create("test.txt")
+	require.NoError(t, err)
+	fmt.Fprint(fileInArchive, "testtest")
+	archive.Close()
+	f.Close()
+
+	fileServer := httptest.NewServer(http.FileServer(http.Dir(tempDir)))
+	defer fileServer.Close()
+
+	response := testEntryServer(t, fileServer.URL+"/archive.zip", "test.txt")
+
+	require.Equal(t, 200, response.Code)
+
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Type",
+		"text/plain; charset=utf-8")
+	testhelper.RequireResponseHeader(t, response,
+		"Content-Disposition",
+		"attachment; filename=\"test.txt\"")
+
+	testhelper.RequireResponseBody(t, response, "testtest")
+}
+
+func TestDownloadingNonExistingFile(t *testing.T) {
+	tempFile, err := ioutil.TempFile("", "uploads")
+	require.NoError(t, err)
+	defer tempFile.Close()
+	defer os.Remove(tempFile.Name())
+
+	archive := zip.NewWriter(tempFile)
+	defer archive.Close()
+	archive.Close()
+
+	response := testEntryServer(t, tempFile.Name(), "test")
+	require.Equal(t, 404, response.Code)
+}
+
+func TestDownloadingFromInvalidArchive(t *testing.T) {
+	response := testEntryServer(t, "path/to/non/existing/file", "test")
+	require.Equal(t, 404, response.Code)
+}
+
+func TestIncompleteApiResponse(t *testing.T) {
+	response := testEntryServer(t, "", "")
+	require.Equal(t, 500, response.Code)
+}
+
+func TestDownloadingFromNonExistingHTTPArchive(t *testing.T) {
+	tempDir, err := ioutil.TempDir("", "uploads")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDir)
+
+	fileServer := httptest.NewServer(http.FileServer(http.Dir(tempDir)))
+	defer fileServer.Close()
+
+	response := testEntryServer(t, fileServer.URL+"/not-existing-archive-file.zip", "test.txt")
+
+	require.Equal(t, 404, response.Code)
+}
diff --git a/workhorse/internal/artifacts/escape_quotes.go b/workhorse/internal/artifacts/escape_quotes.go
new file mode 100644
index 00000000000..94db2be39b7
--- /dev/null
+++ b/workhorse/internal/artifacts/escape_quotes.go
@@ -0,0 +1,10 @@
+package artifacts
+
+import "strings"
+
+// taken from mime/multipart/writer.go
+var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
+
+func escapeQuotes(s string) string {
+	return quoteEscaper.Replace(s)
+}