diff options
Diffstat (limited to 'workhorse/internal/git')
-rw-r--r-- | workhorse/internal/git/archive.go | 216 | ||||
-rw-r--r-- | workhorse/internal/git/archive_test.go | 87 | ||||
-rw-r--r-- | workhorse/internal/git/blob.go | 47 | ||||
-rw-r--r-- | workhorse/internal/git/blob_test.go | 17 | ||||
-rw-r--r-- | workhorse/internal/git/diff.go | 48 | ||||
-rw-r--r-- | workhorse/internal/git/error.go | 4 | ||||
-rw-r--r-- | workhorse/internal/git/format-patch.go | 48 | ||||
-rw-r--r-- | workhorse/internal/git/git-http.go | 100 | ||||
-rw-r--r-- | workhorse/internal/git/info-refs.go | 76 | ||||
-rw-r--r-- | workhorse/internal/git/pktline.go | 59 | ||||
-rw-r--r-- | workhorse/internal/git/pktline_test.go | 39 | ||||
-rw-r--r-- | workhorse/internal/git/receive-pack.go | 33 | ||||
-rw-r--r-- | workhorse/internal/git/responsewriter.go | 75 | ||||
-rw-r--r-- | workhorse/internal/git/snapshot.go | 64 | ||||
-rw-r--r-- | workhorse/internal/git/upload-pack.go | 57 | ||||
-rw-r--r-- | workhorse/internal/git/upload-pack_test.go | 85 |
16 files changed, 1055 insertions, 0 deletions
diff --git a/workhorse/internal/git/archive.go b/workhorse/internal/git/archive.go new file mode 100644 index 00000000000..b7575be2c02 --- /dev/null +++ b/workhorse/internal/git/archive.go @@ -0,0 +1,216 @@ +/* +In this file we handle 'git archive' downloads +*/ + +package git + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "path/filepath" + "regexp" + "time" + + "github.com/golang/protobuf/proto" //lint:ignore SA1019 https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/274 + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata" +) + +type archive struct{ senddata.Prefix } +type archiveParams struct { + ArchivePath string + ArchivePrefix string + CommitId string + GitalyServer gitaly.Server + GitalyRepository gitalypb.Repository + DisableCache bool + GetArchiveRequest []byte +} + +var ( + SendArchive = &archive{"git-archive:"} + gitArchiveCache = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "gitlab_workhorse_git_archive_cache", + Help: "Cache hits and misses for 'git archive' streaming", + }, + []string{"result"}, + ) +) + +func (a *archive) Inject(w http.ResponseWriter, r *http.Request, sendData string) { + var params archiveParams + if err := a.Unpack(¶ms, sendData); err != nil { + helper.Fail500(w, r, fmt.Errorf("SendArchive: unpack sendData: %v", err)) + return + } + + urlPath := r.URL.Path + format, ok := parseBasename(filepath.Base(urlPath)) + if !ok { + helper.Fail500(w, r, fmt.Errorf("SendArchive: invalid format: %s", urlPath)) + return + } + + cacheEnabled := !params.DisableCache + archiveFilename := path.Base(params.ArchivePath) + + if cacheEnabled { + cachedArchive, err := os.Open(params.ArchivePath) + if err == nil { + defer cachedArchive.Close() + gitArchiveCache.WithLabelValues("hit").Inc() + setArchiveHeaders(w, format, archiveFilename) + // Even if somebody deleted the cachedArchive from disk since we opened + // the file, Unix file semantics guarantee we can still read from the + // open file in this process. + http.ServeContent(w, r, "", time.Unix(0, 0), cachedArchive) + return + } + } + + gitArchiveCache.WithLabelValues("miss").Inc() + + var tempFile *os.File + var err error + + if cacheEnabled { + // We assume the tempFile has a unique name so that concurrent requests are + // safe. We create the tempfile in the same directory as the final cached + // archive we want to create so that we can use an atomic link(2) operation + // to finalize the cached archive. + tempFile, err = prepareArchiveTempfile(path.Dir(params.ArchivePath), archiveFilename) + if err != nil { + helper.Fail500(w, r, fmt.Errorf("SendArchive: create tempfile: %v", err)) + return + } + defer tempFile.Close() + defer os.Remove(tempFile.Name()) + } + + var archiveReader io.Reader + + archiveReader, err = handleArchiveWithGitaly(r, params, format) + if err != nil { + helper.Fail500(w, r, fmt.Errorf("operations.GetArchive: %v", err)) + return + } + + reader := archiveReader + if cacheEnabled { + reader = io.TeeReader(archiveReader, tempFile) + } + + // Start writing the response + setArchiveHeaders(w, format, archiveFilename) + w.WriteHeader(200) // Don't bother with HTTP 500 from this point on, just return + if _, err := io.Copy(w, reader); err != nil { + helper.LogError(r, ©Error{fmt.Errorf("SendArchive: copy 'git archive' output: %v", err)}) + return + } + + if cacheEnabled { + err := finalizeCachedArchive(tempFile, params.ArchivePath) + if err != nil { + helper.LogError(r, fmt.Errorf("SendArchive: finalize cached archive: %v", err)) + return + } + } +} + +func handleArchiveWithGitaly(r *http.Request, params archiveParams, format gitalypb.GetArchiveRequest_Format) (io.Reader, error) { + var request *gitalypb.GetArchiveRequest + ctx, c, err := gitaly.NewRepositoryClient(r.Context(), params.GitalyServer) + if err != nil { + return nil, err + } + + if params.GetArchiveRequest != nil { + request = &gitalypb.GetArchiveRequest{} + + if err := proto.Unmarshal(params.GetArchiveRequest, request); err != nil { + return nil, fmt.Errorf("unmarshal GetArchiveRequest: %v", err) + } + } else { + request = &gitalypb.GetArchiveRequest{ + Repository: ¶ms.GitalyRepository, + CommitId: params.CommitId, + Prefix: params.ArchivePrefix, + Format: format, + } + } + + return c.ArchiveReader(ctx, request) +} + +func setArchiveHeaders(w http.ResponseWriter, format gitalypb.GetArchiveRequest_Format, archiveFilename string) { + w.Header().Del("Content-Length") + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, archiveFilename)) + // Caching proxies usually don't cache responses with Set-Cookie header + // present because it implies user-specific data, which is not the case + // for repository archives. + w.Header().Del("Set-Cookie") + if format == gitalypb.GetArchiveRequest_ZIP { + w.Header().Set("Content-Type", "application/zip") + } else { + w.Header().Set("Content-Type", "application/octet-stream") + } + w.Header().Set("Content-Transfer-Encoding", "binary") +} + +func prepareArchiveTempfile(dir string, prefix string) (*os.File, error) { + if err := os.MkdirAll(dir, 0700); err != nil { + return nil, err + } + return ioutil.TempFile(dir, prefix) +} + +func finalizeCachedArchive(tempFile *os.File, archivePath string) error { + if err := tempFile.Close(); err != nil { + return err + } + if err := os.Link(tempFile.Name(), archivePath); err != nil && !os.IsExist(err) { + return err + } + + return nil +} + +var ( + patternZip = regexp.MustCompile(`\.zip$`) + patternTar = regexp.MustCompile(`\.tar$`) + patternTarGz = regexp.MustCompile(`\.(tar\.gz|tgz|gz)$`) + patternTarBz2 = regexp.MustCompile(`\.(tar\.bz2|tbz|tbz2|tb2|bz2)$`) +) + +func parseBasename(basename string) (gitalypb.GetArchiveRequest_Format, bool) { + var format gitalypb.GetArchiveRequest_Format + + switch { + case (basename == "archive"): + format = gitalypb.GetArchiveRequest_TAR_GZ + case patternZip.MatchString(basename): + format = gitalypb.GetArchiveRequest_ZIP + case patternTar.MatchString(basename): + format = gitalypb.GetArchiveRequest_TAR + case patternTarGz.MatchString(basename): + format = gitalypb.GetArchiveRequest_TAR_GZ + case patternTarBz2.MatchString(basename): + format = gitalypb.GetArchiveRequest_TAR_BZ2 + default: + return format, false + } + + return format, true +} diff --git a/workhorse/internal/git/archive_test.go b/workhorse/internal/git/archive_test.go new file mode 100644 index 00000000000..4b0753499e5 --- /dev/null +++ b/workhorse/internal/git/archive_test.go @@ -0,0 +1,87 @@ +package git + +import ( + "io/ioutil" + "net/http/httptest" + "testing" + + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper" + + "github.com/stretchr/testify/require" +) + +func TestParseBasename(t *testing.T) { + for _, testCase := range []struct { + in string + out gitalypb.GetArchiveRequest_Format + }{ + {"archive", gitalypb.GetArchiveRequest_TAR_GZ}, + {"master.tar.gz", gitalypb.GetArchiveRequest_TAR_GZ}, + {"foo-master.tgz", gitalypb.GetArchiveRequest_TAR_GZ}, + {"foo-v1.2.1.gz", gitalypb.GetArchiveRequest_TAR_GZ}, + {"foo.tar.bz2", gitalypb.GetArchiveRequest_TAR_BZ2}, + {"archive.tbz", gitalypb.GetArchiveRequest_TAR_BZ2}, + {"archive.tbz2", gitalypb.GetArchiveRequest_TAR_BZ2}, + {"archive.tb2", gitalypb.GetArchiveRequest_TAR_BZ2}, + {"archive.bz2", gitalypb.GetArchiveRequest_TAR_BZ2}, + } { + basename := testCase.in + out, ok := parseBasename(basename) + if !ok { + t.Fatalf("parseBasename did not recognize %q", basename) + } + + if out != testCase.out { + t.Fatalf("expected %q, got %q", testCase.out, out) + } + } +} + +func TestFinalizeArchive(t *testing.T) { + tempFile, err := ioutil.TempFile("", "gitlab-workhorse-test") + if err != nil { + t.Fatal(err) + } + defer tempFile.Close() + + // Deliberately cause an EEXIST error: we know tempFile.Name() already exists + err = finalizeCachedArchive(tempFile, tempFile.Name()) + if err != nil { + t.Fatalf("expected nil from finalizeCachedArchive, received %v", err) + } +} + +func TestSetArchiveHeaders(t *testing.T) { + for _, testCase := range []struct { + in gitalypb.GetArchiveRequest_Format + out string + }{ + {gitalypb.GetArchiveRequest_ZIP, "application/zip"}, + {gitalypb.GetArchiveRequest_TAR, "application/octet-stream"}, + {gitalypb.GetArchiveRequest_TAR_GZ, "application/octet-stream"}, + {gitalypb.GetArchiveRequest_TAR_BZ2, "application/octet-stream"}, + } { + w := httptest.NewRecorder() + + // These should be replaced, not appended to + w.Header().Set("Content-Type", "test") + w.Header().Set("Content-Length", "test") + w.Header().Set("Content-Disposition", "test") + + // This should be deleted + w.Header().Set("Set-Cookie", "test") + + // This should be preserved + w.Header().Set("Cache-Control", "public, max-age=3600") + + setArchiveHeaders(w, testCase.in, "filename") + + testhelper.RequireResponseHeader(t, w, "Content-Type", testCase.out) + testhelper.RequireResponseHeader(t, w, "Content-Length") + testhelper.RequireResponseHeader(t, w, "Content-Disposition", `attachment; filename="filename"`) + testhelper.RequireResponseHeader(t, w, "Cache-Control", "public, max-age=3600") + require.Empty(t, w.Header().Get("Set-Cookie"), "remove Set-Cookie") + } +} diff --git a/workhorse/internal/git/blob.go b/workhorse/internal/git/blob.go new file mode 100644 index 00000000000..472f5d0bc96 --- /dev/null +++ b/workhorse/internal/git/blob.go @@ -0,0 +1,47 @@ +package git + +import ( + "fmt" + "net/http" + + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata" +) + +type blob struct{ senddata.Prefix } +type blobParams struct { + GitalyServer gitaly.Server + GetBlobRequest gitalypb.GetBlobRequest +} + +var SendBlob = &blob{"git-blob:"} + +func (b *blob) Inject(w http.ResponseWriter, r *http.Request, sendData string) { + var params blobParams + if err := b.Unpack(¶ms, sendData); err != nil { + helper.Fail500(w, r, fmt.Errorf("SendBlob: unpack sendData: %v", err)) + return + } + + ctx, blobClient, err := gitaly.NewBlobClient(r.Context(), params.GitalyServer) + if err != nil { + helper.Fail500(w, r, fmt.Errorf("blob.GetBlob: %v", err)) + return + } + + setBlobHeaders(w) + if err := blobClient.SendBlob(ctx, w, ¶ms.GetBlobRequest); err != nil { + helper.Fail500(w, r, fmt.Errorf("blob.GetBlob: %v", err)) + return + } +} + +func setBlobHeaders(w http.ResponseWriter) { + // Caching proxies usually don't cache responses with Set-Cookie header + // present because it implies user-specific data, which is not the case + // for blobs. + w.Header().Del("Set-Cookie") +} diff --git a/workhorse/internal/git/blob_test.go b/workhorse/internal/git/blob_test.go new file mode 100644 index 00000000000..ec28c2adb2f --- /dev/null +++ b/workhorse/internal/git/blob_test.go @@ -0,0 +1,17 @@ +package git + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSetBlobHeaders(t *testing.T) { + w := httptest.NewRecorder() + w.Header().Set("Set-Cookie", "gitlab_cookie=123456") + + setBlobHeaders(w) + + require.Empty(t, w.Header().Get("Set-Cookie"), "remove Set-Cookie") +} diff --git a/workhorse/internal/git/diff.go b/workhorse/internal/git/diff.go new file mode 100644 index 00000000000..b1a1c17a650 --- /dev/null +++ b/workhorse/internal/git/diff.go @@ -0,0 +1,48 @@ +package git + +import ( + "fmt" + "net/http" + + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata" +) + +type diff struct{ senddata.Prefix } +type diffParams struct { + GitalyServer gitaly.Server + RawDiffRequest string +} + +var SendDiff = &diff{"git-diff:"} + +func (d *diff) Inject(w http.ResponseWriter, r *http.Request, sendData string) { + var params diffParams + if err := d.Unpack(¶ms, sendData); err != nil { + helper.Fail500(w, r, fmt.Errorf("SendDiff: unpack sendData: %v", err)) + return + } + + request := &gitalypb.RawDiffRequest{} + if err := gitaly.UnmarshalJSON(params.RawDiffRequest, request); err != nil { + helper.Fail500(w, r, fmt.Errorf("diff.RawDiff: %v", err)) + return + } + + ctx, diffClient, err := gitaly.NewDiffClient(r.Context(), params.GitalyServer) + if err != nil { + helper.Fail500(w, r, fmt.Errorf("diff.RawDiff: %v", err)) + return + } + + if err := diffClient.SendRawDiff(ctx, w, request); err != nil { + helper.LogError( + r, + ©Error{fmt.Errorf("diff.RawDiff: request=%v, err=%v", request, err)}, + ) + return + } +} diff --git a/workhorse/internal/git/error.go b/workhorse/internal/git/error.go new file mode 100644 index 00000000000..2b7cad6bb64 --- /dev/null +++ b/workhorse/internal/git/error.go @@ -0,0 +1,4 @@ +package git + +// For cosmetic purposes in Sentry +type copyError struct{ error } diff --git a/workhorse/internal/git/format-patch.go b/workhorse/internal/git/format-patch.go new file mode 100644 index 00000000000..db96029b07e --- /dev/null +++ b/workhorse/internal/git/format-patch.go @@ -0,0 +1,48 @@ +package git + +import ( + "fmt" + "net/http" + + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata" +) + +type patch struct{ senddata.Prefix } +type patchParams struct { + GitalyServer gitaly.Server + RawPatchRequest string +} + +var SendPatch = &patch{"git-format-patch:"} + +func (p *patch) Inject(w http.ResponseWriter, r *http.Request, sendData string) { + var params patchParams + if err := p.Unpack(¶ms, sendData); err != nil { + helper.Fail500(w, r, fmt.Errorf("SendPatch: unpack sendData: %v", err)) + return + } + + request := &gitalypb.RawPatchRequest{} + if err := gitaly.UnmarshalJSON(params.RawPatchRequest, request); err != nil { + helper.Fail500(w, r, fmt.Errorf("diff.RawPatch: %v", err)) + return + } + + ctx, diffClient, err := gitaly.NewDiffClient(r.Context(), params.GitalyServer) + if err != nil { + helper.Fail500(w, r, fmt.Errorf("diff.RawPatch: %v", err)) + return + } + + if err := diffClient.SendRawPatch(ctx, w, request); err != nil { + helper.LogError( + r, + ©Error{fmt.Errorf("diff.RawPatch: request=%v, err=%v", request, err)}, + ) + return + } +} diff --git a/workhorse/internal/git/git-http.go b/workhorse/internal/git/git-http.go new file mode 100644 index 00000000000..5df20a68bb7 --- /dev/null +++ b/workhorse/internal/git/git-http.go @@ -0,0 +1,100 @@ +/* +In this file we handle the Git 'smart HTTP' protocol +*/ + +package git + +import ( + "fmt" + "io" + "net/http" + "path/filepath" + "sync" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/api" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" +) + +const ( + // We have to use a negative transfer.hideRefs since this is the only way + // to undo an already set parameter: https://www.spinics.net/lists/git/msg256772.html + GitConfigShowAllRefs = "transfer.hideRefs=!refs" +) + +func ReceivePack(a *api.API) http.Handler { + return postRPCHandler(a, "handleReceivePack", handleReceivePack) +} + +func UploadPack(a *api.API) http.Handler { + return postRPCHandler(a, "handleUploadPack", handleUploadPack) +} + +func gitConfigOptions(a *api.Response) []string { + var out []string + + if a.ShowAllRefs { + out = append(out, GitConfigShowAllRefs) + } + + return out +} + +func postRPCHandler(a *api.API, name string, handler func(*HttpResponseWriter, *http.Request, *api.Response) error) http.Handler { + return repoPreAuthorizeHandler(a, func(rw http.ResponseWriter, r *http.Request, ar *api.Response) { + cr := &countReadCloser{ReadCloser: r.Body} + r.Body = cr + + w := NewHttpResponseWriter(rw) + defer func() { + w.Log(r, cr.Count()) + }() + + if err := handler(w, r, ar); err != nil { + // If the handler already wrote a response this WriteHeader call is a + // no-op. It never reaches net/http because GitHttpResponseWriter calls + // WriteHeader on its underlying ResponseWriter at most once. + w.WriteHeader(500) + helper.LogError(r, fmt.Errorf("%s: %v", name, err)) + } + }) +} + +func repoPreAuthorizeHandler(myAPI *api.API, handleFunc api.HandleFunc) http.Handler { + return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) { + handleFunc(w, r, a) + }, "") +} + +func writePostRPCHeader(w http.ResponseWriter, action string) { + w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-result", action)) + w.Header().Set("Cache-Control", "no-cache") +} + +func getService(r *http.Request) string { + if r.Method == "GET" { + return r.URL.Query().Get("service") + } + return filepath.Base(r.URL.Path) +} + +type countReadCloser struct { + n int64 + io.ReadCloser + sync.Mutex +} + +func (c *countReadCloser) Read(p []byte) (n int, err error) { + n, err = c.ReadCloser.Read(p) + + c.Lock() + defer c.Unlock() + c.n += int64(n) + + return n, err +} + +func (c *countReadCloser) Count() int64 { + c.Lock() + defer c.Unlock() + return c.n +} diff --git a/workhorse/internal/git/info-refs.go b/workhorse/internal/git/info-refs.go new file mode 100644 index 00000000000..e5491a7b733 --- /dev/null +++ b/workhorse/internal/git/info-refs.go @@ -0,0 +1,76 @@ +package git + +import ( + "compress/gzip" + "context" + "fmt" + "io" + "net/http" + + "github.com/golang/gddo/httputil" + + "gitlab.com/gitlab-org/labkit/log" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/api" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" +) + +func GetInfoRefsHandler(a *api.API) http.Handler { + return repoPreAuthorizeHandler(a, handleGetInfoRefs) +} + +func handleGetInfoRefs(rw http.ResponseWriter, r *http.Request, a *api.Response) { + responseWriter := NewHttpResponseWriter(rw) + // Log 0 bytes in because we ignore the request body (and there usually is none anyway). + defer responseWriter.Log(r, 0) + + rpc := getService(r) + if !(rpc == "git-upload-pack" || rpc == "git-receive-pack") { + // The 'dumb' Git HTTP protocol is not supported + http.Error(responseWriter, "Not Found", 404) + return + } + + responseWriter.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc)) + responseWriter.Header().Set("Cache-Control", "no-cache") + + gitProtocol := r.Header.Get("Git-Protocol") + + offers := []string{"gzip", "identity"} + encoding := httputil.NegotiateContentEncoding(r, offers) + + if err := handleGetInfoRefsWithGitaly(r.Context(), responseWriter, a, rpc, gitProtocol, encoding); err != nil { + helper.Fail500(responseWriter, r, fmt.Errorf("handleGetInfoRefs: %v", err)) + } +} + +func handleGetInfoRefsWithGitaly(ctx context.Context, responseWriter *HttpResponseWriter, a *api.Response, rpc, gitProtocol, encoding string) error { + ctx, smarthttp, err := gitaly.NewSmartHTTPClient(ctx, a.GitalyServer) + if err != nil { + return fmt.Errorf("GetInfoRefsHandler: %v", err) + } + + infoRefsResponseReader, err := smarthttp.InfoRefsResponseReader(ctx, &a.Repository, rpc, gitConfigOptions(a), gitProtocol) + if err != nil { + return fmt.Errorf("GetInfoRefsHandler: %v", err) + } + + var w io.Writer + + if encoding == "gzip" { + gzWriter := gzip.NewWriter(responseWriter) + w = gzWriter + defer gzWriter.Close() + + responseWriter.Header().Set("Content-Encoding", "gzip") + } else { + w = responseWriter + } + + if _, err = io.Copy(w, infoRefsResponseReader); err != nil { + log.WithError(err).Error("GetInfoRefsHandler: error copying gitaly response") + } + + return nil +} diff --git a/workhorse/internal/git/pktline.go b/workhorse/internal/git/pktline.go new file mode 100644 index 00000000000..e970f60182d --- /dev/null +++ b/workhorse/internal/git/pktline.go @@ -0,0 +1,59 @@ +package git + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strconv" +) + +func scanDeepen(body io.Reader) bool { + scanner := bufio.NewScanner(body) + scanner.Split(pktLineSplitter) + for scanner.Scan() { + if bytes.HasPrefix(scanner.Bytes(), []byte("deepen")) && scanner.Err() == nil { + return true + } + } + + return false +} + +func pktLineSplitter(data []byte, atEOF bool) (advance int, token []byte, err error) { + if len(data) < 4 { + if atEOF && len(data) > 0 { + return 0, nil, fmt.Errorf("pktLineSplitter: incomplete length prefix on %q", data) + } + return 0, nil, nil // want more data + } + + if bytes.HasPrefix(data, []byte("0000")) { + // special case: "0000" terminator packet: return empty token + return 4, data[:0], nil + } + + // We have at least 4 bytes available so we can decode the 4-hex digit + // length prefix of the packet line. + pktLength64, err := strconv.ParseInt(string(data[:4]), 16, 0) + if err != nil { + return 0, nil, fmt.Errorf("pktLineSplitter: decode length: %v", err) + } + + // Cast is safe because we requested an int-size number from strconv.ParseInt + pktLength := int(pktLength64) + + if pktLength < 0 { + return 0, nil, fmt.Errorf("pktLineSplitter: invalid length: %d", pktLength) + } + + if len(data) < pktLength { + if atEOF { + return 0, nil, fmt.Errorf("pktLineSplitter: less than %d bytes in input %q", pktLength, data) + } + return 0, nil, nil // want more data + } + + // return "pkt" token without length prefix + return pktLength, data[4:pktLength], nil +} diff --git a/workhorse/internal/git/pktline_test.go b/workhorse/internal/git/pktline_test.go new file mode 100644 index 00000000000..d4be8634538 --- /dev/null +++ b/workhorse/internal/git/pktline_test.go @@ -0,0 +1,39 @@ +package git + +import ( + "bytes" + "testing" +) + +func TestSuccessfulScanDeepen(t *testing.T) { + examples := []struct { + input string + output bool + }{ + {"000dsomething000cdeepen 10000", true}, + {"000dsomething0000000cdeepen 1", true}, + {"000dsomething0000", false}, + } + + for _, example := range examples { + hasDeepen := scanDeepen(bytes.NewReader([]byte(example.input))) + + if hasDeepen != example.output { + t.Fatalf("scanDeepen %q: expected %v, got %v", example.input, example.output, hasDeepen) + } + } +} + +func TestFailedScanDeepen(t *testing.T) { + examples := []string{ + "invalid data", + "deepen", + "000cdeepen", + } + + for _, example := range examples { + if scanDeepen(bytes.NewReader([]byte(example))) { + t.Fatalf("scanDeepen %q: expected result to be false, got true", example) + } + } +} diff --git a/workhorse/internal/git/receive-pack.go b/workhorse/internal/git/receive-pack.go new file mode 100644 index 00000000000..e72d8be5174 --- /dev/null +++ b/workhorse/internal/git/receive-pack.go @@ -0,0 +1,33 @@ +package git + +import ( + "fmt" + "net/http" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/api" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" +) + +// Will not return a non-nil error after the response body has been +// written to. +func handleReceivePack(w *HttpResponseWriter, r *http.Request, a *api.Response) error { + action := getService(r) + writePostRPCHeader(w, action) + + cr, cw := helper.NewWriteAfterReader(r.Body, w) + defer cw.Flush() + + gitProtocol := r.Header.Get("Git-Protocol") + + ctx, smarthttp, err := gitaly.NewSmartHTTPClient(r.Context(), a.GitalyServer) + if err != nil { + return fmt.Errorf("smarthttp.ReceivePack: %v", err) + } + + if err := smarthttp.ReceivePack(ctx, &a.Repository, a.GL_ID, a.GL_USERNAME, a.GL_REPOSITORY, a.GitConfigOptions, cr, cw, gitProtocol); err != nil { + return fmt.Errorf("smarthttp.ReceivePack: %v", err) + } + + return nil +} diff --git a/workhorse/internal/git/responsewriter.go b/workhorse/internal/git/responsewriter.go new file mode 100644 index 00000000000..c4d4ac252d4 --- /dev/null +++ b/workhorse/internal/git/responsewriter.go @@ -0,0 +1,75 @@ +package git + +import ( + "net/http" + "strconv" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" +) + +const ( + directionIn = "in" + directionOut = "out" +) + +var ( + gitHTTPSessionsActive = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "gitlab_workhorse_git_http_sessions_active", + Help: "Number of Git HTTP request-response cycles currently being handled by gitlab-workhorse.", + }) + + gitHTTPRequests = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "gitlab_workhorse_git_http_requests", + Help: "How many Git HTTP requests have been processed by gitlab-workhorse, partitioned by request type and agent.", + }, + []string{"method", "code", "service", "agent"}, + ) + + gitHTTPBytes = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "gitlab_workhorse_git_http_bytes", + Help: "How many Git HTTP bytes have been sent by gitlab-workhorse, partitioned by request type, agent and direction.", + }, + []string{"method", "code", "service", "agent", "direction"}, + ) +) + +type HttpResponseWriter struct { + helper.CountingResponseWriter +} + +func NewHttpResponseWriter(rw http.ResponseWriter) *HttpResponseWriter { + gitHTTPSessionsActive.Inc() + return &HttpResponseWriter{ + CountingResponseWriter: helper.NewCountingResponseWriter(rw), + } +} + +func (w *HttpResponseWriter) Log(r *http.Request, writtenIn int64) { + service := getService(r) + agent := getRequestAgent(r) + + gitHTTPSessionsActive.Dec() + gitHTTPRequests.WithLabelValues(r.Method, strconv.Itoa(w.Status()), service, agent).Inc() + gitHTTPBytes.WithLabelValues(r.Method, strconv.Itoa(w.Status()), service, agent, directionIn). + Add(float64(writtenIn)) + gitHTTPBytes.WithLabelValues(r.Method, strconv.Itoa(w.Status()), service, agent, directionOut). + Add(float64(w.Count())) +} + +func getRequestAgent(r *http.Request) string { + u, _, ok := r.BasicAuth() + if !ok { + return "anonymous" + } + + if u == "gitlab-ci-token" { + return "gitlab-ci" + } + + return "logged" +} diff --git a/workhorse/internal/git/snapshot.go b/workhorse/internal/git/snapshot.go new file mode 100644 index 00000000000..eb38becbd06 --- /dev/null +++ b/workhorse/internal/git/snapshot.go @@ -0,0 +1,64 @@ +package git + +import ( + "fmt" + "io" + "net/http" + + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata" +) + +type snapshot struct { + senddata.Prefix +} + +type snapshotParams struct { + GitalyServer gitaly.Server + GetSnapshotRequest string +} + +var ( + SendSnapshot = &snapshot{"git-snapshot:"} +) + +func (s *snapshot) Inject(w http.ResponseWriter, r *http.Request, sendData string) { + var params snapshotParams + + if err := s.Unpack(¶ms, sendData); err != nil { + helper.Fail500(w, r, fmt.Errorf("SendSnapshot: unpack sendData: %v", err)) + return + } + + request := &gitalypb.GetSnapshotRequest{} + if err := gitaly.UnmarshalJSON(params.GetSnapshotRequest, request); err != nil { + helper.Fail500(w, r, fmt.Errorf("SendSnapshot: unmarshal GetSnapshotRequest: %v", err)) + return + } + + ctx, c, err := gitaly.NewRepositoryClient(r.Context(), params.GitalyServer) + if err != nil { + helper.Fail500(w, r, fmt.Errorf("SendSnapshot: gitaly.NewRepositoryClient: %v", err)) + return + } + + reader, err := c.SnapshotReader(ctx, request) + if err != nil { + helper.Fail500(w, r, fmt.Errorf("SendSnapshot: client.SnapshotReader: %v", err)) + return + } + + w.Header().Del("Content-Length") + w.Header().Set("Content-Disposition", `attachment; filename="snapshot.tar"`) + w.Header().Set("Content-Type", "application/x-tar") + w.Header().Set("Content-Transfer-Encoding", "binary") + w.Header().Set("Cache-Control", "private") + w.WriteHeader(http.StatusOK) // Errors aren't detectable beyond this point + + if _, err := io.Copy(w, reader); err != nil { + helper.LogError(r, fmt.Errorf("SendSnapshot: copy gitaly output: %v", err)) + } +} diff --git a/workhorse/internal/git/upload-pack.go b/workhorse/internal/git/upload-pack.go new file mode 100644 index 00000000000..a3dbf2f2e02 --- /dev/null +++ b/workhorse/internal/git/upload-pack.go @@ -0,0 +1,57 @@ +package git + +import ( + "context" + "fmt" + "io" + "net/http" + "time" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/api" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper" +) + +var ( + uploadPackTimeout = 10 * time.Minute +) + +// Will not return a non-nil error after the response body has been +// written to. +func handleUploadPack(w *HttpResponseWriter, r *http.Request, a *api.Response) error { + ctx := r.Context() + + // Prevent the client from holding the connection open indefinitely. A + // transfer rate of 17KiB/sec is sufficient to send 10MiB of data in + // ten minutes, which seems adequate. Most requests will be much smaller. + // This mitigates a use-after-check issue. + // + // We can't reliably interrupt the read from a http handler, but we can + // ensure the request will (eventually) fail: https://github.com/golang/go/issues/16100 + readerCtx, cancel := context.WithTimeout(ctx, uploadPackTimeout) + defer cancel() + + limited := helper.NewContextReader(readerCtx, r.Body) + cr, cw := helper.NewWriteAfterReader(limited, w) + defer cw.Flush() + + action := getService(r) + writePostRPCHeader(w, action) + + gitProtocol := r.Header.Get("Git-Protocol") + + return handleUploadPackWithGitaly(ctx, a, cr, cw, gitProtocol) +} + +func handleUploadPackWithGitaly(ctx context.Context, a *api.Response, clientRequest io.Reader, clientResponse io.Writer, gitProtocol string) error { + ctx, smarthttp, err := gitaly.NewSmartHTTPClient(ctx, a.GitalyServer) + if err != nil { + return fmt.Errorf("smarthttp.UploadPack: %v", err) + } + + if err := smarthttp.UploadPack(ctx, &a.Repository, clientRequest, clientResponse, gitConfigOptions(a), gitProtocol); err != nil { + return fmt.Errorf("smarthttp.UploadPack: %v", err) + } + + return nil +} diff --git a/workhorse/internal/git/upload-pack_test.go b/workhorse/internal/git/upload-pack_test.go new file mode 100644 index 00000000000..c198939d5df --- /dev/null +++ b/workhorse/internal/git/upload-pack_test.go @@ -0,0 +1,85 @@ +package git + +import ( + "fmt" + "io/ioutil" + "net" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" + + "gitlab.com/gitlab-org/gitlab-workhorse/internal/api" + "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly" +) + +var ( + originalUploadPackTimeout = uploadPackTimeout +) + +type fakeReader struct { + n int + err error +} + +func (f *fakeReader) Read(b []byte) (int, error) { + return f.n, f.err +} + +type smartHTTPServiceServer struct { + gitalypb.UnimplementedSmartHTTPServiceServer + PostUploadPackFunc func(gitalypb.SmartHTTPService_PostUploadPackServer) error +} + +func (srv *smartHTTPServiceServer) PostUploadPack(s gitalypb.SmartHTTPService_PostUploadPackServer) error { + return srv.PostUploadPackFunc(s) +} + +func TestUploadPackTimesOut(t *testing.T) { + uploadPackTimeout = time.Millisecond + defer func() { uploadPackTimeout = originalUploadPackTimeout }() + + addr, cleanUp := startSmartHTTPServer(t, &smartHTTPServiceServer{ + PostUploadPackFunc: func(stream gitalypb.SmartHTTPService_PostUploadPackServer) error { + _, err := stream.Recv() // trigger a read on the client request body + require.NoError(t, err) + return nil + }, + }) + defer cleanUp() + + body := &fakeReader{n: 0, err: nil} + + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/", body) + a := &api.Response{GitalyServer: gitaly.Server{Address: addr}} + + err := handleUploadPack(NewHttpResponseWriter(w), r, a) + require.EqualError(t, err, "smarthttp.UploadPack: busyReader: context deadline exceeded") +} + +func startSmartHTTPServer(t testing.TB, s gitalypb.SmartHTTPServiceServer) (string, func()) { + tmp, err := ioutil.TempDir("", "") + require.NoError(t, err) + + socket := filepath.Join(tmp, "gitaly.sock") + ln, err := net.Listen("unix", socket) + require.NoError(t, err) + + srv := grpc.NewServer() + gitalypb.RegisterSmartHTTPServiceServer(srv, s) + go func() { + require.NoError(t, srv.Serve(ln)) + }() + + return fmt.Sprintf("%s://%s", ln.Addr().Network(), ln.Addr().String()), func() { + srv.GracefulStop() + require.NoError(t, os.RemoveAll(tmp), "error removing temp dir %q", tmp) + } +} |