summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--GITLAB_SHELL_VERSION2
-rw-r--r--app/assets/javascripts/jobs/components/log/line.vue32
-rw-r--r--changelogs/unreleased/update-gitlab-shell-13-11-0.yml5
-rw-r--r--doc/api/repositories.md4
-rw-r--r--doc/development/lfs.md71
-rw-r--r--doc/topics/git/lfs/index.md46
-rw-r--r--doc/user/project/repository/index.md1
-rw-r--r--package.json1
-rw-r--r--spec/frontend/jobs/components/log/line_spec.js65
-rw-r--r--[-rwxr-xr-x]vendor/gitignore/C++.gitignore0
-rw-r--r--[-rwxr-xr-x]vendor/gitignore/Java.gitignore0
-rw-r--r--yarn.lock5
12 files changed, 145 insertions, 87 deletions
diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION
index c554e7e8652..fbda4b8d751 100644
--- a/GITLAB_SHELL_VERSION
+++ b/GITLAB_SHELL_VERSION
@@ -1 +1 @@
-13.10.0
+13.11.0
diff --git a/app/assets/javascripts/jobs/components/log/line.vue b/app/assets/javascripts/jobs/components/log/line.vue
index 791664c05d9..e68d5b8eda4 100644
--- a/app/assets/javascripts/jobs/components/log/line.vue
+++ b/app/assets/javascripts/jobs/components/log/line.vue
@@ -1,24 +1,6 @@
<script>
-import linkifyHtml from 'linkifyjs/html';
-import { sanitize } from '~/lib/dompurify';
-import { isAbsolute } from '~/lib/utils/url_utility';
import LineNumber from './line_number.vue';
-const linkifyOptions = {
- attributes: {
- // eslint-disable-next-line @gitlab/require-i18n-strings
- rel: 'nofollow noopener',
- },
- className: 'gl-reset-color!',
- defaultProtocol: 'https',
- validate: {
- email: false,
- url(value) {
- return isAbsolute(value);
- },
- },
-};
-
export default {
functional: true,
props: {
@@ -35,15 +17,13 @@ export default {
const { line, path } = props;
const chars = line.content.map(content => {
- const linkfied = linkifyHtml(content.text, linkifyOptions);
- return h('span', {
- class: ['gl-white-space-pre-wrap', content.style],
- domProps: {
- innerHTML: sanitize(linkfied, {
- ALLOWED_TAGS: ['a'],
- }),
+ return h(
+ 'span',
+ {
+ class: ['gl-white-space-pre-wrap', content.style],
},
- });
+ content.text,
+ );
});
return h('div', { class: 'js-line log-line' }, [
diff --git a/changelogs/unreleased/update-gitlab-shell-13-11-0.yml b/changelogs/unreleased/update-gitlab-shell-13-11-0.yml
new file mode 100644
index 00000000000..53dbd63b8cb
--- /dev/null
+++ b/changelogs/unreleased/update-gitlab-shell-13-11-0.yml
@@ -0,0 +1,5 @@
+---
+title: Update GitLab Shell to v13.11.0
+merge_request: 45660
+author:
+type: other
diff --git a/doc/api/repositories.md b/doc/api/repositories.md
index 7e94ff7b7f2..e880c2d95e1 100644
--- a/doc/api/repositories.md
+++ b/doc/api/repositories.md
@@ -112,10 +112,12 @@ Parameters:
## Get file archive
+> Support for [including Git LFS blobs](../topics/git/lfs/index.md#lfs-objects-in-project-archives) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15079) in GitLab 13.5.
+
Get an archive of the repository. This endpoint can be accessed without
authentication if the repository is publicly accessible.
-This endpoint has a rate limit threshold of 5 requests per minute.
+This endpoint has a rate limit threshold of 5 requests per minute for GitLab.com users.
```plaintext
GET /projects/:id/repository/archive[.format]
diff --git a/doc/development/lfs.md b/doc/development/lfs.md
index 32e2e3d1bde..3ba81e6a140 100644
--- a/doc/development/lfs.md
+++ b/doc/development/lfs.md
@@ -10,3 +10,74 @@ and the slides on [Google Slides](https://docs.google.com/presentation/d/1E-aw6-
and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/07a89257a140db067bdfb484aecd35e1/Git_LFS_Deep_Dive__Create_.pdf).
Everything covered in this deep dive was accurate as of GitLab 11.10, and while specific
details may have changed since then, it should still serve as a good introduction.
+
+## Including LFS blobs in project archives
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15079) in GitLab 13.5.
+
+The following diagram illustrates how GitLab resolves LFS files for project archives:
+
+```mermaid
+sequenceDiagram
+ autonumber
+ Client->>+Workhorse: GET /group/project/-/archive/master.zip
+ Workhorse->>+Rails: GET /group/project/-/archive/master.zip
+ Rails->>+Workhorse: Gitlab-Workhorse-Send-Data git-archive
+ Workhorse->>Gitaly: SendArchiveRequest
+ Gitaly->>Git: git archive master
+ Git->>Smudge: OID 12345
+ Smudge->>+Workhorse: GET /internal/api/v4/lfs?oid=12345&gl_repository=project-1234
+ Workhorse->>+Rails: GET /internal/api/v4/lfs?oid=12345&gl_repository=project-1234
+ Rails->>+Workhorse: Gitlab-Workhorse-Send-Data send-url
+ Workhorse->>Smudge: <LFS data>
+ Smudge->>Git: <LFS data>
+ Git->>Gitaly: <streamed data>
+ Gitaly->>Workhorse: <streamed data>
+ Workhorse->>Client: master.zip
+```
+
+1. The user requests the project archive from the UI.
+1. Workhorse forwards this request to Rails.
+1. If the user is authorized to download the archive, Rails replies with
+an HTTP header of `Gitlab-Workhorse-Send-Data` with a base64-encoded
+JSON payload prefaced with `git-archive`. This payload includes the
+`SendArchiveRequest` binary message, which is encoded again in base64.
+1. Workhorse decodes the `Gitlab-Workhorse-Send-Data` payload. If the
+archive already exists in the archive cache, Workhorse sends that
+file. Otherwise, Workhorse sends the `SendArchiveRequest` to the
+appropriate Gitaly server.
+1. The Gitaly server will call `git archive <ref>` to begin generating
+the Git archive on-the-fly. If the `include_lfs_blobs` flag is enabled,
+Gitaly enables a custom LFS smudge filter via the `-c
+filter.lfs.smudge=/path/to/gitaly-lfs-smudge` Git option.
+1. When `git` identifies a possible LFS pointer using the
+`.gitattributes` file, `git` calls `gitaly-lfs-smudge` and provides the
+LFS pointer via the standard input. Gitaly provides `GL_PROJECT_PATH`
+and `GL_INTERNAL_CONFIG` as environment variables to enable lookup of
+the LFS object.
+1. If a valid LFS pointer is decoded, `gitaly-lfs-smudge` makes an
+internal API call to Workhorse to download the LFS object from GitLab.
+1. Workhorse forwards this request to Rails. If the LFS object exists
+and is associated with the project, Rails sends `ArchivePath` either
+with a path where the LFS object resides (for local disk) or a
+pre-signed URL (when object storage is enabled) via the
+`Gitlab-Workhorse-Send-Data` HTTP header with a payload prefaced with
+`send-url`.
+1. Workhorse retrieves the file and send it to the `gitaly-lfs-smudge`
+process, which writes the contents to the standard output.
+1. `git` reads this output and sends it back to the Gitaly process.
+1. Gitaly sends the data back to Rails.
+1. The archive data is sent back to the client.
+
+In step 7, the `gitaly-lfs-smudge` filter must talk to Workhorse, not to
+Rails, or an invalid LFS blob will be saved. To support this, GitLab
+13.5 [changed the default Omnibus configuration to have Gitaly talk to
+the Workhorse](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4592)
+instead of Rails.
+
+One side effect of this change: the correlation ID of the original
+request is not preserved for the internal API requests made by Gitaly
+(or `gitaly-lfs-smudge`), such as the one made in step 8. The
+correlation IDs for those API requests will be random values until [this
+Workhorse issue](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/309) is
+resolved.
diff --git a/doc/topics/git/lfs/index.md b/doc/topics/git/lfs/index.md
index 11997f46255..7235ba07d0a 100644
--- a/doc/topics/git/lfs/index.md
+++ b/doc/topics/git/lfs/index.md
@@ -42,7 +42,6 @@ Documentation for GitLab instance administrators is under [LFS administration do
credentials store is recommended
- Git LFS always assumes HTTPS so if you have GitLab server on HTTP you will have
to add the URL to Git configuration manually (see [troubleshooting](#troubleshooting))
-- Files added using Git LFS are [not included in the archives created using "download zip" functionality](https://gitlab.com/gitlab-org/gitlab/-/issues/15079)
NOTE: **Note:**
With 8.12 GitLab added LFS support to SSH. The Git LFS communication
@@ -112,6 +111,51 @@ To remove objects from LFS:
See the documentation on [File Locking](../../../user/project/file_lock.md).
+## LFS objects in project archives
+
+> - Support for including Git LFS blobs inside [project source downloads](../../../user/project/repository/index.md) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15079) in GitLab 13.5.
+> - It's [deployed behind a feature flag](../../../user/feature_flags.md), disabled by default.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-lfs-objects-in-project-archives). **(CORE ONLY)**
+
+CAUTION: **Warning:**
+This feature might not be available to you. Check the **version history** note above for details.
+
+Prior to GitLab 13.5, [project source
+downloads](../../../user/project/repository/index.md) would include Git
+LFS pointers instead of the actual objects. For example, LFS pointers
+look like the following:
+
+```markdown
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ea5dd307f195f449f0e08234183b82e92c3d5f4cff11c2a6bb014f9e0de12aa
+size 177735
+```
+
+Starting with GitLab 13.5, these pointers are converted to the uploaded
+LFS object if the `include_lfs_blobs_in_archive` feature flag is
+enabled.
+
+Technical details about how this works can be found in the [development documentation for LFS](../../../development/lfs.md#including-lfs-blobs-in-project-archives).
+
+### Enable or disable LFS objects in project archives
+
+_LFS objects in project archives_ is under development and not ready for production use. It is
+deployed behind a feature flag that is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md)
+can enable it.
+
+To enable it:
+
+```ruby
+Feature.enable(:include_lfs_blobs_in_archive)
+```
+
+To disable it:
+
+```ruby
+Feature.disable(:include_lfs_blobs_in_archive)
+```
+
## Troubleshooting
### error: Repository or object not found
diff --git a/doc/user/project/repository/index.md b/doc/user/project/repository/index.md
index 5473439a162..40bf40a3dba 100644
--- a/doc/user/project/repository/index.md
+++ b/doc/user/project/repository/index.md
@@ -247,6 +247,7 @@ used for cloning your project. The button is only shown on macOS.
## Download Source Code
> Support for directory download was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/24704) in GitLab 11.11.
+> Support for [including Git LFS blobs](../../../topics/git/lfs#lfs-objects-in-project-archives) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15079) in GitLab 13.5.
The source code stored in a repository can be downloaded from the UI.
By clicking the download icon, a dropdown will open with links to download the following:
diff --git a/package.json b/package.json
index eed43d4eb53..fe3b4526e21 100644
--- a/package.json
+++ b/package.json
@@ -105,7 +105,6 @@
"jszip": "^3.1.3",
"jszip-utils": "^0.0.2",
"katex": "^0.10.0",
- "linkifyjs": "^2.1.9",
"lodash": "^4.17.20",
"marked": "^0.3.12",
"mermaid": "^8.5.2",
diff --git a/spec/frontend/jobs/components/log/line_spec.js b/spec/frontend/jobs/components/log/line_spec.js
index 1a30921fece..c2412a807c3 100644
--- a/spec/frontend/jobs/components/log/line_spec.js
+++ b/spec/frontend/jobs/components/log/line_spec.js
@@ -2,25 +2,21 @@ import { shallowMount } from '@vue/test-utils';
import Line from '~/jobs/components/log/line.vue';
import LineNumber from '~/jobs/components/log/line_number.vue';
-const httpUrl = 'http://example.com';
-const httpsUrl = 'https://example.com';
-
-const mockProps = ({ text = 'Running with gitlab-runner 12.1.0 (de7731dd)' } = {}) => ({
- line: {
- content: [
- {
- text,
- style: 'term-fg-l-green',
- },
- ],
- lineNumber: 0,
- },
- path: '/jashkenas/underscore/-/jobs/335',
-});
-
describe('Job Log Line', () => {
let wrapper;
- let data;
+
+ const data = {
+ line: {
+ content: [
+ {
+ text: 'Running with gitlab-runner 12.1.0 (de7731dd)',
+ style: 'term-fg-l-green',
+ },
+ ],
+ lineNumber: 0,
+ },
+ path: '/jashkenas/underscore/-/jobs/335',
+ };
const createComponent = (props = {}) => {
wrapper = shallowMount(Line, {
@@ -31,7 +27,6 @@ describe('Job Log Line', () => {
};
beforeEach(() => {
- data = mockProps();
createComponent(data);
});
@@ -50,38 +45,4 @@ describe('Job Log Line', () => {
it('renders the provided style as a class attribute', () => {
expect(wrapper.find('span').classes()).toContain(data.line.content[0].style);
});
-
- describe('when the line contains a link', () => {
- const findLink = () => wrapper.find('span a');
-
- it('renders an http link', () => {
- createComponent(mockProps({ text: httpUrl }));
-
- expect(findLink().text()).toBe(httpUrl);
- expect(findLink().attributes().href).toEqual(httpUrl);
- });
-
- it('renders an https link', () => {
- createComponent(mockProps({ text: httpsUrl }));
-
- expect(findLink().text()).toBe(httpsUrl);
- expect(findLink().attributes().href).toEqual(httpsUrl);
- });
-
- it('renders a link with rel nofollow and noopener', () => {
- createComponent(mockProps({ text: httpsUrl }));
-
- expect(findLink().attributes().rel).toBe('nofollow noopener');
- });
-
- test.each`
- type | text
- ${'ftp'} | ${'ftp://example.com/file'}
- ${'email'} | ${'email@example.com'}
- ${'no scheme'} | ${'example.com/page'}
- `('does not render a $type link', ({ text }) => {
- createComponent(mockProps({ text }));
- expect(findLink().exists()).toBe(false);
- });
- });
});
diff --git a/vendor/gitignore/C++.gitignore b/vendor/gitignore/C++.gitignore
index 259148fa18f..259148fa18f 100755..100644
--- a/vendor/gitignore/C++.gitignore
+++ b/vendor/gitignore/C++.gitignore
diff --git a/vendor/gitignore/Java.gitignore b/vendor/gitignore/Java.gitignore
index a1c2a238a96..a1c2a238a96 100755..100644
--- a/vendor/gitignore/Java.gitignore
+++ b/vendor/gitignore/Java.gitignore
diff --git a/yarn.lock b/yarn.lock
index 731828e3049..851794067f1 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7550,11 +7550,6 @@ linkify-it@^2.0.0:
dependencies:
uc.micro "^1.0.1"
-linkifyjs@^2.1.9:
- version "2.1.9"
- resolved "https://registry.yarnpkg.com/linkifyjs/-/linkifyjs-2.1.9.tgz#af06e45a2866ff06c4766582590d098a4d584702"
- integrity sha512-74ivurkK6WHvHFozVaGtQWV38FzBwSTGNmJolEgFp7QgR2bl6ArUWlvT4GcHKbPe1z3nWYi+VUdDZk16zDOVug==
-
load-json-file@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0"