diff options
355 files changed, 8125 insertions, 2356 deletions
@@ -1,20 +1,20 @@ { - "presets": [ - ["latest", { "es2015": { "modules": false } }], - "stage-2" - ], + "presets": [["latest", { "es2015": { "modules": false } }], "stage-2"], "env": { "coverage": { "plugins": [ - ["istanbul", { - "exclude": [ - "spec/javascripts/**/*", - "app/assets/javascripts/locale/**/app.js" - ] - }], - ["transform-define", { - "process.env.BABEL_ENV": "coverage" - }] + [ + "istanbul", + { + "exclude": ["spec/javascripts/**/*", "app/assets/javascripts/locale/**/app.js"] + } + ], + [ + "transform-define", + { + "process.env.BABEL_ENV": "coverage" + } + ] ] } } diff --git a/.eslintignore b/.eslintignore index 1623b996213..33a8186fade 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,11 +1,12 @@ +/app/assets/javascripts/locale/**/app.js +/config/ /builds/ /coverage/ /coverage-javascript/ /node_modules/ /public/ +/scripts/ /tmp/ /vendor/ karma.config.js webpack.config.js -svg.config.js -/app/assets/javascripts/locale/**/app.js diff --git a/.prettierignore b/.prettierignore index 62fe6a45b95..b674ccd50cf 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,6 +1,5 @@ -/config/ +/app/assets/javascripts/locale/**/app.js /node_modules/ /public/ /vendor/ -karma.config.js -webpack.config.js +/tmp/ diff --git a/.prettierrc b/.prettierrc index a20502b7f06..3384551aea5 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,4 +1,13 @@ { + "printWidth": 100, "singleQuote": true, - "trailingComma": "all" + "trailingComma": "es5", + "overrides": [ + { + "files": ["**/app/**/*", "**/spec/**/*"], + "options": { + "trailingComma": "all" + } + } + ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index e737c38e35c..adb0ec9f5b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ documentation](doc/development/changelog.md) for instructions on adding your own entry. +## 10.6.1 (2018-03-27) + +### Security (1 change) + +- Bump rails-html-sanitizer to 1.0.4. + +### Fixed (2 changes) + +- Prevent auto-retry AccessDenied error from stopping transition to failed. !17862 +- Fix 500 error when trying to resolve non-ASCII conflicts in the editor. !17962 + +### Performance (1 change) + +- Add indexes for user activity queries. !17890 + +### Other (1 change) + +- Add documentation for runner IP address (#44232). !17837 + + ## 10.6.0 (2018-03-22) ### Security (4 changes) @@ -168,13 +188,17 @@ entry. - Add one group board to Libre. - Add support for filtering by source and target branch to merge requests API. -### Other (14 changes, 3 of them are from the community) +### Other (18 changes, 7 of them are from the community) +- Group MRs on issue page by project and namespace. !8494 (Jeff Stubler) +- Make oauth provider login generic. !8809 (Horatiu Eugen Vlad) +- Add email button to new issue by email. !10942 (Islam Wazery) - Update vue component naming guidelines. !17018 (George Tsiolis) - Added new design for promotion modals. !17197 - Update to github-linguist 5.3.x. !17241 (Ken Ding) - update toml-rb to 1.0.0. !17259 (Ken Ding) - Keep track of projects a user interacted with. !17327 +- Moved o_auth/saml/ldap modules under gitlab/auth. !17359 (Horatiu Eugen Vlad) - Enables eslint in codeclimate job. !17392 - Port Labels Select dropdown to Vue. !17411 - Add NOT NULL constraint to projects.namespace_id. !17448 diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index a3fcc7121bb..21c8c7b46b8 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -7.1.0 +7.1.1 @@ -49,9 +49,10 @@ gem 'omniauth-kerberos', '~> 0.3.0', group: :kerberos gem 'omniauth-oauth2-generic', '~> 0.2.2' gem 'omniauth-saml', '~> 1.10' gem 'omniauth-shibboleth', '~> 1.2.0' -gem 'omniauth-twitter', '~> 1.2.0' +gem 'omniauth-twitter', '~> 1.4' gem 'omniauth_crowd', '~> 2.2.0' gem 'omniauth-authentiq', '~> 0.3.1' +gem 'omniauth-jwt', '~> 0.0.2' gem 'rack-oauth2', '~> 1.2.1' gem 'jwt', '~> 1.5.6' @@ -162,7 +163,7 @@ group :unicorn do end # State machine -gem 'state_machines-activerecord', '~> 0.4.0' +gem 'state_machines-activerecord', '~> 0.5.1' # Issue tags gem 'acts-as-taggable-on', '~> 5.0' @@ -231,7 +232,7 @@ gem 'sanitize', '~> 2.0' gem 'babosa', '~> 1.0.2' # Sanitizes SVG input -gem 'loofah', '~> 2.0.3' +gem 'loofah', '~> 2.2' # Working with license gem 'licensee', '~> 8.9' diff --git a/Gemfile.lock b/Gemfile.lock index aed9f1d6b30..fcffe45acbf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -143,6 +143,7 @@ GEM connection_pool (2.2.1) crack (0.4.3) safe_yaml (~> 1.0.0) + crass (1.0.3) creole (0.5.0) css_parser (1.5.0) addressable @@ -356,7 +357,7 @@ GEM signet (~> 0.7) gpgme (2.0.13) mini_portile2 (~> 2.1) - grape (1.0.0) + grape (1.0.2) activesupport builder mustermann-grape (~> 1.0.0) @@ -485,7 +486,8 @@ GEM actionpack (>= 4, < 5.2) activesupport (>= 4, < 5.2) railties (>= 4, < 5.2) - loofah (2.0.3) + loofah (2.2.2) + crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.0) mini_mime (>= 0.1.1) @@ -505,7 +507,7 @@ GEM multi_json (1.13.1) multi_xml (0.6.0) multipart-post (2.0.0) - mustermann (1.0.0) + mustermann (1.0.2) mustermann-grape (1.0.0) mustermann (~> 1.0.0) mysql2 (0.4.10) @@ -515,7 +517,7 @@ GEM nokogiri (1.8.2) mini_portile2 (~> 2.3.0) numerizer (0.1.1) - oauth (0.5.1) + oauth (0.5.4) oauth2 (1.4.0) faraday (>= 0.8, < 0.13) jwt (~> 1.0) @@ -552,6 +554,9 @@ GEM multi_json (~> 1.3) omniauth (>= 1.1.1) omniauth-oauth2 (>= 1.3.1) + omniauth-jwt (0.0.2) + jwt + omniauth (~> 1.1) omniauth-kerberos (0.3.0) omniauth-multipassword timfel-krb5-auth (~> 0.8) @@ -570,9 +575,9 @@ GEM ruby-saml (~> 1.7) omniauth-shibboleth (1.2.1) omniauth (>= 1.0.0) - omniauth-twitter (1.2.1) - json (~> 1.3) + omniauth-twitter (1.4.0) omniauth-oauth (~> 1.1) + rack omniauth_crowd (2.2.3) activesupport nokogiri (>= 1.4.4) @@ -679,8 +684,8 @@ GEM activesupport (>= 4.2.0, < 5.0) nokogiri (~> 1.6) rails-deprecated_sanitizer (>= 1.0.1) - rails-html-sanitizer (1.0.3) - loofah (~> 2.0) + rails-html-sanitizer (1.0.4) + loofah (~> 2.2, >= 2.2.2) rails-i18n (4.0.9) i18n (~> 0.7) railties (~> 4.0) @@ -895,13 +900,13 @@ GEM sqlite3 (1.3.13) sshkey (1.9.0) stackprof (0.2.10) - state_machines (0.4.0) - state_machines-activemodel (0.4.0) - activemodel (>= 4.1, < 5.1) - state_machines (>= 0.4.0) - state_machines-activerecord (0.4.0) - activerecord (>= 4.1, < 5.1) - state_machines-activemodel (>= 0.3.0) + state_machines (0.5.0) + state_machines-activemodel (0.5.1) + activemodel (>= 4.1, < 6.0) + state_machines (>= 0.5.0) + state_machines-activerecord (0.5.1) + activerecord (>= 4.1, < 6.0) + state_machines-activemodel (>= 0.5.0) stringex (2.7.1) sys-filesystem (1.1.6) ffi @@ -1093,7 +1098,7 @@ DEPENDENCIES license_finder (~> 3.1) licensee (~> 8.9) lograge (~> 0.5) - loofah (~> 2.0.3) + loofah (~> 2.2) mail_room (~> 0.9.1) method_source (~> 0.8) minitest (~> 5.7.0) @@ -1113,11 +1118,12 @@ DEPENDENCIES omniauth-github (~> 1.1.1) omniauth-gitlab (~> 1.0.2) omniauth-google-oauth2 (~> 0.5.2) + omniauth-jwt (~> 0.0.2) omniauth-kerberos (~> 0.3.0) omniauth-oauth2-generic (~> 0.2.2) omniauth-saml (~> 1.10) omniauth-shibboleth (~> 1.2.0) - omniauth-twitter (~> 1.2.0) + omniauth-twitter (~> 1.4) omniauth_crowd (~> 2.2.0) org-ruby (~> 0.9.12) peek (~> 1.0.1) @@ -1192,7 +1198,7 @@ DEPENDENCIES sprockets (~> 3.7.0) sshkey (~> 1.9.0) stackprof (~> 0.2.10) - state_machines-activerecord (~> 0.4.0) + state_machines-activerecord (~> 0.5.1) sys-filesystem (~> 1.1.6) test-prof (~> 0.2.5) test_after_commit (~> 1.1) diff --git a/Gemfile.rails5.lock b/Gemfile.rails5.lock index 85bf28ef8dd..86a22dbe550 100644 --- a/Gemfile.rails5.lock +++ b/Gemfile.rails5.lock @@ -43,8 +43,8 @@ GEM i18n (~> 0.7) minitest (~> 5.1) tzinfo (~> 1.1) - acts-as-taggable-on (4.0.0) - activerecord (>= 4.0) + acts-as-taggable-on (5.0.0) + activerecord (>= 4.2.8) adamantium (0.2.0) ice_nine (~> 0.11.0) memoizable (~> 0.4.0) @@ -178,10 +178,10 @@ GEM docile (1.1.5) domain_name (0.5.20170404) unf (>= 0.0.5, < 1.0.0) - doorkeeper (4.2.6) + doorkeeper (4.3.1) railties (>= 4.2) - doorkeeper-openid_connect (1.2.0) - doorkeeper (~> 4.0) + doorkeeper-openid_connect (1.3.0) + doorkeeper (~> 4.3) json-jwt (~> 1.6) dropzonejs-rails (0.7.4) rails (> 3.1) @@ -220,13 +220,13 @@ GEM path_expander (~> 1.0) ruby_parser (~> 3.0) sexp_processor (~> 4.0) - flipper (0.11.0) - flipper-active_record (0.11.0) + flipper (0.13.0) + flipper-active_record (0.13.0) activerecord (>= 3.2, < 6) - flipper (~> 0.11.0) - flipper-active_support_cache_store (0.11.0) + flipper (~> 0.13.0) + flipper-active_support_cache_store (0.13.0) activesupport (>= 3.2, < 6) - flipper (~> 0.11.0) + flipper (~> 0.13.0) flowdock (0.7.1) httparty (~> 0.7) multi_json @@ -235,7 +235,7 @@ GEM fog-json (~> 1.0) ipaddress (~> 0.8) xml-simple (~> 1.1) - fog-aws (1.4.1) + fog-aws (2.0.1) fog-core (~> 1.38) fog-json (~> 1.0) fog-xml (~> 0.1) @@ -267,7 +267,7 @@ GEM nokogiri (>= 1.5.11, < 2.0.0) font-awesome-rails (4.7.0.3) railties (>= 3.2, < 5.2) - foreman (0.78.0) + foreman (0.84.0) thor (~> 0.19.1) formatador (0.2.5) fuubar (2.2.0) @@ -283,7 +283,7 @@ GEM text (>= 1.3.0) gettext_i18n_rails (1.8.0) fast_gettext (>= 0.9.0) - gettext_i18n_rails_js (1.2.0) + gettext_i18n_rails_js (1.3.0) gettext (>= 3.0.2) gettext_i18n_rails (>= 0.7.1) po_to_json (>= 1.0.0) @@ -337,9 +337,9 @@ GEM json multi_json request_store (>= 1.0) - google-api-client (0.13.6) + google-api-client (0.19.8) addressable (~> 2.5, >= 2.5.1) - googleauth (~> 0.5) + googleauth (>= 0.5, < 0.7.0) httpclient (>= 2.8.1, < 3.0) mime-types (~> 3.0) representable (~> 3.0) @@ -404,7 +404,7 @@ GEM html2text (0.2.1) nokogiri (~> 1.6) htmlentities (4.3.4) - http (0.9.8) + http (2.2.2) addressable (~> 2.3) http-cookie (~> 1.0) http-form_data (~> 1.0.1) @@ -427,10 +427,6 @@ GEM multipart-post oauth (~> 0.5, >= 0.5.0) jquery-atwho-rails (1.3.2) - jquery-rails (4.3.1) - rails-dom-testing (>= 1, < 3) - railties (>= 4.2.0) - thor (>= 0.14, < 2.0) json (1.8.6) json-jwt (1.9.2) activesupport @@ -454,13 +450,12 @@ GEM kaminari-core (= 1.1.1) kaminari-core (1.1.1) kgio (2.11.2) - knapsack (1.11.1) + knapsack (1.16.0) rake - timecop (>= 0.1.0) - kubeclient (2.2.0) - http (= 0.9.8) - recursive-open-struct (= 1.0.0) - rest-client + kubeclient (3.0.0) + http (~> 2.2.2) + recursive-open-struct (~> 1.0.4) + rest-client (~> 2.0) launchy (2.4.3) addressable (~> 2.3) letter_opener (1.6.0) @@ -477,7 +472,7 @@ GEM toml (= 0.1.2) with_env (> 1.0) xml-simple - licensee (8.7.0) + licensee (8.9.2) rugged (~> 0.24) little-plugger (1.1.4) locale (2.1.2) @@ -514,7 +509,7 @@ GEM mustermann (~> 1.0.0) mysql2 (0.4.10) net-ldap (0.16.1) - net-ssh (4.1.0) + net-ssh (4.2.0) netrc (0.11.0) nio4r (2.2.0) nokogiri (1.8.2) @@ -527,11 +522,10 @@ GEM multi_json (~> 1.3) multi_xml (~> 0.5) rack (>= 1.2, < 3) - octokit (4.6.2) + octokit (4.8.0) sawyer (~> 0.8.0, >= 0.5.3) - oj (2.17.5) - omniauth (1.4.3) - hashie (>= 1.2, < 4) + omniauth (1.8.1) + hashie (>= 3.4.6, < 3.6.0) rack (>= 1.6.2, < 3) omniauth-auth0 (1.4.2) omniauth-oauth2 (~> 1.1) @@ -570,9 +564,9 @@ GEM omniauth (~> 1.2) omniauth-oauth2-generic (0.2.4) omniauth-oauth2 (~> 1.0) - omniauth-saml (1.7.0) - omniauth (~> 1.3) - ruby-saml (~> 1.4) + omniauth-saml (1.10.0) + omniauth (~> 1.3, >= 1.3.2) + ruby-saml (~> 1.7) omniauth-shibboleth (1.2.1) omniauth (>= 1.0.0) omniauth-twitter (1.2.1) @@ -598,8 +592,6 @@ GEM railties (>= 4.0.0) peek-gc (0.0.2) peek - peek-host (1.0.0) - peek peek-mysql2 (1.1.0) atomic (>= 1.0.0) mysql2 @@ -713,7 +705,7 @@ GEM re2 (1.1.1) recaptcha (3.4.0) json - recursive-open-struct (1.0.0) + recursive-open-struct (1.0.5) redcarpet (3.4.0) redis (3.3.5) redis-actionpack (5.0.2) @@ -805,7 +797,7 @@ GEM i18n ruby-fogbugz (0.2.1) crack (~> 0.4) - ruby-prof (0.16.2) + ruby-prof (0.17.0) ruby-progressbar (1.9.0) ruby-saml (1.7.2) nokogiri (>= 1.5.10) @@ -846,7 +838,7 @@ GEM selenium-webdriver (3.11.0) childprocess (~> 0.5) rubyzip (~> 1.2) - sentry-raven (2.5.3) + sentry-raven (2.7.2) faraday (>= 0.7.6, < 1.0) settingslogic (2.0.9) sexp_processor (4.10.1) @@ -903,12 +895,12 @@ GEM sshkey (1.9.0) stackprof (0.2.11) state_machines (0.5.0) - state_machines-activemodel (0.5.0) - activemodel (>= 4.1, < 5.2) + state_machines-activemodel (0.5.1) + activemodel (>= 4.1, < 6.0) state_machines (>= 0.5.0) - state_machines-activerecord (0.4.1) - activerecord (>= 4.1, < 5.2) - state_machines-activemodel (>= 0.3.0) + state_machines-activerecord (0.5.1) + activerecord (>= 4.1, < 6.0) + state_machines-activemodel (>= 0.5.0) stringex (2.8.4) sys-filesystem (1.1.9) ffi @@ -998,7 +990,7 @@ DEPENDENCIES RedCloth (~> 4.3.2) ace-rails-ap (~> 4.1.0) activerecord_sane_schema_dumper (= 1.0) - acts-as-taggable-on (~> 4.0) + acts-as-taggable-on (~> 5.0) addressable (~> 2.5.2) akismet (~> 2.0) allocations (~> 1.0) @@ -1038,8 +1030,8 @@ DEPENDENCIES devise (~> 4.2) devise-two-factor (~> 3.0.0) diffy (~> 3.1.0) - doorkeeper (~> 4.2.0) - doorkeeper-openid_connect (~> 1.2.0) + doorkeeper (~> 4.3) + doorkeeper-openid_connect (~> 1.3) dropzonejs-rails (~> 0.7.1) email_reply_trimmer (~> 0.1) email_spec (~> 1.6.0) @@ -1048,24 +1040,24 @@ DEPENDENCIES fast_blank ffaker (~> 2.4) flay (~> 2.10.0) - flipper (~> 0.11.0) - flipper-active_record (~> 0.11.0) - flipper-active_support_cache_store (~> 0.11.0) + flipper (~> 0.13.0) + flipper-active_record (~> 0.13.0) + flipper-active_support_cache_store (~> 0.13.0) fog-aliyun (~> 0.2.0) - fog-aws (~> 1.4) + fog-aws (~> 2.0) fog-core (~> 1.44) fog-google (~> 0.5) fog-local (~> 0.3) fog-openstack (~> 0.1) fog-rackspace (~> 0.1.1) font-awesome-rails (~> 4.7) - foreman (~> 0.78.0) + foreman (~> 0.84.0) fuubar (~> 2.2.0) gemnasium-gitlab-service (~> 0.2) gemojione (~> 3.3) gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) - gettext_i18n_rails_js (~> 1.2.0) + gettext_i18n_rails_js (~> 1.3) gitaly-proto (~> 0.88.0) github-linguist (~> 5.3.3) gitlab-flowdock-git-hook (~> 1.0.1) @@ -1075,7 +1067,7 @@ DEPENDENCIES gollum-lib (~> 4.2) gollum-rugged_adapter (~> 0.4.4) gon (~> 6.1.0) - google-api-client (~> 0.13.6) + google-api-client (~> 0.19.8) google-protobuf (= 3.5.1) gpgme grape (~> 1.0) @@ -1094,15 +1086,14 @@ DEPENDENCIES influxdb (~> 0.2) jira-ruby (~> 1.4) jquery-atwho-rails (~> 1.3.2) - jquery-rails (~> 4.3.1) json-schema (~> 2.8.0) jwt (~> 1.5.6) kaminari (~> 1.0) - knapsack (~> 1.11.0) - kubeclient (~> 2.2.0) + knapsack (~> 1.16) + kubeclient (~> 3.0) letter_opener_web (~> 1.3.0) license_finder (~> 3.1) - licensee (~> 8.7.0) + licensee (~> 8.9) lograge (~> 0.5) loofah (~> 2.0.3) mail_room (~> 0.9.1) @@ -1111,12 +1102,11 @@ DEPENDENCIES mousetrap-rails (~> 1.4.6) mysql2 (~> 0.4.10) net-ldap - net-ssh (~> 4.1.0) + net-ssh (~> 4.2.0) nokogiri (~> 1.8.2) oauth2 (~> 1.4) - octokit (~> 4.6.2) - oj (~> 2.17.4) - omniauth (~> 1.4.2) + octokit (~> 4.8) + omniauth (~> 1.8) omniauth-auth0 (~> 1.4.1) omniauth-authentiq (~> 0.3.1) omniauth-azure-oauth2 (~> 0.0.9) @@ -1127,14 +1117,13 @@ DEPENDENCIES omniauth-google-oauth2 (~> 0.5.2) omniauth-kerberos (~> 0.3.0) omniauth-oauth2-generic (~> 0.2.2) - omniauth-saml (~> 1.7.0) + omniauth-saml (~> 1.10) omniauth-shibboleth (~> 1.2.0) omniauth-twitter (~> 1.2.0) omniauth_crowd (~> 2.2.0) org-ruby (~> 0.9.12) peek (~> 1.0.1) peek-gc (~> 0.0.2) - peek-host (~> 1.0.0) peek-mysql2 (~> 1.1.0) peek-performance_bar (~> 1.3.0) peek-pg (~> 1.3.0) @@ -1177,7 +1166,7 @@ DEPENDENCIES rubocop (~> 0.52.1) rubocop-rspec (~> 1.22.1) ruby-fogbugz (~> 0.2.1) - ruby-prof (~> 0.16.2) + ruby-prof (~> 0.17.0) ruby_parser (~> 3.8) rufus-scheduler (~> 3.4) rugged (~> 0.26.0) @@ -1187,7 +1176,7 @@ DEPENDENCIES seed-fu (~> 2.3.7) select2-rails (~> 3.5.9) selenium-webdriver (~> 3.5) - sentry-raven (~> 2.5.3) + sentry-raven (~> 2.7) settingslogic (~> 2.0.9) sham_rack (~> 1.3.6) shoulda-matchers (~> 3.1.2) @@ -1205,7 +1194,7 @@ DEPENDENCIES sprockets (~> 3.7.0) sshkey (~> 1.9.0) stackprof (~> 0.2.10) - state_machines-activerecord (~> 0.4.0) + state_machines-activerecord (~> 0.5.1) sys-filesystem (~> 1.1.6) test-prof (~> 0.2.5) test_after_commit (~> 1.1) diff --git a/app/assets/javascripts/ci_variable_list/ci_variable_list.js b/app/assets/javascripts/ci_variable_list/ci_variable_list.js index 745f3404295..e177a3bfdc7 100644 --- a/app/assets/javascripts/ci_variable_list/ci_variable_list.js +++ b/app/assets/javascripts/ci_variable_list/ci_variable_list.js @@ -33,7 +33,7 @@ export default class VariableList { selector: '.js-ci-variable-input-key', default: '', }, - value: { + secret_value: { selector: '.js-ci-variable-input-value', default: '', }, @@ -105,7 +105,7 @@ export default class VariableList { setupToggleButtons($row[0]); // Reset the resizable textarea - $row.find(this.inputMap.value.selector).css('height', ''); + $row.find(this.inputMap.secret_value.selector).css('height', ''); const $environmentSelect = $row.find('.js-variable-environment-toggle'); if ($environmentSelect.length) { diff --git a/app/assets/javascripts/commons/polyfills.js b/app/assets/javascripts/commons/polyfills.js index 46232726510..d62d3c23654 100644 --- a/app/assets/javascripts/commons/polyfills.js +++ b/app/assets/javascripts/commons/polyfills.js @@ -1,4 +1,5 @@ // ECMAScript polyfills +import 'core-js/fn/array/fill'; import 'core-js/fn/array/find'; import 'core-js/fn/array/find-index'; import 'core-js/fn/array/from'; diff --git a/app/assets/javascripts/groups/components/app.vue b/app/assets/javascripts/groups/components/app.vue index 63bb5832bd0..22eb7bd44c5 100644 --- a/app/assets/javascripts/groups/components/app.vue +++ b/app/assets/javascripts/groups/components/app.vue @@ -4,7 +4,7 @@ import $ from 'jquery'; import { s__ } from '~/locale'; import loadingIcon from '~/vue_shared/components/loading_icon.vue'; -import modal from '~/vue_shared/components/modal.vue'; +import DeprecatedModal from '~/vue_shared/components/deprecated_modal.vue'; import { getParameterByName } from '~/lib/utils/common_utils'; import { mergeUrlParams } from '~/lib/utils/url_utility'; @@ -15,7 +15,7 @@ import groupsComponent from './groups.vue'; export default { components: { loadingIcon, - modal, + DeprecatedModal, groupsComponent, }, props: { @@ -52,8 +52,9 @@ export default { }, }, created() { - this.searchEmptyMessage = this.hideProjects ? - COMMON_STR.GROUP_SEARCH_EMPTY : COMMON_STR.GROUP_PROJECT_SEARCH_EMPTY; + this.searchEmptyMessage = this.hideProjects + ? COMMON_STR.GROUP_SEARCH_EMPTY + : COMMON_STR.GROUP_PROJECT_SEARCH_EMPTY; eventHub.$on('fetchPage', this.fetchPage); eventHub.$on('toggleChildren', this.toggleChildren); @@ -72,22 +73,30 @@ export default { eventHub.$off('updateGroups', this.updateGroups); }, methods: { - fetchGroups({ parentId, page, filterGroupsBy, sortBy, archived, updatePagination }) { - return this.service.getGroups(parentId, page, filterGroupsBy, sortBy, archived) - .then((res) => { - if (updatePagination) { - this.updatePagination(res.headers); - } + fetchGroups({ + parentId, + page, + filterGroupsBy, + sortBy, + archived, + updatePagination, + }) { + return this.service + .getGroups(parentId, page, filterGroupsBy, sortBy, archived) + .then(res => { + if (updatePagination) { + this.updatePagination(res.headers); + } - return res; - }) - .then(res => res.json()) - .catch(() => { - this.isLoading = false; - $.scrollTo(0); + return res; + }) + .then(res => res.json()) + .catch(() => { + this.isLoading = false; + $.scrollTo(0); - Flash(COMMON_STR.FAILURE); - }); + Flash(COMMON_STR.FAILURE); + }); }, fetchAllGroups() { const page = getParameterByName('page') || null; @@ -103,7 +112,7 @@ export default { sortBy, archived, updatePagination: true, - }).then((res) => { + }).then(res => { this.isLoading = false; this.updateGroups(res, Boolean(filterGroupsBy)); }); @@ -118,14 +127,18 @@ export default { sortBy, archived, updatePagination: true, - }).then((res) => { + }).then(res => { this.isLoading = false; $.scrollTo(0); const currentPath = mergeUrlParams({ page }, window.location.href); - window.history.replaceState({ - page: currentPath, - }, document.title, currentPath); + window.history.replaceState( + { + page: currentPath, + }, + document.title, + currentPath, + ); this.updateGroups(res); }); @@ -138,11 +151,13 @@ export default { // eslint-disable-next-line promise/catch-or-return this.fetchGroups({ parentId: parentGroup.id, - }).then((res) => { - this.store.setGroupChildren(parentGroup, res); - }).catch(() => { - parentGroup.isChildrenLoading = false; - }); + }) + .then(res => { + this.store.setGroupChildren(parentGroup, res); + }) + .catch(() => { + parentGroup.isChildrenLoading = false; + }); } else { parentGroup.isOpen = true; } @@ -154,7 +169,11 @@ export default { this.targetGroup = group; this.targetParentGroup = parentGroup; this.showModal = true; - this.groupLeaveConfirmationMessage = s__(`GroupsTree|Are you sure you want to leave the "${group.fullName}" group?`); + this.groupLeaveConfirmationMessage = s__( + `GroupsTree|Are you sure you want to leave the "${ + group.fullName + }" group?`, + ); }, hideLeaveGroupModal() { this.showModal = false; @@ -162,14 +181,15 @@ export default { leaveGroup() { this.showModal = false; this.targetGroup.isBeingRemoved = true; - this.service.leaveGroup(this.targetGroup.leavePath) + this.service + .leaveGroup(this.targetGroup.leavePath) .then(res => res.json()) - .then((res) => { + .then(res => { $.scrollTo(0); this.store.removeGroup(this.targetGroup, this.targetParentGroup); Flash(res.notice, 'notice'); }) - .catch((err) => { + .catch(err => { let message = COMMON_STR.FAILURE; if (err.status === 403) { message = COMMON_STR.LEAVE_FORBIDDEN; @@ -208,8 +228,8 @@ export default { :search-empty-message="searchEmptyMessage" :page-info="pageInfo" /> - <modal - v-if="showModal" + <deprecated-modal + v-show="showModal" kind="warning" :primary-button-label="__('Leave')" :title="__('Are you sure?')" diff --git a/app/assets/javascripts/ide/components/new_dropdown/modal.vue b/app/assets/javascripts/ide/components/new_dropdown/modal.vue index 5723891d130..4b5a50785b6 100644 --- a/app/assets/javascripts/ide/components/new_dropdown/modal.vue +++ b/app/assets/javascripts/ide/components/new_dropdown/modal.vue @@ -1,75 +1,75 @@ <script> - import { __ } from '~/locale'; - import modal from '~/vue_shared/components/modal.vue'; +import { __ } from '~/locale'; +import DeprecatedModal from '~/vue_shared/components/deprecated_modal.vue'; - export default { - components: { - modal, +export default { + components: { + DeprecatedModal, + }, + props: { + branchId: { + type: String, + required: true, }, - props: { - branchId: { - type: String, - required: true, - }, - type: { - type: String, - required: true, - }, - path: { - type: String, - required: true, - }, + type: { + type: String, + required: true, }, - data() { - return { - entryName: this.path !== '' ? `${this.path}/` : '', - }; + path: { + type: String, + required: true, }, - computed: { - modalTitle() { - if (this.type === 'tree') { - return __('Create new directory'); - } + }, + data() { + return { + entryName: this.path !== '' ? `${this.path}/` : '', + }; + }, + computed: { + modalTitle() { + if (this.type === 'tree') { + return __('Create new directory'); + } - return __('Create new file'); - }, - buttonLabel() { - if (this.type === 'tree') { - return __('Create directory'); - } - - return __('Create file'); - }, - formLabelName() { - if (this.type === 'tree') { - return __('Directory name'); - } + return __('Create new file'); + }, + buttonLabel() { + if (this.type === 'tree') { + return __('Create directory'); + } - return __('File name'); - }, + return __('Create file'); }, - mounted() { - this.$refs.fieldName.focus(); + formLabelName() { + if (this.type === 'tree') { + return __('Directory name'); + } + + return __('File name'); }, - methods: { - createEntryInStore() { - this.$emit('create', { - branchId: this.branchId, - name: this.entryName, - type: this.type, - }); + }, + mounted() { + this.$refs.fieldName.focus(); + }, + methods: { + createEntryInStore() { + this.$emit('create', { + branchId: this.branchId, + name: this.entryName, + type: this.type, + }); - this.hideModal(); - }, - hideModal() { - this.$emit('hide'); - }, + this.hideModal(); + }, + hideModal() { + this.$emit('hide'); }, - }; + }, +}; </script> <template> - <modal + <deprecated-modal :title="modalTitle" :primary-button-label="buttonLabel" kind="success" @@ -95,5 +95,5 @@ </div> </fieldset> </form> - </modal> + </deprecated-modal> </template> diff --git a/app/assets/javascripts/ide/components/repo_commit_section.vue b/app/assets/javascripts/ide/components/repo_commit_section.vue index d772cab2d0e..d885ed5e301 100644 --- a/app/assets/javascripts/ide/components/repo_commit_section.vue +++ b/app/assets/javascripts/ide/components/repo_commit_section.vue @@ -2,7 +2,7 @@ import { mapState, mapActions, mapGetters } from 'vuex'; import tooltip from '~/vue_shared/directives/tooltip'; import icon from '~/vue_shared/components/icon.vue'; -import modal from '~/vue_shared/components/modal.vue'; +import DeprecatedModal from '~/vue_shared/components/deprecated_modal.vue'; import LoadingButton from '~/vue_shared/components/loading_button.vue'; import commitFilesList from './commit_sidebar/list.vue'; import * as consts from '../stores/modules/commit/constants'; @@ -10,7 +10,7 @@ import Actions from './commit_sidebar/actions.vue'; export default { components: { - modal, + DeprecatedModal, icon, commitFilesList, Actions, @@ -37,23 +37,20 @@ export default { 'lastCommitMsg', 'changedFiles', ]), - ...mapState('commit', [ - 'commitMessage', - 'submitCommitLoading', - ]), + ...mapState('commit', ['commitMessage', 'submitCommitLoading']), ...mapGetters('commit', [ 'commitButtonDisabled', 'discardDraftButtonDisabled', 'branchName', ]), statusSvg() { - return this.lastCommitMsg ? this.committedStateSvgPath : this.noChangesStateSvgPath; + return this.lastCommitMsg + ? this.committedStateSvgPath + : this.noChangesStateSvgPath; }, }, methods: { - ...mapActions([ - 'setPanelCollapsedStatus', - ]), + ...mapActions(['setPanelCollapsedStatus']), ...mapActions('commit', [ 'updateCommitMessage', 'discardDraft', @@ -67,8 +64,9 @@ export default { }); }, forceCreateNewBranch() { - return this.updateCommitAction(consts.COMMIT_TO_NEW_BRANCH) - .then(() => this.commitChanges()); + return this.updateCommitAction(consts.COMMIT_TO_NEW_BRANCH).then(() => + this.commitChanges(), + ); }, }, }; @@ -81,7 +79,7 @@ export default { 'multi-file-commit-empty-state-container': !changedFiles.length }" > - <modal + <deprecated-modal id="ide-create-branch-modal" :primary-button-label="__('Create new branch')" kind="success" @@ -92,7 +90,7 @@ export default { {{ __(`This branch has changed since you started editing. Would you like to create a new branch?`) }} </template> - </modal> + </deprecated-modal> <commit-files-list title="Staged" :file-list="changedFiles" diff --git a/app/assets/javascripts/ide/lib/editor.js b/app/assets/javascripts/ide/lib/editor.js index 38de2fe2b27..887dd7e39b1 100644 --- a/app/assets/javascripts/ide/lib/editor.js +++ b/app/assets/javascripts/ide/lib/editor.js @@ -65,6 +65,10 @@ export default class Editor { (this.instance = this.monaco.editor.createDiffEditor(domElement, { ...defaultEditorOptions, readOnly: true, + quickSuggestions: false, + occurrencesHighlight: false, + renderLineHighlight: 'none', + hideCursorInOverviewRuler: true, })), ); diff --git a/app/assets/javascripts/notes.js b/app/assets/javascripts/notes.js index 09f0ea37103..b0573510ff9 100644 --- a/app/assets/javascripts/notes.js +++ b/app/assets/javascripts/notes.js @@ -1809,9 +1809,11 @@ export default class Notes { } } + $closeBtn.text($closeBtn.data('originalText')); + /* eslint-disable promise/catch-or-return */ // Make request to submit comment on server - axios + return axios .post(`${formAction}?html=true`, formData) .then(res => { const note = res.data; @@ -1928,8 +1930,6 @@ export default class Notes { this.reenableTargetFormSubmitButton(e); this.addNoteError($form); }); - - return $closeBtn.text($closeBtn.data('originalText')); } /** diff --git a/app/assets/javascripts/pages/admin/application_settings/index.js b/app/assets/javascripts/pages/admin/application_settings/index.js new file mode 100644 index 00000000000..48d75f5443b --- /dev/null +++ b/app/assets/javascripts/pages/admin/application_settings/index.js @@ -0,0 +1,6 @@ +import initSettingsPanels from '~/settings_panels'; + +document.addEventListener('DOMContentLoaded', () => { + // Initialize expandable settings panels + initSettingsPanels(); +}); diff --git a/app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue b/app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue index 14315d5492e..343c65edb37 100644 --- a/app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue +++ b/app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue @@ -1,11 +1,11 @@ <script> import _ from 'underscore'; - import modal from '~/vue_shared/components/modal.vue'; + import DeprecatedModal from '~/vue_shared/components/deprecated_modal.vue'; import { s__, sprintf } from '~/locale'; export default { components: { - modal, + DeprecatedModal, }, props: { deleteProjectUrl: { @@ -79,7 +79,7 @@ </script> <template> - <modal + <deprecated-modal id="delete-project-modal" :title="title" :text="text" @@ -121,5 +121,5 @@ /> </form> </template> - </modal> + </deprecated-modal> </template> diff --git a/app/assets/javascripts/pages/admin/users/components/delete_user_modal.vue b/app/assets/javascripts/pages/admin/users/components/delete_user_modal.vue index 7b5e333011e..0e3ac636661 100644 --- a/app/assets/javascripts/pages/admin/users/components/delete_user_modal.vue +++ b/app/assets/javascripts/pages/admin/users/components/delete_user_modal.vue @@ -1,11 +1,11 @@ <script> import _ from 'underscore'; - import modal from '~/vue_shared/components/modal.vue'; + import DeprecatedModal from '~/vue_shared/components/deprecated_modal.vue'; import { s__, sprintf } from '~/locale'; export default { components: { - modal, + DeprecatedModal, }, props: { deleteUserUrl: { @@ -113,7 +113,7 @@ </script> <template> - <modal + <deprecated-modal id="delete-user-modal" :title="title" :text="text" @@ -170,5 +170,5 @@ {{ secondaryButtonLabel }} </button> </template> - </modal> + </deprecated-modal> </template> diff --git a/app/assets/javascripts/pages/milestones/shared/components/delete_milestone_modal.vue b/app/assets/javascripts/pages/milestones/shared/components/delete_milestone_modal.vue index c43e0a0490f..16f792d635a 100644 --- a/app/assets/javascripts/pages/milestones/shared/components/delete_milestone_modal.vue +++ b/app/assets/javascripts/pages/milestones/shared/components/delete_milestone_modal.vue @@ -2,14 +2,14 @@ import axios from '~/lib/utils/axios_utils'; import Flash from '~/flash'; - import modal from '~/vue_shared/components/modal.vue'; + import DeprecatedModal from '~/vue_shared/components/deprecated_modal.vue'; import { n__, s__, sprintf } from '~/locale'; import { redirectTo } from '~/lib/utils/url_utility'; import eventHub from '../event_hub'; export default { components: { - modal, + DeprecatedModal, }, props: { issueCount: { @@ -92,7 +92,7 @@ Once deleted, it cannot be undone or recovered.`), </script> <template> - <modal + <deprecated-modal id="delete-milestone-modal" :title="title" :text="text" @@ -106,5 +106,5 @@ Once deleted, it cannot be undone or recovered.`), <p v-html="props.text"></p> </template> - </modal> + </deprecated-modal> </template> diff --git a/app/assets/javascripts/performance_bar/services/performance_bar_service.js b/app/assets/javascripts/performance_bar/services/performance_bar_service.js index d8e792446c3..3ebfaa87a4e 100644 --- a/app/assets/javascripts/performance_bar/services/performance_bar_service.js +++ b/app/assets/javascripts/performance_bar/services/performance_bar_service.js @@ -1,11 +1,28 @@ +import Vue from 'vue'; +import _ from 'underscore'; import axios from '../../lib/utils/axios_utils'; +let vueResourceInterceptor; + export default class PerformanceBarService { static fetchRequestDetails(peekUrl, requestId) { return axios.get(peekUrl, { params: { request_id: requestId } }); } static registerInterceptor(peekUrl, callback) { + vueResourceInterceptor = (request, next) => { + next(response => { + const requestId = response.headers['x-request-id']; + const requestUrl = response.url; + + if (requestUrl !== peekUrl && requestId) { + callback(requestId, requestUrl); + } + }); + }; + + Vue.http.interceptors.push(vueResourceInterceptor); + return axios.interceptors.response.use(response => { const requestId = response.headers['x-request-id']; const requestUrl = response.config.url; @@ -20,5 +37,9 @@ export default class PerformanceBarService { static removeInterceptor(interceptor) { axios.interceptors.response.eject(interceptor); + Vue.http.interceptors = _.without( + Vue.http.interceptors, + vueResourceInterceptor, + ); } } diff --git a/app/assets/javascripts/pipelines/components/pipelines_table.vue b/app/assets/javascripts/pipelines/components/pipelines_table.vue index c9028952ddd..714aed1333e 100644 --- a/app/assets/javascripts/pipelines/components/pipelines_table.vue +++ b/app/assets/javascripts/pipelines/components/pipelines_table.vue @@ -1,5 +1,5 @@ <script> - import modal from '~/vue_shared/components/modal.vue'; + import DeprecatedModal from '~/vue_shared/components/deprecated_modal.vue'; import { s__, sprintf } from '~/locale'; import pipelinesTableRowComponent from './pipelines_table_row.vue'; import eventHub from '../event_hub'; @@ -12,7 +12,7 @@ export default { components: { pipelinesTableRowComponent, - modal, + DeprecatedModal, }, props: { pipelines: { @@ -120,7 +120,7 @@ :auto-devops-help-path="autoDevopsHelpPath" :view-type="viewType" /> - <modal + <deprecated-modal id="confirmation-modal" :title="modalTitle" :text="modalText" @@ -134,6 +134,6 @@ > <p v-html="props.text"></p> </template> - </modal> + </deprecated-modal> </div> </template> diff --git a/app/assets/javascripts/profile/account/components/delete_account_modal.vue b/app/assets/javascripts/profile/account/components/delete_account_modal.vue index 1ffe482d782..f50002afbf2 100644 --- a/app/assets/javascripts/profile/account/components/delete_account_modal.vue +++ b/app/assets/javascripts/profile/account/components/delete_account_modal.vue @@ -1,11 +1,11 @@ <script> - import modal from '~/vue_shared/components/modal.vue'; + import DeprecatedModal from '~/vue_shared/components/deprecated_modal.vue'; import { __, s__, sprintf } from '~/locale'; import csrf from '~/lib/utils/csrf'; export default { components: { - modal, + DeprecatedModal, }, props: { actionUrl: { @@ -76,7 +76,7 @@ Once you confirm %{deleteAccount}, it cannot be undone or recovered.`), </script> <template> - <modal + <deprecated-modal id="delete-account-modal" :title="s__('Profiles|Delete your account?')" :text="text" @@ -131,5 +131,5 @@ Once you confirm %{deleteAccount}, it cannot be undone or recovered.`), </form> </template> - </modal> + </deprecated-modal> </template> diff --git a/app/assets/javascripts/vue_merge_request_widget/components/memory_usage.vue b/app/assets/javascripts/vue_merge_request_widget/components/memory_usage.vue index a16f9055a6d..95c8b0a4c55 100644 --- a/app/assets/javascripts/vue_merge_request_widget/components/memory_usage.vue +++ b/app/assets/javascripts/vue_merge_request_widget/components/memory_usage.vue @@ -1,4 +1,5 @@ <script> +import { sprintf, s__ } from '~/locale'; import statusCodes from '../../lib/utils/http_status'; import { bytesToMiB } from '../../lib/utils/number_utils'; import { backOff } from '../../lib/utils/common_utils'; @@ -45,17 +46,28 @@ export default { shouldShowMetricsUnavailable() { return !this.loadingMetrics && !this.hasMetrics && !this.loadFailed; }, - memoryChangeType() { + memoryChangeMessage() { + const messageProps = { + memoryFrom: this.memoryFrom, + memoryTo: this.memoryTo, + metricsLinkStart: `<a href="${this.metricsMonitoringUrl}">`, + metricsLinkEnd: '</a>', + emphasisStart: '<b>', + emphasisEnd: '</b>', + }; const memoryTo = Number(this.memoryTo); const memoryFrom = Number(this.memoryFrom); + let memoryUsageMsg = ''; if (memoryTo > memoryFrom) { - return 'increased'; + memoryUsageMsg = sprintf(s__('mrWidget|%{metricsLinkStart} Memory %{metricsLinkEnd} usage %{emphasisStart} increased %{emphasisEnd} from %{memoryFrom}MB to %{memoryTo}MB'), messageProps, false); } else if (memoryTo < memoryFrom) { - return 'decreased'; + memoryUsageMsg = sprintf(s__('mrWidget|%{metricsLinkStart} Memory %{metricsLinkEnd} usage %{emphasisStart} decreased %{emphasisEnd} from %{memoryFrom}MB to %{memoryTo}MB'), messageProps, false); + } else { + memoryUsageMsg = sprintf(s__('mrWidget|%{metricsLinkStart} Memory %{metricsLinkEnd} usage is %{emphasisStart} unchanged %{emphasisEnd} at %{memoryFrom}MB'), messageProps, false); } - return 'unchanged'; + return memoryUsageMsg; }, }, mounted() { @@ -130,24 +142,22 @@ export default { <i class="fa fa-spinner fa-spin usage-info-load-spinner" aria-hidden="true"> - </i>Loading deployment statistics + </i>{{ s__('mrWidget|Loading deployment statistics') }} </p> <p v-if="shouldShowMemoryGraph" class="usage-info js-usage-info"> - <a - :href="metricsMonitoringUrl" - >Memory</a> usage <b>{{ memoryChangeType }}</b> from {{ memoryFrom }}MB to {{ memoryTo }}MB + {{ memoryChangeMessage }} </p> <p v-if="shouldShowLoadFailure" class="usage-info js-usage-info usage-info-failed"> - Failed to load deployment statistics + {{ s__('mrWidget|Failed to load deployment statistics') }} </p> <p v-if="shouldShowMetricsUnavailable" class="usage-info js-usage-info usage-info-unavailable"> - Deployment statistics are not available currently + {{ s__('mrWidget|Deployment statistics are not available currently') }} </p> <memory-graph v-if="shouldShowMemoryGraph" diff --git a/app/assets/javascripts/vue_shared/components/modal.vue b/app/assets/javascripts/vue_shared/components/deprecated_modal.vue index 5f1364421aa..dcf1489b37c 100644 --- a/app/assets/javascripts/vue_shared/components/modal.vue +++ b/app/assets/javascripts/vue_shared/components/deprecated_modal.vue @@ -1,7 +1,7 @@ <script> /* eslint-disable vue/require-default-prop */ export default { - name: 'Modal', + name: 'DeprecatedModal', // use GlModal instead props: { id: { diff --git a/app/assets/javascripts/vue_shared/components/recaptcha_modal.vue b/app/assets/javascripts/vue_shared/components/recaptcha_modal.vue index c35621c9ef3..21ffdc1dc86 100644 --- a/app/assets/javascripts/vue_shared/components/recaptcha_modal.vue +++ b/app/assets/javascripts/vue_shared/components/recaptcha_modal.vue @@ -1,11 +1,11 @@ <script> - import modal from './modal.vue'; + import DeprecatedModal from './deprecated_modal.vue'; export default { name: 'RecaptchaModal', components: { - modal, + DeprecatedModal, }, props: { @@ -65,7 +65,7 @@ </script> <template> - <modal + <deprecated-modal kind="warning" class="recaptcha-modal js-recaptcha-modal" :hide-footer="true" @@ -82,5 +82,5 @@ > </div> </div> - </modal> + </deprecated-modal> </template> diff --git a/app/assets/stylesheets/pages/repo.scss b/app/assets/stylesheets/pages/repo.scss index 7a8fbfc517d..57b995adb64 100644 --- a/app/assets/stylesheets/pages/repo.scss +++ b/app/assets/stylesheets/pages/repo.scss @@ -290,6 +290,10 @@ .margin-view-overlays .delete-sign { opacity: 0.4; } + + .cursors-layer { + display: none; + } } } diff --git a/app/assets/stylesheets/performance_bar.scss b/app/assets/stylesheets/performance_bar.scss index d69b390ac27..45ae94abaff 100644 --- a/app/assets/stylesheets/performance_bar.scss +++ b/app/assets/stylesheets/performance_bar.scss @@ -15,6 +15,10 @@ line-height: $performance-bar-height; color: $perf-bar-text; + select { + width: 200px; + } + &.disabled { display: none; } diff --git a/app/controllers/admin/application_controller.rb b/app/controllers/admin/application_controller.rb index c27f2ee3c09..a4648b33cfa 100644 --- a/app/controllers/admin/application_controller.rb +++ b/app/controllers/admin/application_controller.rb @@ -3,23 +3,9 @@ # Automatically sets the layout and ensures an administrator is logged in class Admin::ApplicationController < ApplicationController before_action :authenticate_admin! - before_action :display_read_only_information layout 'admin' def authenticate_admin! render_404 unless current_user.admin? end - - def display_read_only_information - return unless Gitlab::Database.read_only? - - flash.now[:notice] = read_only_message - end - - private - - # Overridden in EE - def read_only_message - _('You are on a read-only GitLab instance.') - end end diff --git a/app/controllers/concerns/send_file_upload.rb b/app/controllers/concerns/send_file_upload.rb new file mode 100644 index 00000000000..55011c89886 --- /dev/null +++ b/app/controllers/concerns/send_file_upload.rb @@ -0,0 +1,17 @@ +module SendFileUpload + def send_upload(file_upload, send_params: {}, redirect_params: {}, attachment: nil, disposition: 'attachment') + if attachment + redirect_params[:query] = { "response-content-disposition" => "#{disposition};filename=#{attachment.inspect}" } + send_params.merge!(filename: attachment, disposition: disposition) + end + + if file_upload.file_storage? + send_file file_upload.path, send_params + elsif file_upload.class.proxy_download_enabled? + headers.store(*Gitlab::Workhorse.send_url(file_upload.url(**redirect_params))) + head :ok + else + redirect_to file_upload.url(**redirect_params) + end + end +end diff --git a/app/controllers/concerns/uploads_actions.rb b/app/controllers/concerns/uploads_actions.rb index 3dbfabcae8a..b9b9b6e4e88 100644 --- a/app/controllers/concerns/uploads_actions.rb +++ b/app/controllers/concerns/uploads_actions.rb @@ -1,5 +1,6 @@ module UploadsActions include Gitlab::Utils::StrongMemoize + include SendFileUpload UPLOAD_MOUNTS = %w(avatar attachment file logo header_logo).freeze @@ -26,14 +27,11 @@ module UploadsActions def show return render_404 unless uploader&.exists? - if uploader.file_storage? - disposition = uploader.image_or_video? ? 'inline' : 'attachment' - expires_in 0.seconds, must_revalidate: true, private: true + expires_in 0.seconds, must_revalidate: true, private: true - send_file uploader.file.path, disposition: disposition - else - redirect_to uploader.url - end + disposition = uploader.image_or_video? ? 'inline' : 'attachment' + + send_upload(uploader, attachment: uploader.filename, disposition: disposition) end private @@ -62,19 +60,27 @@ module UploadsActions end def build_uploader_from_upload - return nil unless params[:secret] && params[:filename] + return unless uploader = build_uploader - upload_path = uploader_class.upload_path(params[:secret], params[:filename]) - upload = Upload.find_by(uploader: uploader_class.to_s, path: upload_path) + upload_paths = uploader.upload_paths(params[:filename]) + upload = Upload.find_by(uploader: uploader_class.to_s, path: upload_paths) upload&.build_uploader end def build_uploader_from_params + return unless uploader = build_uploader + + uploader.retrieve_from_store!(params[:filename]) + uploader + end + + def build_uploader + return unless params[:secret] && params[:filename] + uploader = uploader_class.new(model, secret: params[:secret]) - return nil unless uploader.model_valid? + return unless uploader.model_valid? - uploader.retrieve_from_store!(params[:filename]) uploader end diff --git a/app/controllers/groups/variables_controller.rb b/app/controllers/groups/variables_controller.rb index cb8771bc97e..6142e75b4c1 100644 --- a/app/controllers/groups/variables_controller.rb +++ b/app/controllers/groups/variables_controller.rb @@ -39,7 +39,7 @@ module Groups end def variable_params_attributes - %i[id key value protected _destroy] + %i[id key secret_value protected _destroy] end def authorize_admin_build! diff --git a/app/controllers/omniauth_callbacks_controller.rb b/app/controllers/omniauth_callbacks_controller.rb index fff249577a2..5e6676ea513 100644 --- a/app/controllers/omniauth_callbacks_controller.rb +++ b/app/controllers/omniauth_callbacks_controller.rb @@ -18,6 +18,18 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController end end + # Extend the standard implementation to also increment + # the number of failed sign in attempts + def failure + if params[:username].present? && AuthHelper.form_based_provider?(failed_strategy.name) + user = User.by_login(params[:username]) + + user&.increment_failed_attempts! + end + + super + end + # Extend the standard message generation to accept our custom exception def failure_message exception = env["omniauth.error"] diff --git a/app/controllers/projects/artifacts_controller.rb b/app/controllers/projects/artifacts_controller.rb index 0837451cc49..abc283d7aa9 100644 --- a/app/controllers/projects/artifacts_controller.rb +++ b/app/controllers/projects/artifacts_controller.rb @@ -1,6 +1,7 @@ class Projects::ArtifactsController < Projects::ApplicationController include ExtractsPath include RendersBlob + include SendFileUpload layout 'project' before_action :authorize_read_build! @@ -10,11 +11,7 @@ class Projects::ArtifactsController < Projects::ApplicationController before_action :entry, only: [:file] def download - if artifacts_file.file_storage? - send_file artifacts_file.path, disposition: 'attachment' - else - redirect_to artifacts_file.url - end + send_upload(artifacts_file, attachment: artifacts_file.filename) end def browse @@ -45,8 +42,7 @@ class Projects::ArtifactsController < Projects::ApplicationController end def raw - path = Gitlab::Ci::Build::Artifacts::Path - .new(params[:path]) + path = Gitlab::Ci::Build::Artifacts::Path.new(params[:path]) send_artifacts_entry(build, path) end @@ -75,7 +71,7 @@ class Projects::ArtifactsController < Projects::ApplicationController end def validate_artifacts! - render_404 unless build && build.artifacts? + render_404 unless build&.artifacts? end def build diff --git a/app/controllers/projects/jobs_controller.rb b/app/controllers/projects/jobs_controller.rb index 8b54ba3ad7c..85e972d9731 100644 --- a/app/controllers/projects/jobs_controller.rb +++ b/app/controllers/projects/jobs_controller.rb @@ -1,4 +1,6 @@ class Projects::JobsController < Projects::ApplicationController + include SendFileUpload + before_action :build, except: [:index, :cancel_all] before_action :authorize_read_build!, @@ -117,11 +119,17 @@ class Projects::JobsController < Projects::ApplicationController end def raw - build.trace.read do |stream| - if stream.file? - send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline' - else - render_404 + if trace_artifact_file + send_upload(trace_artifact_file, + send_params: raw_send_params, + redirect_params: raw_redirect_params) + else + build.trace.read do |stream| + if stream.file? + send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline' + else + render_404 + end end end end @@ -136,9 +144,21 @@ class Projects::JobsController < Projects::ApplicationController return access_denied! unless can?(current_user, :erase_build, build) end + def raw_send_params + { type: 'text/plain; charset=utf-8', disposition: 'inline' } + end + + def raw_redirect_params + { query: { 'response-content-type' => 'text/plain; charset=utf-8', 'response-content-disposition' => 'inline' } } + end + + def trace_artifact_file + @trace_artifact_file ||= build.job_artifacts_trace&.file + end + def build @build ||= project.builds.find(params[:id]) - .present(current_user: current_user) + .present(current_user: current_user) end def build_path(build) diff --git a/app/controllers/projects/lfs_storage_controller.rb b/app/controllers/projects/lfs_storage_controller.rb index 941638db427..6b16f1ccbbb 100644 --- a/app/controllers/projects/lfs_storage_controller.rb +++ b/app/controllers/projects/lfs_storage_controller.rb @@ -1,6 +1,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController include LfsRequest include WorkhorseRequest + include SendFileUpload skip_before_action :verify_workhorse_api!, only: [:download, :upload_finalize] @@ -11,7 +12,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController return end - send_file lfs_object.file.path, content_type: "application/octet-stream" + send_upload(lfs_object.file, send_params: { content_type: "application/octet-stream" }) end def upload_authorize @@ -70,10 +71,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController end def move_tmp_file_to_storage(object, path) - File.open(path) do |f| - object.file = f - end - + object.file = File.open(path) object.file.store! object.save end diff --git a/app/controllers/projects/pages_controller.rb b/app/controllers/projects/pages_controller.rb index d421b1a8eb5..cae6e2c40b8 100644 --- a/app/controllers/projects/pages_controller.rb +++ b/app/controllers/projects/pages_controller.rb @@ -21,4 +21,26 @@ class Projects::PagesController < Projects::ApplicationController end end end + + def update + result = Projects::UpdateService.new(@project, current_user, project_params).execute + + respond_to do |format| + format.html do + if result[:status] == :success + flash[:notice] = 'Your changes have been saved' + else + flash[:alert] = 'Something went wrong on our end' + end + + redirect_to project_pages_path(@project) + end + end + end + + private + + def project_params + params.require(:project).permit(:pages_https_only) + end end diff --git a/app/controllers/projects/pipeline_schedules_controller.rb b/app/controllers/projects/pipeline_schedules_controller.rb index b478e7b5e05..fa258f3d9af 100644 --- a/app/controllers/projects/pipeline_schedules_controller.rb +++ b/app/controllers/projects/pipeline_schedules_controller.rb @@ -92,7 +92,7 @@ class Projects::PipelineSchedulesController < Projects::ApplicationController def schedule_params params.require(:schedule) .permit(:description, :cron, :cron_timezone, :ref, :active, - variables_attributes: [:id, :key, :value, :_destroy] ) + variables_attributes: [:id, :key, :secret_value, :_destroy] ) end def authorize_play_pipeline_schedule! diff --git a/app/controllers/projects/pipelines_settings_controller.rb b/app/controllers/projects/pipelines_settings_controller.rb index 06ce7328fb5..557671ab186 100644 --- a/app/controllers/projects/pipelines_settings_controller.rb +++ b/app/controllers/projects/pipelines_settings_controller.rb @@ -10,10 +10,7 @@ class Projects::PipelinesSettingsController < Projects::ApplicationController if service.execute flash[:notice] = "Pipelines settings for '#{@project.name}' were successfully updated." - if service.run_auto_devops_pipeline? - CreatePipelineWorker.perform_async(project.id, current_user.id, project.default_branch, :web, ignore_skip_ci: true, save_on_errors: false) - flash[:success] = "A new Auto DevOps pipeline has been created, go to <a href=\"#{project_pipelines_path(@project)}\">Pipelines page</a> for details".html_safe - end + run_autodevops_pipeline(service) redirect_to project_settings_ci_cd_path(@project) else @@ -24,6 +21,18 @@ class Projects::PipelinesSettingsController < Projects::ApplicationController private + def run_autodevops_pipeline(service) + return unless service.run_auto_devops_pipeline? + + if @project.empty_repo? + flash[:warning] = "This repository is currently empty. A new Auto DevOps pipeline will be created after a new file has been pushed to a branch." + return + end + + CreatePipelineWorker.perform_async(project.id, current_user.id, project.default_branch, :web, ignore_skip_ci: true, save_on_errors: false) + flash[:success] = "A new Auto DevOps pipeline has been created, go to <a href=\"#{project_pipelines_path(@project)}\">Pipelines page</a> for details".html_safe + end + def update_params params.require(:project).permit( :runners_token, :builds_enabled, :build_allow_git_fetch, diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb index a02cc477e08..9bc774b7636 100644 --- a/app/controllers/projects/raw_controller.rb +++ b/app/controllers/projects/raw_controller.rb @@ -2,6 +2,7 @@ class Projects::RawController < Projects::ApplicationController include ExtractsPath include BlobHelper + include SendFileUpload before_action :require_non_empty_project before_action :assign_ref_vars @@ -31,7 +32,7 @@ class Projects::RawController < Projects::ApplicationController lfs_object = find_lfs_object if lfs_object && lfs_object.project_allowed_access?(@project) - send_file lfs_object.file.path, filename: @blob.name, disposition: 'attachment' + send_upload(lfs_object.file, attachment: @blob.name) else render_404 end diff --git a/app/controllers/projects/variables_controller.rb b/app/controllers/projects/variables_controller.rb index 7eb509e2e64..517d0b026c2 100644 --- a/app/controllers/projects/variables_controller.rb +++ b/app/controllers/projects/variables_controller.rb @@ -36,6 +36,6 @@ class Projects::VariablesController < Projects::ApplicationController end def variable_params_attributes - %i[id key value protected _destroy] + %i[id key secret_value protected _destroy] end end diff --git a/app/controllers/root_controller.rb b/app/controllers/root_controller.rb index 8acefd58e77..651b82f04f4 100644 --- a/app/controllers/root_controller.rb +++ b/app/controllers/root_controller.rb @@ -42,6 +42,10 @@ class RootController < Dashboard::ProjectsController redirect_to(dashboard_groups_path) when 'todos' redirect_to(dashboard_todos_path) + when 'issues' + redirect_to(issues_dashboard_path(assignee_id: current_user.id)) + when 'merge_requests' + redirect_to(merge_requests_dashboard_path(assignee_id: current_user.id)) end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 3ddf8eb3369..701be97ee96 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -323,4 +323,11 @@ module ApplicationHelper def locale_path asset_path("locale/#{Gitlab::I18n.locale}/app.js") end + + # Overridden in EE + def read_only_message + return unless Gitlab::Database.read_only? + + _('You are on a read-only GitLab instance.') + end end diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 63e4a5dc45c..b3b080e6dcf 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -96,7 +96,7 @@ module ApplicationSettingsHelper def repository_storages_options_for_select(selected) options = Gitlab.config.repositories.storages.map do |name, storage| - ["#{name} - #{storage['path']}", name] + ["#{name} - #{storage['gitaly_address']}", name] end options_for_select(options, selected) diff --git a/app/helpers/preferences_helper.rb b/app/helpers/preferences_helper.rb index 373dfd457f7..fb523cb865b 100644 --- a/app/helpers/preferences_helper.rb +++ b/app/helpers/preferences_helper.rb @@ -9,12 +9,14 @@ module PreferencesHelper # Maps `dashboard` values to more user-friendly option text DASHBOARD_CHOICES = { - projects: 'Your Projects (default)', - stars: 'Starred Projects', - project_activity: "Your Projects' Activity", - starred_project_activity: "Starred Projects' Activity", - groups: "Your Groups", - todos: "Your Todos" + projects: _("Your Projects (default)"), + stars: _("Starred Projects"), + project_activity: _("Your Projects' Activity"), + starred_project_activity: _("Starred Projects' Activity"), + groups: _("Your Groups"), + todos: _("Your Todos"), + issues: _("Assigned Issues"), + merge_requests: _("Assigned Merge Requests") }.with_indifferent_access.freeze # Returns an Array usable by a select field for more user-friendly option text diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index da9fe734f1c..15f48e43a28 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -531,4 +531,22 @@ module ProjectsHelper def can_show_last_commit_in_list?(project) can?(current_user, :read_cross_project) && project.commit end + + def pages_https_only_disabled? + !@project.pages_domains.all?(&:https?) + end + + def pages_https_only_title + return unless pages_https_only_disabled? + + "You must enable HTTPS for all your domains first" + end + + def pages_https_only_label_class + if pages_https_only_disabled? + "list-label disabled" + else + "list-label" + end + end end diff --git a/app/mailers/emails/merge_requests.rb b/app/mailers/emails/merge_requests.rb index 5fe09cea83f..be99f3780cc 100644 --- a/app/mailers/emails/merge_requests.rb +++ b/app/mailers/emails/merge_requests.rb @@ -11,6 +11,14 @@ module Emails mail_answer_thread(@merge_request, merge_request_thread_options(updated_by_user_id, recipient_id, reason)) end + def push_to_merge_request_email(recipient_id, merge_request_id, updated_by_user_id, reason = nil, new_commits: [], existing_commits: []) + setup_merge_request_mail(merge_request_id, recipient_id) + @new_commits = new_commits + @existing_commits = existing_commits + + mail_answer_thread(@merge_request, merge_request_thread_options(updated_by_user_id, recipient_id, reason)) + end + def reassigned_merge_request_email(recipient_id, merge_request_id, previous_assignee_id, updated_by_user_id, reason = nil) setup_merge_request_mail(merge_request_id, recipient_id) diff --git a/app/models/appearance.rb b/app/models/appearance.rb index dcd14c08f3c..2a6406d63c7 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -1,5 +1,7 @@ class Appearance < ActiveRecord::Base include CacheMarkdownField + include AfterCommitQueue + include ObjectStorage::BackgroundMove cache_markdown_field :description cache_markdown_field :new_project_guidelines diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 1e066b69c6e..08bb5915d10 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -3,6 +3,7 @@ module Ci prepend ArtifactMigratable include TokenAuthenticatable include AfterCommitQueue + include ObjectStorage::BackgroundMove include Presentable include Importable @@ -45,6 +46,7 @@ module Ci where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)', '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive) end + scope :with_artifacts_stored_locally, -> { with_artifacts_archive.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) } scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } @@ -365,13 +367,19 @@ module Ci project.running_or_pending_build_count(force: true) end + def browsable_artifacts? + artifacts_metadata? + end + def artifacts_metadata_entry(path, **options) - metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( - artifacts_metadata.path, - path, - **options) + artifacts_metadata.use_file do |metadata_path| + metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( + metadata_path, + path, + **options) - metadata.to_entry + metadata.to_entry + end end def erase_artifacts! diff --git a/app/models/ci/group_variable.rb b/app/models/ci/group_variable.rb index 1dd0e050ba9..62d768cc6cf 100644 --- a/app/models/ci/group_variable.rb +++ b/app/models/ci/group_variable.rb @@ -6,6 +6,8 @@ module Ci belongs_to :group + alias_attribute :secret_value, :value + validates :key, uniqueness: { scope: :group_id, message: "(%{value}) has already been taken" diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 0a599f72bc7..df57b4f65e3 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -1,5 +1,7 @@ module Ci class JobArtifact < ActiveRecord::Base + include AfterCommitQueue + include ObjectStorage::BackgroundMove extend Gitlab::Ci::Model belongs_to :project @@ -7,9 +9,11 @@ module Ci before_save :set_size, if: :file_changed? + scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } + mount_uploader :file, JobArtifactUploader - delegate :open, :exists?, to: :file + delegate :exists?, :open, to: :file enum file_type: { archive: 1, @@ -21,6 +25,10 @@ module Ci self.where(project: project).sum(:size) end + def local_store? + [nil, ::JobArtifactUploader::Store::LOCAL].include?(self.file_store) + end + def set_size self.size = file.size end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 44f9bdf111e..434b9b64c65 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -6,6 +6,7 @@ module Ci include AfterCommitQueue include Presentable include Gitlab::OptimisticLocking + include Gitlab::Utils::StrongMemoize belongs_to :project, inverse_of: :pipelines belongs_to :user @@ -361,21 +362,23 @@ module Ci def stage_seeds return [] unless config_processor - @stage_seeds ||= config_processor.stage_seeds(self) + strong_memoize(:stage_seeds) do + seeds = config_processor.stages_attributes.map do |attributes| + Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes) + end + + seeds.select(&:included?) + end end def seeds_size - @seeds_size ||= stage_seeds.sum(&:size) + stage_seeds.sum(&:size) end def has_kubernetes_active? project.deployment_platform&.active? end - def has_stage_seeds? - stage_seeds.any? - end - def has_warnings? builds.latest.failed_but_allowed.any? end @@ -388,6 +391,9 @@ module Ci end end + ## + # TODO, setting yaml_errors should be moved to the pipeline creation chain. + # def config_processor return unless ci_yaml_file return @config_processor if defined?(@config_processor) @@ -472,6 +478,14 @@ module Ci end end + def protected_ref? + strong_memoize(:protected_ref) { project.protected_for?(ref) } + end + + def legacy_trigger + strong_memoize(:legacy_trigger) { trigger_requests.first } + end + def predefined_variables Gitlab::Ci::Variables::Collection.new .append(key: 'CI_PIPELINE_ID', value: id.to_s) diff --git a/app/models/ci/pipeline_schedule_variable.rb b/app/models/ci/pipeline_schedule_variable.rb index af989fb14b4..03df4e3e638 100644 --- a/app/models/ci/pipeline_schedule_variable.rb +++ b/app/models/ci/pipeline_schedule_variable.rb @@ -5,6 +5,8 @@ module Ci belongs_to :pipeline_schedule + alias_attribute :secret_value, :value + validates :key, uniqueness: { scope: :pipeline_schedule_id } end end diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index 7c71291de84..452cb910bca 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -6,6 +6,8 @@ module Ci belongs_to :project + alias_attribute :secret_value, :value + validates :key, uniqueness: { scope: [:project_id, :environment_scope], message: "(%{value}) has already been taken" diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 49eb069016a..bfdfc5ae6fe 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -10,6 +10,7 @@ module Clusters Applications::Prometheus.application_name => Applications::Prometheus, Applications::Runner.application_name => Applications::Runner }.freeze + DEFAULT_ENVIRONMENT = '*'.freeze belongs_to :user @@ -50,6 +51,7 @@ module Clusters scope :enabled, -> { where(enabled: true) } scope :disabled, -> { where(enabled: false) } + scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) } def status_name if provider diff --git a/app/models/commit.rb b/app/models/commit.rb index cceae5efb72..b64462fb768 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -175,7 +175,7 @@ class Commit if safe_message.blank? no_commit_message else - safe_message.split("\n", 2).first + safe_message.split(/[\r\n]/, 2).first end end diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index d35e37935fb..7677891b9ce 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -3,6 +3,7 @@ module Avatarable included do prepend ShadowMethods + include ObjectStorage::BackgroundMove validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validates :avatar, file_size: { maximum: 200.kilobytes.to_i } @@ -21,7 +22,7 @@ module Avatarable def avatar_type unless self.avatar.image? - self.errors.add :avatar, "only images allowed" + errors.add :avatar, "file format is not supported. Please try one of the following supported formats: #{AvatarUploader::IMAGE_EXT.join(', ')}" end end diff --git a/app/models/concerns/deployment_platform.rb b/app/models/concerns/deployment_platform.rb index faa94204e33..52851b3d0b2 100644 --- a/app/models/concerns/deployment_platform.rb +++ b/app/models/concerns/deployment_platform.rb @@ -1,16 +1,24 @@ module DeploymentPlatform - # EE would override this and utilize the extra argument + # EE would override this and utilize environment argument + # rubocop:disable Gitlab/ModuleWithInstanceVariables def deployment_platform(environment: nil) - @deployment_platform ||= - find_cluster_platform_kubernetes || - find_kubernetes_service_integration || - build_cluster_and_deployment_platform + @deployment_platform ||= {} + + @deployment_platform[environment] ||= find_deployment_platform(environment) end private - def find_cluster_platform_kubernetes - clusters.find_by(enabled: true)&.platform_kubernetes + def find_deployment_platform(environment) + find_cluster_platform_kubernetes(environment: environment) || + find_kubernetes_service_integration || + build_cluster_and_deployment_platform + end + + # EE would override this and utilize environment argument + def find_cluster_platform_kubernetes(environment: nil) + clusters.enabled.default_environment + .last&.platform_kubernetes end def find_kubernetes_service_integration diff --git a/app/models/event.rb b/app/models/event.rb index 17a198d52c7..3805f6cf857 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -52,12 +52,12 @@ class Event < ActiveRecord::Base belongs_to :target, -> { # If the association for "target" defines an "author" association we want to # eager-load this so Banzai & friends don't end up performing N+1 queries to - # get the authors of notes, issues, etc. - if reflections['events'].active_record.reflect_on_association(:author) - includes(:author) - else - self + # get the authors of notes, issues, etc. (likewise for "noteable"). + incs = %i(author noteable).select do |a| + reflections['events'].active_record.reflect_on_association(a) end + + incs.reduce(self) { |obj, a| obj.includes(a) } }, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations has_one :push_event_payload diff --git a/app/models/group.rb b/app/models/group.rb index f669b1a7009..d99af79b5fe 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -189,12 +189,6 @@ class Group < Namespace owners.include?(user) && owners.size == 1 end - def avatar_type - unless self.avatar.image? - self.errors.add :avatar, "only images allowed" - end - end - def post_create_hook Gitlab::AppLogger.info("Group \"#{name}\" was created") diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index b444812a4cf..64e88d5a6a2 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -1,7 +1,12 @@ class LfsObject < ActiveRecord::Base + include AfterCommitQueue + include ObjectStorage::BackgroundMove + has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :lfs_objects_projects + scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) } + validates :oid, presence: true, uniqueness: true mount_uploader :file, LfsObjectUploader @@ -10,6 +15,10 @@ class LfsObject < ActiveRecord::Base projects.exists?(project.lfs_storage_project.id) end + def local_store? + [nil, LfsObjectUploader::Store::LOCAL].include?(self.file_store) + end + def self.destroy_unreferenced joins("LEFT JOIN lfs_objects_projects ON lfs_objects_projects.lfs_object_id = #{table_name}.id") .where(lfs_objects_projects: { id: nil }) diff --git a/app/models/notification_recipient.rb b/app/models/notification_recipient.rb index e95655e19f8..b3ffad00a07 100644 --- a/app/models/notification_recipient.rb +++ b/app/models/notification_recipient.rb @@ -48,7 +48,7 @@ class NotificationRecipient when :custom custom_enabled? || %i[participating mention].include?(@type) when :watch, :participating - !excluded_watcher_action? + !action_excluded? when :mention @type == :mention else @@ -96,13 +96,22 @@ class NotificationRecipient end end + def action_excluded? + excluded_watcher_action? || excluded_participating_action? + end + def excluded_watcher_action? - return false unless @custom_action - return false if notification_level == :custom + return false unless @custom_action && notification_level == :watch NotificationSetting::EXCLUDED_WATCHER_EVENTS.include?(@custom_action) end + def excluded_participating_action? + return false unless @custom_action && notification_level == :participating + + NotificationSetting::EXCLUDED_PARTICIPATING_EVENTS.include?(@custom_action) + end + private def read_ability diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 245f8dddcf9..f6d9b0215fc 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -33,6 +33,7 @@ class NotificationSetting < ActiveRecord::Base :close_issue, :reassign_issue, :new_merge_request, + :push_to_merge_request, :reopen_merge_request, :close_merge_request, :reassign_merge_request, @@ -41,10 +42,14 @@ class NotificationSetting < ActiveRecord::Base :success_pipeline ].freeze - EXCLUDED_WATCHER_EVENTS = [ + EXCLUDED_PARTICIPATING_EVENTS = [ :success_pipeline ].freeze + EXCLUDED_WATCHER_EVENTS = [ + :push_to_merge_request + ].push(*EXCLUDED_PARTICIPATING_EVENTS).freeze + def self.find_or_create_for(source) setting = find_or_initialize_by(source: source) diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 588bd50ed77..2e478a24778 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -6,8 +6,10 @@ class PagesDomain < ActiveRecord::Base validates :domain, hostname: { allow_numeric_hostname: true } validates :domain, uniqueness: { case_sensitive: false } - validates :certificate, certificate: true, allow_nil: true, allow_blank: true - validates :key, certificate_key: true, allow_nil: true, allow_blank: true + validates :certificate, presence: { message: 'must be present if HTTPS-only is enabled' }, if: ->(domain) { domain.project&.pages_https_only? } + validates :certificate, certificate: true, if: ->(domain) { domain.certificate.present? } + validates :key, presence: { message: 'must be present if HTTPS-only is enabled' }, if: ->(domain) { domain.project&.pages_https_only? } + validates :key, certificate_key: true, if: ->(domain) { domain.key.present? } validates :verification_code, presence: true, allow_blank: false validate :validate_pages_domain @@ -46,6 +48,10 @@ class PagesDomain < ActiveRecord::Base !Gitlab::CurrentSettings.pages_domain_verification_enabled? || enabled_until.present? end + def https? + certificate.present? + end + def to_param domain end diff --git a/app/models/project.rb b/app/models/project.rb index 250680e2a2c..6a420663644 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -267,6 +267,7 @@ class Project < ActiveRecord::Base validate :visibility_level_allowed_by_group validate :visibility_level_allowed_as_fork validate :check_wiki_path_conflict + validate :validate_pages_https_only, if: -> { changes.has_key?(:pages_https_only) } validates :repository_storage, presence: true, inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } @@ -503,7 +504,7 @@ class Project < ActiveRecord::Base end def repository_storage_path - Gitlab.config.repositories.storages[repository_storage].try(:[], 'path') + Gitlab.config.repositories.storages[repository_storage]&.legacy_disk_path end def team @@ -737,6 +738,26 @@ class Project < ActiveRecord::Base end end + def pages_https_only + return false unless Gitlab.config.pages.external_https + + super + end + + def pages_https_only? + return false unless Gitlab.config.pages.external_https + + super + end + + def validate_pages_https_only + return unless pages_https_only? + + unless pages_domains.all?(&:https?) + errors.add(:pages_https_only, "cannot be enabled unless all domains have TLS certificates") + end + end + def to_param if persisted? && errors.include?(:path) path_was diff --git a/app/models/upload.rb b/app/models/upload.rb index 99ad37dc892..cf71a7b76fc 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -9,6 +9,8 @@ class Upload < ActiveRecord::Base validates :model, presence: true validates :uploader, presence: true + scope :with_files_stored_locally, -> { where(store: [nil, ObjectStorage::Store::LOCAL]) } + before_save :calculate_checksum!, if: :foreground_checksummable? after_commit :schedule_checksum, if: :checksummable? @@ -21,6 +23,7 @@ class Upload < ActiveRecord::Base end def absolute_path + raise ObjectStorage::RemoteStoreError, "Remote object has no absolute path." unless local? return path unless relative_path? uploader_class.absolute_path(self) @@ -30,11 +33,11 @@ class Upload < ActiveRecord::Base self.checksum = nil return unless checksummable? - self.checksum = self.class.hexdigest(absolute_path) + self.checksum = Digest::SHA256.file(absolute_path).hexdigest end - def build_uploader - uploader_class.new(model, mount_point, **uploader_context).tap do |uploader| + def build_uploader(mounted_as = nil) + uploader_class.new(model, mounted_as || mount_point).tap do |uploader| uploader.upload = self uploader.retrieve_from_store!(identifier) end @@ -51,6 +54,12 @@ class Upload < ActiveRecord::Base }.compact end + def local? + return true if store.nil? + + store == ObjectStorage::Store::LOCAL + end + private def delete_file! @@ -61,10 +70,6 @@ class Upload < ActiveRecord::Base checksum.nil? && local? && exist? end - def local? - true - end - def foreground_checksummable? checksummable? && size <= CHECKSUM_THRESHOLD end diff --git a/app/models/user.rb b/app/models/user.rb index b8c55205ab8..187878f4fb5 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -187,7 +187,7 @@ class User < ActiveRecord::Base # User's Dashboard preference # Note: When adding an option, it MUST go on the end of the array. - enum dashboard: [:projects, :stars, :project_activity, :starred_project_activity, :groups, :todos] + enum dashboard: [:projects, :stars, :project_activity, :starred_project_activity, :groups, :todos, :issues, :merge_requests] # User's Project preference # Note: When adding an option, it MUST go on the end of the array. @@ -623,9 +623,7 @@ class User < ActiveRecord::Base end def owned_projects - @owned_projects ||= - Project.where('namespace_id IN (?) OR namespace_id = ?', - owned_groups.select(:id), namespace.id).joins(:namespace) + @owned_projects ||= Project.from("(#{owned_projects_union.to_sql}) AS projects") end # Returns projects which user can admin issues on (for example to move an issue to that project). @@ -1196,6 +1194,15 @@ class User < ActiveRecord::Base private + def owned_projects_union + Gitlab::SQL::Union.new([ + Project.where(namespace: namespace), + Project.joins(:project_authorizations) + .where("projects.namespace_id <> ?", namespace.id) + .where(project_authorizations: { user_id: id, access_level: Gitlab::Access::OWNER }) + ], remove_duplicates: false) + end + def ci_projects_union scope = { access_level: [Gitlab::Access::MASTER, Gitlab::Access::OWNER] } groups = groups_projects.where(members: scope) diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index ad27f320853..6ce86983287 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -7,6 +7,7 @@ module Ci Gitlab::Ci::Pipeline::Chain::Validate::Repository, Gitlab::Ci::Pipeline::Chain::Validate::Config, Gitlab::Ci::Pipeline::Chain::Skip, + Gitlab::Ci::Pipeline::Chain::Populate, Gitlab::Ci::Pipeline::Chain::Create].freeze def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil, &block) diff --git a/app/services/ci/create_pipeline_stages_service.rb b/app/services/ci/create_pipeline_stages_service.rb deleted file mode 100644 index f2c175adee6..00000000000 --- a/app/services/ci/create_pipeline_stages_service.rb +++ /dev/null @@ -1,20 +0,0 @@ -module Ci - class CreatePipelineStagesService < BaseService - def execute(pipeline) - pipeline.stage_seeds.each do |seed| - seed.user = current_user - - seed.create! do |build| - ## - # Create the environment before the build starts. This sets its slug and - # makes it available as an environment variable - # - if build.has_environment? - environment_name = build.expanded_environment_name - project.environments.find_or_create_by(name: environment_name) - end - end - end - end - end -end diff --git a/app/services/ci/pipeline_trigger_service.rb b/app/services/ci/pipeline_trigger_service.rb index a9813d774bb..85533a1cbdb 100644 --- a/app/services/ci/pipeline_trigger_service.rb +++ b/app/services/ci/pipeline_trigger_service.rb @@ -16,8 +16,8 @@ module Ci pipeline = Ci::CreatePipelineService.new(project, trigger.owner, ref: params[:ref]) .execute(:trigger, ignore_skip_ci: true) do |pipeline| - pipeline.trigger_requests.create!(trigger: trigger) - create_pipeline_variables!(pipeline) + pipeline.trigger_requests.build(trigger: trigger) + pipeline.variables.build(variables) end if pipeline.persisted? @@ -33,14 +33,10 @@ module Ci end end - def create_pipeline_variables!(pipeline) - return unless params[:variables] - - variables = params[:variables].map do |key, value| + def variables + params[:variables].to_h.map do |key, value| { key: key, value: value } end - - pipeline.variables.create!(variables) end end end diff --git a/app/services/merge_requests/refresh_service.rb b/app/services/merge_requests/refresh_service.rb index 18c40ce8992..1fb1796b56c 100644 --- a/app/services/merge_requests/refresh_service.rb +++ b/app/services/merge_requests/refresh_service.rb @@ -21,7 +21,7 @@ module MergeRequests comment_mr_branch_presence_changed end - comment_mr_with_commits + notify_about_push mark_mr_as_wip_from_commits execute_mr_web_hooks @@ -141,8 +141,8 @@ module MergeRequests end end - # Add comment about pushing new commits to merge requests - def comment_mr_with_commits + # Add comment about pushing new commits to merge requests and send nofitication emails + def notify_about_push return unless @commits.present? merge_requests_for_source_branch.each do |merge_request| @@ -155,6 +155,8 @@ module MergeRequests SystemNoteService.add_commits(merge_request, merge_request.project, @current_user, new_commits, existing_commits, @oldrev) + + notification_service.push_to_merge_request(merge_request, @current_user, new_commits: new_commits, existing_commits: existing_commits) end end diff --git a/app/services/notification_service.rb b/app/services/notification_service.rb index d7d2cde1004..f94c76cf3ac 100644 --- a/app/services/notification_service.rb +++ b/app/services/notification_service.rb @@ -113,6 +113,16 @@ class NotificationService new_resource_email(merge_request, :new_merge_request_email) end + def push_to_merge_request(merge_request, current_user, new_commits: [], existing_commits: []) + new_commits = new_commits.map { |c| { short_id: c.short_id, title: c.title } } + existing_commits = existing_commits.map { |c| { short_id: c.short_id, title: c.title } } + recipients = NotificationRecipientService.build_recipients(merge_request, current_user, action: "push_to") + + recipients.each do |recipient| + mailer.send(:push_to_merge_request_email, recipient.user.id, merge_request.id, current_user.id, recipient.reason, new_commits: new_commits, existing_commits: existing_commits).deliver_later + end + end + # When merge request text is updated, we should send an email to: # # * newly mentioned project team members with notification level higher than Participating diff --git a/app/services/projects/update_pages_configuration_service.rb b/app/services/projects/update_pages_configuration_service.rb index 52ff64cc938..25017c5cbe3 100644 --- a/app/services/projects/update_pages_configuration_service.rb +++ b/app/services/projects/update_pages_configuration_service.rb @@ -18,7 +18,8 @@ module Projects def pages_config { - domains: pages_domains_config + domains: pages_domains_config, + https_only: project.pages_https_only? } end @@ -27,7 +28,8 @@ module Projects { domain: domain.domain, certificate: domain.certificate, - key: domain.key + key: domain.key, + https_only: project.pages_https_only? && domain.https? } end end diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index 00fdd047208..5bf8208e035 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -81,11 +81,13 @@ module Projects end def extract_tar_archive!(temp_path) - results = Open3.pipeline(%W(gunzip -c #{artifacts}), - %W(dd bs=#{BLOCK_SIZE} count=#{blocks}), - %W(tar -x -C #{temp_path} #{SITE_PATH}), - err: '/dev/null') - raise FailedToExtractError, 'pages failed to extract' unless results.compact.all?(&:success?) + build.artifacts_file.use_file do |artifacts_path| + results = Open3.pipeline(%W(gunzip -c #{artifacts_path}), + %W(dd bs=#{BLOCK_SIZE} count=#{blocks}), + %W(tar -x -C #{temp_path} #{SITE_PATH}), + err: '/dev/null') + raise FailedToExtractError, 'pages failed to extract' unless results.compact.all?(&:success?) + end end def extract_zip_archive!(temp_path) @@ -103,8 +105,10 @@ module Projects # -n never overwrite existing files # We add * to end of SITE_PATH, because we want to extract SITE_PATH and all subdirectories site_path = File.join(SITE_PATH, '*') - unless system(*%W(unzip -qq -n #{artifacts} #{site_path} -d #{temp_path})) - raise FailedToExtractError, 'pages failed to extract' + build.artifacts_file.use_file do |artifacts_path| + unless system(*%W(unzip -n #{artifacts_path} #{site_path} -d #{temp_path})) + raise FailedToExtractError, 'pages failed to extract' + end end end diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb index 5f2615a2c01..679f4a9cb62 100644 --- a/app/services/projects/update_service.rb +++ b/app/services/projects/update_service.rb @@ -24,6 +24,8 @@ module Projects system_hook_service.execute_hooks_for(project, :update) end + update_pages_config if changing_pages_https_only? + success else model_errors = project.errors.full_messages.to_sentence @@ -67,5 +69,13 @@ module Projects log_error("Could not create wiki for #{project.full_name}") Gitlab::Metrics.counter(:wiki_can_not_be_created_total, 'Counts the times we failed to create a wiki') end + + def update_pages_config + Projects::UpdatePagesConfigurationService.new(project).execute + end + + def changing_pages_https_only? + project.previous_changes.include?(:pages_https_only) + end end end diff --git a/app/uploaders/attachment_uploader.rb b/app/uploaders/attachment_uploader.rb index 4930fb2fca7..cd819dc9bff 100644 --- a/app/uploaders/attachment_uploader.rb +++ b/app/uploaders/attachment_uploader.rb @@ -1,8 +1,8 @@ class AttachmentUploader < GitlabUploader - include UploaderHelper include RecordsUploads::Concern - - storage :file + include ObjectStorage::Concern + prepend ObjectStorage::Extension::RecordsUploads + include UploaderHelper private diff --git a/app/uploaders/avatar_uploader.rb b/app/uploaders/avatar_uploader.rb index 5c8e1cea62e..5848e6c6994 100644 --- a/app/uploaders/avatar_uploader.rb +++ b/app/uploaders/avatar_uploader.rb @@ -1,18 +1,18 @@ class AvatarUploader < GitlabUploader include UploaderHelper include RecordsUploads::Concern - - storage :file + include ObjectStorage::Concern + prepend ObjectStorage::Extension::RecordsUploads def exists? model.avatar.file && model.avatar.file.present? end - def move_to_cache + def move_to_store false end - def move_to_store + def move_to_cache false end diff --git a/app/uploaders/file_mover.rb b/app/uploaders/file_mover.rb index 8f56f09c9f7..bd7736ad74e 100644 --- a/app/uploaders/file_mover.rb +++ b/app/uploaders/file_mover.rb @@ -10,7 +10,11 @@ class FileMover def execute move - uploader.record_upload if update_markdown + + if update_markdown + uploader.record_upload + uploader.schedule_background_upload + end end private @@ -24,11 +28,8 @@ class FileMover updated_text = model.read_attribute(update_field) .gsub(temp_file_uploader.markdown_link, uploader.markdown_link) model.update_attribute(update_field, updated_text) - - true rescue revert - false end diff --git a/app/uploaders/file_uploader.rb b/app/uploaders/file_uploader.rb index bde1161dfa8..133fdf6684d 100644 --- a/app/uploaders/file_uploader.rb +++ b/app/uploaders/file_uploader.rb @@ -9,14 +9,18 @@ class FileUploader < GitlabUploader include UploaderHelper include RecordsUploads::Concern + include ObjectStorage::Concern + prepend ObjectStorage::Extension::RecordsUploads MARKDOWN_PATTERN = %r{\!?\[.*?\]\(/uploads/(?<secret>[0-9a-f]{32})/(?<file>.*?)\)} DYNAMIC_PATH_PATTERN = %r{(?<secret>\h{32})/(?<identifier>.*)} - storage :file - after :remove, :prune_store_dir + # FileUploader do not run in a model transaction, so we can simply + # enqueue a job after the :store hook. + after :store, :schedule_background_upload + def self.root File.join(options.storage_path, 'uploads') end @@ -28,8 +32,11 @@ class FileUploader < GitlabUploader ) end - def self.base_dir(model) - model_path_segment(model) + def self.base_dir(model, store = Store::LOCAL) + decorated_model = model + decorated_model = Storage::HashedProject.new(model) if store == Store::REMOTE + + model_path_segment(decorated_model) end # used in migrations and import/exports @@ -47,21 +54,24 @@ class FileUploader < GitlabUploader # # Returns a String without a trailing slash def self.model_path_segment(model) - if model.hashed_storage?(:attachments) - model.disk_path + case model + when Storage::HashedProject then model.disk_path else - model.full_path + model.hashed_storage?(:attachments) ? model.disk_path : model.full_path end end - def self.upload_path(secret, identifier) - File.join(secret, identifier) - end - def self.generate_secret SecureRandom.hex end + def upload_paths(filename) + [ + File.join(secret, filename), + File.join(base_dir(Store::REMOTE), secret, filename) + ] + end + attr_accessor :model def initialize(model, mounted_as = nil, **uploader_context) @@ -71,8 +81,10 @@ class FileUploader < GitlabUploader apply_context!(uploader_context) end - def base_dir - self.class.base_dir(@model) + # enforce the usage of Hashed storage when storing to + # remote store as the FileMover doesn't support OS + def base_dir(store = nil) + self.class.base_dir(@model, store || object_store) end # we don't need to know the actual path, an uploader instance should be @@ -82,15 +94,19 @@ class FileUploader < GitlabUploader end def upload_path - self.class.upload_path(dynamic_segment, identifier) - end - - def model_path_segment - self.class.model_path_segment(@model) + if file_storage? + # Legacy path relative to project.full_path + File.join(dynamic_segment, identifier) + else + File.join(store_dir, identifier) + end end - def store_dir - File.join(base_dir, dynamic_segment) + def store_dirs + { + Store::LOCAL => File.join(base_dir, dynamic_segment), + Store::REMOTE => File.join(base_dir(ObjectStorage::Store::REMOTE), dynamic_segment) + } end def markdown_link diff --git a/app/uploaders/gitlab_uploader.rb b/app/uploaders/gitlab_uploader.rb index 010100f2da1..f12f0466a1d 100644 --- a/app/uploaders/gitlab_uploader.rb +++ b/app/uploaders/gitlab_uploader.rb @@ -37,12 +37,10 @@ class GitlabUploader < CarrierWave::Uploader::Base cache_storage.is_a?(CarrierWave::Storage::File) end - # Reduce disk IO def move_to_cache file_storage? end - # Reduce disk IO def move_to_store file_storage? end @@ -51,10 +49,6 @@ class GitlabUploader < CarrierWave::Uploader::Base file.present? end - def store_dir - File.join(base_dir, dynamic_segment) - end - def cache_dir File.join(root, base_dir, 'tmp/cache') end @@ -76,6 +70,10 @@ class GitlabUploader < CarrierWave::Uploader::Base # Designed to be overridden by child uploaders that have a dynamic path # segment -- that is, a path that changes based on mutable attributes of its # associated model + # + # For example, `FileUploader` builds the storage path based on the associated + # project model's `path_with_namespace` value, which can change when the + # project or its containing namespace is moved or renamed. def dynamic_segment raise(NotImplementedError) end diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb index ad5385f45a4..ef0f8acefd6 100644 --- a/app/uploaders/job_artifact_uploader.rb +++ b/app/uploaders/job_artifact_uploader.rb @@ -1,5 +1,6 @@ class JobArtifactUploader < GitlabUploader extend Workhorse::UploadPath + include ObjectStorage::Concern storage_options Gitlab.config.artifacts @@ -14,9 +15,11 @@ class JobArtifactUploader < GitlabUploader end def open - raise 'Only File System is supported' unless file_storage? - - File.open(path, "rb") if path + if file_storage? + File.open(path, "rb") if path + else + ::Gitlab::Ci::Trace::HttpIO.new(url, size) if url + end end private diff --git a/app/uploaders/legacy_artifact_uploader.rb b/app/uploaders/legacy_artifact_uploader.rb index 28c458d3ff1..b726b053493 100644 --- a/app/uploaders/legacy_artifact_uploader.rb +++ b/app/uploaders/legacy_artifact_uploader.rb @@ -1,5 +1,6 @@ class LegacyArtifactUploader < GitlabUploader extend Workhorse::UploadPath + include ObjectStorage::Concern storage_options Gitlab.config.artifacts diff --git a/app/uploaders/lfs_object_uploader.rb b/app/uploaders/lfs_object_uploader.rb index e04c97ce179..eb521a22ebc 100644 --- a/app/uploaders/lfs_object_uploader.rb +++ b/app/uploaders/lfs_object_uploader.rb @@ -1,10 +1,6 @@ class LfsObjectUploader < GitlabUploader extend Workhorse::UploadPath - - # LfsObject are in `tmp/upload` instead of `tmp/uploads` - def self.workhorse_upload_path - File.join(root, 'tmp/upload') - end + include ObjectStorage::Concern storage_options Gitlab.config.lfs diff --git a/app/uploaders/namespace_file_uploader.rb b/app/uploaders/namespace_file_uploader.rb index 993e85fbc13..1085ecb1700 100644 --- a/app/uploaders/namespace_file_uploader.rb +++ b/app/uploaders/namespace_file_uploader.rb @@ -4,7 +4,7 @@ class NamespaceFileUploader < FileUploader options.storage_path end - def self.base_dir(model) + def self.base_dir(model, _store = nil) File.join(options.base_dir, 'namespace', model_path_segment(model)) end @@ -14,6 +14,13 @@ class NamespaceFileUploader < FileUploader # Re-Override def store_dir - File.join(base_dir, dynamic_segment) + store_dirs[object_store] + end + + def store_dirs + { + Store::LOCAL => File.join(base_dir, dynamic_segment), + Store::REMOTE => File.join('namespace', self.class.model_path_segment(model), dynamic_segment) + } end end diff --git a/app/uploaders/object_storage.rb b/app/uploaders/object_storage.rb new file mode 100644 index 00000000000..7218cb0a0fc --- /dev/null +++ b/app/uploaders/object_storage.rb @@ -0,0 +1,335 @@ +require 'fog/aws' +require 'carrierwave/storage/fog' + +# +# This concern should add object storage support +# to the GitlabUploader class +# +module ObjectStorage + RemoteStoreError = Class.new(StandardError) + UnknownStoreError = Class.new(StandardError) + ObjectStorageUnavailable = Class.new(StandardError) + + module Store + LOCAL = 1 + REMOTE = 2 + end + + module Extension + # this extension is the glue between the ObjectStorage::Concern and RecordsUploads::Concern + module RecordsUploads + extend ActiveSupport::Concern + + def prepended(base) + raise "#{base} must include ObjectStorage::Concern to use extensions." unless base < Concern + + base.include(RecordsUploads::Concern) + end + + def retrieve_from_store!(identifier) + paths = store_dirs.map { |store, path| File.join(path, identifier) } + + unless current_upload_satisfies?(paths, model) + # the upload we already have isn't right, find the correct one + self.upload = uploads.find_by(model: model, path: paths) + end + + super + end + + def build_upload + super.tap do |upload| + upload.store = object_store + end + end + + def upload=(upload) + return unless upload + + self.object_store = upload.store + super + end + + def schedule_background_upload(*args) + return unless schedule_background_upload? + return unless upload + + ObjectStorage::BackgroundMoveWorker.perform_async(self.class.name, + upload.class.to_s, + mounted_as, + upload.id) + end + + private + + def current_upload_satisfies?(paths, model) + return false unless upload + return false unless model + + paths.include?(upload.path) && + upload.model_id == model.id && + upload.model_type == model.class.base_class.sti_name + end + end + end + + # Add support for automatic background uploading after the file is stored. + # + module BackgroundMove + extend ActiveSupport::Concern + + def background_upload(mount_points = []) + return unless mount_points.any? + + run_after_commit do + mount_points.each { |mount| send(mount).schedule_background_upload } # rubocop:disable GitlabSecurity/PublicSend + end + end + + def changed_mounts + self.class.uploaders.select do |mount, uploader_class| + mounted_as = uploader_class.serialization_column(self.class, mount) + uploader = send(:"#{mounted_as}") # rubocop:disable GitlabSecurity/PublicSend + + next unless uploader + next unless uploader.exists? + next unless send(:"#{mounted_as}_changed?") # rubocop:disable GitlabSecurity/PublicSend + + mount + end.keys + end + + included do + after_save on: [:create, :update] do + background_upload(changed_mounts) + end + end + end + + module Concern + extend ActiveSupport::Concern + + included do |base| + base.include(ObjectStorage) + + after :migrate, :delete_migrated_file + end + + class_methods do + def object_store_options + options.object_store + end + + def object_store_enabled? + object_store_options.enabled + end + + def background_upload_enabled? + object_store_options.background_upload + end + + def proxy_download_enabled? + object_store_options.proxy_download + end + + def direct_download_enabled? + !proxy_download_enabled? + end + + def object_store_credentials + object_store_options.connection.to_hash.deep_symbolize_keys + end + + def remote_store_path + object_store_options.remote_directory + end + + def serialization_column(model_class, mount_point) + model_class.uploader_options.dig(mount_point, :mount_on) || mount_point + end + end + + def file_storage? + storage.is_a?(CarrierWave::Storage::File) + end + + def file_cache_storage? + cache_storage.is_a?(CarrierWave::Storage::File) + end + + def object_store + @object_store ||= model.try(store_serialization_column) || Store::LOCAL + end + + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def object_store=(value) + @object_store = value || Store::LOCAL + @storage = storage_for(object_store) + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + + # Return true if the current file is part or the model (i.e. is mounted in the model) + # + def persist_object_store? + model.respond_to?(:"#{store_serialization_column}=") + end + + # Save the current @object_store to the model <mounted_as>_store column + def persist_object_store! + return unless persist_object_store? + + updated = model.update_column(store_serialization_column, object_store) + raise 'Failed to update object store' unless updated + end + + def use_file + if file_storage? + return yield path + end + + begin + cache_stored_file! + yield cache_path + ensure + cache_storage.delete_dir!(cache_path(nil)) + end + end + + def filename + super || file&.filename + end + + # + # Move the file to another store + # + # new_store: Enum (Store::LOCAL, Store::REMOTE) + # + def migrate!(new_store) + uuid = Gitlab::ExclusiveLease.new(exclusive_lease_key, timeout: 1.hour.to_i).try_obtain + raise 'Already running' unless uuid + + unsafe_migrate!(new_store) + ensure + Gitlab::ExclusiveLease.cancel(exclusive_lease_key, uuid) + end + + def schedule_background_upload(*args) + return unless schedule_background_upload? + + ObjectStorage::BackgroundMoveWorker.perform_async(self.class.name, + model.class.name, + mounted_as, + model.id) + end + + def fog_directory + self.class.remote_store_path + end + + def fog_credentials + self.class.object_store_credentials + end + + def fog_public + false + end + + def delete_migrated_file(migrated_file) + migrated_file.delete if exists? + end + + def exists? + file.present? + end + + def store_dir(store = nil) + store_dirs[store || object_store] + end + + def store_dirs + { + Store::LOCAL => File.join(base_dir, dynamic_segment), + Store::REMOTE => File.join(dynamic_segment) + } + end + + private + + def schedule_background_upload? + self.class.object_store_enabled? && + self.class.background_upload_enabled? && + self.file_storage? + end + + # this is a hack around CarrierWave. The #migrate method needs to be + # able to force the current file to the migrated file upon success. + def file=(file) + @file = file # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + def serialization_column + self.class.serialization_column(model.class, mounted_as) + end + + # Returns the column where the 'store' is saved + # defaults to 'store' + def store_serialization_column + [serialization_column, 'store'].compact.join('_').to_sym + end + + def storage + @storage ||= storage_for(object_store) + end + + def storage_for(store) + case store + when Store::REMOTE + raise 'Object Storage is not enabled' unless self.class.object_store_enabled? + + CarrierWave::Storage::Fog.new(self) + when Store::LOCAL + CarrierWave::Storage::File.new(self) + else + raise UnknownStoreError + end + end + + def exclusive_lease_key + "object_storage_migrate:#{model.class}:#{model.id}" + end + + # + # Move the file to another store + # + # new_store: Enum (Store::LOCAL, Store::REMOTE) + # + def unsafe_migrate!(new_store) + return unless object_store != new_store + return unless file + + new_file = nil + file_to_delete = file + from_object_store = object_store + self.object_store = new_store # changes the storage and file + + cache_stored_file! if file_storage? + + with_callbacks(:migrate, file_to_delete) do + with_callbacks(:store, file_to_delete) do # for #store_versions! + new_file = storage.store!(file) + persist_object_store! + self.file = new_file + end + end + + file + rescue => e + # in case of failure delete new file + new_file.delete unless new_file.nil? + # revert back to the old file + self.object_store = from_object_store + self.file = file_to_delete + raise e + end + end +end diff --git a/app/uploaders/personal_file_uploader.rb b/app/uploaders/personal_file_uploader.rb index f2ad0badd53..e3898b07730 100644 --- a/app/uploaders/personal_file_uploader.rb +++ b/app/uploaders/personal_file_uploader.rb @@ -4,7 +4,7 @@ class PersonalFileUploader < FileUploader options.storage_path end - def self.base_dir(model) + def self.base_dir(model, _store = nil) File.join(options.base_dir, model_path_segment(model)) end @@ -14,6 +14,12 @@ class PersonalFileUploader < FileUploader File.join(model.class.to_s.underscore, model.id.to_s) end + def object_store + return Store::LOCAL unless model + + super + end + # model_path_segment does not require a model to be passed, so we can always # generate a path, even when there's no model. def model_valid? @@ -22,7 +28,14 @@ class PersonalFileUploader < FileUploader # Revert-Override def store_dir - File.join(base_dir, dynamic_segment) + store_dirs[object_store] + end + + def store_dirs + { + Store::LOCAL => File.join(base_dir, dynamic_segment), + Store::REMOTE => File.join(self.class.model_path_segment(model), dynamic_segment) + } end private diff --git a/app/uploaders/records_uploads.rb b/app/uploaders/records_uploads.rb index 458928bc067..89c74a78835 100644 --- a/app/uploaders/records_uploads.rb +++ b/app/uploaders/records_uploads.rb @@ -24,8 +24,7 @@ module RecordsUploads uploads.where(path: upload_path).delete_all upload.destroy! if upload - self.upload = build_upload - upload.save! + self.upload = build_upload.tap(&:save!) end end diff --git a/app/validators/certificate_validator.rb b/app/validators/certificate_validator.rb index 5239e70a326..b0c9a1b92a4 100644 --- a/app/validators/certificate_validator.rb +++ b/app/validators/certificate_validator.rb @@ -16,8 +16,6 @@ class CertificateValidator < ActiveModel::EachValidator private def valid_certificate_pem?(value) - return false unless value - OpenSSL::X509::Certificate.new(value).present? rescue OpenSSL::X509::CertificateError false diff --git a/app/views/admin/application_settings/_account_and_limit.html.haml b/app/views/admin/application_settings/_account_and_limit.html.haml new file mode 100644 index 00000000000..dd86c9ed2eb --- /dev/null +++ b/app/views/admin/application_settings/_account_and_limit.html.haml @@ -0,0 +1,39 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :gravatar_enabled do + = f.check_box :gravatar_enabled + Gravatar enabled + .form-group + = f.label :default_projects_limit, class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :default_projects_limit, class: 'form-control' + .form-group + = f.label :max_attachment_size, 'Maximum attachment size (MB)', class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :max_attachment_size, class: 'form-control' + .form-group + = f.label :session_expire_delay, 'Session duration (minutes)', class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :session_expire_delay, class: 'form-control' + %span.help-block#session_expire_delay_help_block GitLab restart is required to apply changes + .form-group + = f.label :user_oauth_applications, 'User OAuth applications', class: 'control-label col-sm-2' + .col-sm-10 + .checkbox + = f.label :user_oauth_applications do + = f.check_box :user_oauth_applications + Allow users to register any application to use GitLab as an OAuth provider + .form-group + = f.label :user_default_external, 'New users set to external', class: 'control-label col-sm-2' + .col-sm-10 + .checkbox + = f.label :user_default_external do + = f.check_box :user_default_external + Newly registered users will by default be external + + = f.submit 'Save changes', class: 'btn btn-success' diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index 54b39df8cf3..0f75db3f6ae 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -2,254 +2,6 @@ = form_errors(@application_setting) %fieldset - %legend Visibility and Access Controls - .form-group - = f.label :default_branch_protection, class: 'control-label col-sm-2' - .col-sm-10 - = f.select :default_branch_protection, options_for_select(Gitlab::Access.protection_options, @application_setting.default_branch_protection), {}, class: 'form-control' - .form-group.visibility-level-setting - = f.label :default_project_visibility, class: 'control-label col-sm-2' - .col-sm-10 - = render('shared/visibility_radios', model_method: :default_project_visibility, form: f, selected_level: @application_setting.default_project_visibility, form_model: Project.new) - .form-group.visibility-level-setting - = f.label :default_snippet_visibility, class: 'control-label col-sm-2' - .col-sm-10 - = render('shared/visibility_radios', model_method: :default_snippet_visibility, form: f, selected_level: @application_setting.default_snippet_visibility, form_model: ProjectSnippet.new) - .form-group.visibility-level-setting - = f.label :default_group_visibility, class: 'control-label col-sm-2' - .col-sm-10 - = render('shared/visibility_radios', model_method: :default_group_visibility, form: f, selected_level: @application_setting.default_group_visibility, form_model: Group.new) - .form-group - = f.label :restricted_visibility_levels, class: 'control-label col-sm-2' - .col-sm-10 - - checkbox_name = 'application_setting[restricted_visibility_levels][]' - = hidden_field_tag(checkbox_name) - - restricted_level_checkboxes('restricted-visibility-help', checkbox_name).each do |level| - .checkbox - = level - %span.help-block#restricted-visibility-help - Selected levels cannot be used by non-admin users for projects or snippets. - If the public level is restricted, user profiles are only visible to logged in users. - .form-group - = f.label :import_sources, class: 'control-label col-sm-2' - .col-sm-10 - - import_sources_checkboxes('import-sources-help').each do |source| - .checkbox= source - %span.help-block#import-sources-help - Enabled sources for code import during project creation. OmniAuth must be configured for GitHub - = link_to "(?)", help_page_path("integration/github") - , Bitbucket - = link_to "(?)", help_page_path("integration/bitbucket") - and GitLab.com - = link_to "(?)", help_page_path("integration/gitlab") - - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :project_export_enabled do - = f.check_box :project_export_enabled - Project export enabled - - .form-group - %label.control-label.col-sm-2 Enabled Git access protocols - .col-sm-10 - = select(:application_setting, :enabled_git_access_protocol, [['Both SSH and HTTP(S)', nil], ['Only SSH', 'ssh'], ['Only HTTP(S)', 'http']], {}, class: 'form-control') - %span.help-block#clone-protocol-help - Allow only the selected protocols to be used for Git access. - - - ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| - - field_name = :"#{type}_key_restriction" - .form-group - = f.label field_name, "#{type.upcase} SSH keys", class: 'control-label col-sm-2' - .col-sm-10 - = f.select field_name, key_restriction_options_for_select(type), {}, class: 'form-control' - - %fieldset - %legend Account and Limit Settings - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :gravatar_enabled do - = f.check_box :gravatar_enabled - Gravatar enabled - .form-group - = f.label :default_projects_limit, class: 'control-label col-sm-2' - .col-sm-10 - = f.number_field :default_projects_limit, class: 'form-control' - .form-group - = f.label :max_attachment_size, 'Maximum attachment size (MB)', class: 'control-label col-sm-2' - .col-sm-10 - = f.number_field :max_attachment_size, class: 'form-control' - .form-group - = f.label :session_expire_delay, 'Session duration (minutes)', class: 'control-label col-sm-2' - .col-sm-10 - = f.number_field :session_expire_delay, class: 'form-control' - %span.help-block#session_expire_delay_help_block GitLab restart is required to apply changes - .form-group - = f.label :user_oauth_applications, 'User OAuth applications', class: 'control-label col-sm-2' - .col-sm-10 - .checkbox - = f.label :user_oauth_applications do - = f.check_box :user_oauth_applications - Allow users to register any application to use GitLab as an OAuth provider - .form-group - = f.label :user_default_external, 'New users set to external', class: 'control-label col-sm-2' - .col-sm-10 - .checkbox - = f.label :user_default_external do - = f.check_box :user_default_external - Newly registered users will by default be external - - %fieldset - %legend Sign-up Restrictions - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :signup_enabled do - = f.check_box :signup_enabled - Sign-up enabled - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :send_user_confirmation_email do - = f.check_box :send_user_confirmation_email - Send confirmation email on sign-up - .form-group - = f.label :domain_whitelist, 'Whitelisted domains for sign-ups', class: 'control-label col-sm-2' - .col-sm-10 - = f.text_area :domain_whitelist_raw, placeholder: 'domain.com', class: 'form-control', rows: 8 - .help-block ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com - .form-group - = f.label :domain_blacklist_enabled, 'Domain Blacklist', class: 'control-label col-sm-2' - .col-sm-10 - .checkbox - = f.label :domain_blacklist_enabled do - = f.check_box :domain_blacklist_enabled - Enable domain blacklist for sign ups - .form-group - .col-sm-offset-2.col-sm-10 - .radio - = label_tag :blacklist_type_file do - = radio_button_tag :blacklist_type, :file - .option-title - Upload blacklist file - .radio - = label_tag :blacklist_type_raw do - = radio_button_tag :blacklist_type, :raw, @application_setting.domain_blacklist.present? || @application_setting.domain_blacklist.blank? - .option-title - Enter blacklist manually - .form-group.blacklist-file - = f.label :domain_blacklist_file, 'Blacklist file', class: 'control-label col-sm-2' - .col-sm-10 - = f.file_field :domain_blacklist_file, class: 'form-control', accept: '.txt,.conf' - .help-block Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines or commas for multiple entries. - .form-group.blacklist-raw - = f.label :domain_blacklist, 'Blacklisted domains for sign-ups', class: 'control-label col-sm-2' - .col-sm-10 - = f.text_area :domain_blacklist_raw, placeholder: 'domain.com', class: 'form-control', rows: 8 - .help-block Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com - - .form-group - = f.label :after_sign_up_text, class: 'control-label col-sm-2' - .col-sm-10 - = f.text_area :after_sign_up_text, class: 'form-control', rows: 4 - .help-block Markdown enabled - - %fieldset - %legend Sign-in Restrictions - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :password_authentication_enabled_for_web do - = f.check_box :password_authentication_enabled_for_web - Password authentication enabled for web interface - .help-block - When disabled, an external authentication provider must be used. - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :password_authentication_enabled_for_git do - = f.check_box :password_authentication_enabled_for_git - Password authentication enabled for Git over HTTP(S) - .help-block - When disabled, a Personal Access Token - - if Gitlab::Auth::LDAP::Config.enabled? - or LDAP password - must be used to authenticate. - - if omniauth_enabled? && button_based_providers.any? - .form-group - = f.label :enabled_oauth_sign_in_sources, 'Enabled OAuth sign-in sources', class: 'control-label col-sm-2' - .col-sm-10 - .btn-group{ data: { toggle: 'buttons' } } - - oauth_providers_checkboxes.each do |source| - = source - .form-group - = f.label :two_factor_authentication, 'Two-factor authentication', class: 'control-label col-sm-2' - .col-sm-10 - .checkbox - = f.label :require_two_factor_authentication do - = f.check_box :require_two_factor_authentication - Require all users to setup Two-factor authentication - .form-group - = f.label :two_factor_authentication, 'Two-factor grace period (hours)', class: 'control-label col-sm-2' - .col-sm-10 - = f.number_field :two_factor_grace_period, min: 0, class: 'form-control', placeholder: '0' - .help-block Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication - .form-group - = f.label :home_page_url, 'Home page URL', class: 'control-label col-sm-2' - .col-sm-10 - = f.text_field :home_page_url, class: 'form-control', placeholder: 'http://company.example.com', :'aria-describedby' => 'home_help_block' - %span.help-block#home_help_block We will redirect non-logged in users to this page - .form-group - = f.label :after_sign_out_path, class: 'control-label col-sm-2' - .col-sm-10 - = f.text_field :after_sign_out_path, class: 'form-control', placeholder: 'http://company.example.com', :'aria-describedby' => 'after_sign_out_path_help_block' - %span.help-block#after_sign_out_path_help_block We will redirect users to this page after they sign out - .form-group - = f.label :sign_in_text, class: 'control-label col-sm-2' - .col-sm-10 - = f.text_area :sign_in_text, class: 'form-control', rows: 4 - .help-block Markdown enabled - - %fieldset - %legend Help Page - .form-group - = f.label :help_page_text, class: 'control-label col-sm-2' - .col-sm-10 - = f.text_area :help_page_text, class: 'form-control', rows: 4 - .help-block Markdown enabled - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :help_page_hide_commercial_content do - = f.check_box :help_page_hide_commercial_content - Hide marketing-related entries from help - .form-group - = f.label :help_page_support_url, 'Support page URL', class: 'control-label col-sm-2' - .col-sm-10 - = f.text_field :help_page_support_url, class: 'form-control', placeholder: 'http://company.example.com/getting-help', :'aria-describedby' => 'support_help_block' - %span.help-block#support_help_block Alternate support URL for help page - - %fieldset - %legend Pages - .form-group - = f.label :max_pages_size, 'Maximum size of pages (MB)', class: 'control-label col-sm-2' - .col-sm-10 - = f.number_field :max_pages_size, class: 'form-control' - .help-block 0 for unlimited - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :pages_domain_verification_enabled do - = f.check_box :pages_domain_verification_enabled - Require users to prove ownership of custom domains - .help-block - Domain verification is an essential security measure for public GitLab - sites. Users are required to demonstrate they control a domain before - it is enabled - = link_to icon('question-circle'), help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') - - %fieldset %legend Continuous Integration and Deployment .form-group .col-sm-offset-2.col-sm-10 diff --git a/app/views/admin/application_settings/_help_page.html.haml b/app/views/admin/application_settings/_help_page.html.haml new file mode 100644 index 00000000000..3bc101ddf04 --- /dev/null +++ b/app/views/admin/application_settings/_help_page.html.haml @@ -0,0 +1,22 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + = f.label :help_page_text, class: 'control-label col-sm-2' + .col-sm-10 + = f.text_area :help_page_text, class: 'form-control', rows: 4 + .help-block Markdown enabled + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :help_page_hide_commercial_content do + = f.check_box :help_page_hide_commercial_content + Hide marketing-related entries from help + .form-group + = f.label :help_page_support_url, 'Support page URL', class: 'control-label col-sm-2' + .col-sm-10 + = f.text_field :help_page_support_url, class: 'form-control', placeholder: 'http://company.example.com/getting-help', :'aria-describedby' => 'support_help_block' + %span.help-block#support_help_block Alternate support URL for help page + + = f.submit 'Save changes', class: "btn btn-success" diff --git a/app/views/admin/application_settings/_pages.html.haml b/app/views/admin/application_settings/_pages.html.haml new file mode 100644 index 00000000000..b28ecf9a039 --- /dev/null +++ b/app/views/admin/application_settings/_pages.html.haml @@ -0,0 +1,22 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + = f.label :max_pages_size, 'Maximum size of pages (MB)', class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :max_pages_size, class: 'form-control' + .help-block 0 for unlimited + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :pages_domain_verification_enabled do + = f.check_box :pages_domain_verification_enabled + Require users to prove ownership of custom domains + .help-block + Domain verification is an essential security measure for public GitLab + sites. Users are required to demonstrate they control a domain before + it is enabled + = link_to icon('question-circle'), help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record') + + = f.submit 'Save changes', class: "btn btn-success" diff --git a/app/views/admin/application_settings/_signin.html.haml b/app/views/admin/application_settings/_signin.html.haml new file mode 100644 index 00000000000..864e64b5fa9 --- /dev/null +++ b/app/views/admin/application_settings/_signin.html.haml @@ -0,0 +1,59 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :password_authentication_enabled_for_web do + = f.check_box :password_authentication_enabled_for_web + Password authentication enabled for web interface + .help-block + When disabled, an external authentication provider must be used. + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :password_authentication_enabled_for_git do + = f.check_box :password_authentication_enabled_for_git + Password authentication enabled for Git over HTTP(S) + .help-block + When disabled, a Personal Access Token + - if Gitlab::Auth::LDAP::Config.enabled? + or LDAP password + must be used to authenticate. + - if omniauth_enabled? && button_based_providers.any? + .form-group + = f.label :enabled_oauth_sign_in_sources, 'Enabled OAuth sign-in sources', class: 'control-label col-sm-2' + .col-sm-10 + .btn-group{ data: { toggle: 'buttons' } } + - oauth_providers_checkboxes.each do |source| + = source + .form-group + = f.label :two_factor_authentication, 'Two-factor authentication', class: 'control-label col-sm-2' + .col-sm-10 + .checkbox + = f.label :require_two_factor_authentication do + = f.check_box :require_two_factor_authentication + Require all users to setup Two-factor authentication + .form-group + = f.label :two_factor_authentication, 'Two-factor grace period (hours)', class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :two_factor_grace_period, min: 0, class: 'form-control', placeholder: '0' + .help-block Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication + .form-group + = f.label :home_page_url, 'Home page URL', class: 'control-label col-sm-2' + .col-sm-10 + = f.text_field :home_page_url, class: 'form-control', placeholder: 'http://company.example.com', :'aria-describedby' => 'home_help_block' + %span.help-block#home_help_block We will redirect non-logged in users to this page + .form-group + = f.label :after_sign_out_path, class: 'control-label col-sm-2' + .col-sm-10 + = f.text_field :after_sign_out_path, class: 'form-control', placeholder: 'http://company.example.com', :'aria-describedby' => 'after_sign_out_path_help_block' + %span.help-block#after_sign_out_path_help_block We will redirect users to this page after they sign out + .form-group + = f.label :sign_in_text, class: 'control-label col-sm-2' + .col-sm-10 + = f.text_area :sign_in_text, class: 'form-control', rows: 4 + .help-block Markdown enabled + + = f.submit 'Save changes', class: "btn btn-success" diff --git a/app/views/admin/application_settings/_signup.html.haml b/app/views/admin/application_settings/_signup.html.haml new file mode 100644 index 00000000000..85f311dd894 --- /dev/null +++ b/app/views/admin/application_settings/_signup.html.haml @@ -0,0 +1,58 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :signup_enabled do + = f.check_box :signup_enabled + Sign-up enabled + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :send_user_confirmation_email do + = f.check_box :send_user_confirmation_email + Send confirmation email on sign-up + .form-group + = f.label :domain_whitelist, 'Whitelisted domains for sign-ups', class: 'control-label col-sm-2' + .col-sm-10 + = f.text_area :domain_whitelist_raw, placeholder: 'domain.com', class: 'form-control', rows: 8 + .help-block ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com + .form-group + = f.label :domain_blacklist_enabled, 'Domain Blacklist', class: 'control-label col-sm-2' + .col-sm-10 + .checkbox + = f.label :domain_blacklist_enabled do + = f.check_box :domain_blacklist_enabled + Enable domain blacklist for sign ups + .form-group + .col-sm-offset-2.col-sm-10 + .radio + = label_tag :blacklist_type_file do + = radio_button_tag :blacklist_type, :file + .option-title + Upload blacklist file + .radio + = label_tag :blacklist_type_raw do + = radio_button_tag :blacklist_type, :raw, @application_setting.domain_blacklist.present? || @application_setting.domain_blacklist.blank? + .option-title + Enter blacklist manually + .form-group.blacklist-file + = f.label :domain_blacklist_file, 'Blacklist file', class: 'control-label col-sm-2' + .col-sm-10 + = f.file_field :domain_blacklist_file, class: 'form-control', accept: '.txt,.conf' + .help-block Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines or commas for multiple entries. + .form-group.blacklist-raw + = f.label :domain_blacklist, 'Blacklisted domains for sign-ups', class: 'control-label col-sm-2' + .col-sm-10 + = f.text_area :domain_blacklist_raw, placeholder: 'domain.com', class: 'form-control', rows: 8 + .help-block Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com + + .form-group + = f.label :after_sign_up_text, class: 'control-label col-sm-2' + .col-sm-10 + = f.text_area :after_sign_up_text, class: 'form-control', rows: 4 + .help-block Markdown enabled + + = f.submit 'Save changes', class: "btn btn-success" diff --git a/app/views/admin/application_settings/_visibility_and_access.html.haml b/app/views/admin/application_settings/_visibility_and_access.html.haml new file mode 100644 index 00000000000..cbc779548f6 --- /dev/null +++ b/app/views/admin/application_settings/_visibility_and_access.html.haml @@ -0,0 +1,66 @@ += form_for @application_setting, url: admin_application_settings_path, html: { class: 'form-horizontal fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + = f.label :default_branch_protection, class: 'control-label col-sm-2' + .col-sm-10 + = f.select :default_branch_protection, options_for_select(Gitlab::Access.protection_options, @application_setting.default_branch_protection), {}, class: 'form-control' + .form-group.visibility-level-setting + = f.label :default_project_visibility, class: 'control-label col-sm-2' + .col-sm-10 + = render('shared/visibility_radios', model_method: :default_project_visibility, form: f, selected_level: @application_setting.default_project_visibility, form_model: Project.new) + .form-group.visibility-level-setting + = f.label :default_snippet_visibility, class: 'control-label col-sm-2' + .col-sm-10 + = render('shared/visibility_radios', model_method: :default_snippet_visibility, form: f, selected_level: @application_setting.default_snippet_visibility, form_model: ProjectSnippet.new) + .form-group.visibility-level-setting + = f.label :default_group_visibility, class: 'control-label col-sm-2' + .col-sm-10 + = render('shared/visibility_radios', model_method: :default_group_visibility, form: f, selected_level: @application_setting.default_group_visibility, form_model: Group.new) + .form-group + = f.label :restricted_visibility_levels, class: 'control-label col-sm-2' + .col-sm-10 + - checkbox_name = 'application_setting[restricted_visibility_levels][]' + = hidden_field_tag(checkbox_name) + - restricted_level_checkboxes('restricted-visibility-help', checkbox_name).each do |level| + .checkbox + = level + %span.help-block#restricted-visibility-help + Selected levels cannot be used by non-admin users for projects or snippets. + If the public level is restricted, user profiles are only visible to logged in users. + .form-group + = f.label :import_sources, class: 'control-label col-sm-2' + .col-sm-10 + - import_sources_checkboxes('import-sources-help').each do |source| + .checkbox= source + %span.help-block#import-sources-help + Enabled sources for code import during project creation. OmniAuth must be configured for GitHub + = link_to "(?)", help_page_path("integration/github") + , Bitbucket + = link_to "(?)", help_page_path("integration/bitbucket") + and GitLab.com + = link_to "(?)", help_page_path("integration/gitlab") + + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :project_export_enabled do + = f.check_box :project_export_enabled + Project export enabled + + .form-group + %label.control-label.col-sm-2 Enabled Git access protocols + .col-sm-10 + = select(:application_setting, :enabled_git_access_protocol, [['Both SSH and HTTP(S)', nil], ['Only SSH', 'ssh'], ['Only HTTP(S)', 'http']], {}, class: 'form-control') + %span.help-block#clone-protocol-help + Allow only the selected protocols to be used for Git access. + + - ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| + - field_name = :"#{type}_key_restriction" + .form-group + = f.label field_name, "#{type.upcase} SSH keys", class: 'control-label col-sm-2' + .col-sm-10 + = f.select field_name, key_restriction_options_for_select(type), {}, class: 'form-control' + + = f.submit 'Save changes', class: "btn btn-success" diff --git a/app/views/admin/application_settings/show.html.haml b/app/views/admin/application_settings/show.html.haml index ecc46d86afe..82d97f90248 100644 --- a/app/views/admin/application_settings/show.html.haml +++ b/app/views/admin/application_settings/show.html.haml @@ -1,5 +1,73 @@ +- breadcrumb_title "Settings" - page_title "Settings" +- @content_class = "limit-container-width" unless fluid_layout +- expanded = Rails.env.test? -%h3.page-title Settings -%hr -= render 'form' +%section.settings.as-visibility-access.no-animate#js-visibility-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Visibility and access controls') + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + = _('Set default and restrict visibility levels. Configure import sources and git access protocol.') + .settings-content + = render 'visibility_and_access' + +%section.settings.as-account-limit.no-animate#js-account-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Account and limit settings') + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + = _('Session expiration, projects limit and attachment size.') + .settings-content + = render 'account_and_limit' + +%section.settings.as-signup.no-animate#js-signup-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Sign-up restrictions') + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + = _('Configure the way a user creates a new account.') + .settings-content + = render 'signup' + +%section.settings.as-signin.no-animate#js-signin-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Sign-in restrictions') + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + = _('Set requirements for a user to sign-in. Enable mandatory two-factor authentication.') + .settings-content + = render 'signin' + +%section.settings.as-help-page.no-animate#js-help-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Help page') + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + = _('Help page text and support page url.') + .settings-content + = render 'help_page' + +%section.settings.as-pages.no-animate#js-pages-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Pages') + %button.btn.js-settings-toggle + = expanded ? 'Collapse' : 'Expand' + %p + = _('Size and domain settings for static websites') + .settings-content + = render 'pages' + +.prepend-top-20 + = render 'form' diff --git a/app/views/ci/variables/_variable_row.html.haml b/app/views/ci/variables/_variable_row.html.haml index 15201780451..5d4229c80af 100644 --- a/app/views/ci/variables/_variable_row.html.haml +++ b/app/views/ci/variables/_variable_row.html.haml @@ -10,7 +10,7 @@ - id_input_name = "#{form_field}[variables_attributes][][id]" - destroy_input_name = "#{form_field}[variables_attributes][][_destroy]" - key_input_name = "#{form_field}[variables_attributes][][key]" -- value_input_name = "#{form_field}[variables_attributes][][value]" +- value_input_name = "#{form_field}[variables_attributes][][secret_value]" - protected_input_name = "#{form_field}[variables_attributes][][protected]" %li.js-row.ci-variable-row{ data: { is_persisted: "#{!id.nil?}" } } diff --git a/app/views/layouts/_page.html.haml b/app/views/layouts/_page.html.haml index f0963cf9da8..f67a8878c80 100644 --- a/app/views/layouts/_page.html.haml +++ b/app/views/layouts/_page.html.haml @@ -6,6 +6,7 @@ .mobile-overlay .alert-wrapper = render "layouts/broadcast" + = render 'layouts/header/read_only_banner' = yield :flash_message - unless @hide_breadcrumbs = render "layouts/nav/breadcrumbs" diff --git a/app/views/layouts/header/_read_only_banner.html.haml b/app/views/layouts/header/_read_only_banner.html.haml new file mode 100644 index 00000000000..f3d563c362f --- /dev/null +++ b/app/views/layouts/header/_read_only_banner.html.haml @@ -0,0 +1,7 @@ +- message = read_only_message +- if message + .flash-container.flash-container-page + .flash-notice + %div{ class: (container_class) } + %span + = message diff --git a/app/views/notify/push_to_merge_request_email.html.haml b/app/views/notify/push_to_merge_request_email.html.haml new file mode 100644 index 00000000000..5cc6f21c0f3 --- /dev/null +++ b/app/views/notify/push_to_merge_request_email.html.haml @@ -0,0 +1,26 @@ +%h3 + New commits were pushed to the merge request + = link_to(@merge_request.to_reference, project_merge_request_url(@merge_request.target_project, @merge_request)) + by #{@current_user.name} + +- if @existing_commits.any? + - count = @existing_commits.size + %ul + %li + - if count.one? + - commit_id = @existing_commits.first[:short_id] + = link_to(commit_id, project_commit_url(@merge_request.target_project, commit_id)) + - else + = link_to(project_compare_url(@merge_request.target_project, from: @existing_commits.first[:short_id], to: @existing_commits.last[:short_id])) do + #{@existing_commits.first[:short_id]}...#{@existing_commits.last[:short_id]} + = precede ' - ' do + - commits_text = "#{count} commit".pluralize(count) + #{commits_text} from branch `#{@merge_request.target_branch}` + +- if @new_commits.any? + %ul + - @new_commits.each do |commit| + %li + = link_to(commit[:short_id], project_commit_url(@merge_request.target_project, commit[:short_id])) + = precede ' - ' do + #{commit[:title]} diff --git a/app/views/notify/push_to_merge_request_email.text.haml b/app/views/notify/push_to_merge_request_email.text.haml new file mode 100644 index 00000000000..d7722e5f41f --- /dev/null +++ b/app/views/notify/push_to_merge_request_email.text.haml @@ -0,0 +1,13 @@ +New commits were pushed to the merge request #{@merge_request.to_reference} by #{@current_user.name} +\ +#{url_for(project_merge_request_url(@merge_request.target_project, @merge_request))} +\ +- if @existing_commits.any? + - count = @existing_commits.size + - commits_id = count.one? ? @existing_commits.first[:short_id] : "#{@existing_commits.first[:short_id]}...#{@existing_commits.last[:short_id]}" + - commits_text = "#{count} commit".pluralize(count) + + * #{commits_id} - #{commits_text} from branch `#{@merge_request.target_branch}` +\ +- @new_commits.each do |commit| + * #{commit[:short_id]} - #{raw commit[:title]} diff --git a/app/views/projects/jobs/_sidebar.html.haml b/app/views/projects/jobs/_sidebar.html.haml index e779473c239..ecf186e3dc8 100644 --- a/app/views/projects/jobs/_sidebar.html.haml +++ b/app/views/projects/jobs/_sidebar.html.haml @@ -35,7 +35,7 @@ = link_to download_project_job_artifacts_path(@project, @build), rel: 'nofollow', download: '', class: 'btn btn-sm btn-default' do Download - - if @build.artifacts_metadata? + - if @build.browsable_artifacts? = link_to browse_project_job_artifacts_path(@project, @build), class: 'btn btn-sm btn-default' do Browse diff --git a/app/views/projects/pages/_https_only.html.haml b/app/views/projects/pages/_https_only.html.haml new file mode 100644 index 00000000000..6a3ffce949f --- /dev/null +++ b/app/views/projects/pages/_https_only.html.haml @@ -0,0 +1,10 @@ += form_for @project, url: namespace_project_pages_path(@project.namespace.becomes(Namespace), @project), html: { class: 'inline', title: pages_https_only_title } do |f| + = f.check_box :pages_https_only, class: 'pull-left', disabled: pages_https_only_disabled? + + .prepend-left-20 + = f.label :pages_https_only, class: pages_https_only_label_class do + %strong Force domains with SSL certificates to use HTTPS + + - unless pages_https_only_disabled? + .prepend-top-10 + = f.submit 'Save', class: 'btn btn-success' diff --git a/app/views/projects/pages/show.html.haml b/app/views/projects/pages/show.html.haml index 04e647c0dc6..f17d9d24db6 100644 --- a/app/views/projects/pages/show.html.haml +++ b/app/views/projects/pages/show.html.haml @@ -13,6 +13,9 @@ Combined with the power of GitLab CI and the help of GitLab Runner you can deploy static pages for your individual projects, your user or your group. +- if Gitlab.config.pages.external_https + = render 'https_only' + %hr.clearfix = render 'access' diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index f65e8385ac8..9a11cdb121e 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -39,6 +39,10 @@ - github_importer:github_import_stage_import_pull_requests - github_importer:github_import_stage_import_repository +- object_storage_upload +- object_storage:object_storage_background_move +- object_storage:object_storage_migrate_uploads + - pipeline_cache:expire_job_cache - pipeline_cache:expire_pipeline_cache - pipeline_creation:create_pipeline diff --git a/app/workers/concerns/object_storage_queue.rb b/app/workers/concerns/object_storage_queue.rb new file mode 100644 index 00000000000..a80f473a6d4 --- /dev/null +++ b/app/workers/concerns/object_storage_queue.rb @@ -0,0 +1,8 @@ +# Concern for setting Sidekiq settings for the various GitLab ObjectStorage workers. +module ObjectStorageQueue + extend ActiveSupport::Concern + + included do + queue_namespace :object_storage + end +end diff --git a/app/workers/object_storage/background_move_worker.rb b/app/workers/object_storage/background_move_worker.rb new file mode 100644 index 00000000000..9c4d72e0ecf --- /dev/null +++ b/app/workers/object_storage/background_move_worker.rb @@ -0,0 +1,29 @@ +module ObjectStorage + class BackgroundMoveWorker + include ApplicationWorker + include ObjectStorageQueue + + sidekiq_options retry: 5 + + def perform(uploader_class_name, subject_class_name, file_field, subject_id) + uploader_class = uploader_class_name.constantize + subject_class = subject_class_name.constantize + + return unless uploader_class < ObjectStorage::Concern + return unless uploader_class.object_store_enabled? + return unless uploader_class.background_upload_enabled? + + subject = subject_class.find(subject_id) + uploader = build_uploader(subject, file_field&.to_sym) + uploader.migrate!(ObjectStorage::Store::REMOTE) + end + + def build_uploader(subject, mount_point) + case subject + when Upload then subject.build_uploader(mount_point) + else + subject.send(mount_point) # rubocop:disable GitlabSecurity/PublicSend + end + end + end +end diff --git a/app/workers/object_storage/migrate_uploads_worker.rb b/app/workers/object_storage/migrate_uploads_worker.rb new file mode 100644 index 00000000000..01ed123e6c8 --- /dev/null +++ b/app/workers/object_storage/migrate_uploads_worker.rb @@ -0,0 +1,202 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Style/Documentation + +module ObjectStorage + class MigrateUploadsWorker + include ApplicationWorker + include ObjectStorageQueue + + SanityCheckError = Class.new(StandardError) + + class Upload < ActiveRecord::Base + # Upper limit for foreground checksum processing + CHECKSUM_THRESHOLD = 100.megabytes + + belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations + + validates :size, presence: true + validates :path, presence: true + validates :model, presence: true + validates :uploader, presence: true + + before_save :calculate_checksum!, if: :foreground_checksummable? + after_commit :schedule_checksum, if: :checksummable? + + scope :stored_locally, -> { where(store: [nil, ObjectStorage::Store::LOCAL]) } + scope :stored_remotely, -> { where(store: ObjectStorage::Store::REMOTE) } + + def self.hexdigest(path) + Digest::SHA256.file(path).hexdigest + end + + def absolute_path + raise ObjectStorage::RemoteStoreError, "Remote object has no absolute path." unless local? + return path unless relative_path? + + uploader_class.absolute_path(self) + end + + def calculate_checksum! + self.checksum = nil + return unless checksummable? + + self.checksum = self.class.hexdigest(absolute_path) + end + + def build_uploader(mounted_as = nil) + uploader_class.new(model, mounted_as).tap do |uploader| + uploader.upload = self + uploader.retrieve_from_store!(identifier) + end + end + + def exist? + File.exist?(absolute_path) + end + + def local? + return true if store.nil? + + store == ObjectStorage::Store::LOCAL + end + + private + + def checksummable? + checksum.nil? && local? && exist? + end + + def foreground_checksummable? + checksummable? && size <= CHECKSUM_THRESHOLD + end + + def schedule_checksum + UploadChecksumWorker.perform_async(id) + end + + def relative_path? + !path.start_with?('/') + end + + def identifier + File.basename(path) + end + + def uploader_class + Object.const_get(uploader) + end + end + + class MigrationResult + attr_reader :upload + attr_accessor :error + + def initialize(upload, error = nil) + @upload, @error = upload, error + end + + def success? + error.nil? + end + + def to_s + success? ? "Migration successful." : "Error while migrating #{upload.id}: #{error.message}" + end + end + + module Report + class MigrationFailures < StandardError + attr_reader :errors + + def initialize(errors) + @errors = errors + end + + def message + errors.map(&:message).join("\n") + end + end + + def report!(results) + success, failures = results.partition(&:success?) + + Rails.logger.info header(success, failures) + Rails.logger.warn failures(failures) + + raise MigrationFailures.new(failures.map(&:error)) if failures.any? + end + + def header(success, failures) + "Migrated #{success.count}/#{success.count + failures.count} files." + end + + def failures(failures) + failures.map { |f| "\t#{f}" }.join('\n') + end + end + + include Report + + def self.enqueue!(uploads, mounted_as, to_store) + sanity_check!(uploads, mounted_as) + + perform_async(uploads.ids, mounted_as, to_store) + end + + # We need to be sure all the uploads are for the same uploader and model type + # and that the mount point exists if provided. + # + def self.sanity_check!(uploads, mounted_as) + upload = uploads.first + + uploader_class = upload.uploader.constantize + model_class = uploads.first.model_type.constantize + + uploader_types = uploads.map(&:uploader).uniq + model_types = uploads.map(&:model_type).uniq + model_has_mount = mounted_as.nil? || model_class.uploaders[mounted_as] == uploader_class + + raise(SanityCheckError, "Multiple uploaders found: #{uploader_types}") unless uploader_types.count == 1 + raise(SanityCheckError, "Multiple model types found: #{model_types}") unless model_types.count == 1 + raise(SanityCheckError, "Mount point #{mounted_as} not found in #{model_class}.") unless model_has_mount + end + + def perform(ids, mounted_as, to_store) + @mounted_as = mounted_as&.to_sym + @to_store = to_store + + uploads = Upload.preload(:model).where(id: ids) + + sanity_check!(uploads) + results = migrate(uploads) + + report!(results) + rescue SanityCheckError => e + # do not retry: the job is insane + Rails.logger.warn "#{self.class}: Sanity check error (#{e.message})" + end + + def sanity_check!(uploads) + self.class.sanity_check!(uploads, @mounted_as) + end + + def build_uploaders(uploads) + uploads.map { |upload| upload.build_uploader(@mounted_as) } + end + + def migrate(uploads) + build_uploaders(uploads).map(&method(:process_uploader)) + end + + def process_uploader(uploader) + MigrationResult.new(uploader.upload).tap do |result| + begin + uploader.migrate!(@to_store) + rescue => e + result.error = e + end + end + end + end +end diff --git a/app/workers/object_storage_upload_worker.rb b/app/workers/object_storage_upload_worker.rb new file mode 100644 index 00000000000..5c80f34069c --- /dev/null +++ b/app/workers/object_storage_upload_worker.rb @@ -0,0 +1,21 @@ +# @Deprecated - remove once the `object_storage_upload` queue is empty +# The queue has been renamed `object_storage:object_storage_background_upload` +# +class ObjectStorageUploadWorker + include ApplicationWorker + + sidekiq_options retry: 5 + + def perform(uploader_class_name, subject_class_name, file_field, subject_id) + uploader_class = uploader_class_name.constantize + subject_class = subject_class_name.constantize + + return unless uploader_class < ObjectStorage::Concern + return unless uploader_class.object_store_enabled? + return unless uploader_class.background_upload_enabled? + + subject = subject_class.find(subject_id) + uploader = subject.public_send(file_field) # rubocop:disable GitlabSecurity/PublicSend + uploader.migrate!(ObjectStorage::Store::REMOTE) + end +end diff --git a/bin/rails b/bin/rails index 0138d79b751..228f812ccaf 100755 --- a/bin/rails +++ b/bin/rails @@ -1,9 +1,14 @@ #!/usr/bin/env ruby -begin - load File.expand_path('../spring', __FILE__) -rescue LoadError => e - raise unless e.message.include?('spring') + +# Remove this block when upgraded to rails 5.0. +unless %w[1 true].include?(ENV["RAILS5"]) + begin + load File.expand_path('../spring', __FILE__) + rescue LoadError => e + raise unless e.message.include?('spring') + end end -APP_PATH = File.expand_path('../../config/application', __FILE__) + +APP_PATH = File.expand_path('../config/application', __dir__) require_relative '../config/boot' require 'rails/commands' @@ -1,9 +1,14 @@ #!/usr/bin/env ruby -begin - load File.expand_path('../spring', __FILE__) -rescue LoadError => e - raise unless e.message.include?('spring') + +# Remove this block when upgraded to rails 5.0. +unless %w[1 true].include?(ENV["RAILS5"]) + begin + load File.expand_path('../spring', __FILE__) + rescue LoadError => e + raise unless e.message.include?('spring') + end end + require_relative '../config/boot' require 'rake' Rake.application.run diff --git a/bin/setup b/bin/setup index 6cb2d7f1e3a..c60c1267e06 100755 --- a/bin/setup +++ b/bin/setup @@ -1,29 +1,61 @@ #!/usr/bin/env ruby -require 'pathname' + +def rails5? + %w[1 true].include?(ENV["RAILS5"]) +end + +require "pathname" # path to your application root. -APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) +APP_ROOT = Pathname.new File.expand_path("../../", __FILE__) + +if rails5? + def system!(*args) + system(*args) || abort("\n== Command #{args} failed ==") + end +end Dir.chdir APP_ROOT do # This script is a starting point to setup your application. # Add necessary setup steps to this file: puts "== Installing dependencies ==" - system "gem install bundler --conservative" - system "bundle check || bundle install" + + if rails5? + system! "gem install bundler --conservative" + system("bundle check") || system!("bundle install") + else + system "gem install bundler --conservative" + system "bundle check || bundle install" + end # puts "\n== Copying sample files ==" # unless File.exist?("config/database.yml") - # system "cp config/database.yml.sample config/database.yml" + # cp "config/database.yml.sample", "config/database.yml" # end puts "\n== Preparing database ==" - system "bin/rake db:reset" + + if rails5? + system! "bin/rails db:setup" + else + system "bin/rake db:reset" + end puts "\n== Removing old logs and tempfiles ==" - system "rm -f log/*" - system "rm -rf tmp/cache" + + if rails5? + system! "bin/rails log:clear tmp:clear" + else + system "rm -f log/*" + system "rm -rf tmp/cache" + end puts "\n== Restarting application server ==" - system "touch tmp/restart.txt" + + if rails5? + system! "bin/rails restart" + else + system "touch tmp/restart.txt" + end end diff --git a/bin/update b/bin/update new file mode 100755 index 00000000000..a8e4462f203 --- /dev/null +++ b/bin/update @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +require 'pathname' +require 'fileutils' +include FileUtils + +# path to your application root. +APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) + +def system!(*args) + system(*args) || abort("\n== Command #{args} failed ==") +end + +chdir APP_ROOT do + # This script is a way to update your development environment automatically. + # Add necessary update steps to this file. + + puts '== Installing dependencies ==' + system! 'gem install bundler --conservative' + system('bundle check') || system!('bundle install') + + puts "\n== Updating database ==" + system! 'bin/rails db:migrate' + + puts "\n== Removing old logs and tempfiles ==" + system! 'bin/rails log:clear tmp:clear' + + puts "\n== Restarting application server ==" + system! 'bin/rails restart' +end diff --git a/changelogs/unreleased-ee/39118-dynamic-pipeline-variables-fe.yml b/changelogs/unreleased-ee/39118-dynamic-pipeline-variables-fe.yml deleted file mode 100644 index a38b447e345..00000000000 --- a/changelogs/unreleased-ee/39118-dynamic-pipeline-variables-fe.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Update CI/CD secret variables list to be dynamic and save without reloading - the page -merge_request: 4110 -author: -type: added diff --git a/changelogs/unreleased-ee/4378-fix-cluster-js-not-running-on-update-page.yml b/changelogs/unreleased-ee/4378-fix-cluster-js-not-running-on-update-page.yml deleted file mode 100644 index bbb6cbd05be..00000000000 --- a/changelogs/unreleased-ee/4378-fix-cluster-js-not-running-on-update-page.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix JavaScript bundle running on Cluster update/destroy pages -merge_request: -author: -type: fixed diff --git a/changelogs/unreleased-ee/bvl-external-policy-classification.yml b/changelogs/unreleased-ee/bvl-external-policy-classification.yml deleted file mode 100644 index 074629c8c12..00000000000 --- a/changelogs/unreleased-ee/bvl-external-policy-classification.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Authorize project access with an external service -merge_request: 4675 -author: -type: added diff --git a/changelogs/unreleased/23460-send-email-when-pushing-more-commits-to-the-merge-request.yml b/changelogs/unreleased/23460-send-email-when-pushing-more-commits-to-the-merge-request.yml new file mode 100644 index 00000000000..a62137ea2c9 --- /dev/null +++ b/changelogs/unreleased/23460-send-email-when-pushing-more-commits-to-the-merge-request.yml @@ -0,0 +1,5 @@ +--- +title: Send notification emails when push to a merge request +merge_request: 7610 +author: YarNayar +type: feature diff --git a/changelogs/unreleased/40781-os-to-ce.yml b/changelogs/unreleased/40781-os-to-ce.yml new file mode 100644 index 00000000000..4a364292c60 --- /dev/null +++ b/changelogs/unreleased/40781-os-to-ce.yml @@ -0,0 +1,5 @@ +--- +title: Add object storage support for LFS objects, CI artifacts, and uploads. +merge_request: 17358 +author: +type: added diff --git a/changelogs/unreleased/43316-controller-parameters-handling-sensitive-information-should-use-a-more-specific-name.yml b/changelogs/unreleased/43316-controller-parameters-handling-sensitive-information-should-use-a-more-specific-name.yml new file mode 100644 index 00000000000..de1cee6e436 --- /dev/null +++ b/changelogs/unreleased/43316-controller-parameters-handling-sensitive-information-should-use-a-more-specific-name.yml @@ -0,0 +1,5 @@ +--- +title: Use specific names for filtered CI variable controller parameters +merge_request: 17796 +author: +type: other diff --git a/changelogs/unreleased/43482-enabling-auto-devops-on-an-empty-project-gives-you-wrong-information.yml b/changelogs/unreleased/43482-enabling-auto-devops-on-an-empty-project-gives-you-wrong-information.yml new file mode 100644 index 00000000000..889fd008bad --- /dev/null +++ b/changelogs/unreleased/43482-enabling-auto-devops-on-an-empty-project-gives-you-wrong-information.yml @@ -0,0 +1,5 @@ +--- +title: Add empty repo check before running AutoDevOps pipeline +merge_request: 17605 +author: +type: changed diff --git a/changelogs/unreleased/43512-add-support-for-omniauth-jwt-provider.yml b/changelogs/unreleased/43512-add-support-for-omniauth-jwt-provider.yml new file mode 100644 index 00000000000..039d3de7168 --- /dev/null +++ b/changelogs/unreleased/43512-add-support-for-omniauth-jwt-provider.yml @@ -0,0 +1,5 @@ +--- +title: Adds support for OmniAuth JWT provider +merge_request: 17774 +author: +type: added diff --git a/changelogs/unreleased/43525-limit-number-of-failed-logins-using-ldap.yml b/changelogs/unreleased/43525-limit-number-of-failed-logins-using-ldap.yml new file mode 100644 index 00000000000..f30fea3c4a7 --- /dev/null +++ b/changelogs/unreleased/43525-limit-number-of-failed-logins-using-ldap.yml @@ -0,0 +1,5 @@ +--- +title: Limit the number of failed logins when using LDAP for authentication +merge_request: 43525 +author: +type: added diff --git a/changelogs/unreleased/43552-user-owned-projects-query-performance-improvement.yml b/changelogs/unreleased/43552-user-owned-projects-query-performance-improvement.yml new file mode 100644 index 00000000000..39f92c281ad --- /dev/null +++ b/changelogs/unreleased/43552-user-owned-projects-query-performance-improvement.yml @@ -0,0 +1,5 @@ +--- +title: Improves the performance of projects list page +merge_request: 17934 +author: +type: performance diff --git a/changelogs/unreleased/43771-improve-avatar-error-message.yml b/changelogs/unreleased/43771-improve-avatar-error-message.yml new file mode 100644 index 00000000000..1fae10f4d1f --- /dev/null +++ b/changelogs/unreleased/43771-improve-avatar-error-message.yml @@ -0,0 +1,5 @@ +--- +title: Change avatar error message to include allowed file formats +merge_request: 17747 +author: Fabian Schneider +type: changed diff --git a/changelogs/unreleased/44218-add-internationalization-support-for-the-prometheus-merge-request-widget.yml b/changelogs/unreleased/44218-add-internationalization-support-for-the-prometheus-merge-request-widget.yml new file mode 100644 index 00000000000..12c73281998 --- /dev/null +++ b/changelogs/unreleased/44218-add-internationalization-support-for-the-prometheus-merge-request-widget.yml @@ -0,0 +1,5 @@ +--- +title: Added i18n support for the prometheus memory widget +merge_request: 17753 +author: +type: other diff --git a/changelogs/unreleased/44232-docs-for-runner-ip-address.yml b/changelogs/unreleased/44232-docs-for-runner-ip-address.yml deleted file mode 100644 index 82485d31b24..00000000000 --- a/changelogs/unreleased/44232-docs-for-runner-ip-address.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Add documentation for runner IP address (#44232) -merge_request: 17837 -author: -type: other diff --git a/changelogs/unreleased/44587-autolinking-includes-trailing-exclamation-marks.yml b/changelogs/unreleased/44587-autolinking-includes-trailing-exclamation-marks.yml new file mode 100644 index 00000000000..636fde601ee --- /dev/null +++ b/changelogs/unreleased/44587-autolinking-includes-trailing-exclamation-marks.yml @@ -0,0 +1,5 @@ +--- +title: Don't capture trailing punctuation when autolinking +merge_request: 17965 +author: +type: fixed diff --git a/changelogs/unreleased/44608-Cloning-a-repository-over-HTTPS-with-LDAP-credentials-causes-a-HTTP-401-Access-denied.yml b/changelogs/unreleased/44608-Cloning-a-repository-over-HTTPS-with-LDAP-credentials-causes-a-HTTP-401-Access-denied.yml new file mode 100644 index 00000000000..5afb1e3d908 --- /dev/null +++ b/changelogs/unreleased/44608-Cloning-a-repository-over-HTTPS-with-LDAP-credentials-causes-a-HTTP-401-Access-denied.yml @@ -0,0 +1,5 @@ +--- +title: 'Cloning a repository over HTTPS with LDAP credentials causes a HTTP 401 Access denied' +merge_request: !17988 +author: Horatiu Eugen Vlad +type: fixed diff --git a/changelogs/unreleased/ab-43150-users-controller-show-query-limit.yml b/changelogs/unreleased/ab-43150-users-controller-show-query-limit.yml new file mode 100644 index 00000000000..502c1176d2d --- /dev/null +++ b/changelogs/unreleased/ab-43150-users-controller-show-query-limit.yml @@ -0,0 +1,5 @@ +--- +title: Remove N+1 query for Noteable association. +merge_request: 17956 +author: +type: performance diff --git a/changelogs/unreleased/ab-44446-add-indexes-for-user-activity-queries.yml b/changelogs/unreleased/ab-44446-add-indexes-for-user-activity-queries.yml deleted file mode 100644 index 0f89c06fcee..00000000000 --- a/changelogs/unreleased/ab-44446-add-indexes-for-user-activity-queries.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Add indexes for user activity queries. -merge_request: 17890 -author: -type: performance diff --git a/changelogs/unreleased/blackst0ne-rails5-update-state_machines-activerecord-gem.yml b/changelogs/unreleased/blackst0ne-rails5-update-state_machines-activerecord-gem.yml new file mode 100644 index 00000000000..a9c6fcbf428 --- /dev/null +++ b/changelogs/unreleased/blackst0ne-rails5-update-state_machines-activerecord-gem.yml @@ -0,0 +1,5 @@ +--- +title: Bump `state_machines-activerecord` to 0.5.1 +merge_request: 17924 +author: blackst0ne +type: other diff --git a/changelogs/unreleased/dashboard-view-user-choices-issues-merge-requests.yml b/changelogs/unreleased/dashboard-view-user-choices-issues-merge-requests.yml new file mode 100644 index 00000000000..92a03070d78 --- /dev/null +++ b/changelogs/unreleased/dashboard-view-user-choices-issues-merge-requests.yml @@ -0,0 +1,5 @@ +--- +title: Add 'Assigned Issues' and 'Assigned Merge Requests' as dashboard view choices for users +merge_request: 17860 +author: Elias Werberich +type: added diff --git a/changelogs/unreleased/fix-40798-namespace-forking.yml b/changelogs/unreleased/fix-40798-namespace-forking.yml new file mode 100644 index 00000000000..095235725f8 --- /dev/null +++ b/changelogs/unreleased/fix-40798-namespace-forking.yml @@ -0,0 +1,5 @@ +--- +title: Fix forking to subgroup via API when namespace is given by name +merge_request: 17815 +author: Jan Beckmann +type: fixed diff --git a/changelogs/unreleased/fix-ci-job-auto-retry.yml b/changelogs/unreleased/fix-ci-job-auto-retry.yml deleted file mode 100644 index 442126461f0..00000000000 --- a/changelogs/unreleased/fix-ci-job-auto-retry.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Prevent auto-retry AccessDenied error from stopping transition to failed -merge_request: 17862 -author: -type: fixed diff --git a/changelogs/unreleased/increase-unicorn-memory-killer-limits.yml b/changelogs/unreleased/increase-unicorn-memory-killer-limits.yml new file mode 100644 index 00000000000..6d7d2df4f4a --- /dev/null +++ b/changelogs/unreleased/increase-unicorn-memory-killer-limits.yml @@ -0,0 +1,5 @@ +--- +title: Increase the memory limits used in the unicorn killer +merge_request: 17948 +author: +type: other diff --git a/changelogs/unreleased/jprovazn-issueref.yml b/changelogs/unreleased/jprovazn-issueref.yml new file mode 100644 index 00000000000..ee19cac7b19 --- /dev/null +++ b/changelogs/unreleased/jprovazn-issueref.yml @@ -0,0 +1,6 @@ +--- +title: Display state indicator for issuable references in non-project scope (e.g. + when referencing issuables from group scope). +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/pages_force_https.yml b/changelogs/unreleased/pages_force_https.yml new file mode 100644 index 00000000000..da7e29087f3 --- /dev/null +++ b/changelogs/unreleased/pages_force_https.yml @@ -0,0 +1,5 @@ +--- +title: Add HTTPS-only pages +merge_request: 16273 +author: rfwatson +type: added diff --git a/changelogs/unreleased/poc-upload-hashing-path.yml b/changelogs/unreleased/poc-upload-hashing-path.yml new file mode 100644 index 00000000000..7970405bea1 --- /dev/null +++ b/changelogs/unreleased/poc-upload-hashing-path.yml @@ -0,0 +1,5 @@ +--- +title: File uploads in remote storage now support project renaming. +merge_request: 4597 +author: +type: fixed diff --git a/changelogs/unreleased/tc-re-add-read-only-banner.yml b/changelogs/unreleased/tc-re-add-read-only-banner.yml new file mode 100644 index 00000000000..35bcd7e184e --- /dev/null +++ b/changelogs/unreleased/tc-re-add-read-only-banner.yml @@ -0,0 +1,5 @@ +--- +title: Add read-only banner to all pages +merge_request: 17798 +author: +type: fixed diff --git a/changelogs/unreleased/winh-deprecate-old-modal.yml b/changelogs/unreleased/winh-deprecate-old-modal.yml new file mode 100644 index 00000000000..4fae1fafbea --- /dev/null +++ b/changelogs/unreleased/winh-deprecate-old-modal.yml @@ -0,0 +1,5 @@ +--- +title: Rename modal.vue to deprecated_modal.vue +merge_request: 17438 +author: +type: other diff --git a/changelogs/unreleased/workhorse-gitaly-mandatory.yml b/changelogs/unreleased/workhorse-gitaly-mandatory.yml new file mode 100644 index 00000000000..dc33aba48aa --- /dev/null +++ b/changelogs/unreleased/workhorse-gitaly-mandatory.yml @@ -0,0 +1,5 @@ +--- +title: Make all workhorse gitaly calls opt-out +merge_request: 18002 +author: +type: other diff --git a/config.ru b/config.ru index 7b15939c6ff..405d01863ac 100644 --- a/config.ru +++ b/config.ru @@ -7,8 +7,8 @@ if defined?(Unicorn) # Unicorn self-process killer require 'unicorn/worker_killer' - min = (ENV['GITLAB_UNICORN_MEMORY_MIN'] || 300 * 1 << 20).to_i - max = (ENV['GITLAB_UNICORN_MEMORY_MAX'] || 350 * 1 << 20).to_i + min = (ENV['GITLAB_UNICORN_MEMORY_MIN'] || 400 * 1 << 20).to_i + max = (ENV['GITLAB_UNICORN_MEMORY_MAX'] || 650 * 1 << 20).to_i # Max memory size (RSS) per worker use Unicorn::WorkerKiller::Oom, min, max diff --git a/config/boot.rb b/config/boot.rb index f2830ae3166..84f390f3228 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -1,6 +1,11 @@ -require 'rubygems' +def rails5? + %w[1 true].include?(ENV["RAILS5"]) +end -# Set up gems listed in the Gemfile. -ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) +require 'rubygems' unless rails5? + +gemfile = rails5? ? "Gemfile.rails5" : "Gemfile" +ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../#{gemfile}", __dir__) +# Set up gems listed in the Gemfile. require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE']) diff --git a/config/environment.rb b/config/environment.rb index df3006d349c..487a4564b47 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -1,5 +1,11 @@ # Load the rails application -require File.expand_path('../application', __FILE__) + +# Remove this condition when upgraded to rails 5.0. +if %w[1 true].include?(ENV["RAILS5"]) + require_relative 'application' +else + require File.expand_path('../application', __FILE__) +end # Initialize the rails application Rails.application.initialize! diff --git a/config/environments/production.rb b/config/environments/production.rb index c5cbfcf64cf..9941987929c 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -9,7 +9,11 @@ Rails.application.configure do config.action_controller.perform_caching = true # Disable Rails's static asset server (Apache or nginx will already do this) - config.serve_static_files = false + if Gitlab.rails5? + config.public_file_server.enabled = false + else + config.serve_static_files = false + end # Compress JavaScripts and CSS. config.assets.js_compressor = :uglifier diff --git a/config/environments/test.rb b/config/environments/test.rb index d09e51e766a..1849c984351 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -18,7 +18,13 @@ Rails.application.configure do # Configure static asset server for tests with Cache-Control for performance config.assets.compile = false if ENV['CI'] - config.serve_static_files = true + + if Gitlab.rails5? + config.public_file_server.enabled = true + else + config.serve_static_files = true + end + config.static_cache_control = "public, max-age=3600" # Show full error reports and disable caching diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index bd696a7f2c5..a23145de3e5 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -145,18 +145,55 @@ production: &base enabled: true # The location where build artifacts are stored (default: shared/artifacts). # path: shared/artifacts + # object_store: + # enabled: false + # remote_directory: artifacts # The bucket name + # background_upload: false # Temporary option to limit automatic upload (Default: true) + # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage + # connection: + # provider: AWS # Only AWS supported at the moment + # aws_access_key_id: AWS_ACCESS_KEY_ID + # aws_secret_access_key: AWS_SECRET_ACCESS_KEY + # region: eu-central-1 ## Git LFS lfs: enabled: true # The location where LFS objects are stored (default: shared/lfs-objects). # storage_path: shared/lfs-objects + object_store: + enabled: false + remote_directory: lfs-objects # Bucket name + # background_upload: false # Temporary option to limit automatic upload (Default: true) + # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage + connection: + provider: AWS + aws_access_key_id: AWS_ACCESS_KEY_ID + aws_secret_access_key: AWS_SECRET_ACCESS_KEY + region: eu-central-1 + # Use the following options to configure an AWS compatible host + # host: 'localhost' # default: s3.amazonaws.com + # endpoint: 'http://127.0.0.1:9000' # default: nil + # path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object' ## Uploads (attachments, avatars, etc...) uploads: # The location where uploads objects are stored (default: public/). # storage_path: public/ # base_dir: uploads/-/system + object_store: + enabled: false + # remote_directory: uploads # Bucket name + # background_upload: false # Temporary option to limit automatic upload (Default: true) + # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage + # connection: + # provider: AWS + # aws_access_key_id: AWS_ACCESS_KEY_ID + # aws_secret_access_key: AWS_SECRET_ACCESS_KEY + # region: eu-central-1 + # host: 'localhost' # default: s3.amazonaws.com + # endpoint: 'http://127.0.0.1:9000' # default: nil + # path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object' ## GitLab Pages pages: @@ -481,7 +518,17 @@ production: &base # - { name: 'twitter', # app_id: 'YOUR_APP_ID', # app_secret: 'YOUR_APP_SECRET' } - # + # - { name: 'jwt', + # app_secret: 'YOUR_APP_SECRET', + # args: { + # algorithm: 'HS256', + # uid_claim: 'email', + # required_claims: ["name", "email"], + # info_map: { name: "name", email: "email" }, + # auth_url: 'https://example.com/', + # valid_within: nil, + # } + # } # - { name: 'saml', # label: 'Our SAML Provider', # groups_attribute: 'Groups', @@ -655,10 +702,39 @@ test: enabled: true lfs: enabled: false + # The location where LFS objects are stored (default: shared/lfs-objects). + # storage_path: shared/lfs-objects + object_store: + enabled: false + remote_directory: lfs-objects # The bucket name + connection: + provider: AWS # Only AWS supported at the moment + aws_access_key_id: AWS_ACCESS_KEY_ID + aws_secret_access_key: AWS_SECRET_ACCESS_KEY + region: eu-central-1 artifacts: path: tmp/tests/artifacts + enabled: true + # The location where build artifacts are stored (default: shared/artifacts). + # path: shared/artifacts + object_store: + enabled: false + remote_directory: artifacts # The bucket name + background_upload: false + connection: + provider: AWS # Only AWS supported at the moment + aws_access_key_id: AWS_ACCESS_KEY_ID + aws_secret_access_key: AWS_SECRET_ACCESS_KEY + region: eu-central-1 uploads: storage_path: tmp/tests/public + object_store: + enabled: false + connection: + provider: AWS # Only AWS supported at the moment + aws_access_key_id: AWS_ACCESS_KEY_ID + aws_secret_access_key: AWS_SECRET_ACCESS_KEY + region: eu-central-1 gitlab: host: localhost port: 80 @@ -733,6 +809,17 @@ test: - { name: 'twitter', app_id: 'YOUR_APP_ID', app_secret: 'YOUR_APP_SECRET' } + - { name: 'jwt', + app_secret: 'YOUR_APP_SECRET', + args: { + algorithm: 'HS256', + uid_claim: 'email', + required_claims: ["name", "email"], + info_map: { name: "name", email: "email" }, + auth_url: 'https://example.com/', + valid_within: nil, + } + } - { name: 'auth0', args: { client_id: 'YOUR_AUTH0_CLIENT_ID', diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index ea0dee7af53..906ae8b6180 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -305,6 +305,13 @@ Settings.artifacts['storage_path'] = Settings.absolute(Settings.artifacts.values # Settings.artifact['path'] is deprecated, use `storage_path` instead Settings.artifacts['path'] = Settings.artifacts['storage_path'] Settings.artifacts['max_size'] ||= 100 # in megabytes +Settings.artifacts['object_store'] ||= Settingslogic.new({}) +Settings.artifacts['object_store']['enabled'] = false if Settings.artifacts['object_store']['enabled'].nil? +Settings.artifacts['object_store']['remote_directory'] ||= nil +Settings.artifacts['object_store']['background_upload'] = true if Settings.artifacts['object_store']['background_upload'].nil? +Settings.artifacts['object_store']['proxy_download'] = false if Settings.artifacts['object_store']['proxy_download'].nil? +# Convert upload connection settings to use string keys, to make Fog happy +Settings.artifacts['object_store']['connection']&.deep_stringify_keys! # # Registry @@ -340,6 +347,13 @@ Settings.pages['artifacts_server'] ||= Settings.pages['enabled'] if Settings.pa Settings['lfs'] ||= Settingslogic.new({}) Settings.lfs['enabled'] = true if Settings.lfs['enabled'].nil? Settings.lfs['storage_path'] = Settings.absolute(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects")) +Settings.lfs['object_store'] ||= Settingslogic.new({}) +Settings.lfs['object_store']['enabled'] = false if Settings.lfs['object_store']['enabled'].nil? +Settings.lfs['object_store']['remote_directory'] ||= nil +Settings.lfs['object_store']['background_upload'] = true if Settings.lfs['object_store']['background_upload'].nil? +Settings.lfs['object_store']['proxy_download'] = false if Settings.lfs['object_store']['proxy_download'].nil? +# Convert upload connection settings to use string keys, to make Fog happy +Settings.lfs['object_store']['connection']&.deep_stringify_keys! # # Uploads @@ -347,6 +361,13 @@ Settings.lfs['storage_path'] = Settings.absolute(Settings.lfs['storage_path'] || Settings['uploads'] ||= Settingslogic.new({}) Settings.uploads['storage_path'] = Settings.absolute(Settings.uploads['storage_path'] || 'public') Settings.uploads['base_dir'] = Settings.uploads['base_dir'] || 'uploads/-/system' +Settings.uploads['object_store'] ||= Settingslogic.new({}) +Settings.uploads['object_store']['enabled'] = false if Settings.uploads['object_store']['enabled'].nil? +Settings.uploads['object_store']['remote_directory'] ||= 'uploads' +Settings.uploads['object_store']['background_upload'] = true if Settings.uploads['object_store']['background_upload'].nil? +Settings.uploads['object_store']['proxy_download'] = false if Settings.uploads['object_store']['proxy_download'].nil? +# Convert upload connection settings to use string keys, to make Fog happy +Settings.uploads['object_store']['connection']&.deep_stringify_keys! # # Mattermost @@ -467,12 +488,7 @@ unless Settings.repositories.storages['default'] end Settings.repositories.storages.each do |key, storage| - storage = Settingslogic.new(storage) - - # Expand relative paths - storage['path'] = Settings.absolute(storage['path']) - - Settings.repositories.storages[key] = storage + Settings.repositories.storages[key] = Gitlab::GitalyClient::StorageSettings.new(storage) end # @@ -486,7 +502,7 @@ repositories_storages = Settings.repositories.storages.values repository_downloads_path = Settings.gitlab['repository_downloads_path'].to_s.gsub(%r{/$}, '') repository_downloads_full_path = File.expand_path(repository_downloads_path, Settings.gitlab['user_home']) -if repository_downloads_path.blank? || repositories_storages.any? { |rs| [repository_downloads_path, repository_downloads_full_path].include?(rs['path'].gsub(%r{/$}, '')) } +if repository_downloads_path.blank? || repositories_storages.any? { |rs| [repository_downloads_path, repository_downloads_full_path].include?(rs.legacy_disk_path.gsub(%r{/$}, '')) } Settings.gitlab['repository_downloads_path'] = File.join(Settings.shared['path'], 'cache/archive') end diff --git a/config/initializers/6_validations.rb b/config/initializers/6_validations.rb index f8e67ce04c9..d92cdb97766 100644 --- a/config/initializers/6_validations.rb +++ b/config/initializers/6_validations.rb @@ -5,7 +5,7 @@ end def find_parent_path(name, path) parent = Pathname.new(path).realpath.parent Gitlab.config.repositories.storages.detect do |n, rs| - name != n && Pathname.new(rs['path']).realpath == parent + name != n && Pathname.new(rs.legacy_disk_path).realpath == parent end rescue Errno::EIO, Errno::ENOENT => e warning = "WARNING: couldn't verify #{path} (#{name}). "\ @@ -33,7 +33,7 @@ def validate_storages_config "If you're using the Gitlab Development Kit, you can update your configuration running `gdk reconfigure`.\n" end - if !repository_storage.is_a?(Hash) || repository_storage['path'].nil? + if !repository_storage.is_a?(Gitlab::GitalyClient::StorageSettings) || repository_storage.legacy_disk_path.nil? storage_validation_error("#{name} is not a valid storage, because it has no `path` key. Refer to gitlab.yml.example for an updated example") end @@ -50,7 +50,7 @@ end def validate_storages_paths Gitlab.config.repositories.storages.each do |name, repository_storage| - parent_name, _parent_path = find_parent_path(name, repository_storage['path']) + parent_name, _parent_path = find_parent_path(name, repository_storage.legacy_disk_path) if parent_name storage_validation_error("#{name} is a nested path of #{parent_name}. Nested paths are not supported for repository storages") end diff --git a/config/initializers/application_controller_renderer.rb b/config/initializers/application_controller_renderer.rb new file mode 100644 index 00000000000..a65f8aecf9e --- /dev/null +++ b/config/initializers/application_controller_renderer.rb @@ -0,0 +1,12 @@ +# Remove this `if` condition when upgraded to rails 5.0. +# The body must be kept. +if Gitlab.rails5? + # Be sure to restart your server when you modify this file. + + # ActiveSupport::Reloader.to_prepare do + # ApplicationController.renderer.defaults.merge!( + # http_host: 'example.org', + # https: false + # ) + # end +end diff --git a/config/initializers/fog_google_https_private_urls.rb b/config/initializers/fog_google_https_private_urls.rb new file mode 100644 index 00000000000..f92e623a5d2 --- /dev/null +++ b/config/initializers/fog_google_https_private_urls.rb @@ -0,0 +1,20 @@ +# +# Monkey patching the https support for private urls +# See https://gitlab.com/gitlab-org/gitlab-ee/issues/4879 +# +module Fog + module Storage + class GoogleXML + class File < Fog::Model + module MonkeyPatch + def url(expires) + requires :key + collection.get_https_url(key, expires) + end + end + + prepend MonkeyPatch + end + end + end +end diff --git a/config/initializers/new_framework_defaults.rb b/config/initializers/new_framework_defaults.rb new file mode 100644 index 00000000000..2d130bc0bf8 --- /dev/null +++ b/config/initializers/new_framework_defaults.rb @@ -0,0 +1,29 @@ +# Remove this `if` condition when upgraded to rails 5.0. +# The body must be kept. +if Gitlab.rails5? + # Be sure to restart your server when you modify this file. + # + # This file contains migration options to ease your Rails 5.0 upgrade. + # + # Once upgraded flip defaults one by one to migrate to the new default. + # + # Read the Guide for Upgrading Ruby on Rails for more info on each option. + + Rails.application.config.action_controller.raise_on_unfiltered_parameters = true + + # Enable per-form CSRF tokens. Previous versions had false. + Rails.application.config.action_controller.per_form_csrf_tokens = false + + # Enable origin-checking CSRF mitigation. Previous versions had false. + Rails.application.config.action_controller.forgery_protection_origin_check = false + + # Make Ruby 2.4 preserve the timezone of the receiver when calling `to_time`. + # Previous versions had false. + ActiveSupport.to_time_preserves_timezone = false + + # Require `belongs_to` associations by default. Previous versions had false. + Rails.application.config.active_record.belongs_to_required_by_default = false + + # Do not halt callback chains when a callback returns false. Previous versions had true. + ActiveSupport.halt_callback_chains_on_return_false = true +end diff --git a/config/karma.config.js b/config/karma.config.js index 3d95e1622b2..7ede745b591 100644 --- a/config/karma.config.js +++ b/config/karma.config.js @@ -5,7 +5,7 @@ var ROOT_PATH = path.resolve(__dirname, '..'); // remove problematic plugins if (webpackConfig.plugins) { - webpackConfig.plugins = webpackConfig.plugins.filter(function (plugin) { + webpackConfig.plugins = webpackConfig.plugins.filter(function(plugin) { return !( plugin instanceof webpack.optimize.CommonsChunkPlugin || plugin instanceof webpack.optimize.ModuleConcatenationPlugin || @@ -24,7 +24,7 @@ module.exports = function(config) { var karmaConfig = { basePath: ROOT_PATH, - browsers: ['ChromeHeadlessCustom'], + browsers: ['ChromeHeadlessCustom'], customLaunchers: { ChromeHeadlessCustom: { base: 'ChromeHeadless', @@ -34,7 +34,7 @@ module.exports = function(config) { // escalated kernel privileges (e.g. docker run --cap-add=CAP_SYS_ADMIN) '--no-sandbox', ], - } + }, }, frameworks: ['jasmine'], files: [ @@ -55,7 +55,7 @@ module.exports = function(config) { reports: ['html', 'text-summary'], dir: 'coverage-javascript/', subdir: '.', - fixWebpackSourcePaths: true + fixWebpackSourcePaths: true, }; karmaConfig.browserNoActivityTimeout = 60000; // 60 seconds } diff --git a/config/routes/project.rb b/config/routes/project.rb index c803737d40b..f50b9aded8d 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -52,7 +52,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end - resource :pages, only: [:show, :destroy] do + resource :pages, only: [:show, :update, :destroy] do resources :domains, except: :index, controller: 'pages_domains', constraints: { id: %r{[^/]+} } do member do post :verify diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 554502c5d83..c811034b29d 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -68,5 +68,7 @@ - [project_migrate_hashed_storage, 1] - [storage_migrator, 1] - [pages_domain_verification, 1] + - [object_storage_upload, 1] + - [object_storage, 1] - [plugin, 1] - [pipeline_background, 1] diff --git a/config/spring.rb b/config/spring.rb new file mode 100644 index 00000000000..c9119b40c08 --- /dev/null +++ b/config/spring.rb @@ -0,0 +1,6 @@ +%w( + .ruby-version + .rbenv-vars + tmp/restart.txt + tmp/caching-dev.txt +).each { |path| Spring.watch(path) } diff --git a/config/webpack.config.js b/config/webpack.config.js index 42fe4b345e1..b74d9dde494 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -1,5 +1,3 @@ -'use strict'; - const crypto = require('crypto'); const fs = require('fs'); const path = require('path'); @@ -27,10 +25,10 @@ let watchAutoEntries = []; function generateEntries() { // generate automatic entry points const autoEntries = {}; - const pageEntries = glob.sync('pages/**/index.js', { cwd: path.join(ROOT_PATH, 'app/assets/javascripts') }); - watchAutoEntries = [ - path.join(ROOT_PATH, 'app/assets/javascripts/pages/'), - ]; + const pageEntries = glob.sync('pages/**/index.js', { + cwd: path.join(ROOT_PATH, 'app/assets/javascripts'), + }); + watchAutoEntries = [path.join(ROOT_PATH, 'app/assets/javascripts/pages/')]; function generateAutoEntries(path, prefix = '.') { const chunkPath = path.replace(/\/index\.js$/, ''); @@ -38,16 +36,16 @@ function generateEntries() { autoEntries[chunkName] = `${prefix}/${path}`; } - pageEntries.forEach(( path ) => generateAutoEntries(path)); + pageEntries.forEach(path => generateAutoEntries(path)); autoEntriesCount = Object.keys(autoEntries).length; const manualEntries = { - common: './commons/index.js', - main: './main.js', - raven: './raven/index.js', - webpack_runtime: './webpack.js', - ide: './ide/index.js', + common: './commons/index.js', + main: './main.js', + raven: './raven/index.js', + webpack_runtime: './webpack.js', + ide: './ide/index.js', }; return Object.assign(manualEntries, autoEntries); @@ -91,8 +89,8 @@ const config = { { loader: 'worker-loader', options: { - inline: true - } + inline: true, + }, }, { loader: 'babel-loader' }, ], @@ -103,7 +101,7 @@ const config = { loader: 'file-loader', options: { name: '[name].[hash].[ext]', - } + }, }, { test: /katex.css$/, @@ -113,8 +111,8 @@ const config = { { loader: 'css-loader', options: { - name: '[name].[hash].[ext]' - } + name: '[name].[hash].[ext]', + }, }, ], }, @@ -124,7 +122,7 @@ const config = { loader: 'file-loader', options: { name: '[name].[hash].[ext]', - } + }, }, { test: /monaco-editor\/\w+\/vs\/loader\.js$/, @@ -132,7 +130,7 @@ const config = { { loader: 'exports-loader', options: 'l.global' }, { loader: 'imports-loader', options: 'l=>{},this=>l,AMDLoader=>this,module=>undefined' }, ], - } + }, ], noParse: [/monaco-editor\/\w+\/vs\//], @@ -150,10 +148,10 @@ const config = { source: false, chunks: false, modules: false, - assets: true + assets: true, }); return JSON.stringify(stats, null, 2); - } + }, }), // prevent pikaday from including moment.js @@ -170,7 +168,7 @@ const config = { new NameAllModulesPlugin(), // assign deterministic chunk ids - new webpack.NamedChunksPlugin((chunk) => { + new webpack.NamedChunksPlugin(chunk => { if (chunk.name) { return chunk.name; } @@ -187,9 +185,12 @@ const config = { const pagesBase = path.join(ROOT_PATH, 'app/assets/javascripts/pages'); if (m.resource.indexOf(pagesBase) === 0) { - moduleNames.push(path.relative(pagesBase, m.resource) - .replace(/\/index\.[a-z]+$/, '') - .replace(/\//g, '__')); + moduleNames.push( + path + .relative(pagesBase, m.resource) + .replace(/\/index\.[a-z]+$/, '') + .replace(/\//g, '__') + ); } else { moduleNames.push(path.relative(m.context, m.resource)); } @@ -197,7 +198,8 @@ const config = { chunk.forEachModule(collectModuleNames); - const hash = crypto.createHash('sha256') + const hash = crypto + .createHash('sha256') .update(moduleNames.join('_')) .digest('hex'); @@ -212,7 +214,10 @@ const config = { // copy pre-compiled vendor libraries verbatim new CopyWebpackPlugin([ { - from: path.join(ROOT_PATH, `node_modules/monaco-editor/${IS_PRODUCTION ? 'min' : 'dev'}/vs`), + from: path.join( + ROOT_PATH, + `node_modules/monaco-editor/${IS_PRODUCTION ? 'min' : 'dev'}/vs` + ), to: 'monaco-editor/vs', transform: function(content, path) { if (/\.js$/.test(path) && !/worker/i.test(path) && !/typescript/i.test(path)) { @@ -225,23 +230,23 @@ const config = { ); } return content; - } - } + }, + }, ]), ], resolve: { extensions: ['.js'], alias: { - '~': path.join(ROOT_PATH, 'app/assets/javascripts'), - 'emojis': path.join(ROOT_PATH, 'fixtures/emojis'), - 'empty_states': path.join(ROOT_PATH, 'app/views/shared/empty_states'), - 'icons': path.join(ROOT_PATH, 'app/views/shared/icons'), - 'images': path.join(ROOT_PATH, 'app/assets/images'), - 'vendor': path.join(ROOT_PATH, 'vendor/assets/javascripts'), - 'vue$': 'vue/dist/vue.esm.js', - 'spec': path.join(ROOT_PATH, 'spec/javascripts'), - } + '~': path.join(ROOT_PATH, 'app/assets/javascripts'), + emojis: path.join(ROOT_PATH, 'fixtures/emojis'), + empty_states: path.join(ROOT_PATH, 'app/views/shared/empty_states'), + icons: path.join(ROOT_PATH, 'app/views/shared/icons'), + images: path.join(ROOT_PATH, 'app/assets/images'), + vendor: path.join(ROOT_PATH, 'vendor/assets/javascripts'), + vue$: 'vue/dist/vue.esm.js', + spec: path.join(ROOT_PATH, 'spec/javascripts'), + }, }, // sqljs requires fs @@ -256,14 +261,14 @@ if (IS_PRODUCTION) { new webpack.NoEmitOnErrorsPlugin(), new webpack.LoaderOptionsPlugin({ minimize: true, - debug: false + debug: false, }), new webpack.optimize.ModuleConcatenationPlugin(), new webpack.optimize.UglifyJsPlugin({ - sourceMap: true + sourceMap: true, }), new webpack.DefinePlugin({ - 'process.env': { NODE_ENV: JSON.stringify('production') } + 'process.env': { NODE_ENV: JSON.stringify('production') }, }) ); @@ -282,7 +287,7 @@ if (IS_DEV_SERVER) { headers: { 'Access-Control-Allow-Origin': '*' }, stats: 'errors-only', hot: DEV_SERVER_LIVERELOAD, - inline: DEV_SERVER_LIVERELOAD + inline: DEV_SERVER_LIVERELOAD, }; config.plugins.push( // watch node_modules for changes if we encounter a missing module compile error @@ -298,10 +303,12 @@ if (IS_DEV_SERVER) { ]; // report our auto-generated bundle count - console.log(`${autoEntriesCount} entries from '/pages' automatically added to webpack output.`); + console.log( + `${autoEntriesCount} entries from '/pages' automatically added to webpack output.` + ); callback(); - }) + }); }, } ); diff --git a/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb b/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb index bcdae272209..a96ea7d9db4 100644 --- a/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb +++ b/db/migrate/20161124141322_migrate_process_commit_worker_jobs.rb @@ -12,7 +12,7 @@ class MigrateProcessCommitWorkerJobs < ActiveRecord::Migration end def repository_storage_path - Gitlab.config.repositories.storages[repository_storage]['path'] + Gitlab.config.repositories.storages[repository_storage].legacy_disk_path end def repository_path diff --git a/db/migrate/20161220141214_remove_dot_git_from_group_names.rb b/db/migrate/20161220141214_remove_dot_git_from_group_names.rb index 8fb1f9d5e73..bddc234db25 100644 --- a/db/migrate/20161220141214_remove_dot_git_from_group_names.rb +++ b/db/migrate/20161220141214_remove_dot_git_from_group_names.rb @@ -60,7 +60,7 @@ class RemoveDotGitFromGroupNames < ActiveRecord::Migration def move_namespace(group_id, path_was, path) repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row| - Gitlab.config.repositories.storages[row['repository_storage']]['path'] + Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path end.compact # Move the namespace directory in all storages paths used by member projects diff --git a/db/migrate/20161226122833_remove_dot_git_from_usernames.rb b/db/migrate/20161226122833_remove_dot_git_from_usernames.rb index 61dcc8c54f5..7c28d934c29 100644 --- a/db/migrate/20161226122833_remove_dot_git_from_usernames.rb +++ b/db/migrate/20161226122833_remove_dot_git_from_usernames.rb @@ -71,7 +71,7 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration route_exists = route_exists?(path) Gitlab.config.repositories.storages.each_value do |storage| - if route_exists || path_exists?(path, storage['path']) + if route_exists || path_exists?(path, storage.legacy_disk_path) counter += 1 path = "#{base}#{counter}" @@ -84,7 +84,7 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration def move_namespace(namespace_id, path_was, path) repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row| - Gitlab.config.repositories.storages[row['repository_storage']]['path'] + Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path end.compact # Move the namespace directory in all storages paths used by member projects diff --git a/db/migrate/20170601163708_add_artifacts_store_to_ci_build.rb b/db/migrate/20170601163708_add_artifacts_store_to_ci_build.rb new file mode 100644 index 00000000000..e82109190a7 --- /dev/null +++ b/db/migrate/20170601163708_add_artifacts_store_to_ci_build.rb @@ -0,0 +1,10 @@ +class AddArtifactsStoreToCiBuild < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column(:ci_builds, :artifacts_file_store, :integer) + add_column(:ci_builds, :artifacts_metadata_store, :integer) + end +end diff --git a/db/migrate/20170825015534_add_file_store_to_lfs_objects.rb b/db/migrate/20170825015534_add_file_store_to_lfs_objects.rb new file mode 100644 index 00000000000..41bb031014f --- /dev/null +++ b/db/migrate/20170825015534_add_file_store_to_lfs_objects.rb @@ -0,0 +1,31 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddFileStoreToLfsObjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index", "remove_concurrent_index" or + # "add_column_with_default" you must disable the use of transactions + # as these methods can not run in an existing transaction. + # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure + # that either of them is the _only_ method called in the migration, + # any other changes should go in a separate migration. + # This ensures that upon failure _only_ the index creation or removing fails + # and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + add_column(:lfs_objects, :file_store, :integer) + end +end diff --git a/db/migrate/20170918072949_add_file_store_job_artifacts.rb b/db/migrate/20170918072949_add_file_store_job_artifacts.rb new file mode 100644 index 00000000000..b1f1bea6deb --- /dev/null +++ b/db/migrate/20170918072949_add_file_store_job_artifacts.rb @@ -0,0 +1,10 @@ +class AddFileStoreJobArtifacts < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + DOWNTIME = false + + def change + add_column(:ci_job_artifacts, :file_store, :integer) + end +end diff --git a/db/migrate/20171214144320_add_store_column_to_uploads.rb b/db/migrate/20171214144320_add_store_column_to_uploads.rb new file mode 100644 index 00000000000..e35798e2c41 --- /dev/null +++ b/db/migrate/20171214144320_add_store_column_to_uploads.rb @@ -0,0 +1,12 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddStoreColumnToUploads < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column(:uploads, :store, :integer) + end +end diff --git a/db/migrate/20180102220145_add_pages_https_only_to_projects.rb b/db/migrate/20180102220145_add_pages_https_only_to_projects.rb new file mode 100644 index 00000000000..ef6bc6896c0 --- /dev/null +++ b/db/migrate/20180102220145_add_pages_https_only_to_projects.rb @@ -0,0 +1,9 @@ +class AddPagesHttpsOnlyToProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :projects, :pages_https_only, :boolean + end +end diff --git a/db/migrate/20180109183319_change_default_value_for_pages_https_only.rb b/db/migrate/20180109183319_change_default_value_for_pages_https_only.rb new file mode 100644 index 00000000000..c242e1b0d24 --- /dev/null +++ b/db/migrate/20180109183319_change_default_value_for_pages_https_only.rb @@ -0,0 +1,13 @@ +class ChangeDefaultValueForPagesHttpsOnly < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + change_column_default :projects, :pages_https_only, true + end + + def down + change_column_default :projects, :pages_https_only, nil + end +end diff --git a/db/migrate/20180323150945_add_push_to_merge_request_to_notification_settings.rb b/db/migrate/20180323150945_add_push_to_merge_request_to_notification_settings.rb new file mode 100644 index 00000000000..12b8875d8dc --- /dev/null +++ b/db/migrate/20180323150945_add_push_to_merge_request_to_notification_settings.rb @@ -0,0 +1,9 @@ +class AddPushToMergeRequestToNotificationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :notification_settings, :push_to_merge_request, :boolean + end +end diff --git a/db/schema.rb b/db/schema.rb index 56116a2d241..b3b2d5b0da9 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180320182229) do +ActiveRecord::Schema.define(version: 20180323150945) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -307,6 +307,8 @@ ActiveRecord::Schema.define(version: 20180320182229) do t.integer "auto_canceled_by_id" t.boolean "retried" t.integer "stage_id" + t.integer "artifacts_file_store" + t.integer "artifacts_metadata_store" t.boolean "protected" t.integer "failure_reason" end @@ -345,6 +347,7 @@ ActiveRecord::Schema.define(version: 20180320182229) do t.integer "project_id", null: false t.integer "job_id", null: false t.integer "file_type", null: false + t.integer "file_store" t.integer "size", limit: 8 t.datetime_with_timezone "created_at", null: false t.datetime_with_timezone "updated_at", null: false @@ -1009,6 +1012,7 @@ ActiveRecord::Schema.define(version: 20180320182229) do t.datetime "created_at" t.datetime "updated_at" t.string "file" + t.integer "file_store" end add_index "lfs_objects", ["oid"], name: "index_lfs_objects_on_oid", unique: true, using: :btree @@ -1296,6 +1300,7 @@ ActiveRecord::Schema.define(version: 20180320182229) do t.boolean "merge_merge_request" t.boolean "failed_pipeline" t.boolean "success_pipeline" + t.boolean "push_to_merge_request" end add_index "notification_settings", ["source_id", "source_type"], name: "index_notification_settings_on_source_id_and_source_type", using: :btree @@ -1513,6 +1518,7 @@ ActiveRecord::Schema.define(version: 20180320182229) do t.boolean "merge_requests_ff_only_enabled", default: false t.boolean "merge_requests_rebase_enabled", default: false, null: false t.integer "jobs_cache_index" + t.boolean "pages_https_only", default: true end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1822,6 +1828,7 @@ ActiveRecord::Schema.define(version: 20180320182229) do t.datetime "created_at", null: false t.string "mount_point" t.string "secret" + t.integer "store" end add_index "uploads", ["checksum"], name: "index_uploads_on_checksum", using: :btree diff --git a/doc/README.md b/doc/README.md index 05fa444657c..be805a2ccc4 100644 --- a/doc/README.md +++ b/doc/README.md @@ -11,35 +11,62 @@ GitLab offers the most scalable Git-based fully integrated platform for software development, with flexible products and subscriptions. To understand what features you have access to, check the [GitLab subscriptions](#gitlab-subscriptions) below. -## Shortcuts to GitLab's most visited docs +**Shortcuts to GitLab's most visited docs:** -| [GitLab CI/CD](ci/README.md) | Other | +| General documentation | GitLab CI/CD docs | | :----- | :----- | -| [Quick start guide](ci/quick_start/README.md) | [API](api/README.md) | -| [Configuring `.gitlab-ci.yml`](ci/yaml/README.md) | [SSH authentication](ssh/README.md) | -| [Using Docker images](ci/docker/using_docker_images.md) | [GitLab Pages](user/project/pages/index.md) | +| [User documentation](user/index.md) | [GitLab CI/CD](ci/README.md) | +| [Administrator documentation](administration/index.md) | [GitLab CI/CD quick start guide](ci/quick_start/README.md) | +| [Contributor documentation](#contributor-documentation) | [Configuring `.gitlab-ci.yml`](ci/yaml/README.md) | +| [Getting started with GitLab](#getting-started-with-gitlab) | [Using Docker images](ci/docker/using_docker_images.md) | +| [API](api/README.md) | [Auto DevOps](topics/autodevops/index.md) | +| [SSH authentication](ssh/README.md) | [Kubernetes integration](user/project/clusters/index.md)| +| [GitLab Pages](user/project/pages/index.md) | [GitLab Container Registry](user/project/container_registry.md) | + +## Complete DevOps with GitLab + +GitLab is the first single application for software development, security, +and operations that enables Concurrent DevOps, making the software lifecycle +three times faster and radically improving the speed of business. GitLab +provides solutions for all the stages of the DevOps lifecycle: +[plan](#plan), [create](#create), [verify](#verify), [package](#package), +[release](#release), [configure](#configure), [monitor](#monitor). + + + +### Plan + +Whether you use Waterfall, Agile, or Conversational Development, +GitLab streamlines your collaborative workflows. Visualize, prioritize, +coordinate, and track your progress your way with GitLab’s flexible project +management tools. + +- Chat operations + - [Mattermost slash commands](user/project/integrations/mattermost_slash_commands.md) + - [Slack slash commands](user/project/integrations/slack_slash_commands.md) +- [Discussions](user/discussions/index.md): Threads, comments, and resolvable discussions in issues, commits, and merge requests. +- [Issues](user/project/issues/index.md) +- [Project Issue Board](user/project/issue_board.md) +- [Issues and merge requests templates](user/project/description_templates.md): Create templates for submitting new issues and merge requests. +- [Labels](user/project/labels.md): Categorize your issues or merge requests based on descriptive titles. +- [Milestones](user/project/milestones/index.md): Organize issues and merge requests into a cohesive group, optionally setting a due date. +- [Todos](workflow/todos.md): A chronological list of to-dos that are waiting for your input, all in a simple dashboard. +- [GitLab Quick Actions](user/project/quick_actions.md): Textual shortcuts for common actions on issues or merge requests that are usually done by clicking buttons or dropdowns in GitLab's UI. -- [User documentation](user/index.md) -- [Administrator documentation](administration/index.md) -- [Contributor documentation](#contributor-documentation) +#### Migrate and import your projects from other platforms -## Getting started with GitLab +- [Importing to GitLab](user/project/import/index.md): Import your projects from GitHub, Bitbucket, GitLab.com, FogBugz and SVN into GitLab. +- [Migrating from SVN](workflow/importing/migrating_from_svn.md): Convert a SVN repository to Git and GitLab. -- [GitLab Basics](gitlab-basics/README.md): Start working on your command line and on GitLab. -- [GitLab Workflow](workflow/README.md): Enhance your workflow with the best of GitLab Workflow. - - See also [GitLab Workflow - an overview](https://about.gitlab.com/2016/10/25/gitlab-workflow-an-overview/). -- [GitLab Markdown](user/markdown.md): GitLab's advanced formatting system (GitLab Flavored Markdown). -- [GitLab Quick Actions](user/project/quick_actions.md): Textual shortcuts for common actions on issues or merge requests that are usually done by clicking buttons or dropdowns in GitLab's UI. -- [Auto DevOps](topics/autodevops/index.md) +### Create -### User account +Consolidate source code into a single [DVCS](https://en.wikipedia.org/wiki/Distributed_version_control) +that’s easily managed and controlled without disrupting your workflow. +GitLab’s git repositories come complete with branching tools and access +controls, providing a scalable, single source of truth for collaborating +on projects and code. -- [User account](user/profile/index.md): Manage your account - - [Authentication](topics/authentication/index.md): Account security with two-factor authentication, setup your ssh keys and deploy keys for secure access to your projects. - - [Profile settings](user/profile/index.md#profile-settings): Manage your profile settings, two factor authentication and more. -- [User permissions](user/permissions.md): Learn what each role in a project (external/guest/reporter/developer/master/owner) can do. - -### Projects and groups +#### Projects and groups - [Projects](user/project/index.md): - [Project settings](user/project/settings/index.md) @@ -54,7 +81,7 @@ To understand what features you have access to, check the [GitLab subscriptions] - [Snippets](user/snippets.md): Snippets allow you to create little bits of code. - [Wikis](user/project/wiki/index.md): Enhance your repository documentation with built-in wikis. -### Repository +#### Repositories Manage your [repositories](user/project/repository/index.md) from the UI (user interface): @@ -72,51 +99,88 @@ Manage your [repositories](user/project/repository/index.md) from the UI (user i - [Commits](user/project/repository/index.md#commits) - [Signing commits](user/project/repository/gpg_signed_commits/index.md): use GPG to sign your commits. -### Issues and Merge Requests (MRs) +#### Integrations + +- [Project Services](user/project/integrations/project_services.md): Integrate a project with external services, such as CI and chat. +- [GitLab Integration](integration/README.md): Integrate with multiple third-party services with GitLab to allow external issue trackers and external authentication. +- [Trello Power-Up](integration/trello_power_up.md): Integrate with GitLab's Trello Power-Up + +#### Automation + +- [API](api/README.md): Automate GitLab via a simple and powerful API. +- [GitLab Webhooks](user/project/integrations/webhooks.md): Let GitLab notify you when new code has been pushed to your project. + +### Verify + +Spot errors sooner and shorten feedback cycles with built-in code review, code testing, +Code Quality, and Review Apps. Customize your approval workflow controls, automatically +test the quality of your code, and spin up a staging environment for every code change. +GitLab Continuous Integration is the most popular next generation testing system that +auto scales to run your tests faster. -- [Discussions](user/discussions/index.md): Threads, comments, and resolvable discussions in issues, commits, and merge requests. -- [Issues](user/project/issues/index.md) -- [Project issue Board](user/project/issue_board.md) -- [Group Issue Board](user/project/issue_board.md#group-issue-board) -- [Issues and merge requests templates](user/project/description_templates.md): Create templates for submitting new issues and merge requests. -- [Labels](user/project/labels.md): Categorize your issues or merge requests based on descriptive titles. - [Merge Requests](user/project/merge_requests/index.md) - [Work In Progress Merge Requests](user/project/merge_requests/work_in_progress_merge_requests.md) - [Merge Request discussion resolution](user/discussions/index.md#moving-a-single-discussion-to-a-new-issue): Resolve discussions, move discussions in a merge request to an issue, only allow merge requests to be merged if all discussions are resolved. - [Checkout merge requests locally](user/project/merge_requests/index.md#checkout-merge-requests-locally) - [Cherry-pick](user/project/merge_requests/cherry_pick_changes.md) -- [Milestones](user/project/milestones/index.md): Organize issues and merge requests into a cohesive group, optionally setting a due date. -- [Todos](workflow/todos.md): A chronological list of to-dos that are waiting for your input, all in a simple dashboard. +- [Review Apps](ci/review_apps/index.md): Preview changes to your app right from a merge request. -### Git and GitLab +### Package -- [Git](topics/git/index.md): Getting started with Git, branching strategies, Git LFS, advanced use. -- [Git cheatsheet](https://gitlab.com/gitlab-com/marketing/raw/master/design/print/git-cheatsheet/print-pdf/git-cheatsheet.pdf): Download a PDF describing the most used Git operations. -- [GitLab Flow](workflow/gitlab_flow.md): explore the best of Git with the GitLab Flow strategy. +GitLab Container Registry gives you the enhanced security and access controls of +custom Docker images without 3rd party add-ons. Easily upload and download images +from GitLab CI/CD with full Git repository management integration. -### Migrate and import your projects from other platforms +- [GitLab CI/CD](ci/README.md): Explore the features and capabilities of Continuous Integration, Continuous Delivery, and Continuous Deployment with GitLab. +- [GitLab Container Registry](user/project/container_registry.md): Learn how to use GitLab's built-in Container Registry. -- [Importing to GitLab](user/project/import/index.md): Import your projects from GitHub, Bitbucket, GitLab.com, FogBugz and SVN into GitLab. -- [Migrating from SVN](workflow/importing/migrating_from_svn.md): Convert a SVN repository to Git and GitLab. +### Release + +Spend less time configuring your tools, and more time creating. Whether you’re +deploying to one server or thousands, build, test, and release your code +confidently and securely with GitLab’s built-in Continuous Delivery and Deployment. + +- [GitLab Pages](user/project/pages/index.md): Build, test, and deploy a static site directly from GitLab. +- [Auto Deploy](topics/autodevops/index.md#auto-deploy): Configure GitLab CI for the deployment of your application. +- [Environments and deployments](ci/environments.md): With environments, you can control the continuous deployment of your software within GitLab. -### Continuous Integration, Delivery, and Deployment +### Configure + +Automate your entire workflow from build to deploy and monitoring with GitLab +Auto Devops. Best practice templates get you started with minimal to zero +configuration. Then customize everything from buildpacks to CI/CD. + +- [Auto DevOps](topics/autodevops/index.md) + +### Monitor + +Measure how long it takes to go from planning to monitoring and ensure your +applications are always responsive and available. GitLab collects and displays +performance metrics for deployed apps using Prometheus so you can know in an +instant how code changes impact your production environment. -- [GitLab CI](ci/README.md): Explore the features and capabilities of Continuous Integration, Continuous Delivery, and Continuous Deployment with GitLab. - - [Auto Deploy](ci/autodeploy/index.md): Configure GitLab CI for the deployment of your application. - - [Review Apps](ci/review_apps/index.md): Preview changes to your app right from a merge request. - [GitLab Cycle Analytics](user/project/cycle_analytics.md): Cycle Analytics measures the time it takes to go from an [idea to production](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) for each project you have. -- [GitLab Container Registry](user/project/container_registry.md): Learn how to use GitLab's built-in Container Registry. +- [GitLab Performance Monitoring](administration/monitoring/performance/index.md) -### Automation +## Getting started with GitLab -- [API](api/README.md): Automate GitLab via a simple and powerful API. -- [GitLab Webhooks](user/project/integrations/webhooks.md): Let GitLab notify you when new code has been pushed to your project. +- [GitLab Basics](gitlab-basics/README.md): Start working on your command line and on GitLab. +- [GitLab Workflow](workflow/README.md): Enhance your workflow with the best of GitLab Workflow. + - See also [GitLab Workflow - an overview](https://about.gitlab.com/2016/10/25/gitlab-workflow-an-overview/). +- [GitLab Markdown](user/markdown.md): GitLab's advanced formatting system (GitLab Flavored Markdown). -### Integrations +### User account -- [Project Services](user/project/integrations/project_services.md): Integrate a project with external services, such as CI and chat. -- [GitLab Integration](integration/README.md): Integrate with multiple third-party services with GitLab to allow external issue trackers and external authentication. -- [Trello Power-Up](integration/trello_power_up.md): Integrate with GitLab's Trello Power-Up +- [User account](user/profile/index.md): Manage your account + - [Authentication](topics/authentication/index.md): Account security with two-factor authentication, setup your ssh keys and deploy keys for secure access to your projects. + - [Profile settings](user/profile/index.md#profile-settings): Manage your profile settings, two factor authentication and more. +- [User permissions](user/permissions.md): Learn what each role in a project (external/guest/reporter/developer/master/owner) can do. + +### Git and GitLab + +- [Git](topics/git/index.md): Getting started with Git, branching strategies, Git LFS, advanced use. +- [Git cheatsheet](https://gitlab.com/gitlab-com/marketing/raw/master/design/print/git-cheatsheet/print-pdf/git-cheatsheet.pdf): Download a PDF describing the most used Git operations. +- [GitLab Flow](workflow/gitlab_flow.md): explore the best of Git with the GitLab Flow strategy. ## Administrator documentation diff --git a/doc/administration/auth/jwt.md b/doc/administration/auth/jwt.md new file mode 100644 index 00000000000..b51e705ab52 --- /dev/null +++ b/doc/administration/auth/jwt.md @@ -0,0 +1,72 @@ +# JWT OmniAuth provider + +To enable the JWT OmniAuth provider, you must register your application with JWT. +JWT will provide you with a secret key for you to use. + +1. On your GitLab server, open the configuration file. + + For Omnibus GitLab: + + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` + + For installations from source: + + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` + +1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings. +1. Add the provider configuration. + + For Omnibus GitLab: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { name: 'jwt', + app_secret: 'YOUR_APP_SECRET', + args: { + algorithm: 'HS256', + uid_claim: 'email', + required_claims: ["name", "email"], + info_maps: { name: "name", email: "email" }, + auth_url: 'https://example.com/', + valid_within: nil, + } + } + ] + ``` + + For installation from source: + + ``` + - { name: 'jwt', + app_secret: 'YOUR_APP_SECRET', + args: { + algorithm: 'HS256', + uid_claim: 'email', + required_claims: ["name", "email"], + info_map: { name: "name", email: "email" }, + auth_url: 'https://example.com/', + valid_within: nil, + } + } + ``` + + NOTE: **Note:** For more information on each configuration option refer to + the [OmniAuth JWT usage documentation](https://github.com/mbleigh/omniauth-jwt#usage). + +1. Change `YOUR_APP_SECRET` to the client secret and set `auth_url` to your redirect URL. +1. Save the configuration file. +1. [Reconfigure GitLab][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. + +On the sign in page there should now be a JWT icon below the regular sign in form. +Click the icon to begin the authentication process. JWT will ask the user to +sign in and authorize the GitLab application. If everything goes well, the user +will be redirected to GitLab and will be signed in. + +[reconfigure GitLab]: ../restart_gitlab.md#omnibus-gitlab-reconfigure +[restart GitLab]: ../restart_gitlab.md#installations-from-source diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index d86a54daadd..ac3a12930c3 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -87,10 +87,124 @@ _The artifacts are stored by default in ### Using object storage +>**Notes:** +- [Introduced][ee-1762] in [GitLab Premium][eep] 9.4. +- Since version 9.5, artifacts are [browsable], when object storage is enabled. + 9.4 lacks this feature. > Available in [GitLab Premium](https://about.gitlab.com/products/) and [GitLab.com Silver](https://about.gitlab.com/gitlab-com/). +> Since version 10.6, available in [GitLab CE](https://about.gitlab.com/products/) + +If you don't want to use the local disk where GitLab is installed to store the +artifacts, you can use an object storage like AWS S3 instead. +This configuration relies on valid AWS credentials to be configured already. +Use an [Object storage option][os] like AWS S3 to store job artifacts. + +### Object Storage Settings + +For source installations the following settings are nested under `artifacts:` and then `object_store:`. On omnibus installs they are prefixed by `artifacts_object_store_`. + +| Setting | Description | Default | +|---------|-------------|---------| +| `enabled` | Enable/disable object storage | `false` | +| `remote_directory` | The bucket name where Artfacts will be stored| | +| `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` | +| `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | +| `connection` | Various connection options described below | | + +#### S3 compatible connection settings + +The connection settings match those provided by [Fog](https://github.com/fog), and are as follows: + +| Setting | Description | Default | +|---------|-------------|---------| +| `provider` | Always `AWS` for compatible hosts | AWS | +| `aws_access_key_id` | AWS credentials, or compatible | | +| `aws_secret_access_key` | AWS credentials, or compatible | | +| `region` | AWS region | us-east-1 | +| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | +| `endpoint` | Can be used when configuring an S3 compatible service such as [Minio](https://www.minio.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | +| `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false | + +**In Omnibus installations:** + +_The artifacts are stored by default in +`/var/opt/gitlab/gitlab-rails/shared/artifacts`._ + +1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with + the values you want: + + ```ruby + gitlab_rails['artifacts_enabled'] = true + gitlab_rails['artifacts_object_store_enabled'] = true + gitlab_rails['artifacts_object_store_remote_directory'] = "artifacts" + gitlab_rails['artifacts_object_store_connection'] = { + 'provider' => 'AWS', + 'region' => 'eu-central-1', + 'aws_access_key_id' => 'AWS_ACCESS_KEY_ID', + 'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY' + } + ``` + + NOTE: For GitLab 9.4+, if you are using AWS IAM profiles, be sure to omit the + AWS access key and secret acces key/value pairs. For example: + + ```ruby + gitlab_rails['artifacts_object_store_connection'] = { + 'provider' => 'AWS', + 'region' => 'eu-central-1', + 'use_iam_profile' => true + } + ``` + +1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Migrate any existing local artifacts to the object storage: + + ```bash + gitlab-rake gitlab:artifacts:migrate + ``` + + Currently this has to be executed manually and it will allow you to + migrate the existing artifacts to the object storage, but all new + artifacts will still be stored on the local disk. In the future + you will be given an option to define a default storage artifacts for all + new files. + +--- + +**In installations from source:** + +_The artifacts are stored by default in +`/home/git/gitlab/shared/artifacts`._ + +1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following + lines: + + ```yaml + artifacts: + enabled: true + object_store: + enabled: true + remote_directory: "artifacts" # The bucket name + connection: + provider: AWS # Only AWS supported at the moment + aws_access_key_id: AWS_ACESS_KEY_ID + aws_secret_access_key: AWS_SECRET_ACCESS_KEY + region: eu-central-1 + ``` + +1. Save the file and [restart GitLab][] for the changes to take effect. +1. Migrate any existing local artifacts to the object storage: + + ```bash + sudo -u git -H bundle exec rake gitlab:artifacts:migrate RAILS_ENV=production + ``` -Use an [Object storage option][ee-os] like AWS S3 to store job artifacts. + Currently this has to be executed manually and it will allow you to + migrate the existing artifacts to the object storage, but all new + artifacts will still be stored on the local disk. In the future + you will be given an option to define a default storage artifacts for all + new files. ## Expiring artifacts @@ -194,7 +308,7 @@ When clicking on a specific file, [GitLab Workhorse] extracts it from the archive and the download begins. This implementation saves space, memory and disk I/O. -[reconfigure gitlab]: restart_gitlab.md "How to restart GitLab" -[restart gitlab]: restart_gitlab.md "How to restart GitLab" +[reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab" +[restart gitlab]: restart_gitlab.md#installations-from-source "How to restart GitLab" [gitlab workhorse]: https://gitlab.com/gitlab-org/gitlab-workhorse "GitLab Workhorse repository" -[ee-os]: https://docs.gitlab.com/ee/administration/job_artifacts.html#using-object-storage +[os]: https://docs.gitlab.com/administration/job_artifacts.html#using-object-storage diff --git a/doc/administration/raketasks/uploads/migrate.md b/doc/administration/raketasks/uploads/migrate.md new file mode 100644 index 00000000000..0cd33ffc122 --- /dev/null +++ b/doc/administration/raketasks/uploads/migrate.md @@ -0,0 +1,74 @@ +# Uploads Migrate Rake Task + +## Migrate to Object Storage + +After [configuring the object storage](../../uploads.md#using-object-storage) for GitLab's uploads, you may use this task to migrate existing uploads from the local storage to the remote storage. + +>**Note:** +All of the processing will be done in a background worker and requires **no downtime**. + +This tasks uses 3 parameters to find uploads to migrate. + +>**Note:** +These parameters are mainly internal to GitLab's structure, you may want to refer to the task list instead below. + +Parameter | Type | Description +--------- | ---- | ----------- +`uploader_class` | string | Type of the uploader to migrate from +`model_class` | string | Type of the model to migrate from +`mount_point` | string/symbol | Name of the model's column on which the uploader is mounted on. + +This task also accepts some environment variables which you can use to override +certain values: + +Variable | Type | Description +-------- | ---- | ----------- +`BATCH` | integer | Specifies the size of the batch. Defaults to 200. + +** Omnibus Installation** + +```bash +# gitlab-rake gitlab:uploads:migrate[uploader_class, model_class, mount_point] + +# Avatars +gitlab-rake "gitlab:uploads:migrate[AvatarUploader, Project, :avatar]" +gitlab-rake "gitlab:uploads:migrate[AvatarUploader, Group, :avatar]" +gitlab-rake "gitlab:uploads:migrate[AvatarUploader, User, :avatar]" + +# Attachments +gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Note, :attachment]" +gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :logo]" +gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :header_logo]" + +# Markdown +gitlab-rake "gitlab:uploads:migrate[FileUploader, Project]" +gitlab-rake "gitlab:uploads:migrate[PersonalFileUploader, Snippet]" +gitlab-rake "gitlab:uploads:migrate[NamespaceFileUploader, Snippet]" +gitlab-rake "gitlab:uploads:migrate[FileUploader, MergeRequest]" +``` + +**Source Installation** + +>**Note:** +Use `RAILS_ENV=production` for every task. + +```bash +# sudo -u git -H bundle exec rake gitlab:uploads:migrate + +# Avatars +sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, Project, :avatar]" +sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, Group, :avatar]" +sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, User, :avatar]" + +# Attachments +sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Note, :attachment]" +sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :logo]" +sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :header_logo]" + +# Markdown +sudo -u git -H bundle exec rake "gitlab:uploads:migrate[FileUploader, Project]" +sudo -u git -H bundle exec rake "gitlab:uploads:migrate[PersonalFileUploader, Snippet]" +sudo -u git -H bundle exec rake "gitlab:uploads:migrate[NamespaceFileUploader, Snippet]" +sudo -u git -H bundle exec rake "gitlab:uploads:migrate[FileUploader, MergeRequest]" + +``` diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md new file mode 100644 index 00000000000..a82735cc72c --- /dev/null +++ b/doc/administration/uploads.md @@ -0,0 +1,209 @@ +# Uploads administration + +>**Notes:** +Uploads represent all user data that may be sent to GitLab as a single file. As an example, avatars and notes' attachments are uploads. Uploads are integral to GitLab functionality, and therefore cannot be disabled. + +### Using local storage + +>**Notes:** +This is the default configuration + +To change the location where the uploads are stored locally, follow the steps +below. + +--- + +**In Omnibus installations:** + +>**Notes:** +For historical reasons, uploads are stored into a base directory, which by default is `uploads/-/system`. It is strongly discouraged to change this configuration option on an existing GitLab installation. + +_The uploads are stored by default in `/var/opt/gitlab/gitlab-rails/public/uploads/-/system`._ + +1. To change the storage path for example to `/mnt/storage/uploads`, edit + `/etc/gitlab/gitlab.rb` and add the following line: + + ```ruby + gitlab_rails['uploads_storage_path'] = "/mnt/storage/" + gitlab_rails['uploads_base_dir'] = "uploads" + ``` + +1. Save the file and [reconfigure GitLab][] for the changes to take effect. + +--- + +**In installations from source:** + +_The uploads are stored by default in +`/home/git/gitlab/public/uploads/-/system`._ + +1. To change the storage path for example to `/mnt/storage/uploads`, edit + `/home/git/gitlab/config/gitlab.yml` and add or amend the following lines: + + ```yaml + uploads: + storage_path: /mnt/storage + base_dir: uploads + ``` + +1. Save the file and [restart GitLab][] for the changes to take effect. + +### Using object storage + +>**Notes:** +- [Introduced][ee-3867] in [GitLab Enterprise Edition Premium][eep] 10.5. + +If you don't want to use the local disk where GitLab is installed to store the +uploads, you can use an object storage provider like AWS S3 instead. +This configuration relies on valid AWS credentials to be configured already. + +### Object Storage Settings + +For source installations the following settings are nested under `uploads:` and then `object_store:`. On omnibus installs they are prefixed by `uploads_object_store_`. + +| Setting | Description | Default | +|---------|-------------|---------| +| `enabled` | Enable/disable object storage | `false` | +| `remote_directory` | The bucket name where Uploads will be stored| | +| `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` | +| `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | +| `connection` | Various connection options described below | | + +#### S3 compatible connection settings + +The connection settings match those provided by [Fog](https://github.com/fog), and are as follows: + +| Setting | Description | Default | +|---------|-------------|---------| +| `provider` | Always `AWS` for compatible hosts | AWS | +| `aws_access_key_id` | AWS credentials, or compatible | | +| `aws_secret_access_key` | AWS credentials, or compatible | | +| `region` | AWS region | us-east-1 | +| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | +| `endpoint` | Can be used when configuring an S3 compatible service such as [Minio](https://www.minio.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | +| `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false | + +**In Omnibus installations:** + +_The uploads are stored by default in +`/var/opt/gitlab/gitlab-rails/public/uploads/-/system`._ + +1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with + the values you want: + + ```ruby + gitlab_rails['uploads_object_store_enabled'] = true + gitlab_rails['uploads_object_store_remote_directory'] = "uploads" + gitlab_rails['uploads_object_store_connection'] = { + 'provider' => 'AWS', + 'region' => 'eu-central-1', + 'aws_access_key_id' => 'AWS_ACCESS_KEY_ID', + 'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY' + } + ``` + +>**Note:** +If you are using AWS IAM profiles, be sure to omit the AWS access key and secret acces key/value pairs. + + ```ruby + gitlab_rails['uploads_object_store_connection'] = { + 'provider' => 'AWS', + 'region' => 'eu-central-1', + 'use_iam_profile' => true + } + ``` + +1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Migrate any existing local uploads to the object storage: + +>**Notes:** +These task complies with the `BATCH` environment variable to process uploads in batch (200 by default). All of the processing will be done in a background worker and requires **no downtime**. + + ```bash + # gitlab-rake gitlab:uploads:migrate[uploader_class, model_class, mount_point] + + # Avatars + gitlab-rake "gitlab:uploads:migrate[AvatarUploader, Project, :avatar]" + gitlab-rake "gitlab:uploads:migrate[AvatarUploader, Group, :avatar]" + gitlab-rake "gitlab:uploads:migrate[AvatarUploader, User, :avatar]" + + # Attachments + gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Note, :attachment]" + gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :logo]" + gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :header_logo]" + + # Markdown + gitlab-rake "gitlab:uploads:migrate[FileUploader, Project]" + gitlab-rake "gitlab:uploads:migrate[PersonalFileUploader, Snippet]" + gitlab-rake "gitlab:uploads:migrate[NamespaceFileUploader, Snippet]" + gitlab-rake "gitlab:uploads:migrate[FileUploader, MergeRequest]" + ``` + + Currently this has to be executed manually and it will allow you to + migrate the existing uploads to the object storage, but all new + uploads will still be stored on the local disk. In the future + you will be given an option to define a default storage for all + new files. + +--- + +**In installations from source:** + +_The uploads are stored by default in +`/home/git/gitlab/public/uploads/-/system`._ + +1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following + lines: + + ```yaml + uploads: + object_store: + enabled: true + remote_directory: "uploads" # The bucket name + connection: + provider: AWS # Only AWS supported at the moment + aws_access_key_id: AWS_ACESS_KEY_ID + aws_secret_access_key: AWS_SECRET_ACCESS_KEY + region: eu-central-1 + ``` + +1. Save the file and [restart GitLab][] for the changes to take effect. +1. Migrate any existing local uploads to the object storage: + +>**Notes:** + +- These task comply with the `BATCH` environment variable to process uploads in batch (200 by default). All of the processing will be done in a background worker and requires **no downtime**. + +- To migrate in production use `RAILS_ENV=production` environment variable. + + ```bash + # sudo -u git -H bundle exec rake gitlab:uploads:migrate + + # Avatars + sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, Project, :avatar]" + sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, Group, :avatar]" + sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, User, :avatar]" + + # Attachments + sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Note, :attachment]" + sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :logo]" + sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :header_logo]" + + # Markdown + sudo -u git -H bundle exec rake "gitlab:uploads:migrate[FileUploader, Project]" + sudo -u git -H bundle exec rake "gitlab:uploads:migrate[PersonalFileUploader, Snippet]" + sudo -u git -H bundle exec rake "gitlab:uploads:migrate[NamespaceFileUploader, Snippet]" + sudo -u git -H bundle exec rake "gitlab:uploads:migrate[FileUploader, MergeRequest]" + + ``` + + Currently this has to be executed manually and it will allow you to + migrate the existing uploads to the object storage, but all new + uploads will still be stored on the local disk. In the future + you will be given an option to define a default storage for all + new files. + +[reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab" +[restart gitlab]: restart_gitlab.md#installations-from-source "How to restart GitLab" +[eep]: https://about.gitlab.com/gitlab-ee/ "GitLab Enterprise Edition Premium" +[ee-3867]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3867 diff --git a/doc/api/notification_settings.md b/doc/api/notification_settings.md index 3a2c398e355..f05ae647577 100644 --- a/doc/api/notification_settings.md +++ b/doc/api/notification_settings.md @@ -24,6 +24,7 @@ reopen_issue close_issue reassign_issue new_merge_request +push_to_merge_request reopen_merge_request close_merge_request reassign_merge_request @@ -75,6 +76,7 @@ curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab | `close_issue` | boolean | no | Enable/disable this notification | | `reassign_issue` | boolean | no | Enable/disable this notification | | `new_merge_request` | boolean | no | Enable/disable this notification | +| `push_to_merge_request` | boolean | no | Enable/disable this notification | | `reopen_merge_request` | boolean | no | Enable/disable this notification | | `close_merge_request` | boolean | no | Enable/disable this notification | | `reassign_merge_request` | boolean | no | Enable/disable this notification | @@ -141,6 +143,7 @@ curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab | `close_issue` | boolean | no | Enable/disable this notification | | `reassign_issue` | boolean | no | Enable/disable this notification | | `new_merge_request` | boolean | no | Enable/disable this notification | +| `push_to_merge_request` | boolean | no | Enable/disable this notification | | `reopen_merge_request` | boolean | no | Enable/disable this notification | | `close_merge_request` | boolean | no | Enable/disable this notification | | `reassign_merge_request` | boolean | no | Enable/disable this notification | @@ -164,6 +167,7 @@ Example responses: "close_issue": false, "reassign_issue": false, "new_merge_request": false, + "push_to_merge_request": false, "reopen_merge_request": false, "close_merge_request": false, "reassign_merge_request": false, diff --git a/doc/development/i18n/proofreader.md b/doc/development/i18n/proofreader.md index 960eabd5538..cf62314bc29 100644 --- a/doc/development/i18n/proofreader.md +++ b/doc/development/i18n/proofreader.md @@ -10,6 +10,7 @@ are very appreciative of the work done by translators and proofreaders! - Huang Tao - [GitLab](https://gitlab.com/htve), [Crowdin](https://crowdin.com/profile/htve) - Chinese Traditional - Huang Tao - [GitLab](https://gitlab.com/htve), [Crowdin](https://crowdin.com/profile/htve) + - Weizhe Ding - [GitLab](https://gitlab.com/d.weizhe), [Crowdin](https://crowdin.com/profile/d.weizhe) - Chinese Traditional, Hong Kong - Huang Tao - [GitLab](https://gitlab.com/htve), [Crowdin](https://crowdin.com/profile/htve) - Dutch diff --git a/doc/img/devops_lifecycle.png b/doc/img/devops_lifecycle.png Binary files differnew file mode 100644 index 00000000000..0616be46df8 --- /dev/null +++ b/doc/img/devops_lifecycle.png diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index 20087a981f9..3edde3de83d 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -32,6 +32,7 @@ contains some settings that are common for all providers. - [Auth0](auth0.md) - [Authentiq](../administration/auth/authentiq.md) - [OAuth2Generic](oauth2_generic.md) +- [JWT](../administration/auth/jwt.md) ## Initial OmniAuth Configuration diff --git a/doc/integration/saml.md b/doc/integration/saml.md index f8a7dd6b1dc..3f49432ce93 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -102,9 +102,10 @@ in your SAML IdP: installation to generate the correct value). 1. Change the values of `idp_cert_fingerprint`, `idp_sso_target_url`, - `name_identifier_format` to match your IdP. Check + `name_identifier_format` to match your IdP. If a fingerprint is used it must + be a SHA1 fingerprint; check [the omniauth-saml documentation](https://github.com/omniauth/omniauth-saml) - for details on these options. + for more details on these options. 1. Change the value of `issuer` to a unique name, which will identify the application to the IdP. @@ -311,6 +312,7 @@ need to be validated using a fingerprint, a certificate or a validator. For this you need take the following into account: +- If a fingerprint is used, it must be the SHA1 fingerprint - If no certificate is provided in the settings, a fingerprint or fingerprint validator needs to be provided and the response from the server must contain a certificate (`<ds:KeyInfo><ds:X509Data><ds:X509Certificate>`) diff --git a/doc/raketasks/README.md b/doc/raketasks/README.md index 2f916f5dea7..90187617c41 100644 --- a/doc/raketasks/README.md +++ b/doc/raketasks/README.md @@ -14,3 +14,4 @@ comments: false - [Webhooks](web_hooks.md) - [Import](import.md) of git repositories in bulk - [Rebuild authorized_keys file](http://docs.gitlab.com/ce/raketasks/maintenance.html#rebuild-authorized_keys-file) task for administrators +- [Migrate Uploads](../administration/raketasks/uploads/migrate.md) diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md new file mode 100644 index 00000000000..83eb7a225b2 --- /dev/null +++ b/doc/user/gitlab_com/index.md @@ -0,0 +1,262 @@ +# GitLab.com settings + +In this page you will find information about the settings that are used on +[GitLab.com](https://about.gitlab.com/pricing). + +## SSH host keys fingerprints + +Below are the fingerprints for GitLab.com's SSH host keys. + +| Algorithm | MD5 | SHA256 | +| --------- | --- | ------- | +| DSA | `7a:47:81:3a:ee:89:89:64:33:ca:44:52:3d:30:d4:87` | `p8vZBUOR0XQz6sYiaWSMLmh0t9i8srqYKool/Xfdfqw` | +| ECDSA | `f1:d0:fb:46:73:7a:70:92:5a:ab:5d:ef:43:e2:1c:35` | `HbW3g8zUjNSksFbqTiUWPWg2Bq1x8xdGUrliXFzSnUw` | +| ED25519 | `2e:65:6a:c8:cf:bf:b2:8b:9a:bd:6d:9f:11:5c:12:16` | `eUXGGm1YGsMAS7vkcx6JOJdOGHPem5gQp4taiCfCLB8` | +| RSA | `b6:03:0e:39:97:9e:d0:e7:24:ce:a3:77:3e:01:42:09` | `ROQFvPThGrW4RuWLoL9tq9I9zJ42fK4XywyRtbOz/EQ` | + +## Mail configuration + +GitLab.com sends emails from the `mg.gitlab.com` domain via [Mailgun] and has +its own dedicated IP address (`198.61.254.240`). + +## Alternative SSH port + +GitLab.com can be reached via a [different SSH port][altssh] for `git+ssh`. + +| Setting | Value | +| --------- | ------------------- | +| `Hostname` | `altssh.gitlab.com` | +| `Port` | `443` | + +An example `~/.ssh/config` is the following: + +``` +Host gitlab.com + Hostname altssh.gitlab.com + User git + Port 443 + PreferredAuthentications publickey + IdentityFile ~/.ssh/gitlab +``` + +## GitLab Pages + +Below are the settings for [GitLab Pages]. + +| Setting | GitLab.com | Default | +| ----------------------- | ---------------- | ------------- | +| Domain name | `gitlab.io` | - | +| IP address | `52.167.214.135` | - | +| Custom domains support | yes | no | +| TLS certificates support| yes | no | + +The maximum size of your Pages site is regulated by the artifacts maximum size +which is part of [GitLab CI](#gitlab-ci). + +## GitLab CI/CD + +Below are the current settings regarding [GitLab CI/CD](../../ci/README.md). + +| Setting | GitLab.com | Default | +| ----------- | ----------------- | ------------- | +| Artifacts maximum size | 1G | 100M | + +## Shared Runners + +Shared Runners on GitLab.com run in [autoscale mode] and powered by +DigitalOcean. Autoscaling means reduced waiting times to spin up builds, +and isolated VMs for each project, thus maximizing security. + +They're free to use for public open source projects and limited to 2000 CI +minutes per month per group for private projects. Read about all +[GitLab.com plans](https://about.gitlab.com/pricing/). + +All your builds run on 2GB (RAM) ephemeral instances, with CoreOS and the latest +Docker Engine installed. The default region of the VMs is NYC. + +Below are the shared Runners settings. + +| Setting | GitLab.com | Default | +| ----------- | ----------------- | ---------- | +| [GitLab Runner] | [Runner versions dashboard][ci_version_dashboard] | - | +| Executor | `docker+machine` | - | +| Default Docker image | `ruby:2.1` | - | +| `privileged` (run [Docker in Docker]) | `true` | `false` | + +[ci_version_dashboard]: https://monitor.gitlab.net/dashboard/db/ci?refresh=5m&orgId=1&panelId=12&fullscreen&from=now-1h&to=now&var-runner_type=All&var-cache_server=All&var-gl_monitor_fqdn=postgres-01.db.prd.gitlab.com&var-has_minutes=yes&var-hanging_droplets_cleaner=All&var-droplet_zero_machines_cleaner=All&var-runner_job_failure_reason=All&theme=light + +### `config.toml` + +The full contents of our `config.toml` are: + +```toml +[[runners]] + name = "docker-auto-scale" + limit = X + request_concurrency = X + url = "https://gitlab.com/ci" + token = "SHARED_RUNNER_TOKEN" + executor = "docker+machine" + environment = [ + "DOCKER_DRIVER=overlay2" + ] + [runners.docker] + image = "ruby:2.1" + privileged = true + [runners.machine] + IdleCount = 40 + IdleTime = 1800 + MaxBuilds = 1 + MachineDriver = "digitalocean" + MachineName = "machine-%s-digital-ocean-2gb" + MachineOptions = [ + "digitalocean-image=coreos-stable", + "digitalocean-ssh-user=core", + "digitalocean-access-token=DIGITAL_OCEAN_ACCESS_TOKEN", + "digitalocean-region=nyc1", + "digitalocean-size=2gb", + "digitalocean-private-networking", + "digitalocean-userdata=/etc/gitlab-runner/cloudinit.sh", + "engine-registry-mirror=http://IP_TO_OUR_REGISTRY_MIRROR" + ] + [runners.cache] + Type = "s3" + ServerAddress = "IP_TO_OUR_CACHE_SERVER" + AccessKey = "ACCESS_KEY" + SecretKey = "ACCESS_SECRET_KEY" + BucketName = "runner" + Shared = true +``` + +## Sidekiq + +GitLab.com runs [Sidekiq][sidekiq] with arguments `--timeout=4 --concurrency=4` +and the following environment variables: + +| Setting | GitLab.com | Default | +|-------- |----------- |-------- | +| `SIDEKIQ_MEMORY_KILLER_MAX_RSS` | `1000000` | `1000000` | +| `SIDEKIQ_MEMORY_KILLER_SHUTDOWN_SIGNAL` | `SIGKILL` | - | +| `SIDEKIQ_LOG_ARGUMENTS` | `1` | - | + +## Cron jobs + +Periodically executed jobs by Sidekiq, to self-heal Gitlab, do external +synchronizations, run scheduled pipelines, etc.: + +| Setting | GitLab.com | Default | +|-------- |------------- |------------- | +| `pipeline_schedule_worker` | `19 * * * *` | `19 * * * *` | + +## PostgreSQL + +GitLab.com being a fairly large installation of GitLab means we have changed +various PostgreSQL settings to better suit our needs. For example, we use +streaming replication and servers in hot-standby mode to balance queries across +different database servers. + +The list of GitLab.com specific settings (and their defaults) is as follows: + +| Setting | GitLab.com | Default | +|:------------------------------------|:--------------------------------------------------------------------|:--------------------------------------| +| archive_command | `/usr/bin/envdir /etc/wal-e.d/env /opt/wal-e/bin/wal-e wal-push %p` | empty | +| archive_mode | on | off | +| autovacuum_analyze_scale_factor | 0.01 | 0.01 | +| autovacuum_max_workers | 6 | 3 | +| autovacuum_vacuum_cost_limit | 1000 | -1 | +| autovacuum_vacuum_scale_factor | 0.01 | 0.02 | +| checkpoint_completion_target | 0.7 | 0.9 | +| checkpoint_segments | 32 | 10 | +| effective_cache_size | 338688MB | Based on how much memory is available | +| hot_standby | on | off | +| hot_standby_feedback | on | off | +| log_autovacuum_min_duration | 0 | -1 | +| log_checkpoints | on | off | +| log_line_prefix | `%t [%p]: [%l-1] ` | empty | +| log_min_duration_statement | 1000 | -1 | +| log_temp_files | 0 | -1 | +| maintenance_work_mem | 2048MB | 16 MB | +| max_replication_slots | 5 | 0 | +| max_wal_senders | 32 | 0 | +| max_wal_size | 5GB | 1GB | +| shared_buffers | 112896MB | Based on how much memory is available | +| shared_preload_libraries | pg_stat_statements | empty | +| shmall | 30146560 | Based on the server's capabilities | +| shmmax | 123480309760 | Based on the server's capabilities | +| wal_buffers | 16MB | -1 | +| wal_keep_segments | 512 | 10 | +| wal_level | replica | minimal | +| statement_timeout | 15s | 60s | +| idle_in_transaction_session_timeout | 60s | 60s | + +Some of these settings are in the process being adjusted. For example, the value +for `shared_buffers` is quite high and as such we are looking into adjusting it. +More information on this particular change can be found at +<https://gitlab.com/gitlab-com/infrastructure/issues/1555>. An up to date list +of proposed changes can be found at +<https://gitlab.com/gitlab-com/infrastructure/issues?scope=all&utf8=%E2%9C%93&state=opened&label_name[]=database&label_name[]=change>. + +## Unicorn + +GitLab.com adjusts the memory limits for the [unicorn-worker-killer][unicorn-worker-killer] gem. + +Base default: +* `memory_limit_min` = 750MiB +* `memory_limit_max` = 1024MiB + +Web front-ends: +* `memory_limit_min` = 1024MiB +* `memory_limit_max` = 1280MiB + +## GitLab.com at scale + +In addition to the GitLab Enterprise Edition Omnibus install, GitLab.com uses +the following applications and settings to achieve scale. All settings are +located publicly available [chef cookbooks](https://gitlab.com/gitlab-cookbooks). + +### ELK + +We use Elasticsearch, logstash, and Kibana for part of our monitoring solution: + +- [gitlab-cookbooks / gitlab-elk · GitLab](https://gitlab.com/gitlab-cookbooks/gitlab-elk) +- [gitlab-cookbooks / gitlab_elasticsearch · GitLab](https://gitlab.com/gitlab-cookbooks/gitlab_elasticsearch) + +### Prometheus + +Prometheus complete our monitoring stack: + +- [gitlab-cookbooks / gitlab-prometheus · GitLab](https://gitlab.com/gitlab-cookbooks/gitlab-prometheus) + +### Grafana + +For the visualization of monitoring data: + +- [gitlab-cookbooks / gitlab-grafana · GitLab](https://gitlab.com/gitlab-cookbooks/gitlab-grafana) + +### Sentry + +Open source error tracking: + +- [gitlab-cookbooks / gitlab-sentry · GitLab](https://gitlab.com/gitlab-cookbooks/gitlab-sentry) + +### Consul + +Service discovery: + +- [gitlab-cookbooks / gitlab_consul · GitLab](https://gitlab.com/gitlab-cookbooks/gitlab_consul) + +### Haproxy + +High Performance TCP/HTTP Load Balancer: + +- [gitlab-cookbooks / gitlab-haproxy · GitLab](https://gitlab.com/gitlab-cookbooks/gitlab-haproxy) + +[autoscale mode]: https://docs.gitlab.com/runner/configuration/autoscale.html "How Autoscale works" +[runners-post]: https://about.gitlab.com/2016/04/05/shared-runners/ "Shared Runners on GitLab.com" +[GitLab Runner]: https://gitlab.com/gitlab-org/gitlab-runner +[altssh]: https://about.gitlab.com/2016/02/18/gitlab-dot-com-now-supports-an-alternate-git-plus-ssh-port/ "GitLab.com now supports an alternate git+ssh port" +[GitLab Pages]: https://about.gitlab.com/features/pages "GitLab Pages" +[docker in docker]: https://hub.docker.com/_/docker/ "Docker in Docker at DockerHub" +[mailgun]: https://www.mailgun.com/ "Mailgun website" +[sidekiq]: http://sidekiq.org/ "Sidekiq website" +[unicorn-worker-killer]: https://rubygems.org/gems/unicorn-worker-killer "unicorn-worker-killer" diff --git a/doc/user/profile/preferences.md b/doc/user/profile/preferences.md index 022d6317555..930e506802a 100644 --- a/doc/user/profile/preferences.md +++ b/doc/user/profile/preferences.md @@ -41,7 +41,7 @@ select few, the amount of activity on the default Dashboard page can be overwhelming. Changing this setting allows you to redefine what your default dashboard will be. -You have 6 options here that you can use for your default dashboard view: +You have 8 options here that you can use for your default dashboard view: - Your projects (default) - Starred projects @@ -49,6 +49,8 @@ You have 6 options here that you can use for your default dashboard view: - Starred projects' activity - Your groups - Your [Todos] +- Assigned Issues +- Assigned Merge Requests ### Project home page content diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index bd9bcfadb99..716787532fc 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -71,7 +71,7 @@ You need Master [permissions] and above to access the Kubernetes page. To add an existing Kubernetes cluster to your project: 1. Navigate to your project's **CI/CD > Kubernetes** page. -1. Click on **Add Kuberntes cluster**. +1. Click on **Add Kubernetes cluster**. 1. Click on **Add an existing Kubernetes cluster** and fill in the details: - **Kubernetes cluster name** (required) - The name you wish to give the cluster. - **Environment scope** (required)- The @@ -101,7 +101,7 @@ To add an existing Kubernetes cluster to your project: - If you or someone created a secret specifically for the project, usually with limited permissions, the secret's namespace and project namespace may be the same. -1. Finally, click the **Create Kuberntes cluster** button. +1. Finally, click the **Create Kubernetes cluster** button. After a few moments, your cluster should be created. If something goes wrong, you will be notified. diff --git a/doc/user/project/pipelines/schedules.md b/doc/user/project/pipelines/schedules.md index 34809a2826f..a13b1b4561c 100644 --- a/doc/user/project/pipelines/schedules.md +++ b/doc/user/project/pipelines/schedules.md @@ -12,7 +12,7 @@ month on the 22nd for a certain branch. In order to schedule a pipeline: -1. Navigate to your project's **Pipelines ➔ Schedules** and click the +1. Navigate to your project's **CI / CD ➔ Schedules** and click the **New Schedule** button. 1. Fill in the form 1. Hit **Save pipeline schedule** for the changes to take effect. diff --git a/doc/workflow/lfs/lfs_administration.md b/doc/workflow/lfs/lfs_administration.md index d768b73286d..ca28e0a3304 100644 --- a/doc/workflow/lfs/lfs_administration.md +++ b/doc/workflow/lfs/lfs_administration.md @@ -5,6 +5,7 @@ Documentation on how to use Git LFS are under [Managing large binary files with ## Requirements * Git LFS is supported in GitLab starting with version 8.2. +* Support for object storage, such as AWS S3, was introduced in 10.0. * Users need to install [Git LFS client](https://git-lfs.github.com) version 1.0.1 and up. ## Configuration @@ -12,16 +13,18 @@ Documentation on how to use Git LFS are under [Managing large binary files with Git LFS objects can be large in size. By default, they are stored on the server GitLab is installed on. -There are two configuration options to help GitLab server administrators: +There are various configuration options to help GitLab server administrators: * Enabling/disabling Git LFS support * Changing the location of LFS object storage +* Setting up AWS S3 compatible object storage ### Omnibus packages In `/etc/gitlab/gitlab.rb`: ```ruby +# Change to true to enable lfs gitlab_rails['lfs_enabled'] = false # Optionally, change the storage path location. Defaults to @@ -35,11 +38,114 @@ gitlab_rails['lfs_storage_path'] = "/mnt/storage/lfs-objects" In `config/gitlab.yml`: ```yaml +# Change to true to enable lfs lfs: enabled: false storage_path: /mnt/storage/lfs-objects ``` +## Setting up S3 compatible object storage + +> **Note:** [Introduced][ee-2760] in [GitLab Premium][eep] 10.0. +> Available in [GitLab CE][ce] 10.7 + +It is possible to store LFS objects on remote object storage instead of on a local disk. + +This allows you to offload storage to an external AWS S3 compatible service, freeing up disk space locally. You can also host your own S3 compatible storage decoupled from GitLab, with with a service such as [Minio](https://www.minio.io/). + +Object storage currently transfers files first to GitLab, and then on the object storage in a second stage. This can be done either by using a rake task to transfer existing objects, or in a background job after each file is received. + +### Object Storage Settings + +For source installations the following settings are nested under `lfs:` and then `object_store:`. On omnibus installs they are prefixed by `lfs_object_store_`. + +| Setting | Description | Default | +|---------|-------------|---------| +| `enabled` | Enable/disable object storage | `false` | +| `remote_directory` | The bucket name where LFS objects will be stored| | +| `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` | +| `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | +| `connection` | Various connection options described below | | + +#### S3 compatible connection settings + +The connection settings match those provided by [Fog](https://github.com/fog), and are as follows: + +| Setting | Description | Default | +|---------|-------------|---------| +| `provider` | Always `AWS` for compatible hosts | AWS | +| `aws_access_key_id` | AWS credentials, or compatible | | +| `aws_secret_access_key` | AWS credentials, or compatible | | +| `region` | AWS region | us-east-1 | +| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | +| `endpoint` | Can be used when configuring an S3 compatible service such as [Minio](https://www.minio.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | +| `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false | + + +### From source + +1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following + lines: + + ```yaml + lfs: + enabled: true + object_store: + enabled: false + remote_directory: lfs-objects # Bucket name + connection: + provider: AWS + aws_access_key_id: 1ABCD2EFGHI34JKLM567N + aws_secret_access_key: abcdefhijklmnopQRSTUVwxyz0123456789ABCDE + region: eu-central-1 + # Use the following options to configure an AWS compatible host such as Minio + host: 'localhost' + endpoint: 'http://127.0.0.1:9000' + path_style: true + ``` + +1. Save the file and [restart GitLab][] for the changes to take effect. +1. Migrate any existing local LFS objects to the object storage: + + ```bash + sudo -u git -H bundle exec rake gitlab:lfs:migrate RAILS_ENV=production + ``` + + This will migrate existing LFS objects to object storage. New LFS objects + will be forwarded to object storage unless + `gitlab_rails['lfs_object_store_background_upload']` is set to false. + +### In Omnibus + +1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with + the values you want: + + ```ruby + gitlab_rails['lfs_object_store_enabled'] = true + gitlab_rails['lfs_object_store_remote_directory'] = "lfs-objects" + gitlab_rails['lfs_object_store_connection'] = { + 'provider' => 'AWS', + 'region' => 'eu-central-1', + 'aws_access_key_id' => '1ABCD2EFGHI34JKLM567N', + 'aws_secret_access_key' => 'abcdefhijklmnopQRSTUVwxyz0123456789ABCDE', + # The below options configure an S3 compatible host instead of AWS + 'host' => 'localhost', + 'endpoint' => 'http://127.0.0.1:9000', + 'path_style' => true + } + ``` + +1. Save the file and [reconfigure GitLab]s for the changes to take effect. +1. Migrate any existing local LFS objects to the object storage: + + ```bash + gitlab-rake gitlab:lfs:migrate + ``` + + This will migrate existing LFS objects to object storage. New LFS objects + will be forwarded to object storage unless + `gitlab_rails['lfs_object_store_background_upload']` is set to false. + ## Storage statistics You can see the total storage used for LFS objects on groups and projects @@ -48,10 +154,13 @@ and [projects APIs](../../api/projects.md). ## Known limitations -* Currently, storing GitLab Git LFS objects on a non-local storage (like S3 buckets) - is not supported * Support for removing unreferenced LFS objects was added in 8.14 onwards. * LFS authentications via SSH was added with GitLab 8.12 * Only compatible with the GitLFS client versions 1.1.0 and up, or 1.0.2. * The storage statistics currently count each LFS object multiple times for every project linking to it + +[reconfigure gitlab]: ../../administration/restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab" +[restart gitlab]: ../../administration/restart_gitlab.md#installations-from-source "How to restart GitLab" +[eep]: https://about.gitlab.com/products/ "GitLab Premium" +[ee-2760]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2760 diff --git a/doc/workflow/notifications.md b/doc/workflow/notifications.md index 37265a5b771..c4095ee0f69 100644 --- a/doc/workflow/notifications.md +++ b/doc/workflow/notifications.md @@ -67,7 +67,7 @@ Below is the table of events users can be notified of: ### Issue / Merge request events -In all of the below cases, the notification will be sent to: +In most of the below cases, the notification will be sent to: - Participants: - the author and assignee of the issue/merge request - authors of comments on the issue/merge request @@ -87,6 +87,7 @@ In all of the below cases, the notification will be sent to: | Reassign issue | The above, plus the old assignee | | Reopen issue | | | New merge request | | +| Push to merge request | Participants and Custom notification level with this event selected | | Reassign merge request | The above, plus the old assignee | | Close merge request | | | Reopen merge request | | diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index e4fca77ab5d..e59e8a45908 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -410,7 +410,7 @@ module API ) end - def present_file!(path, filename, content_type = 'application/octet-stream') + def present_disk_file!(path, filename, content_type = 'application/octet-stream') filename ||= File.basename(path) header['Content-Disposition'] = "attachment; filename=#{filename}" header['Content-Transfer-Encoding'] = 'binary' @@ -426,13 +426,17 @@ module API end end - def present_artifacts!(artifacts_file) - return not_found! unless artifacts_file.exists? + def present_carrierwave_file!(file, supports_direct_download: true) + return not_found! unless file.exists? - if artifacts_file.file_storage? - present_file!(artifacts_file.path, artifacts_file.filename) + if file.file_storage? + present_disk_file!(file.path, file.filename) + elsif supports_direct_download && file.class.direct_download_enabled? + redirect(file.url) else - redirect_to(artifacts_file.url) + header(*Gitlab::Workhorse.send_url(file.url)) + status :ok + body end end diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 47e5eeab31d..b1adef49d46 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -28,7 +28,7 @@ module API builds = user_project.latest_successful_builds_for(params[:ref_name]) latest_build = builds.find_by!(name: params[:job]) - present_artifacts!(latest_build.artifacts_file) + present_carrierwave_file!(latest_build.artifacts_file) end desc 'Download the artifacts archive from a job' do @@ -43,7 +43,7 @@ module API build = find_build!(params[:job_id]) - present_artifacts!(build.artifacts_file) + present_carrierwave_file!(build.artifacts_file) end desc 'Download a specific file from artifacts archive' do diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 9c205514b3a..60911c8d733 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -72,7 +72,7 @@ module API present build, with: Entities::Job end - # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace + # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. desc 'Get a trace of a specific job of a project' diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index b0a7fd6f4ab..efc4a33ae1b 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -25,7 +25,7 @@ module API render_api_error!('404 Not found or has expired', 404) unless path - present_file!(path, File.basename(path), 'application/gzip') + present_disk_file!(path, File.basename(path), 'application/gzip') end desc 'Start export' do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index b552b0e0c5d..467bc78dad8 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -228,11 +228,7 @@ module API namespace_id = fork_params[:namespace] if namespace_id.present? - fork_params[:namespace] = if namespace_id =~ /^\d+$/ - Namespace.find_by(id: namespace_id) - else - Namespace.find_by_path_or_name(namespace_id) - end + fork_params[:namespace] = find_namespace(namespace_id) unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) not_found!('Target Namespace') diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index c15c487deb4..33321db46e9 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -52,11 +52,7 @@ module API conflict!("Protected branch '#{params[:name]}' already exists") end - # Replace with `declared(params)` after updating to grape v1.0.2 - # See https://github.com/ruby-grape/grape/pull/1710 - # and https://gitlab.com/gitlab-org/gitlab-ce/issues/40843 - declared_params = params.slice("name", "push_access_level", "merge_access_level", "allowed_to_push", "allowed_to_merge") - + declared_params = declared_params(include_missing: false) api_service = ::ProtectedBranches::ApiService.new(user_project, current_user, declared_params) protected_branch = api_service.create diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 7e6c33ec33d..8da97a97754 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -244,11 +244,12 @@ module API params do requires :id, type: Integer, desc: %q(Job's ID) optional :token, type: String, desc: %q(Job's authentication token) + optional :direct_download, default: false, type: Boolean, desc: %q(Perform direct download from remote storage instead of proxying artifacts) end get '/:id/artifacts' do job = authenticate_job! - present_artifacts!(job.artifacts_file) + present_carrierwave_file!(job.artifacts_file, supports_direct_download: params[:direct_download]) end end end diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index ac76fece931..683b9c993cb 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -85,7 +85,7 @@ module API build = get_build!(params[:build_id]) - present_artifacts!(build.artifacts_file) + present_carrierwave_file!(build.artifacts_file) end desc 'Download the artifacts file from build' do @@ -102,10 +102,10 @@ module API builds = user_project.latest_successful_builds_for(params[:ref_name]) latest_build = builds.find_by!(name: params[:job]) - present_artifacts!(latest_build.artifacts_file) + present_carrierwave_file!(latest_build.artifacts_file) end - # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace + # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. desc 'Get a trace of a specific build of a project' diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 7d8b1f369fe..a2df969d819 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -268,11 +268,7 @@ module API namespace_id = fork_params[:namespace] if namespace_id.present? - fork_params[:namespace] = if namespace_id =~ /^\d+$/ - Namespace.find_by(id: namespace_id) - else - Namespace.find_by_path_or_name(namespace_id) - end + fork_params[:namespace] = find_namespace(namespace_id) unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) not_found!('Target Namespace') diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 6715159a1aa..88a7f2a4235 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -65,7 +65,7 @@ module Backup def restore Gitlab.config.repositories.storages.each do |name, repository_storage| - path = repository_storage['path'] + path = repository_storage.legacy_disk_path next unless File.exist?(path) # Move repos dir to 'repositories.old' dir @@ -200,7 +200,7 @@ module Backup end def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } + Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } end def progress diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 75b64ae9af2..ce401c1c31c 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -21,12 +21,13 @@ module Banzai # # See http://en.wikipedia.org/wiki/URI_scheme # - # The negative lookbehind ensures that users can paste a URL followed by a - # period or comma for punctuation without those characters being included - # in the generated link. + # The negative lookbehind ensures that users can paste a URL followed by + # punctuation without those characters being included in the generated + # link. It matches the behaviour of Rinku 2.0.1: + # https://github.com/vmg/rinku/blob/v2.0.1/ext/rinku/autolink.c#L65 # - # Rubular: http://rubular.com/r/JzPhi6DCZp - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!,|\.)} + # Rubular: http://rubular.com/r/nrL3r9yUiq + LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)} # Text matching LINK_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index 77299abe324..8f541dcfdb2 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -17,7 +17,7 @@ module Banzai issuables.each do |node, issuable| next if !can_read_cross_project? && issuable.project != project - if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project) + if VISIBLE_STATES.include?(issuable.state) && issuable_reference?(node.inner_html, issuable) node.content += " (#{issuable.state})" end end @@ -27,6 +27,10 @@ module Banzai private + def issuable_reference?(text, issuable) + text == issuable.reference_link_text(project || group) + end + def can_read_cross_project? Ability.allowed?(current_user, :read_cross_project) end @@ -38,6 +42,10 @@ module Banzai def project context[:project] end + + def group + context[:group] + end end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index f5ccf952cf9..6af763faf10 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -69,7 +69,11 @@ module Gitlab authenticators.compact! - user if authenticators.find { |auth| auth.login(login, password) } + # return found user that was authenticated first for given login credentials + authenticators.find do |auth| + authenticated_user = auth.login(login, password) + break authenticated_user if authenticated_user + end end end diff --git a/lib/gitlab/auth/database/authentication.rb b/lib/gitlab/auth/database/authentication.rb index 260a77058a4..1234ace0334 100644 --- a/lib/gitlab/auth/database/authentication.rb +++ b/lib/gitlab/auth/database/authentication.rb @@ -8,7 +8,7 @@ module Gitlab def login(login, password) return false unless Gitlab::CurrentSettings.password_authentication_enabled_for_git? - user&.valid_password?(password) + return user if user&.valid_password?(password) end end end diff --git a/lib/gitlab/auth/ldap/authentication.rb b/lib/gitlab/auth/ldap/authentication.rb index e70c3ab6b46..7c134fb6438 100644 --- a/lib/gitlab/auth/ldap/authentication.rb +++ b/lib/gitlab/auth/ldap/authentication.rb @@ -12,30 +12,26 @@ module Gitlab return unless Gitlab::Auth::LDAP::Config.enabled? return unless login.present? && password.present? - auth = nil - # loop through providers until valid bind + # return found user that was authenticated by first provider for given login credentials providers.find do |provider| auth = new(provider) - auth.login(login, password) # true will exit the loop + break auth.user if auth.login(login, password) # true will exit the loop end - - # If (login, password) was invalid for all providers, the value of auth is now the last - # Gitlab::Auth::LDAP::Authentication instance we tried. - auth.user end def self.providers Gitlab::Auth::LDAP::Config.providers end - attr_accessor :ldap_user - def login(login, password) - @ldap_user = adapter.bind_as( + result = adapter.bind_as( filter: user_filter(login), size: 1, password: password ) + return unless result + + @user = Gitlab::Auth::LDAP::User.find_by_uid_and_provider(result.dn, provider) end def adapter @@ -56,12 +52,6 @@ module Gitlab filter end - - def user - return unless ldap_user - - Gitlab::Auth::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) - end end end end diff --git a/lib/gitlab/auth/o_auth/authentication.rb b/lib/gitlab/auth/o_auth/authentication.rb index ed03b9f8b40..d4e7f35c857 100644 --- a/lib/gitlab/auth/o_auth/authentication.rb +++ b/lib/gitlab/auth/o_auth/authentication.rb @@ -12,6 +12,7 @@ module Gitlab @user = user end + # Implementation must return user object if login successful def login(login, password) raise NotImplementedError end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d5e17a123df..f4c8d5342c1 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -9,11 +9,16 @@ module Gitlab ::Ci::Pipeline.transaction do pipeline.save! - @command.seeds_block&.call(pipeline) - - ::Ci::CreatePipelineStagesService - .new(project, current_user) - .execute(pipeline) + ## + # Create environments before the pipeline starts. + # + pipeline.builds.each do |build| + if build.has_environment? + project.environments.find_or_create_by( + name: build.expanded_environment_name + ) + end + end end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb new file mode 100644 index 00000000000..b2b00c8cb4b --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -0,0 +1,47 @@ +module Gitlab + module Ci + module Pipeline + module Chain + class Populate < Chain::Base + include Chain::Helpers + + PopulateError = Class.new(StandardError) + + def perform! + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) + + ## + # Populate pipeline with all stages and builds from pipeline seeds. + # + pipeline.stage_seeds.each do |stage| + stage.user = current_user + + pipeline.stages << stage.to_resource + + stage.seeds.each do |build| + pipeline.builds << build.to_resource + end + end + + if pipeline.stages.none? + return error('No stages / jobs for this pipeline.') + end + + if pipeline.invalid? + return error('Failed to build the pipeline!') + end + + raise Populate::PopulateError if pipeline.persisted? + end + + def break? + pipeline.errors.any? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb index 075504bcce5..a3bd2a5a23a 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/config.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -16,11 +16,7 @@ module Gitlab @pipeline.drop!(:config_error) end - return error(@pipeline.yaml_errors) - end - - unless @pipeline.has_stage_seeds? - return error('No stages / jobs for this pipeline.') + error(@pipeline.yaml_errors) end end diff --git a/lib/gitlab/ci/pipeline/seed/base.rb b/lib/gitlab/ci/pipeline/seed/base.rb new file mode 100644 index 00000000000..db9706924bb --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/base.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Pipeline + module Seed + class Base + def attributes + raise NotImplementedError + end + + def included? + raise NotImplementedError + end + + def to_resource + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb new file mode 100644 index 00000000000..7cd7c864448 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -0,0 +1,52 @@ +module Gitlab + module Ci + module Pipeline + module Seed + class Build < Seed::Base + include Gitlab::Utils::StrongMemoize + + delegate :dig, to: :@attributes + + def initialize(pipeline, attributes) + @pipeline = pipeline + @attributes = attributes + + @only = attributes.delete(:only) + @except = attributes.delete(:except) + end + + def user=(current_user) + @attributes.merge!(user: current_user) + end + + def included? + strong_memoize(:inclusion) do + only_specs = Gitlab::Ci::Build::Policy.fabricate(@only) + except_specs = Gitlab::Ci::Build::Policy.fabricate(@except) + + only_specs.all? { |spec| spec.satisfied_by?(@pipeline) } && + except_specs.none? { |spec| spec.satisfied_by?(@pipeline) } + end + end + + def attributes + @attributes.merge( + pipeline: @pipeline, + project: @pipeline.project, + ref: @pipeline.ref, + tag: @pipeline.tag, + trigger_request: @pipeline.legacy_trigger, + protected: @pipeline.protected_ref? + ) + end + + def to_resource + strong_memoize(:resource) do + ::Ci::Build.new(attributes) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb new file mode 100644 index 00000000000..1fcbdc1b15a --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -0,0 +1,51 @@ +module Gitlab + module Ci + module Pipeline + module Seed + class Stage < Seed::Base + include Gitlab::Utils::StrongMemoize + + delegate :size, to: :seeds + delegate :dig, to: :seeds + + def initialize(pipeline, attributes) + @pipeline = pipeline + @attributes = attributes + + @builds = attributes.fetch(:builds).map do |attributes| + Seed::Build.new(@pipeline, attributes) + end + end + + def user=(current_user) + @builds.each { |seed| seed.user = current_user } + end + + def attributes + { name: @attributes.fetch(:name), + pipeline: @pipeline, + project: @pipeline.project } + end + + def seeds + strong_memoize(:seeds) do + @builds.select(&:included?) + end + end + + def included? + seeds.any? + end + + def to_resource + strong_memoize(:stage) do + ::Ci::Stage.new(attributes).tap do |stage| + seeds.each { |seed| stage.builds << seed.to_resource } + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb deleted file mode 100644 index f33c87f554d..00000000000 --- a/lib/gitlab/ci/stage/seed.rb +++ /dev/null @@ -1,62 +0,0 @@ -module Gitlab - module Ci - module Stage - class Seed - include ::Gitlab::Utils::StrongMemoize - - attr_reader :pipeline - - delegate :project, to: :pipeline - delegate :size, to: :@jobs - - def initialize(pipeline, stage, jobs) - @pipeline = pipeline - @stage = { name: stage } - @jobs = jobs.to_a.dup - end - - def user=(current_user) - @jobs.map! do |attributes| - attributes.merge(user: current_user) - end - end - - def stage - @stage.merge(project: project) - end - - def builds - trigger = pipeline.trigger_requests.first - - @jobs.map do |attributes| - attributes.merge(project: project, - ref: pipeline.ref, - tag: pipeline.tag, - trigger_request: trigger, - protected: protected_ref?) - end - end - - def create! - pipeline.stages.create!(stage).tap do |stage| - builds_attributes = builds.map do |attributes| - attributes.merge(stage_id: stage.id) - end - - pipeline.builds.create!(builds_attributes).each do |build| - yield build if block_given? - end - end - end - - private - - def protected_ref? - strong_memoize(:protected_ref) do - project.protected_for?(pipeline.ref) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb new file mode 100644 index 00000000000..ac4308f4e2c --- /dev/null +++ b/lib/gitlab/ci/trace/http_io.rb @@ -0,0 +1,187 @@ +## +# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) +# source: https://gitlab.com/snippets/1685610 +module Gitlab + module Ci + class Trace + class HttpIO + BUFFER_SIZE = 128.kilobytes + + InvalidURLError = Class.new(StandardError) + FailedToGetChunkError = Class.new(StandardError) + + attr_reader :uri, :size + attr_reader :tell + attr_reader :chunk, :chunk_range + + alias_method :pos, :tell + + def initialize(url, size) + raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url) + + @uri = URI(url) + @size = size + @tell = 0 + end + + def close + # no-op + end + + def binmode + # no-op + end + + def binmode? + true + end + + def path + nil + end + + def url + @uri.to_s + end + + def seek(pos, where = IO::SEEK_SET) + new_pos = + case where + when IO::SEEK_END + size + pos + when IO::SEEK_SET + pos + when IO::SEEK_CUR + tell + pos + else + -1 + end + + raise 'new position is outside of file' if new_pos < 0 || new_pos > size + + @tell = new_pos + end + + def eof? + tell == size + end + + def each_line + until eof? + line = readline + break if line.nil? + + yield(line) + end + end + + def read(length = nil) + out = "" + + until eof? || (length && out.length >= length) + data = get_chunk + break if data.empty? + + out << data + @tell += data.bytesize + end + + out = out[0, length] if length && out.length > length + + out + end + + def readline + out = "" + + until eof? + data = get_chunk + new_line = data.index("\n") + + if !new_line.nil? + out << data[0..new_line] + @tell += new_line + 1 + break + else + out << data + @tell += data.bytesize + end + end + + out + end + + def write(data) + raise NotImplementedError + end + + def truncate(offset) + raise NotImplementedError + end + + def flush + raise NotImplementedError + end + + def present? + true + end + + private + + ## + # The below methods are not implemented in IO class + # + def in_range? + @chunk_range&.include?(tell) + end + + def get_chunk + unless in_range? + response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http| + http.request(request) + end + + raise FailedToGetChunkError unless response.code == '200' || response.code == '206' + + @chunk = response.body.force_encoding(Encoding::BINARY) + @chunk_range = response.content_range + + ## + # Note: If provider does not return content_range, then we set it as we requested + # Provider: minio + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: AWS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # Provider: GCS + # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200 + @chunk_range ||= (chunk_start...(chunk_start + @chunk.length)) + end + + @chunk[chunk_offset..BUFFER_SIZE] + end + + def request + Net::HTTP::Get.new(uri).tap do |request| + request.set_range(chunk_start, BUFFER_SIZE) + end + end + + def chunk_offset + tell % BUFFER_SIZE + end + + def chunk_start + (tell / BUFFER_SIZE) * BUFFER_SIZE + end + + def chunk_end + [chunk_start + BUFFER_SIZE, size].min + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index d52194f688b..b3fe3ef1c4d 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :stream - delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true + delegate :close, :tell, :seek, :size, :path, :url, :truncate, to: :stream, allow_nil: true delegate :valid?, to: :stream, as: :present?, allow_nil: true diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index a7285ac8f9d..bc2a6f98dae 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -27,7 +27,7 @@ module Gitlab end def build_attributes(name) - job = @jobs[name.to_sym] || {} + job = @jobs.fetch(name.to_sym, {}) { stage_idx: @stages.index(job[:stage]), stage: job[:stage], @@ -53,30 +53,24 @@ module Gitlab }.compact } end - def pipeline_stage_builds(stage, pipeline) - selected_jobs = @jobs.select do |_, job| - next unless job[:stage] == stage - - only_specs = Gitlab::Ci::Build::Policy - .fabricate(job.fetch(:only, {})) - except_specs = Gitlab::Ci::Build::Policy - .fabricate(job.fetch(:except, {})) - - only_specs.all? { |spec| spec.satisfied_by?(pipeline) } && - except_specs.none? { |spec| spec.satisfied_by?(pipeline) } - end - - selected_jobs.map { |_, job| build_attributes(job[:name]) } + def stage_builds_attributes(stage) + @jobs.values + .select { |job| job[:stage] == stage } + .map { |job| build_attributes(job[:name]) } end - def stage_seeds(pipeline) - seeds = @stages.uniq.map do |stage| - builds = pipeline_stage_builds(stage, pipeline) + def stages_attributes + @stages.uniq.map do |stage| + seeds = stage_builds_attributes(stage).map do |attributes| + job = @jobs.fetch(attributes[:name].to_sym) - Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? - end + attributes + .merge(only: job.fetch(:only, {})) + .merge(except: job.fetch(:except, {})) + end - seeds.compact + { name: stage, index: @stages.index(stage), builds: seeds } + end end def self.validation_message(content) diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 3ccfd9a739d..65a65b67975 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -40,7 +40,10 @@ module Gitlab # when there are no conflict files. files.each(&:lines) files.any? - rescue Gitlab::Git::CommandError, Gitlab::Git::Conflict::Parser::UnresolvableError, Gitlab::Git::Conflict::Resolver::ConflictSideMissing + rescue Gitlab::Git::CommandError, + Gitlab::Git::Conflict::Parser::UnresolvableError, + Gitlab::Git::Conflict::Resolver::ConflictSideMissing, + Gitlab::Git::Conflict::File::UnsupportedEncoding false end cache_method :can_be_resolved_in_ui? diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb index fd4a8832ec2..62d4d0a92a6 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -74,7 +74,7 @@ module Gitlab }.freeze def repository_storage_path - Gitlab.config.repositories.storages[repository_storage]['path'] + Gitlab.config.repositories.storages[repository_storage].legacy_disk_path end # Overridden to have the correct `source_type` for the `route` relation diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 6659efa0961..0b8f6cfe3cb 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -90,7 +90,7 @@ module Gitlab end def clean(message) - message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "".encode("UTF-16BE")) .encode("UTF-8") .gsub("\0".encode("UTF-8"), "") end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index 2a9cf10a068..f08dab59ce4 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -2,17 +2,19 @@ module Gitlab module Git module Conflict class File + UnsupportedEncoding = Class.new(StandardError) + attr_reader :their_path, :our_path, :our_mode, :repository, :commit_oid - attr_accessor :content + attr_accessor :raw_content - def initialize(repository, commit_oid, conflict, content) + def initialize(repository, commit_oid, conflict, raw_content) @repository = repository @commit_oid = commit_oid @their_path = conflict[:theirs][:path] @our_path = conflict[:ours][:path] @our_mode = conflict[:ours][:mode] - @content = content + @raw_content = raw_content end def lines @@ -29,6 +31,14 @@ module Gitlab end end + def content + @content ||= @raw_content.dup.force_encoding('UTF-8') + + raise UnsupportedEncoding unless @content.valid_encoding? + + @content + end + def type lines unless @type diff --git a/lib/gitlab/git/conflict/parser.rb b/lib/gitlab/git/conflict/parser.rb index 3effa9d2d31..fb5717dd556 100644 --- a/lib/gitlab/git/conflict/parser.rb +++ b/lib/gitlab/git/conflict/parser.rb @@ -4,7 +4,6 @@ module Gitlab class Parser UnresolvableError = Class.new(StandardError) UnmergeableFile = Class.new(UnresolvableError) - UnsupportedEncoding = Class.new(UnresolvableError) # Recoverable errors - the conflict can be resolved in an editor, but not with # sections. @@ -75,10 +74,6 @@ module Gitlab def validate_text!(text) raise UnmergeableFile if text.blank? # Typically a binary file raise UnmergeableFile if text.length > 200.kilobytes - - text.force_encoding('UTF-8') - - raise UnsupportedEncoding unless text.valid_encoding? end def validate_delimiter!(condition) diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index a142ed6b2ef..dc0bc8518bc 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -212,7 +212,7 @@ module Gitlab end def shard_name_from_shard_path(shard_path) - Gitlab.config.repositories.storages.find { |_, info| info['path'] == shard_path }&.first || + Gitlab.config.repositories.storages.find { |_, info| info.legacy_disk_path == shard_path }&.first || raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index c15c128507d..20b0647fce9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -93,7 +93,7 @@ module Gitlab @relative_path = relative_path @gl_repository = gl_repository - storage_path = Gitlab.config.repositories.storages[@storage]['path'] + storage_path = Gitlab.config.repositories.storages[@storage].legacy_disk_path @gitlab_projects = Gitlab::Git::GitlabProjects.new( storage_path, relative_path, diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb index d3c37f82101..2f611cef37b 100644 --- a/lib/gitlab/git/storage/checker.rb +++ b/lib/gitlab/git/storage/checker.rb @@ -35,7 +35,7 @@ module Gitlab def initialize(storage, logger = Rails.logger) @storage = storage config = Gitlab.config.repositories.storages[@storage] - @storage_path = config['path'] + @storage_path = config.legacy_disk_path @logger = logger @hostname = Gitlab::Environment.hostname diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb index 898bb1b65be..e35054466ff 100644 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -25,7 +25,7 @@ module Gitlab if !config.present? NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) - elsif !config['path'].present? + elsif !config.legacy_disk_path.present? NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) else new(storage, hostname) diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb index 97c13d1fdb0..c275a065bce 100644 --- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb +++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb @@ -17,7 +17,7 @@ module Gitlab current_file = file_from_gitaly_header(gitaly_file.header) else - current_file.content << gitaly_file.content + current_file.raw_content << gitaly_file.content end end end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb new file mode 100644 index 00000000000..8668caf0c55 --- /dev/null +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -0,0 +1,35 @@ +module Gitlab + module GitalyClient + # This is a chokepoint that is meant to help us stop remove all places + # where production code (app, config, db, lib) touches Git repositories + # directly. + class StorageSettings + DirectPathAccessError = Class.new(StandardError) + + # This class will give easily recognizable NoMethodErrors + Deprecated = Class.new + + attr_reader :legacy_disk_path + + def initialize(storage) + raise "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) + + # Support a nil 'path' field because some of the circuit breaker tests use it. + @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path'] + + storage['path'] = Deprecated + @hash = storage + end + + def gitaly_address + @hash.fetch(:gitaly_address) + end + + private + + def method_missing(m, *args, &block) + @hash.public_send(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + end + end + end +end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index afaa59b1018..6e554383270 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -77,7 +77,7 @@ module Gitlab end def storage_path(storage_name) - storages_paths&.dig(storage_name, 'path') + storages_paths[storage_name]&.legacy_disk_path end # All below test methods use shell commands to perform actions on storage volumes. diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 79265cf952d..1fa2a19b0af 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -21,11 +21,11 @@ module Gitlab result = repo_path storage = Gitlab.config.repositories.storages.values.find do |params| - repo_path.start_with?(params['path']) + repo_path.start_with?(params.legacy_disk_path) end if storage - result = result.sub(storage['path'], '') + result = result.sub(storage.legacy_disk_path, '') elsif fail_on_not_found raise NotFoundError.new("No known storage path matches #{repo_path.inspect}") end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 07d7c91cb5d..e5c02dd8ecc 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -24,7 +24,7 @@ module Gitlab address = val['gitaly_address'] end - storages << { name: key, path: val['path'] } + storages << { name: key, path: val.legacy_disk_path } end if Rails.env.test? diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 3a8f5826818..c8c15b9684a 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -82,7 +82,7 @@ module Gitlab repository.gitaly_repository_client.create_repository true else - repo_path = File.join(Gitlab.config.repositories.storages[storage]['path'], relative_path) + repo_path = File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, relative_path) Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) end end @@ -131,7 +131,7 @@ module Gitlab if is_enabled repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) else - storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] + storage_path = Gitlab.config.repositories.storages[repository.storage].legacy_disk_path local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end end @@ -478,7 +478,7 @@ module Gitlab def gitaly_namespace_client(storage_path) storage, _value = Gitlab.config.repositories.storages.find do |storage, value| - value['path'] == storage_path + value.legacy_disk_path == storage_path end Gitlab::GitalyClient::NamespaceService.new(storage) diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 34bee6fecbe..42be301fd9b 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -129,7 +129,7 @@ module Gitlab def all_repos Gitlab.config.repositories.storages.each_value do |repository_storage| - IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find| + IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find| find.each_line do |path| yield path.chomp end @@ -138,7 +138,7 @@ module Gitlab end def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } + Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } end def user_home diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb index fe51edbdeeb..970e2a7b718 100644 --- a/lib/gitlab/verify/lfs_objects.rb +++ b/lib/gitlab/verify/lfs_objects.rb @@ -12,7 +12,7 @@ module Gitlab private def relation - LfsObject.all + LfsObject.with_files_stored_locally end def expected_checksum(lfs_object) diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb index 6972e517ea5..0ffa71a6d72 100644 --- a/lib/gitlab/verify/uploads.rb +++ b/lib/gitlab/verify/uploads.rb @@ -12,7 +12,7 @@ module Gitlab private def relation - Upload.all + Upload.with_files_stored_locally end def expected_checksum(upload) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 0b0d667d4fd..928957fdb25 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -52,7 +52,7 @@ module Gitlab end def send_git_blob(repository, blob) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'GetBlobRequest' => { @@ -80,7 +80,7 @@ module Gitlab params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) raise "Repository or ref not found" if params.empty? - if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive) + if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive, Gitlab::GitalyClient::MigrationStatus::OPT_OUT) params.merge!( 'GitalyServer' => gitaly_server_hash(repository), 'GitalyRepository' => repository.gitaly_repository.to_h @@ -97,7 +97,7 @@ module Gitlab end def send_git_diff(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff, Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'RawDiffRequest' => Gitaly::RawDiffRequest.new( @@ -115,7 +115,7 @@ module Gitlab end def send_git_patch(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch) + params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch, Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { 'GitalyServer' => gitaly_server_hash(repository), 'RawPatchRequest' => Gitaly::RawPatchRequest.new( diff --git a/lib/system_check/orphans/namespace_check.rb b/lib/system_check/orphans/namespace_check.rb index b8446300f72..b5f443abe06 100644 --- a/lib/system_check/orphans/namespace_check.rb +++ b/lib/system_check/orphans/namespace_check.rb @@ -6,8 +6,8 @@ module SystemCheck def multi_check Gitlab.config.repositories.storages.each do |storage_name, repository_storage| $stdout.puts - $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) - toplevel_namespace_dirs = disk_namespaces(repository_storage['path']) + $stdout.puts "* Storage: #{storage_name} (#{repository_storage.legacy_disk_path})".color(:yellow) + toplevel_namespace_dirs = disk_namespaces(repository_storage.legacy_disk_path) orphans = (toplevel_namespace_dirs - existing_namespaces) print_orphans(orphans, storage_name) diff --git a/lib/system_check/orphans/repository_check.rb b/lib/system_check/orphans/repository_check.rb index 9b6b2429783..5ef0b93ad08 100644 --- a/lib/system_check/orphans/repository_check.rb +++ b/lib/system_check/orphans/repository_check.rb @@ -6,10 +6,12 @@ module SystemCheck def multi_check Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + storage_path = repository_storage.legacy_disk_path + $stdout.puts - $stdout.puts "* Storage: #{storage_name} (#{repository_storage['path']})".color(:yellow) + $stdout.puts "* Storage: #{storage_name} (#{storage_path})".color(:yellow) - repositories = disk_repositories(repository_storage['path']) + repositories = disk_repositories(storage_path) orphans = (repositories - fetch_repositories(storage_name)) print_orphans(orphans, storage_name) diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake new file mode 100644 index 00000000000..bfca4bfb3f7 --- /dev/null +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -0,0 +1,25 @@ +require 'logger' +require 'resolv-replace' + +desc "GitLab | Migrate files for artifacts to comply with new storage format" +namespace :gitlab do + namespace :artifacts do + task migrate: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of artifacts') + + Ci::Build.joins(:project) + .with_artifacts_stored_locally + .find_each(batch_size: 10) do |build| + begin + build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE) + build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE) + + logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage") + rescue => e + logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") + end + end + end + end +end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 2403f57f05a..abef8cd2bcc 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -61,7 +61,7 @@ namespace :gitlab do puts "Repo base directory exists?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " if File.exist?(repo_base_path) @@ -86,7 +86,7 @@ namespace :gitlab do puts "Repo storage directories are symlinks?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " unless File.exist?(repo_base_path) @@ -110,7 +110,7 @@ namespace :gitlab do puts "Repo paths access is drwxrws---?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " unless File.exist?(repo_base_path) @@ -140,7 +140,7 @@ namespace :gitlab do puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage['path'] + repo_base_path = repository_storage.legacy_disk_path print "#{name}... " unless File.exist?(repo_base_path) diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 2453079911d..d6d15285489 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -12,7 +12,7 @@ namespace :gitlab do namespaces = Namespace.pluck(:path) namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored Gitlab.config.repositories.storages.each do |name, repository_storage| - git_base_path = repository_storage['path'] + git_base_path = repository_storage.legacy_disk_path all_dirs = Dir.glob(git_base_path + '/*') puts git_base_path.color(:yellow) @@ -54,7 +54,7 @@ namespace :gitlab do move_suffix = "+orphaned+#{Time.now.to_i}" Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_root = repository_storage['path'] + repo_root = repository_storage.legacy_disk_path # Look for global repos (legacy, depth 1) and normal repos (depth 2) IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| find.each_line do |path| diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 45e9a1a1c72..47ed522aec3 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -68,7 +68,7 @@ namespace :gitlab do puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" puts "Repository storage paths:" Gitlab.config.repositories.storages.each do |name, repository_storage| - puts "- #{name}: \t#{repository_storage['path']}" + puts "- #{name}: \t#{repository_storage.legacy_disk_path}" end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake new file mode 100644 index 00000000000..a45e5ca91e0 --- /dev/null +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -0,0 +1,22 @@ +require 'logger' + +desc "GitLab | Migrate LFS objects to remote storage" +namespace :gitlab do + namespace :lfs do + task migrate: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of LFS files to object storage') + + LfsObject.with_files_stored_locally + .find_each(batch_size: 10) do |lfs_object| + begin + lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) + + logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage") + rescue => e + logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") + end + end + end + end +end diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake new file mode 100644 index 00000000000..c26c3ccb3be --- /dev/null +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -0,0 +1,33 @@ +namespace :gitlab do + namespace :uploads do + desc 'GitLab | Uploads | Migrate the uploaded files to object storage' + task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args| + batch_size = ENV.fetch('BATCH', 200).to_i + @to_store = ObjectStorage::Store::REMOTE + @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym + @uploader_class = args.uploader_class.constantize + @model_class = args.model_class.constantize + + uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches + end + + def enqueue_batch(batch, index) + job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, + @mounted_as, + @to_store) + puts "Enqueued job ##{index}: #{job}" + rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e + # continue for the next batch + puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red) + end + + def uploads + Upload.class_eval { include EachBatch } unless Upload < EachBatch + + Upload + .where.not(store: @to_store) + .where(uploader: @uploader_class.to_s, + model_type: @model_class.base_class.sti_name) + end + end +end diff --git a/package.json b/package.json index 98943997b2d..31edc3a8016 100644 --- a/package.json +++ b/package.json @@ -107,6 +107,7 @@ "eslint-plugin-jasmine": "^2.1.0", "eslint-plugin-promise": "^3.5.0", "eslint-plugin-vue": "^4.0.1", + "ignore": "^3.3.7", "istanbul": "^0.4.5", "jasmine-core": "^2.9.0", "jasmine-jquery": "^2.1.1", diff --git a/scripts/frontend/frontend_script_utils.js b/scripts/frontend/frontend_script_utils.js index 2c06747255c..e42b912d359 100644 --- a/scripts/frontend/frontend_script_utils.js +++ b/scripts/frontend/frontend_script_utils.js @@ -1,4 +1,3 @@ -/* eslint import/no-commonjs: "off" */ const execFileSync = require('child_process').execFileSync; const exec = (command, args) => { @@ -18,12 +17,7 @@ const execGitCmd = args => module.exports = { getStagedFiles: fileExtensionFilter => { - const gitOptions = [ - 'diff', - '--name-only', - '--cached', - '--diff-filter=ACMRTUB', - ]; + const gitOptions = ['diff', '--name-only', '--cached', '--diff-filter=ACMRTUB']; if (fileExtensionFilter) gitOptions.push(...fileExtensionFilter); return execGitCmd(gitOptions); }, diff --git a/scripts/frontend/prettier.js b/scripts/frontend/prettier.js index 863572bf64d..39de77bc333 100644 --- a/scripts/frontend/prettier.js +++ b/scripts/frontend/prettier.js @@ -1,7 +1,8 @@ -/* eslint import/no-commonjs: "off", import/no-extraneous-dependencies: "off", no-console: "off" */ const glob = require('glob'); const prettier = require('prettier'); const fs = require('fs'); +const path = require('path'); +const prettierIgnore = require('ignore')(); const getStagedFiles = require('./frontend_script_utils').getStagedFiles; @@ -11,6 +12,10 @@ const allFiles = mode === 'check-all' || mode === 'save-all'; const config = { patterns: ['**/*.js', '**/*.vue', '**/*.scss'], + /* + * The ignore patterns below are just to reduce search time with glob, as it includes the + * folders with the most ignored assets, the actual `.prettierignore` will be used later on + */ ignore: ['**/node_modules/**', '**/vendor/**', '**/public/**'], parsers: { js: 'babylon', @@ -18,13 +23,25 @@ const config = { scss: 'css', }, }; + +/* + * Unfortunately the prettier API does not expose support for `.prettierignore` files, they however + * use the ignore package, so we do the same. We simply cannot use the glob package, because + * gitignore style is not compatible with globs ignore style. + */ +prettierIgnore.add( + fs + .readFileSync(path.join(__dirname, '../../', '.prettierignore')) + .toString() + .trim() + .split(/\r?\n/) +); + const availableExtensions = Object.keys(config.parsers); console.log(`Loading ${allFiles ? 'All' : 'Staged'} Files ...`); -const stagedFiles = allFiles - ? null - : getStagedFiles(availableExtensions.map(ext => `*.${ext}`)); +const stagedFiles = allFiles ? null : getStagedFiles(availableExtensions.map(ext => `*.${ext}`)); if (stagedFiles) { if (!stagedFiles.length || (stagedFiles.length === 1 && !stagedFiles[0])) { @@ -41,17 +58,14 @@ let files; if (allFiles) { const ignore = config.ignore; const patterns = config.patterns; - const globPattern = - patterns.length > 1 ? `{${patterns.join(',')}}` : `${patterns.join(',')}`; - files = glob - .sync(globPattern, { ignore }) - .filter(f => allFiles || stagedFiles.includes(f)); + const globPattern = patterns.length > 1 ? `{${patterns.join(',')}}` : `${patterns.join(',')}`; + files = glob.sync(globPattern, { ignore }).filter(f => allFiles || stagedFiles.includes(f)); } else { - files = stagedFiles.filter(f => - availableExtensions.includes(f.split('.').pop()), - ); + files = stagedFiles.filter(f => availableExtensions.includes(f.split('.').pop())); } +files = prettierIgnore.filter(files); + if (!files.length) { console.log('No Files found to process with Prettier'); return; @@ -81,7 +95,7 @@ prettier } else if (!prettier.check(input, options)) { if (!didWarn) { console.log( - '\n===============================\nGitLab uses Prettier to format all JavaScript code.\nPlease format each file listed below or run "yarn prettier-staged-save"\n===============================\n', + '\n===============================\nGitLab uses Prettier to format all JavaScript code.\nPlease format each file listed below or run "yarn prettier-staged-save"\n===============================\n' ); didWarn = true; } diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh index e5242fee32b..178b209aacf 100755 --- a/scripts/lint-doc.sh +++ b/scripts/lint-doc.sh @@ -3,7 +3,7 @@ cd "$(dirname "$0")/.." # Use long options (e.g. --header instead of -H) for curl examples in documentation. -echo 'Checking for curl short options...' +echo '=> Checking for cURL short options...' grep --extended-regexp --recursive --color=auto 'curl (.+ )?-[^- ].*' doc/ >/dev/null 2>&1 if [ $? == 0 ] then @@ -15,7 +15,7 @@ fi # Ensure that the CHANGELOG.md does not contain duplicate versions DUPLICATE_CHANGELOG_VERSIONS=$(grep --extended-regexp '^## .+' CHANGELOG.md | sed -E 's| \(.+\)||' | sort -r | uniq -d) -echo 'Checking for CHANGELOG.md duplicate entries...' +echo '=> Checking for CHANGELOG.md duplicate entries...' if [ "${DUPLICATE_CHANGELOG_VERSIONS}" != "" ] then echo '✖ ERROR: Duplicate versions in CHANGELOG.md:' >&2 @@ -25,7 +25,7 @@ fi # Make sure no files in doc/ are executable EXEC_PERM_COUNT=$(find doc/ app/ -type f -perm 755 | wc -l) -echo 'Checking for executable permissions...' +echo '=> Checking for executable permissions...' if [ "${EXEC_PERM_COUNT}" -ne 0 ] then echo '✖ ERROR: Executable permissions should not be used in documentation! Use `chmod 644` to the files in question:' >&2 @@ -33,5 +33,33 @@ then exit 1 fi +# Do not use 'README.md', instead use 'index.md' +# Number of 'README.md's as of 2018-03-26 +NUMBER_READMES_CE=42 +NUMBER_READMES_EE=46 +FIND_READMES=$(find doc/ -name "README.md" | wc -l) +echo '=> Checking for new README.md files...' +if [ "${CI_PROJECT_NAME}" == 'gitlab-ce' ] +then + if [ ${FIND_READMES} -ne ${NUMBER_READMES_CE} ] + then + echo + echo ' ✖ ERROR: New README.md file(s) detected, prefer index.md over README.md.' >&2 + echo ' https://docs.gitlab.com/ee/development/writing_documentation.html#location-and-naming-documents' + echo + exit 1 + fi +elif [ "${CI_PROJECT_NAME}" == 'gitlab-ee' ] +then + if [ ${FIND_READMES} -ne $NUMBER_READMES_EE ] + then + echo + echo ' ✖ ERROR: New README.md file(s) detected, prefer index.md over README.md.' >&2 + echo ' https://docs.gitlab.com/ee/development/writing_documentation.html#location-and-naming-documents' + echo + exit 1 + fi +fi + echo "✔ Linting passed" exit 0 diff --git a/spec/controllers/concerns/send_file_upload_spec.rb b/spec/controllers/concerns/send_file_upload_spec.rb new file mode 100644 index 00000000000..f4c99ea4064 --- /dev/null +++ b/spec/controllers/concerns/send_file_upload_spec.rb @@ -0,0 +1,89 @@ +require 'spec_helper' + +describe SendFileUpload do + let(:uploader_class) do + Class.new(GitlabUploader) do + include ObjectStorage::Concern + + storage_options Gitlab.config.uploads + + private + + # user/:id + def dynamic_segment + File.join(model.class.to_s.underscore, model.id.to_s) + end + end + end + + let(:controller_class) do + Class.new do + include SendFileUpload + end + end + + let(:object) { build_stubbed(:user) } + let(:uploader) { uploader_class.new(object, :file) } + + describe '#send_upload' do + let(:controller) { controller_class.new } + let(:temp_file) { Tempfile.new('test') } + + subject { controller.send_upload(uploader) } + + before do + FileUtils.touch(temp_file) + end + + after do + FileUtils.rm_f(temp_file) + end + + context 'when local file is used' do + before do + uploader.store!(temp_file) + end + + it 'sends a file' do + expect(controller).to receive(:send_file).with(uploader.path, anything) + + subject + end + end + + context 'when remote file is used' do + before do + stub_uploads_object_storage(uploader: uploader_class) + uploader.object_store = ObjectStorage::Store::REMOTE + uploader.store!(temp_file) + end + + context 'and proxying is enabled' do + before do + allow(Gitlab.config.uploads.object_store).to receive(:proxy_download) { true } + end + + it 'sends a file' do + headers = double + expect(headers).to receive(:store).with(Gitlab::Workhorse::SEND_DATA_HEADER, /^send-url:/) + expect(controller).to receive(:headers) { headers } + expect(controller).to receive(:head).with(:ok) + + subject + end + end + + context 'and proxying is disabled' do + before do + allow(Gitlab.config.uploads.object_store).to receive(:proxy_download) { false } + end + + it 'sends a file' do + expect(controller).to receive(:redirect_to).with(/#{uploader.path}/) + + subject + end + end + end + end +end diff --git a/spec/controllers/omniauth_callbacks_controller_spec.rb b/spec/controllers/omniauth_callbacks_controller_spec.rb index 9fd129e4ee9..5f0e8c5eca9 100644 --- a/spec/controllers/omniauth_callbacks_controller_spec.rb +++ b/spec/controllers/omniauth_callbacks_controller_spec.rb @@ -10,83 +10,119 @@ describe OmniauthCallbacksController do stub_omniauth_provider(provider, context: request) end - context 'github' do + context 'when the user is on the last sign in attempt' do let(:extern_uid) { 'my-uid' } - let(:provider) { :github } - it 'allows sign in' do - post provider - - expect(request.env['warden']).to be_authenticated + before do + user.update(failed_attempts: User.maximum_attempts.pred) + subject.response = ActionDispatch::Response.new end - shared_context 'sign_up' do - let(:user) { double(email: 'new@example.com') } + context 'when using a form based provider' do + let(:provider) { :ldap } + + it 'locks the user when sign in fails' do + allow(subject).to receive(:params).and_return(ActionController::Parameters.new(username: user.username)) + request.env['omniauth.error.strategy'] = OmniAuth::Strategies::LDAP.new(nil) + + subject.send(:failure) - before do - stub_omniauth_setting(block_auto_created_users: false) + expect(user.reload).to be_access_locked end end - context 'sign up' do - include_context 'sign_up' + context 'when using a button based provider' do + let(:provider) { :github } - it 'is allowed' do - post provider + it 'does not lock the user when sign in fails' do + request.env['omniauth.error.strategy'] = OmniAuth::Strategies::GitHub.new(nil) - expect(request.env['warden']).to be_authenticated + subject.send(:failure) + + expect(user.reload).not_to be_access_locked end end + end - context 'when OAuth is disabled' do - before do - stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false') - settings = Gitlab::CurrentSettings.current_application_settings - settings.update(disabled_oauth_sign_in_sources: [provider.to_s]) - end + context 'strategies' do + context 'github' do + let(:extern_uid) { 'my-uid' } + let(:provider) { :github } - it 'prevents login via POST' do + it 'allows sign in' do post provider - expect(request.env['warden']).not_to be_authenticated + expect(request.env['warden']).to be_authenticated end - it 'shows warning when attempting login' do - post provider + shared_context 'sign_up' do + let(:user) { double(email: 'new@example.com') } - expect(response).to redirect_to new_user_session_path - expect(flash[:alert]).to eq('Signing in using GitHub has been disabled') + before do + stub_omniauth_setting(block_auto_created_users: false) + end end - it 'allows linking the disabled provider' do - user.identities.destroy_all - sign_in(user) + context 'sign up' do + include_context 'sign_up' + + it 'is allowed' do + post provider - expect { post provider }.to change { user.reload.identities.count }.by(1) + expect(request.env['warden']).to be_authenticated + end end - context 'sign up' do - include_context 'sign_up' + context 'when OAuth is disabled' do + before do + stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false') + settings = Gitlab::CurrentSettings.current_application_settings + settings.update(disabled_oauth_sign_in_sources: [provider.to_s]) + end - it 'is prevented' do + it 'prevents login via POST' do post provider expect(request.env['warden']).not_to be_authenticated end + + it 'shows warning when attempting login' do + post provider + + expect(response).to redirect_to new_user_session_path + expect(flash[:alert]).to eq('Signing in using GitHub has been disabled') + end + + it 'allows linking the disabled provider' do + user.identities.destroy_all + sign_in(user) + + expect { post provider }.to change { user.reload.identities.count }.by(1) + end + + context 'sign up' do + include_context 'sign_up' + + it 'is prevented' do + post provider + + expect(request.env['warden']).not_to be_authenticated + end + end end end - end - context 'auth0' do - let(:extern_uid) { '' } - let(:provider) { :auth0 } + context 'auth0' do + let(:extern_uid) { '' } + let(:provider) { :auth0 } - it 'does not allow sign in without extern_uid' do - post 'auth0' + it 'does not allow sign in without extern_uid' do + post 'auth0' - expect(request.env['warden']).not_to be_authenticated - expect(response.status).to eq(302) - expect(controller).to set_flash[:alert].to('Wrong extern UID provided. Make sure Auth0 is configured correctly.') + expect(request.env['warden']).not_to be_authenticated + expect(response.status).to eq(302) + expect(controller).to set_flash[:alert].to('Wrong extern UID provided. Make sure Auth0 is configured correctly.') + end end end end diff --git a/spec/controllers/projects/artifacts_controller_spec.rb b/spec/controllers/projects/artifacts_controller_spec.rb index 25a2e13fe1a..4ea6f869aa3 100644 --- a/spec/controllers/projects/artifacts_controller_spec.rb +++ b/spec/controllers/projects/artifacts_controller_spec.rb @@ -145,9 +145,23 @@ describe Projects::ArtifactsController do context 'when using local file storage' do it_behaves_like 'a valid file' do let(:job) { create(:ci_build, :success, :artifacts, pipeline: pipeline) } + let(:store) { ObjectStorage::Store::LOCAL } let(:archive_path) { JobArtifactUploader.root } end end + + context 'when using remote file storage' do + before do + stub_artifacts_object_storage + end + + it_behaves_like 'a valid file' do + let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) } + let!(:job) { create(:ci_build, :success, pipeline: pipeline) } + let(:store) { ObjectStorage::Store::REMOTE } + let(:archive_path) { 'https://' } + end + end end end diff --git a/spec/controllers/projects/clusters_controller_spec.rb b/spec/controllers/projects/clusters_controller_spec.rb index 15ce418d0d6..82b20e12850 100644 --- a/spec/controllers/projects/clusters_controller_spec.rb +++ b/spec/controllers/projects/clusters_controller_spec.rb @@ -18,7 +18,7 @@ describe Projects::ClustersController do context 'when project has one or more clusters' do let(:project) { create(:project) } let!(:enabled_cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } - let!(:disabled_cluster) { create(:cluster, :disabled, :provided_by_gcp, projects: [project]) } + let!(:disabled_cluster) { create(:cluster, :disabled, :provided_by_gcp, :production_environment, projects: [project]) } it 'lists available clusters' do go @@ -32,7 +32,7 @@ describe Projects::ClustersController do before do allow(Clusters::Cluster).to receive(:paginates_per).and_return(1) - create_list(:cluster, 2, :provided_by_gcp, projects: [project]) + create_list(:cluster, 2, :provided_by_gcp, :production_environment, projects: [project]) get :index, namespace_id: project.namespace, project_id: project, page: last_page end diff --git a/spec/controllers/projects/jobs_controller_spec.rb b/spec/controllers/projects/jobs_controller_spec.rb index f3e303bb0fe..31046c202e6 100644 --- a/spec/controllers/projects/jobs_controller_spec.rb +++ b/spec/controllers/projects/jobs_controller_spec.rb @@ -1,7 +1,9 @@ +# coding: utf-8 require 'spec_helper' describe Projects::JobsController do include ApiHelpers + include HttpIOHelpers let(:project) { create(:project, :public) } let(:pipeline) { create(:ci_pipeline, project: project) } @@ -203,6 +205,41 @@ describe Projects::JobsController do end end + context 'when trace artifact is in ObjectStorage' do + let!(:job) { create(:ci_build, :success, :trace_artifact, pipeline: pipeline) } + + before do + allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false } + allow_any_instance_of(JobArtifactUploader).to receive(:url) { remote_trace_url } + allow_any_instance_of(JobArtifactUploader).to receive(:size) { remote_trace_size } + end + + context 'when there are no network issues' do + before do + stub_remote_trace_206 + + get_trace + end + + it 'returns a trace' do + expect(response).to have_gitlab_http_status(:ok) + expect(json_response['id']).to eq job.id + expect(json_response['status']).to eq job.status + expect(json_response['html']).to eq(job.trace.html) + end + end + + context 'when there is a network issue' do + before do + stub_remote_trace_500 + end + + it 'returns a trace' do + expect { get_trace }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError) + end + end + end + def get_trace get :trace, namespace_id: project.namespace, project_id: project, @@ -446,14 +483,18 @@ describe Projects::JobsController do end describe 'GET raw' do - before do - get_raw + subject do + post :raw, namespace_id: project.namespace, + project_id: project, + id: job.id end context 'when job has a trace artifact' do let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) } it 'returns a trace' do + response = subject + expect(response).to have_gitlab_http_status(:ok) expect(response.content_type).to eq 'text/plain; charset=utf-8' expect(response.body).to eq job.job_artifacts_trace.open.read @@ -464,6 +505,8 @@ describe Projects::JobsController do let(:job) { create(:ci_build, :trace_live, pipeline: pipeline) } it 'send a trace file' do + response = subject + expect(response).to have_gitlab_http_status(:ok) expect(response.content_type).to eq 'text/plain; charset=utf-8' expect(response.body).to eq 'BUILD TRACE' @@ -474,14 +517,22 @@ describe Projects::JobsController do let(:job) { create(:ci_build, pipeline: pipeline) } it 'returns not_found' do + response = subject + expect(response).to have_gitlab_http_status(:not_found) end end - def get_raw - post :raw, namespace_id: project.namespace, - project_id: project, - id: job.id + context 'when the trace artifact is in ObjectStorage' do + let!(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) } + + before do + allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false } + end + + it 'redirect to the trace file url' do + expect(subject).to redirect_to(job.job_artifacts_trace.file.url) + end end end end diff --git a/spec/controllers/projects/pages_controller_spec.rb b/spec/controllers/projects/pages_controller_spec.rb index 4705c50de7e..11f54eef531 100644 --- a/spec/controllers/projects/pages_controller_spec.rb +++ b/spec/controllers/projects/pages_controller_spec.rb @@ -65,4 +65,41 @@ describe Projects::PagesController do end end end + + describe 'PATCH update' do + let(:request_params) do + { + namespace_id: project.namespace, + project_id: project, + project: { pages_https_only: false } + } + end + + let(:update_service) { double(execute: { status: :success }) } + + before do + allow(Projects::UpdateService).to receive(:new) { update_service } + end + + it 'returns 302 status' do + patch :update, request_params + + expect(response).to have_gitlab_http_status(:found) + end + + it 'redirects back to the pages settings' do + patch :update, request_params + + expect(response).to redirect_to(project_pages_path(project)) + end + + it 'calls the update service' do + expect(Projects::UpdateService) + .to receive(:new) + .with(project, user, request_params[:project]) + .and_return(update_service) + + patch :update, request_params + end + end end diff --git a/spec/controllers/projects/pages_domains_controller_spec.rb b/spec/controllers/projects/pages_domains_controller_spec.rb index 83a3799e883..d4058a5c515 100644 --- a/spec/controllers/projects/pages_domains_controller_spec.rb +++ b/spec/controllers/projects/pages_domains_controller_spec.rb @@ -13,7 +13,7 @@ describe Projects::PagesDomainsController do end let(:pages_domain_params) do - build(:pages_domain, :with_certificate, :with_key, domain: 'my.otherdomain.com').slice(:key, :certificate, :domain) + build(:pages_domain, domain: 'my.otherdomain.com').slice(:key, :certificate, :domain) end before do @@ -68,7 +68,7 @@ describe Projects::PagesDomainsController do end let(:pages_domain_params) do - attributes_for(:pages_domain, :with_certificate, :with_key).slice(:key, :certificate) + attributes_for(:pages_domain).slice(:key, :certificate) end let(:params) do diff --git a/spec/controllers/projects/pipeline_schedules_controller_spec.rb b/spec/controllers/projects/pipeline_schedules_controller_spec.rb index 966ffdf6996..3506305f755 100644 --- a/spec/controllers/projects/pipeline_schedules_controller_spec.rb +++ b/spec/controllers/projects/pipeline_schedules_controller_spec.rb @@ -80,7 +80,7 @@ describe Projects::PipelineSchedulesController do context 'when variables_attributes has one variable' do let(:schedule) do basic_param.merge({ - variables_attributes: [{ key: 'AAA', value: 'AAA123' }] + variables_attributes: [{ key: 'AAA', secret_value: 'AAA123' }] }) end @@ -101,7 +101,8 @@ describe Projects::PipelineSchedulesController do context 'when variables_attributes has two variables and duplicated' do let(:schedule) do basic_param.merge({ - variables_attributes: [{ key: 'AAA', value: 'AAA123' }, { key: 'AAA', value: 'BBB123' }] + variables_attributes: [{ key: 'AAA', secret_value: 'AAA123' }, + { key: 'AAA', secret_value: 'BBB123' }] }) end @@ -152,7 +153,7 @@ describe Projects::PipelineSchedulesController do context 'when params include one variable' do let(:schedule) do basic_param.merge({ - variables_attributes: [{ key: 'AAA', value: 'AAA123' }] + variables_attributes: [{ key: 'AAA', secret_value: 'AAA123' }] }) end @@ -169,7 +170,8 @@ describe Projects::PipelineSchedulesController do context 'when params include two duplicated variables' do let(:schedule) do basic_param.merge({ - variables_attributes: [{ key: 'AAA', value: 'AAA123' }, { key: 'AAA', value: 'BBB123' }] + variables_attributes: [{ key: 'AAA', secret_value: 'AAA123' }, + { key: 'AAA', secret_value: 'BBB123' }] }) end @@ -194,7 +196,7 @@ describe Projects::PipelineSchedulesController do context 'when adds a new variable' do let(:schedule) do basic_param.merge({ - variables_attributes: [{ key: 'AAA', value: 'AAA123' }] + variables_attributes: [{ key: 'AAA', secret_value: 'AAA123' }] }) end @@ -209,7 +211,7 @@ describe Projects::PipelineSchedulesController do context 'when adds a new duplicated variable' do let(:schedule) do basic_param.merge({ - variables_attributes: [{ key: 'CCC', value: 'AAA123' }] + variables_attributes: [{ key: 'CCC', secret_value: 'AAA123' }] }) end @@ -224,7 +226,7 @@ describe Projects::PipelineSchedulesController do context 'when updates a variable' do let(:schedule) do basic_param.merge({ - variables_attributes: [{ id: pipeline_schedule_variable.id, value: 'new_value' }] + variables_attributes: [{ id: pipeline_schedule_variable.id, secret_value: 'new_value' }] }) end @@ -252,7 +254,7 @@ describe Projects::PipelineSchedulesController do let(:schedule) do basic_param.merge({ variables_attributes: [{ id: pipeline_schedule_variable.id, _destroy: true }, - { key: 'CCC', value: 'CCC123' }] + { key: 'CCC', secret_value: 'CCC123' }] }) end diff --git a/spec/controllers/projects/pipelines_settings_controller_spec.rb b/spec/controllers/projects/pipelines_settings_controller_spec.rb index 1cc488bef32..913b9bd804a 100644 --- a/spec/controllers/projects/pipelines_settings_controller_spec.rb +++ b/spec/controllers/projects/pipelines_settings_controller_spec.rb @@ -47,10 +47,32 @@ describe Projects::PipelinesSettingsController do expect_any_instance_of(Projects::UpdateService).to receive(:run_auto_devops_pipeline?).and_return(true) end - it 'queues a CreatePipelineWorker' do - expect(CreatePipelineWorker).to receive(:perform_async).with(project.id, user.id, project.default_branch, :web, any_args) + context 'when the project repository is empty' do + it 'sets a warning flash' do + expect(subject).to set_flash[:warning] + end - subject + it 'does not queue a CreatePipelineWorker' do + expect(CreatePipelineWorker).not_to receive(:perform_async).with(project.id, user.id, project.default_branch, :web, any_args) + + subject + end + end + + context 'when the project repository is not empty' do + let(:project) { create(:project, :repository) } + + it 'sets a success flash' do + allow(CreatePipelineWorker).to receive(:perform_async).with(project.id, user.id, project.default_branch, :web, any_args) + + expect(subject).to set_flash[:success] + end + + it 'queues a CreatePipelineWorker' do + expect(CreatePipelineWorker).to receive(:perform_async).with(project.id, user.id, project.default_branch, :web, any_args) + + subject + end end end diff --git a/spec/controllers/projects/raw_controller_spec.rb b/spec/controllers/projects/raw_controller_spec.rb index b7df42168e0..08e2ccf893a 100644 --- a/spec/controllers/projects/raw_controller_spec.rb +++ b/spec/controllers/projects/raw_controller_spec.rb @@ -8,10 +8,7 @@ describe Projects::RawController do let(:id) { 'master/README.md' } it 'delivers ASCII file' do - get(:show, - namespace_id: public_project.namespace.to_param, - project_id: public_project, - id: id) + get_show(public_project, id) expect(response).to have_gitlab_http_status(200) expect(response.header['Content-Type']).to eq('text/plain; charset=utf-8') @@ -25,10 +22,7 @@ describe Projects::RawController do let(:id) { 'master/files/images/6049019_460s.jpg' } it 'sets image content type header' do - get(:show, - namespace_id: public_project.namespace.to_param, - project_id: public_project, - id: id) + get_show(public_project, id) expect(response).to have_gitlab_http_status(200) expect(response.header['Content-Type']).to eq('image/jpeg') @@ -54,21 +48,40 @@ describe Projects::RawController do it 'serves the file' do expect(controller).to receive(:send_file).with("#{LfsObjectUploader.root}/91/ef/f75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897", filename: 'lfs_object.iso', disposition: 'attachment') - get(:show, - namespace_id: public_project.namespace.to_param, - project_id: public_project, - id: id) + get_show(public_project, id) expect(response).to have_gitlab_http_status(200) end + + context 'and lfs uses object storage' do + before do + lfs_object.file = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png") + lfs_object.save! + stub_lfs_object_storage + lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) + end + + it 'responds with redirect to file' do + get_show(public_project, id) + + expect(response).to have_gitlab_http_status(302) + expect(response.location).to include(lfs_object.reload.file.path) + end + + it 'sets content disposition' do + get_show(public_project, id) + + file_uri = URI.parse(response.location) + params = CGI.parse(file_uri.query) + + expect(params["response-content-disposition"].first).to eq 'attachment;filename="lfs_object.iso"' + end + end end context 'when project does not have access' do it 'does not serve the file' do - get(:show, - namespace_id: public_project.namespace.to_param, - project_id: public_project, - id: id) + get_show(public_project, id) expect(response).to have_gitlab_http_status(404) end @@ -81,10 +94,7 @@ describe Projects::RawController do end it 'delivers ASCII file' do - get(:show, - namespace_id: public_project.namespace.to_param, - project_id: public_project, - id: id) + get_show(public_project, id) expect(response).to have_gitlab_http_status(200) expect(response.header['Content-Type']).to eq('text/plain; charset=utf-8') @@ -95,4 +105,10 @@ describe Projects::RawController do end end end + + def get_show(project, id) + get(:show, namespace_id: project.namespace.to_param, + project_id: project, + id: id) + end end diff --git a/spec/controllers/root_controller_spec.rb b/spec/controllers/root_controller_spec.rb index b32eb39b1fb..7688538a468 100644 --- a/spec/controllers/root_controller_spec.rb +++ b/spec/controllers/root_controller_spec.rb @@ -90,6 +90,30 @@ describe RootController do end end + context 'who has customized their dashboard setting for assigned issues' do + before do + user.dashboard = 'issues' + end + + it 'redirects to their assigned issues' do + get :index + + expect(response).to redirect_to issues_dashboard_path(assignee_id: user.id) + end + end + + context 'who has customized their dashboard setting for assigned merge requests' do + before do + user.dashboard = 'merge_requests' + end + + it 'redirects to their assigned merge requests' do + get :index + + expect(response).to redirect_to merge_requests_dashboard_path(assignee_id: user.id) + end + end + context 'who uses the default dashboard setting' do it 'renders the default dashboard' do get :index diff --git a/spec/factories/appearances.rb b/spec/factories/appearances.rb index 5f9c57c0c8d..18c7453bd1b 100644 --- a/spec/factories/appearances.rb +++ b/spec/factories/appearances.rb @@ -2,8 +2,21 @@ FactoryBot.define do factory :appearance do - title "MepMep" - description "This is my Community Edition instance" + title "GitLab Community Edition" + description "Open source software to collaborate on code" new_project_guidelines "Custom project guidelines" end + + trait :with_logo do + logo { fixture_file_upload('spec/fixtures/dk.png') } + end + + trait :with_header_logo do + header_logo { fixture_file_upload('spec/fixtures/dk.png') } + end + + trait :with_logos do + with_logo + with_header_logo + end end diff --git a/spec/factories/ci/job_artifacts.rb b/spec/factories/ci/job_artifacts.rb index 8544d54ccaa..3d3287d8168 100644 --- a/spec/factories/ci/job_artifacts.rb +++ b/spec/factories/ci/job_artifacts.rb @@ -5,6 +5,10 @@ FactoryBot.define do job factory: :ci_build file_type :archive + trait :remote_store do + file_store JobArtifactUploader::Store::REMOTE + end + after :build do |artifact| artifact.project ||= artifact.job.project end diff --git a/spec/factories/clusters/clusters.rb b/spec/factories/clusters/clusters.rb index 20d5580f0c2..98566f907f9 100644 --- a/spec/factories/clusters/clusters.rb +++ b/spec/factories/clusters/clusters.rb @@ -32,5 +32,9 @@ FactoryBot.define do trait :disabled do enabled false end + + trait :production_environment do + sequence(:environment_scope) { |n| "production#{n}/*" } + end end end diff --git a/spec/factories/lfs_objects.rb b/spec/factories/lfs_objects.rb index caaed4d5246..eaf3a4ed497 100644 --- a/spec/factories/lfs_objects.rb +++ b/spec/factories/lfs_objects.rb @@ -15,4 +15,8 @@ FactoryBot.define do trait :correct_oid do oid 'b804383982bb89b00e828e3f44c038cc991d3d1768009fc39ba8e2c081b9fb75' end + + trait :object_storage do + file_store { LfsObjectUploader::Store::REMOTE } + end end diff --git a/spec/factories/pages_domains.rb b/spec/factories/pages_domains.rb index 35b44e1c52e..20671da016e 100644 --- a/spec/factories/pages_domains.rb +++ b/spec/factories/pages_domains.rb @@ -4,25 +4,7 @@ FactoryBot.define do verified_at { Time.now } enabled_until { 1.week.from_now } - trait :disabled do - verified_at nil - enabled_until nil - end - - trait :unverified do - verified_at nil - end - - trait :reverify do - enabled_until { 1.hour.from_now } - end - - trait :expired do - enabled_until { 1.hour.ago } - end - - trait :with_certificate do - certificate '-----BEGIN CERTIFICATE----- + certificate '-----BEGIN CERTIFICATE----- MIICGzCCAYSgAwIBAgIBATANBgkqhkiG9w0BAQUFADAbMRkwFwYDVQQDExB0ZXN0 LWNlcnRpZmljYXRlMB4XDTE2MDIxMjE0MzIwMFoXDTIwMDQxMjE0MzIwMFowGzEZ MBcGA1UEAxMQdGVzdC1jZXJ0aWZpY2F0ZTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw @@ -36,10 +18,8 @@ joZp2JHYvNlTPkRJ/J4TcXxBTJmArcQgTIuNoBtC+0A/SwdK4MfTCUY4vNWNdese 5A4K65Nb7Oh1AdQieTBHNXXCdyFsva9/ScfQGEl7p55a52jOPs0StPd7g64uvjlg YHi2yesCrOvVXt+lgPTd -----END CERTIFICATE-----' - end - trait :with_key do - key '-----BEGIN PRIVATE KEY----- + key '-----BEGIN PRIVATE KEY----- MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAKS+CfS9GcRSdYSN SzyH5QJQBr5umRL6E+KilOV39iYFO/9oHjUdapTRWkrwnNPCp7qaeck4Jr8iv14t PVNDfNr76eGb6/3YknOAP0QOjLWunoC8kjU+N/JHU52NrUeX3qEy8EKV9LeCDJcB @@ -55,6 +35,30 @@ EPjGlXIT+aW2XiPmK3ZlCDcWIenE+lmtbOpI159Wpk8BGXs/s/xBAkEAlAY3ymgx 63BDJEwvOb2IaP8lDDxNsXx9XJNVvQbv5n15vNsLHbjslHfAhAbxnLQ1fLhUPqSi nNp/xedE1YxutQ== -----END PRIVATE KEY-----' + + trait :disabled do + verified_at nil + enabled_until nil + end + + trait :unverified do + verified_at nil + end + + trait :reverify do + enabled_until { 1.hour.from_now } + end + + trait :expired do + enabled_until { 1.hour.ago } + end + + trait :without_certificate do + certificate nil + end + + trait :without_key do + key nil end trait :with_missing_chain do diff --git a/spec/factories/uploads.rb b/spec/factories/uploads.rb index ff3a2a76acc..b45f6f30e40 100644 --- a/spec/factories/uploads.rb +++ b/spec/factories/uploads.rb @@ -5,6 +5,7 @@ FactoryBot.define do uploader "AvatarUploader" mount_point :avatar secret nil + store ObjectStorage::Store::LOCAL # we should build a mount agnostic upload by default transient do @@ -27,6 +28,10 @@ FactoryBot.define do secret SecureRandom.hex end + trait :object_storage do + store ObjectStorage::Store::REMOTE + end + trait :namespace_upload do model { build(:group) } path { File.join(secret, filename) } diff --git a/spec/features/admin/admin_disables_git_access_protocol_spec.rb b/spec/features/admin/admin_disables_git_access_protocol_spec.rb index 9ea3cfa72c6..9946cc77d1d 100644 --- a/spec/features/admin/admin_disables_git_access_protocol_spec.rb +++ b/spec/features/admin/admin_disables_git_access_protocol_spec.rb @@ -55,14 +55,19 @@ feature 'Admin disables Git access protocol' do end def disable_http_protocol - visit admin_application_settings_path - find('#application_setting_enabled_git_access_protocol').find(:xpath, 'option[2]').select_option - click_on 'Save' + switch_git_protocol(2) end def disable_ssh_protocol + switch_git_protocol(3) + end + + def switch_git_protocol(value) visit admin_application_settings_path - find('#application_setting_enabled_git_access_protocol').find(:xpath, 'option[3]').select_option - click_on 'Save' + + page.within('.as-visibility-access') do + find('#application_setting_enabled_git_access_protocol').find(:xpath, "option[#{value}]").select_option + click_on 'Save' + end end end diff --git a/spec/features/admin/admin_settings_spec.rb b/spec/features/admin/admin_settings_spec.rb index 39b213988f0..34d45aec2fd 100644 --- a/spec/features/admin/admin_settings_spec.rb +++ b/spec/features/admin/admin_settings_spec.rb @@ -10,18 +10,21 @@ feature 'Admin updates settings' do end scenario 'Change visibility settings' do - choose "application_setting_default_project_visibility_20" - click_button 'Save' + page.within('.as-visibility-access') do + choose "application_setting_default_project_visibility_20" + click_button 'Save changes' + end expect(page).to have_content "Application settings saved successfully" end scenario 'Uncheck all restricted visibility levels' do - find('#application_setting_visibility_level_0').set(false) - find('#application_setting_visibility_level_10').set(false) - find('#application_setting_visibility_level_20').set(false) - - click_button 'Save' + page.within('.as-visibility-access') do + find('#application_setting_visibility_level_0').set(false) + find('#application_setting_visibility_level_10').set(false) + find('#application_setting_visibility_level_20').set(false) + click_button 'Save changes' + end expect(page).to have_content "Application settings saved successfully" expect(find('#application_setting_visibility_level_0')).not_to be_checked @@ -29,21 +32,59 @@ feature 'Admin updates settings' do expect(find('#application_setting_visibility_level_20')).not_to be_checked end - scenario 'Change application settings' do - uncheck 'Gravatar enabled' - fill_in 'Home page URL', with: 'https://about.gitlab.com/' - fill_in 'Help page text', with: 'Example text' - check 'Hide marketing-related entries from help' - fill_in 'Support page URL', with: 'http://example.com/help' - uncheck 'Project export enabled' - click_button 'Save' + scenario 'Change Visibility and Access Controls' do + page.within('.as-visibility-access') do + uncheck 'Project export enabled' + click_button 'Save changes' + end + + expect(Gitlab::CurrentSettings.project_export_enabled).to be_falsey + expect(page).to have_content "Application settings saved successfully" + end + + scenario 'Change Account and Limit Settings' do + page.within('.as-account-limit') do + uncheck 'Gravatar enabled' + click_button 'Save changes' + end expect(Gitlab::CurrentSettings.gravatar_enabled).to be_falsey + expect(page).to have_content "Application settings saved successfully" + end + + scenario 'Change Sign-in restrictions' do + page.within('.as-signin') do + fill_in 'Home page URL', with: 'https://about.gitlab.com/' + click_button 'Save changes' + end + expect(Gitlab::CurrentSettings.home_page_url).to eq "https://about.gitlab.com/" + expect(page).to have_content "Application settings saved successfully" + end + + scenario 'Change Help page' do + page.within('.as-help-page') do + fill_in 'Help page text', with: 'Example text' + check 'Hide marketing-related entries from help' + fill_in 'Support page URL', with: 'http://example.com/help' + click_button 'Save changes' + end + expect(Gitlab::CurrentSettings.help_page_text).to eq "Example text" expect(Gitlab::CurrentSettings.help_page_hide_commercial_content).to be_truthy expect(Gitlab::CurrentSettings.help_page_support_url).to eq "http://example.com/help" - expect(Gitlab::CurrentSettings.project_export_enabled).to be_falsey + expect(page).to have_content "Application settings saved successfully" + end + + scenario 'Change Pages settings' do + page.within('.as-pages') do + fill_in 'Maximum size of pages (MB)', with: 15 + check 'Require users to prove ownership of custom domains' + click_button 'Save changes' + end + + expect(Gitlab::CurrentSettings.max_pages_size).to eq 15 + expect(Gitlab::CurrentSettings.pages_domain_verification_enabled?).to be_truthy expect(page).to have_content "Application settings saved successfully" end @@ -83,18 +124,22 @@ feature 'Admin updates settings' do context 'sign-in restrictions', :js do it 'de-activates oauth sign-in source' do - find('input#application_setting_enabled_oauth_sign_in_sources_[value=gitlab]').send_keys(:return) + page.within('.as-signin') do + find('input#application_setting_enabled_oauth_sign_in_sources_[value=gitlab]').send_keys(:return) - expect(find('.btn', text: 'GitLab.com')).not_to have_css('.active') + expect(find('.btn', text: 'GitLab.com')).not_to have_css('.active') + end end end scenario 'Change Keys settings' do - select 'Are forbidden', from: 'RSA SSH keys' - select 'Are allowed', from: 'DSA SSH keys' - select 'Must be at least 384 bits', from: 'ECDSA SSH keys' - select 'Are forbidden', from: 'ED25519 SSH keys' - click_on 'Save' + page.within('.as-visibility-access') do + select 'Are forbidden', from: 'RSA SSH keys' + select 'Are allowed', from: 'DSA SSH keys' + select 'Must be at least 384 bits', from: 'ECDSA SSH keys' + select 'Are forbidden', from: 'ED25519 SSH keys' + click_on 'Save changes' + end forbidden = ApplicationSetting::FORBIDDEN_KEY_VALUE.to_s diff --git a/spec/features/projects/hook_logs/user_reads_log_spec.rb b/spec/features/projects/hook_logs/user_reads_log_spec.rb new file mode 100644 index 00000000000..18e975fa653 --- /dev/null +++ b/spec/features/projects/hook_logs/user_reads_log_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +feature 'Hook logs' do + given(:web_hook_log) { create(:web_hook_log, response_body: '<script>') } + given(:project) { web_hook_log.web_hook.project } + given(:user) { create(:user) } + + before do + project.add_master(user) + + sign_in(user) + end + + scenario 'user reads log without getting XSS' do + visit( + project_hook_hook_log_path( + project, web_hook_log.web_hook, web_hook_log)) + + expect(page).to have_content('<script>') + end +end diff --git a/spec/features/projects/pages_spec.rb b/spec/features/projects/pages_spec.rb index 233d2e67b9d..bdd49f731c7 100644 --- a/spec/features/projects/pages_spec.rb +++ b/spec/features/projects/pages_spec.rb @@ -40,11 +40,6 @@ feature 'Pages' do end context 'when support for external domains is disabled' do - before do - allow(Gitlab.config.pages).to receive(:external_http).and_return(nil) - allow(Gitlab.config.pages).to receive(:external_https).and_return(nil) - end - it 'renders message that support is disabled' do visit project_pages_path(project) @@ -52,7 +47,9 @@ feature 'Pages' do end end - context 'when pages are exposed on external HTTP address' do + context 'when pages are exposed on external HTTP address', :http_pages_enabled do + given(:project) { create(:project, pages_https_only: false) } + shared_examples 'adds new domain' do it 'adds new domain' do visit new_project_pages_domain_path(project) @@ -64,11 +61,6 @@ feature 'Pages' do end end - before do - allow(Gitlab.config.pages).to receive(:external_http).and_return(['1.1.1.1:80']) - allow(Gitlab.config.pages).to receive(:external_https).and_return(nil) - end - it 'allows to add new domain' do visit project_pages_path(project) @@ -80,13 +72,13 @@ feature 'Pages' do context 'when project in group namespace' do it_behaves_like 'adds new domain' do let(:group) { create :group } - let(:project) { create :project, namespace: group } + let(:project) { create(:project, namespace: group, pages_https_only: false) } end end context 'when pages domain is added' do before do - project.pages_domains.create!(domain: 'my.test.domain.com') + create(:pages_domain, project: project, domain: 'my.test.domain.com') visit new_project_pages_domain_path(project) end @@ -104,7 +96,7 @@ feature 'Pages' do end end - context 'when pages are exposed on external HTTPS address' do + context 'when pages are exposed on external HTTPS address', :https_pages_enabled do let(:certificate_pem) do <<~PEM -----BEGIN CERTIFICATE----- @@ -145,11 +137,6 @@ feature 'Pages' do KEY end - before do - allow(Gitlab.config.pages).to receive(:external_http).and_return(['1.1.1.1:80']) - allow(Gitlab.config.pages).to receive(:external_https).and_return(['1.1.1.1:443']) - end - it 'adds new domain with certificate' do visit new_project_pages_domain_path(project) @@ -163,7 +150,7 @@ feature 'Pages' do describe 'updating the certificate for an existing domain' do let!(:domain) do - create(:pages_domain, :with_key, :with_certificate, project: project) + create(:pages_domain, project: project) end it 'allows the certificate to be updated' do @@ -237,6 +224,70 @@ feature 'Pages' do it_behaves_like 'no pages deployed' end + describe 'HTTPS settings', :js, :https_pages_enabled do + background do + project.namespace.update(owner: user) + + allow_any_instance_of(Project).to receive(:pages_deployed?) { true } + end + + scenario 'tries to change the setting' do + visit project_pages_path(project) + expect(page).to have_content("Force domains with SSL certificates to use HTTPS") + + uncheck :project_pages_https_only + + click_button 'Save' + + expect(page).to have_text('Your changes have been saved') + expect(page).not_to have_checked_field('project_pages_https_only') + end + + context 'setting could not be updated' do + let(:service) { instance_double('Projects::UpdateService') } + + before do + allow(Projects::UpdateService).to receive(:new).and_return(service) + allow(service).to receive(:execute).and_return(status: :error) + end + + scenario 'tries to change the setting' do + visit project_pages_path(project) + + uncheck :project_pages_https_only + + click_button 'Save' + + expect(page).to have_text('Something went wrong on our end') + end + end + + context 'non-HTTPS domain exists' do + given(:project) { create(:project, pages_https_only: false) } + + before do + create(:pages_domain, :without_key, :without_certificate, project: project) + end + + scenario 'the setting is disabled' do + visit project_pages_path(project) + + expect(page).to have_field(:project_pages_https_only, disabled: true) + expect(page).not_to have_button('Save') + end + end + + context 'HTTPS pages are disabled', :https_pages_disabled do + scenario 'the setting is unavailable' do + visit project_pages_path(project) + + expect(page).not_to have_field(:project_pages_https_only) + expect(page).not_to have_content('Force domains with SSL certificates to use HTTPS') + expect(page).not_to have_button('Save') + end + end + end + describe 'Remove page' do context 'when user is the owner' do let(:project) { create :project, :repository } diff --git a/spec/features/projects/pipeline_schedules_spec.rb b/spec/features/projects/pipeline_schedules_spec.rb index 65e24862d43..065d00d51d4 100644 --- a/spec/features/projects/pipeline_schedules_spec.rb +++ b/spec/features/projects/pipeline_schedules_spec.rb @@ -160,9 +160,9 @@ feature 'Pipeline Schedules', :js do click_link 'New schedule' fill_in_schedule_form all('[name="schedule[variables_attributes][][key]"]')[0].set('AAA') - all('[name="schedule[variables_attributes][][value]"]')[0].set('AAA123') + all('[name="schedule[variables_attributes][][secret_value]"]')[0].set('AAA123') all('[name="schedule[variables_attributes][][key]"]')[1].set('BBB') - all('[name="schedule[variables_attributes][][value]"]')[1].set('BBB123') + all('[name="schedule[variables_attributes][][secret_value]"]')[1].set('BBB123') save_pipeline_schedule end diff --git a/spec/features/read_only_spec.rb b/spec/features/read_only_spec.rb new file mode 100644 index 00000000000..8bfaf558466 --- /dev/null +++ b/spec/features/read_only_spec.rb @@ -0,0 +1,25 @@ +require 'rails_helper' + +describe 'read-only message' do + set(:user) { create(:user) } + + before do + sign_in(user) + end + + it 'shows read-only banner when database is read-only' do + allow(Gitlab::Database).to receive(:read_only?).and_return(true) + + visit root_dashboard_path + + expect(page).to have_content('You are on a read-only GitLab instance.') + end + + it 'does not show read-only banner when database is able to read-write' do + allow(Gitlab::Database).to receive(:read_only?).and_return(false) + + visit root_dashboard_path + + expect(page).not_to have_content('You are on a read-only GitLab instance.') + end +end diff --git a/spec/finders/clusters_finder_spec.rb b/spec/finders/clusters_finder_spec.rb index c10efac2432..da529e0670f 100644 --- a/spec/finders/clusters_finder_spec.rb +++ b/spec/finders/clusters_finder_spec.rb @@ -6,7 +6,7 @@ describe ClustersFinder do describe '#execute' do let(:enabled_cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } - let(:disabled_cluster) { create(:cluster, :disabled, :provided_by_gcp, projects: [project]) } + let(:disabled_cluster) { create(:cluster, :disabled, :provided_by_gcp, :production_environment, projects: [project]) } subject { described_class.new(project, user, scope).execute } diff --git a/spec/helpers/preferences_helper_spec.rb b/spec/helpers/preferences_helper_spec.rb index e2a0c4322ff..c9d2ec8a4ae 100644 --- a/spec/helpers/preferences_helper_spec.rb +++ b/spec/helpers/preferences_helper_spec.rb @@ -21,7 +21,9 @@ describe PreferencesHelper do ["Your Projects' Activity", 'project_activity'], ["Starred Projects' Activity", 'starred_project_activity'], ["Your Groups", 'groups'], - ["Your Todos", 'todos'] + ["Your Todos", 'todos'], + ["Assigned Issues", 'issues'], + ["Assigned Merge Requests", 'merge_requests'] ] end end diff --git a/spec/initializers/6_validations_spec.rb b/spec/initializers/6_validations_spec.rb index 83283f03940..1dc307ea922 100644 --- a/spec/initializers/6_validations_spec.rb +++ b/spec/initializers/6_validations_spec.rb @@ -15,7 +15,7 @@ describe '6_validations' do describe 'validate_storages_config' do context 'with correct settings' do before do - mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c' }, 'bar' => { 'path' => 'tmp/tests/paths/a/b/d' }) + mock_storages('foo' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/c'), 'bar' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/d')) end it 'passes through' do @@ -25,7 +25,7 @@ describe '6_validations' do context 'when one of the settings is incorrect' do before do - mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c', 'failure_count_threshold' => 'not a number' }) + mock_storages('foo' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/c', 'failure_count_threshold' => 'not a number')) end it 'throws an error' do @@ -35,7 +35,7 @@ describe '6_validations' do context 'with invalid storage names' do before do - mock_storages('name with spaces' => { 'path' => 'tmp/tests/paths/a/b/c' }) + mock_storages('name with spaces' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/c')) end it 'throws an error' do @@ -67,7 +67,7 @@ describe '6_validations' do describe 'validate_storages_paths' do context 'with correct settings' do before do - mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c' }, 'bar' => { 'path' => 'tmp/tests/paths/a/b/d' }) + mock_storages('foo' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/c'), 'bar' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/d')) end it 'passes through' do @@ -77,7 +77,7 @@ describe '6_validations' do context 'with nested storage paths' do before do - mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c' }, 'bar' => { 'path' => 'tmp/tests/paths/a/b/c/d' }) + mock_storages('foo' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/c'), 'bar' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/c/d')) end it 'throws an error' do @@ -87,7 +87,7 @@ describe '6_validations' do context 'with similar but un-nested storage paths' do before do - mock_storages('foo' => { 'path' => 'tmp/tests/paths/a/b/c' }, 'bar' => { 'path' => 'tmp/tests/paths/a/b/c2' }) + mock_storages('foo' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/c'), 'bar' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/paths/a/b/c2')) end it 'passes through' do @@ -97,7 +97,7 @@ describe '6_validations' do describe 'inaccessible storage' do before do - mock_storages('foo' => { 'path' => 'tmp/tests/a/path/that/does/not/exist' }) + mock_storages('foo' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/a/path/that/does/not/exist')) end it 'passes through with a warning' do diff --git a/spec/initializers/fog_google_https_private_urls_spec.rb b/spec/initializers/fog_google_https_private_urls_spec.rb new file mode 100644 index 00000000000..de3c157ab7b --- /dev/null +++ b/spec/initializers/fog_google_https_private_urls_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' + +describe 'Fog::Storage::GoogleXML::File' do + let(:storage) do + Fog.mock! + Fog::Storage.new({ + google_storage_access_key_id: "asdf", + google_storage_secret_access_key: "asdf", + provider: "Google" + }) + end + + let(:file) do + directory = storage.directories.create(key: 'data') + directory.files.create( + body: 'Hello World!', + key: 'hello_world.txt' + ) + end + + it 'delegates to #get_https_url' do + expect(file.url(Time.now)).to start_with("https://") + end +end diff --git a/spec/initializers/settings_spec.rb b/spec/initializers/settings_spec.rb index 838ca9fabef..57f5adbbc40 100644 --- a/spec/initializers/settings_spec.rb +++ b/spec/initializers/settings_spec.rb @@ -1,5 +1,5 @@ require 'spec_helper' -require_relative '../../config/initializers/1_settings' +require_relative '../../config/initializers/1_settings' unless defined?(Settings) describe Settings do describe '#ldap' do diff --git a/spec/javascripts/ci_variable_list/native_form_variable_list_spec.js b/spec/javascripts/ci_variable_list/native_form_variable_list_spec.js index 1ea8d86cb7e..94a0c999d66 100644 --- a/spec/javascripts/ci_variable_list/native_form_variable_list_spec.js +++ b/spec/javascripts/ci_variable_list/native_form_variable_list_spec.js @@ -20,7 +20,7 @@ describe('NativeFormVariableList', () => { it('should clear out the `name` attribute on the inputs for the last empty row on form submission (avoid BE validation)', () => { const $row = $wrapper.find('.js-row'); expect($row.find('.js-ci-variable-input-key').attr('name')).toBe('schedule[variables_attributes][][key]'); - expect($row.find('.js-ci-variable-input-value').attr('name')).toBe('schedule[variables_attributes][][value]'); + expect($row.find('.js-ci-variable-input-value').attr('name')).toBe('schedule[variables_attributes][][secret_value]'); $wrapper.closest('form').trigger('trigger-submit'); diff --git a/spec/javascripts/ide/lib/editor_spec.js b/spec/javascripts/ide/lib/editor_spec.js index d6df35c90e8..3c48d94d17a 100644 --- a/spec/javascripts/ide/lib/editor_spec.js +++ b/spec/javascripts/ide/lib/editor_spec.js @@ -74,6 +74,10 @@ describe('Multi-file editor library', () => { }, readOnly: true, scrollBeyondLastLine: false, + quickSuggestions: false, + occurrencesHighlight: false, + renderLineHighlight: 'none', + hideCursorInOverviewRuler: true, }, ); }); diff --git a/spec/javascripts/notes_spec.js b/spec/javascripts/notes_spec.js index 1858d6b6474..ec56ab0e2f0 100644 --- a/spec/javascripts/notes_spec.js +++ b/spec/javascripts/notes_spec.js @@ -16,15 +16,15 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; window.gl = window.gl || {}; gl.utils = gl.utils || {}; - const htmlEscape = (comment) => { - const escapedString = comment.replace(/["&'<>]/g, (a) => { + const htmlEscape = comment => { + const escapedString = comment.replace(/["&'<>]/g, a => { const escapedToken = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''', - '`': '`' + '`': '`', }[a]; return escapedToken; @@ -39,7 +39,7 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; var commentsTemplate = 'merge_requests/merge_request_with_comment.html.raw'; preloadFixtures(commentsTemplate); - beforeEach(function () { + beforeEach(function() { loadFixtures(commentsTemplate); gl.utils.disableButtonIfEmptyField = _.noop; window.project_uploads_path = 'http://test.host/uploads'; @@ -51,6 +51,17 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; $('body').removeAttr('data-page'); }); + describe('addBinding', () => { + it('calls postComment when comment button is clicked', () => { + spyOn(Notes.prototype, 'postComment'); + this.notes = new Notes('', []); + + $('.js-comment-button').click(); + + expect(Notes.prototype.postComment).toHaveBeenCalled(); + }); + }); + describe('task lists', function() { let mock; @@ -58,7 +69,13 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; spyOn(axios, 'patch').and.callThrough(); mock = new MockAdapter(axios); - mock.onPatch(`${gl.TEST_HOST}/frontend-fixtures/merge-requests-project/merge_requests/1.json`).reply(200, {}); + mock + .onPatch( + `${ + gl.TEST_HOST + }/frontend-fixtures/merge-requests-project/merge_requests/1.json`, + ) + .reply(200, {}); $('.js-comment-button').on('click', function(e) { e.preventDefault(); @@ -73,18 +90,27 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; it('modifies the Markdown field', function() { const changeEvent = document.createEvent('HTMLEvents'); changeEvent.initEvent('change', true, true); - $('input[type=checkbox]').attr('checked', true)[1].dispatchEvent(changeEvent); + $('input[type=checkbox]') + .attr('checked', true)[1] + .dispatchEvent(changeEvent); - expect($('.js-task-list-field.original-task-list').val()).toBe('- [x] Task List Item'); + expect($('.js-task-list-field.original-task-list').val()).toBe( + '- [x] Task List Item', + ); }); it('submits an ajax request on tasklist:changed', function(done) { $('.js-task-list-container').trigger('tasklist:changed'); setTimeout(() => { - expect(axios.patch).toHaveBeenCalledWith(`${gl.TEST_HOST}/frontend-fixtures/merge-requests-project/merge_requests/1.json`, { - note: { note: '' }, - }); + expect(axios.patch).toHaveBeenCalledWith( + `${ + gl.TEST_HOST + }/frontend-fixtures/merge-requests-project/merge_requests/1.json`, + { + note: { note: '' }, + }, + ); done(); }); }); @@ -100,10 +126,10 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; spyOn(this.notes, 'renderNote').and.stub(); $(textarea).data('autosave', { - reset: function() {} + reset: function() {}, }); - $('.js-comment-button').on('click', (e) => { + $('.js-comment-button').on('click', e => { const $form = $(this); e.preventDefault(); this.notes.addNote($form); @@ -149,7 +175,7 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; <div class="note-text">${sampleComment}</div> </li>`, note: sampleComment, - valid: true + valid: true, }; $form = $('form.js-main-target-form'); $notesContainer = $('ul.main-notes-list'); @@ -163,7 +189,7 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; mock.restore(); }); - it('updates note and resets edit form', (done) => { + it('updates note and resets edit form', done => { spyOn(this.notes, 'revertNoteEditForm'); spyOn(this.notes, 'setupNewNote'); @@ -175,7 +201,9 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; updatedNote.note = 'bar'; this.notes.updateNote(updatedNote, $targetNote); - expect(this.notes.revertNoteEditForm).toHaveBeenCalledWith($targetNote); + expect(this.notes.revertNoteEditForm).toHaveBeenCalledWith( + $targetNote, + ); expect(this.notes.setupNewNote).toHaveBeenCalled(); done(); @@ -231,17 +259,14 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; note: 'heya', html: '<div>heya</div>', }; - $notesList = jasmine.createSpyObj('$notesList', [ - 'find', - 'append', - ]); + $notesList = jasmine.createSpyObj('$notesList', ['find', 'append']); notes = jasmine.createSpyObj('notes', [ 'setupNewNote', 'refresh', 'collapseLongCommitList', 'updateNotesCount', - 'putConflictEditWarningInPlace' + 'putConflictEditWarningInPlace', ]); notes.taskList = jasmine.createSpyObj('tasklist', ['init']); notes.note_ids = []; @@ -258,7 +283,10 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; Notes.isNewNote.and.returnValue(true); Notes.prototype.renderNote.call(notes, note, null, $notesList); - expect(Notes.animateAppendNote).toHaveBeenCalledWith(note.html, $notesList); + expect(Notes.animateAppendNote).toHaveBeenCalledWith( + note.html, + $notesList, + ); }); }); @@ -273,7 +301,10 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; Notes.prototype.renderNote.call(notes, note, null, $notesList); - expect(Notes.animateUpdateNote).toHaveBeenCalledWith(note.html, $note); + expect(Notes.animateUpdateNote).toHaveBeenCalledWith( + note.html, + $note, + ); expect(notes.setupNewNote).toHaveBeenCalledWith($newNote); }); @@ -301,7 +332,10 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; $notesList.find.and.returnValue($note); Notes.prototype.renderNote.call(notes, note, null, $notesList); - expect(notes.putConflictEditWarningInPlace).toHaveBeenCalledWith(note, $note); + expect(notes.putConflictEditWarningInPlace).toHaveBeenCalledWith( + note, + $note, + ); }); }); }); @@ -311,11 +345,11 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; it('should consider same note text as the same', () => { const result = Notes.isUpdatedNote( { - note: 'initial' + note: 'initial', }, $(`<div> <div class="original-note-content">initial</div> - </div>`) + </div>`), ); expect(result).toEqual(false); @@ -324,11 +358,11 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; it('should consider same note with trailing newline as the same', () => { const result = Notes.isUpdatedNote( { - note: 'initial\n' + note: 'initial\n', }, $(`<div> <div class="original-note-content">initial\n</div> - </div>`) + </div>`), ); expect(result).toEqual(false); @@ -337,11 +371,11 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; it('should consider different notes as different', () => { const result = Notes.isUpdatedNote( { - note: 'foo' + note: 'foo', }, $(`<div> <div class="original-note-content">bar</div> - </div>`) + </div>`), ); expect(result).toEqual(true); @@ -397,7 +431,10 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; it('should call Notes.animateAppendNote', () => { Notes.prototype.renderDiscussionNote.call(notes, note, $form); - expect(Notes.animateAppendNote).toHaveBeenCalledWith(note.discussion_html, $('.main-notes-list')); + expect(Notes.animateAppendNote).toHaveBeenCalledWith( + note.discussion_html, + $('.main-notes-list'), + ); }); it('should append to row selected with line_code', () => { @@ -428,7 +465,10 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; }); it('should call Notes.animateAppendNote', () => { - expect(Notes.animateAppendNote).toHaveBeenCalledWith(note.html, discussionContainer); + expect(Notes.animateAppendNote).toHaveBeenCalledWith( + note.html, + discussionContainer, + ); }); }); }); @@ -461,9 +501,7 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; beforeEach(() => { noteHTML = '<div></div>'; - $note = jasmine.createSpyObj('$note', [ - 'replaceWith' - ]); + $note = jasmine.createSpyObj('$note', ['replaceWith']); $updatedNote = Notes.animateUpdateNote(noteHTML, $note); }); @@ -501,7 +539,7 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; <div class="note-text">${sampleComment}</div> </li>`, note: sampleComment, - valid: true + valid: true, }; let $form; let $notesContainer; @@ -534,10 +572,12 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; mockNotesPost(); $('.js-comment-button').click(); - expect($notesContainer.find('.note.being-posted').length > 0).toEqual(true); + expect($notesContainer.find('.note.being-posted').length > 0).toEqual( + true, + ); }); - it('should remove placeholder note when new comment is done posting', (done) => { + it('should remove placeholder note when new comment is done posting', done => { mockNotesPost(); $('.js-comment-button').click(); @@ -549,33 +589,44 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; }); }); - it('should disable the submit button when comment button is clicked', (done) => { - expect($form.find('.js-comment-submit-button').is(':disabled')).toEqual(false); - - mockNotesPost(); - $('.js-comment-button').click(); - expect($form.find('.js-comment-submit-button').is(':disabled')).toEqual(true); - - setTimeout(() => { - expect($form.find('.js-comment-submit-button').is(':disabled')).toEqual(false); + describe('postComment', () => { + it('disables the submit button', done => { + const $submitButton = $form.find('.js-comment-submit-button'); + expect($submitButton).not.toBeDisabled(); + const dummyEvent = { + preventDefault() {}, + target: $submitButton, + }; + mock.onPost(NOTES_POST_PATH).replyOnce(() => { + expect($submitButton).toBeDisabled(); + return [200, note]; + }); - done(); + this.notes + .postComment(dummyEvent) + .then(() => { + expect($submitButton).not.toBeDisabled(); + }) + .then(done) + .catch(done.fail); }); }); - it('should show actual note element when new comment is done posting', (done) => { + it('should show actual note element when new comment is done posting', done => { mockNotesPost(); $('.js-comment-button').click(); setTimeout(() => { - expect($notesContainer.find(`#note_${note.id}`).length > 0).toEqual(true); + expect($notesContainer.find(`#note_${note.id}`).length > 0).toEqual( + true, + ); done(); }); }); - it('should reset Form when new comment is done posting', (done) => { + it('should reset Form when new comment is done posting', done => { mockNotesPost(); $('.js-comment-button').click(); @@ -587,19 +638,24 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; }); }); - it('should show flash error message when new comment failed to be posted', (done) => { + it('should show flash error message when new comment failed to be posted', done => { mockNotesPostError(); $('.js-comment-button').click(); setTimeout(() => { - expect($notesContainer.parent().find('.flash-container .flash-text').is(':visible')).toEqual(true); + expect( + $notesContainer + .parent() + .find('.flash-container .flash-text') + .is(':visible'), + ).toEqual(true); done(); }); }); - it('should show flash error message when comment failed to be updated', (done) => { + it('should show flash error message when comment failed to be updated', done => { mockNotesPost(); $('.js-comment-button').click(); @@ -620,7 +676,12 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; .then(() => { const $updatedNoteEl = $notesContainer.find(`#note_${note.id}`); expect($updatedNoteEl.hasClass('.being-posted')).toEqual(false); // Remove being-posted visuals - expect($updatedNoteEl.find('.note-text').text().trim()).toEqual(sampleComment); // See if comment reverted back to original + expect( + $updatedNoteEl + .find('.note-text') + .text() + .trim(), + ).toEqual(sampleComment); // See if comment reverted back to original expect($('.flash-container').is(':visible')).toEqual(true); // Flash error message shown done(); @@ -634,12 +695,12 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; const note = { commands_changes: { assignee_id: 1, - emoji_award: '100' + emoji_award: '100', }, errors: { - commands_only: ['Commands applied'] + commands_only: ['Commands applied'], }, - valid: false + valid: false, }; let $form; let $notesContainer; @@ -654,12 +715,12 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; window.gon.current_user_fullname = 'Administrator'; gl.awardsHandler = { addAwardToEmojiBar: () => {}, - scrollToAwards: () => {} + scrollToAwards: () => {}, }; gl.GfmAutoComplete = { dataSources: { - commands: '/root/test-project/autocomplete_sources/commands' - } + commands: '/root/test-project/autocomplete_sources/commands', + }, }; $form = $('form.js-main-target-form'); $notesContainer = $('ul.main-notes-list'); @@ -670,14 +731,18 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; mock.restore(); }); - it('should remove slash command placeholder when comment with slash commands is done posting', (done) => { + it('should remove slash command placeholder when comment with slash commands is done posting', done => { spyOn(gl.awardsHandler, 'addAwardToEmojiBar').and.callThrough(); $('.js-comment-button').click(); - expect($notesContainer.find('.system-note.being-posted').length).toEqual(1); // Placeholder shown + expect( + $notesContainer.find('.system-note.being-posted').length, + ).toEqual(1); // Placeholder shown setTimeout(() => { - expect($notesContainer.find('.system-note.being-posted').length).toEqual(0); // Placeholder removed + expect( + $notesContainer.find('.system-note.being-posted').length, + ).toEqual(0); // Placeholder removed done(); }); }); @@ -692,7 +757,7 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; <div class="note-text">${sampleComment}</div> </li>`, note: sampleComment, - valid: true + valid: true, }; let $form; let $notesContainer; @@ -714,7 +779,7 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; mock.restore(); }); - it('should not render a script tag', (done) => { + it('should not render a script tag', done => { $('.js-comment-button').click(); setTimeout(() => { @@ -723,8 +788,15 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; $noteEl.find('textarea.js-note-text').html(updatedComment); $noteEl.find('.js-comment-save-button').click(); - const $updatedNoteEl = $notesContainer.find(`#note_${note.id}`).find('.js-task-list-container'); - expect($updatedNoteEl.find('.note-text').text().trim()).toEqual(''); + const $updatedNoteEl = $notesContainer + .find(`#note_${note.id}`) + .find('.js-task-list-container'); + expect( + $updatedNoteEl + .find('.note-text') + .text() + .trim(), + ).toEqual(''); done(); }); @@ -744,7 +816,9 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; it('should return form metadata object from form reference', () => { $form.find('textarea.js-note-text').val(sampleComment); - const { formData, formContent, formAction } = this.notes.getFormData($form); + const { formData, formContent, formAction } = this.notes.getFormData( + $form, + ); expect(formData.indexOf(sampleComment) > -1).toBe(true); expect(formContent).toEqual(sampleComment); @@ -760,7 +834,9 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; const { formContent } = this.notes.getFormData($form); expect(_.escape).toHaveBeenCalledWith(sampleComment); - expect(formContent).toEqual('<script>alert("Boom!");</script>'); + expect(formContent).toEqual( + '<script>alert("Boom!");</script>', + ); }); }); @@ -770,7 +846,8 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; }); it('should return true when comment begins with a quick action', () => { - const sampleComment = '/wip\n/milestone %1.0\n/merge\n/unassign Merging this'; + const sampleComment = + '/wip\n/milestone %1.0\n/merge\n/unassign Merging this'; const hasQuickActions = this.notes.hasQuickActions(sampleComment); expect(hasQuickActions).toBeTruthy(); @@ -794,7 +871,8 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; describe('stripQuickActions', () => { it('should strip quick actions from the comment which begins with a quick action', () => { this.notes = new Notes(); - const sampleComment = '/wip\n/milestone %1.0\n/merge\n/unassign Merging this'; + const sampleComment = + '/wip\n/milestone %1.0\n/merge\n/unassign Merging this'; const stripedComment = this.notes.stripQuickActions(sampleComment); expect(stripedComment).toBe(''); @@ -802,7 +880,8 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; it('should strip quick actions from the comment but leaves plain comment if it is present', () => { this.notes = new Notes(); - const sampleComment = '/wip\n/milestone %1.0\n/merge\n/unassign\nMerging this'; + const sampleComment = + '/wip\n/milestone %1.0\n/merge\n/unassign\nMerging this'; const stripedComment = this.notes.stripQuickActions(sampleComment); expect(stripedComment).toBe('Merging this'); @@ -810,7 +889,8 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; it('should NOT strip string that has slashes within', () => { this.notes = new Notes(); - const sampleComment = 'http://127.0.0.1:3000/root/gitlab-shell/issues/1'; + const sampleComment = + 'http://127.0.0.1:3000/root/gitlab-shell/issues/1'; const stripedComment = this.notes.stripQuickActions(sampleComment); expect(stripedComment).toBe(sampleComment); @@ -821,7 +901,7 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; const availableQuickActions = [ { name: 'close', description: 'Close this issue', params: [] }, { name: 'title', description: 'Change title', params: [{}] }, - { name: 'estimate', description: 'Set time estimate', params: [{}] } + { name: 'estimate', description: 'Set time estimate', params: [{}] }, ]; beforeEach(() => { @@ -830,17 +910,29 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; it('should return executing quick action description when note has single quick action', () => { const sampleComment = '/close'; - expect(this.notes.getQuickActionDescription(sampleComment, availableQuickActions)).toBe('Applying command to close this issue'); + expect( + this.notes.getQuickActionDescription( + sampleComment, + availableQuickActions, + ), + ).toBe('Applying command to close this issue'); }); it('should return generic multiple quick action description when note has multiple quick actions', () => { const sampleComment = '/close\n/title [Duplicate] Issue foobar'; - expect(this.notes.getQuickActionDescription(sampleComment, availableQuickActions)).toBe('Applying multiple commands'); + expect( + this.notes.getQuickActionDescription( + sampleComment, + availableQuickActions, + ), + ).toBe('Applying multiple commands'); }); it('should return generic quick action description when available quick actions list is not populated', () => { const sampleComment = '/close\n/title [Duplicate] Issue foobar'; - expect(this.notes.getQuickActionDescription(sampleComment)).toBe('Applying command'); + expect(this.notes.getQuickActionDescription(sampleComment)).toBe( + 'Applying command', + ); }); }); @@ -870,14 +962,35 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; expect($tempNote.attr('id')).toEqual(uniqueId); expect($tempNote.hasClass('being-posted')).toBeTruthy(); expect($tempNote.hasClass('fade-in-half')).toBeTruthy(); - $tempNote.find('.timeline-icon > a, .note-header-info > a').each(function() { - expect($(this).attr('href')).toEqual(`/${currentUsername}`); - }); - expect($tempNote.find('.timeline-icon .avatar').attr('src')).toEqual(currentUserAvatar); - expect($tempNote.find('.timeline-content').hasClass('discussion')).toBeFalsy(); - expect($tempNoteHeader.find('.hidden-xs').text().trim()).toEqual(currentUserFullname); - expect($tempNoteHeader.find('.note-headline-light').text().trim()).toEqual(`@${currentUsername}`); - expect($tempNote.find('.note-body .note-text p').text().trim()).toEqual(sampleComment); + $tempNote + .find('.timeline-icon > a, .note-header-info > a') + .each(function() { + expect($(this).attr('href')).toEqual(`/${currentUsername}`); + }); + expect($tempNote.find('.timeline-icon .avatar').attr('src')).toEqual( + currentUserAvatar, + ); + expect( + $tempNote.find('.timeline-content').hasClass('discussion'), + ).toBeFalsy(); + expect( + $tempNoteHeader + .find('.hidden-xs') + .text() + .trim(), + ).toEqual(currentUserFullname); + expect( + $tempNoteHeader + .find('.note-headline-light') + .text() + .trim(), + ).toEqual(`@${currentUsername}`); + expect( + $tempNote + .find('.note-body .note-text p') + .text() + .trim(), + ).toEqual(sampleComment); }); it('should return constructed placeholder element for discussion note based on form contents', () => { @@ -886,11 +999,13 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; uniqueId, isDiscussionNote: true, currentUsername, - currentUserFullname + currentUserFullname, }); expect($tempNote.prop('nodeName')).toEqual('LI'); - expect($tempNote.find('.timeline-content').hasClass('discussion')).toBeTruthy(); + expect( + $tempNote.find('.timeline-content').hasClass('discussion'), + ).toBeTruthy(); }); it('should return a escaped user name', () => { @@ -904,7 +1019,12 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; currentUserAvatar, }); const $tempNoteHeader = $tempNote.find('.note-header'); - expect($tempNoteHeader.find('.hidden-xs').text().trim()).toEqual('Foo <script>alert("XSS")</script>'); + expect( + $tempNoteHeader + .find('.hidden-xs') + .text() + .trim(), + ).toEqual('Foo <script>alert("XSS")</script>'); }); }); @@ -927,7 +1047,12 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; expect($tempNote.attr('id')).toEqual(uniqueId); expect($tempNote.hasClass('being-posted')).toBeTruthy(); expect($tempNote.hasClass('fade-in-half')).toBeTruthy(); - expect($tempNote.find('.timeline-content i').text().trim()).toEqual(sampleCommandDescription); + expect( + $tempNote + .find('.timeline-content i') + .text() + .trim(), + ).toEqual(sampleCommandDescription); }); }); @@ -937,7 +1062,11 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; }); it('shows a flash message', () => { - this.notes.addFlash('Error message', FLASH_TYPE_ALERT, this.notes.parentTimeline.get(0)); + this.notes.addFlash( + 'Error message', + FLASH_TYPE_ALERT, + this.notes.parentTimeline.get(0), + ); expect($('.flash-alert').is(':visible')).toBeTruthy(); }); @@ -950,7 +1079,11 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; }); it('hides visible flash message', () => { - this.notes.addFlash('Error message 1', FLASH_TYPE_ALERT, this.notes.parentTimeline.get(0)); + this.notes.addFlash( + 'Error message 1', + FLASH_TYPE_ALERT, + this.notes.parentTimeline.get(0), + ); this.notes.clearFlash(); @@ -958,4 +1091,4 @@ import timeoutPromise from './helpers/set_timeout_promise_helper'; }); }); }); -}).call(window); +}.call(window)); diff --git a/spec/javascripts/vue_mr_widget/components/mr_widget_memory_usage_spec.js b/spec/javascripts/vue_mr_widget/components/mr_widget_memory_usage_spec.js index 31710551399..d9c03296857 100644 --- a/spec/javascripts/vue_mr_widget/components/mr_widget_memory_usage_spec.js +++ b/spec/javascripts/vue_mr_widget/components/mr_widget_memory_usage_spec.js @@ -23,9 +23,7 @@ const metricsMockData = { memory_values: [ { metric: {}, - values: [ - [1493716685, '4.30859375'], - ], + values: [[1493716685, '4.30859375']], }, ], }, @@ -53,7 +51,8 @@ const createComponent = () => { const messages = { loadingMetrics: 'Loading deployment statistics', - hasMetrics: 'Memory usage unchanged from 0MB to 0MB', + hasMetrics: + '<a href="/root/acets-review-apps/environments/15/metrics"> Memory </a> usage is <b> unchanged </b> at 0MB', loadFailed: 'Failed to load deployment statistics', metricsUnavailable: 'Deployment statistics are not available currently', }; @@ -92,26 +91,26 @@ describe('MemoryUsage', () => { }); describe('computed', () => { - describe('memoryChangeType', () => { - it('should return "increased" if memoryFrom value is less than memoryTo value', () => { + describe('memoryChangeMessage', () => { + it('should contain "increased" if memoryFrom value is less than memoryTo value', () => { vm.memoryFrom = 4.28; vm.memoryTo = 9.13; - expect(vm.memoryChangeType).toEqual('increased'); + expect(vm.memoryChangeMessage.indexOf('increased')).not.toEqual('-1'); }); - it('should return "decreased" if memoryFrom value is less than memoryTo value', () => { + it('should contain "decreased" if memoryFrom value is less than memoryTo value', () => { vm.memoryFrom = 9.13; vm.memoryTo = 4.28; - expect(vm.memoryChangeType).toEqual('decreased'); + expect(vm.memoryChangeMessage.indexOf('decreased')).not.toEqual('-1'); }); - it('should return "unchanged" if memoryFrom value equal to memoryTo value', () => { + it('should contain "unchanged" if memoryFrom value equal to memoryTo value', () => { vm.memoryFrom = 1; vm.memoryTo = 1; - expect(vm.memoryChangeType).toEqual('unchanged'); + expect(vm.memoryChangeMessage.indexOf('unchanged')).not.toEqual('-1'); }); }); }); @@ -130,7 +129,13 @@ describe('MemoryUsage', () => { describe('computeGraphData', () => { it('should populate sparkline graph', () => { vm.computeGraphData(metrics, deployment_time); - const { hasMetrics, memoryMetrics, deploymentTime, memoryFrom, memoryTo } = vm; + const { + hasMetrics, + memoryMetrics, + deploymentTime, + memoryFrom, + memoryTo, + } = vm; expect(hasMetrics).toBeTruthy(); expect(memoryMetrics.length > 0).toBeTruthy(); @@ -141,20 +146,26 @@ describe('MemoryUsage', () => { }); describe('loadMetrics', () => { - const returnServicePromise = () => new Promise((resolve) => { - resolve({ - data: metricsMockData, + const returnServicePromise = () => + new Promise(resolve => { + resolve({ + data: metricsMockData, + }); }); - }); - it('should load metrics data using MRWidgetService', (done) => { - spyOn(MRWidgetService, 'fetchMetrics').and.returnValue(returnServicePromise(true)); + it('should load metrics data using MRWidgetService', done => { + spyOn(MRWidgetService, 'fetchMetrics').and.returnValue( + returnServicePromise(true), + ); spyOn(vm, 'computeGraphData'); vm.loadMetrics(); setTimeout(() => { expect(MRWidgetService.fetchMetrics).toHaveBeenCalledWith(url); - expect(vm.computeGraphData).toHaveBeenCalledWith(metrics, deployment_time); + expect(vm.computeGraphData).toHaveBeenCalledWith( + metrics, + deployment_time, + ); done(); }, 333); }); @@ -167,51 +178,67 @@ describe('MemoryUsage', () => { expect(el.querySelector('.js-usage-info')).toBeDefined(); }); - it('should show loading metrics message while metrics are being loaded', (done) => { + it('should show loading metrics message while metrics are being loaded', done => { vm.loadingMetrics = true; vm.hasMetrics = false; vm.loadFailed = false; Vue.nextTick(() => { - expect(el.querySelector('.js-usage-info.usage-info-loading')).toBeDefined(); - expect(el.querySelector('.js-usage-info .usage-info-load-spinner')).toBeDefined(); - expect(el.querySelector('.js-usage-info').innerText).toContain(messages.loadingMetrics); + expect( + el.querySelector('.js-usage-info.usage-info-loading'), + ).toBeDefined(); + expect( + el.querySelector('.js-usage-info .usage-info-load-spinner'), + ).toBeDefined(); + expect(el.querySelector('.js-usage-info').innerText).toContain( + messages.loadingMetrics, + ); done(); }); }); - it('should show deployment memory usage when metrics are loaded', (done) => { + it('should show deployment memory usage when metrics are loaded', done => { vm.loadingMetrics = false; vm.hasMetrics = true; vm.loadFailed = false; Vue.nextTick(() => { expect(el.querySelector('.memory-graph-container')).toBeDefined(); - expect(el.querySelector('.js-usage-info').innerText).toContain(messages.hasMetrics); + expect(el.querySelector('.js-usage-info').innerText).toContain( + messages.hasMetrics, + ); done(); }); }); - it('should show failure message when metrics loading failed', (done) => { + it('should show failure message when metrics loading failed', done => { vm.loadingMetrics = false; vm.hasMetrics = false; vm.loadFailed = true; Vue.nextTick(() => { - expect(el.querySelector('.js-usage-info.usage-info-failed')).toBeDefined(); - expect(el.querySelector('.js-usage-info').innerText).toContain(messages.loadFailed); + expect( + el.querySelector('.js-usage-info.usage-info-failed'), + ).toBeDefined(); + expect(el.querySelector('.js-usage-info').innerText).toContain( + messages.loadFailed, + ); done(); }); }); - it('should show metrics unavailable message when metrics loading failed', (done) => { + it('should show metrics unavailable message when metrics loading failed', done => { vm.loadingMetrics = false; vm.hasMetrics = false; vm.loadFailed = false; Vue.nextTick(() => { - expect(el.querySelector('.js-usage-info.usage-info-unavailable')).toBeDefined(); - expect(el.querySelector('.js-usage-info').innerText).toContain(messages.metricsUnavailable); + expect( + el.querySelector('.js-usage-info.usage-info-unavailable'), + ).toBeDefined(); + expect(el.querySelector('.js-usage-info').innerText).toContain( + messages.metricsUnavailable, + ); done(); }); }); diff --git a/spec/javascripts/vue_shared/components/modal_spec.js b/spec/javascripts/vue_shared/components/deprecated_modal_spec.js index d01a94c25e5..59d4e549a91 100644 --- a/spec/javascripts/vue_shared/components/modal_spec.js +++ b/spec/javascripts/vue_shared/components/deprecated_modal_spec.js @@ -1,11 +1,11 @@ import $ from 'jquery'; import Vue from 'vue'; -import modal from '~/vue_shared/components/modal.vue'; +import DeprecatedModal from '~/vue_shared/components/deprecated_modal.vue'; import mountComponent from 'spec/helpers/vue_mount_component_helper'; -const modalComponent = Vue.extend(modal); +const modalComponent = Vue.extend(DeprecatedModal); -describe('Modal', () => { +describe('DeprecatedModal', () => { let vm; afterEach(() => { diff --git a/spec/lib/backup/repository_spec.rb b/spec/lib/backup/repository_spec.rb index a9b5ed1112a..03573c304aa 100644 --- a/spec/lib/backup/repository_spec.rb +++ b/spec/lib/backup/repository_spec.rb @@ -33,7 +33,7 @@ describe Backup::Repository do let(:timestamp) { Time.utc(2017, 3, 22) } let(:temp_dirs) do Gitlab.config.repositories.storages.map do |name, storage| - File.join(storage['path'], '..', 'repositories.old.' + timestamp.to_i.to_s) + File.join(storage.legacy_disk_path, '..', 'repositories.old.' + timestamp.to_i.to_s) end end diff --git a/spec/lib/banzai/filter/autolink_filter_spec.rb b/spec/lib/banzai/filter/autolink_filter_spec.rb index b502daea418..cbb0089bde7 100644 --- a/spec/lib/banzai/filter/autolink_filter_spec.rb +++ b/spec/lib/banzai/filter/autolink_filter_spec.rb @@ -122,14 +122,10 @@ describe Banzai::Filter::AutolinkFilter do end it 'does not include trailing punctuation' do - doc = filter("See #{link}.") - expect(doc.at_css('a').text).to eq link - - doc = filter("See #{link}, ok?") - expect(doc.at_css('a').text).to eq link - - doc = filter("See #{link}...") - expect(doc.at_css('a').text).to eq link + ['.', ', ok?', '...', '?', '!', ': is that ok?'].each do |trailing_punctuation| + doc = filter("See #{link}#{trailing_punctuation}") + expect(doc.at_css('a').text).to eq link + end end it 'includes trailing punctuation when part of a balanced pair' do diff --git a/spec/lib/banzai/filter/issuable_state_filter_spec.rb b/spec/lib/banzai/filter/issuable_state_filter_spec.rb index 17347768a49..a5373517ac8 100644 --- a/spec/lib/banzai/filter/issuable_state_filter_spec.rb +++ b/spec/lib/banzai/filter/issuable_state_filter_spec.rb @@ -8,6 +8,7 @@ describe Banzai::Filter::IssuableStateFilter do let(:context) { { current_user: user, issuable_state_filter_enabled: true } } let(:closed_issue) { create_issue(:closed) } let(:project) { create(:project, :public) } + let(:group) { create(:group) } let(:other_project) { create(:project, :public) } def create_link(text, data) @@ -77,6 +78,13 @@ describe Banzai::Filter::IssuableStateFilter do expect(doc.css('a').last.text).to eq("#{closed_issue.to_reference(other_project)} (closed)") end + it 'handles references from group scopes' do + link = create_link(closed_issue.to_reference(other_project), issue: closed_issue.id, reference_type: 'issue') + doc = filter(link, context.merge(project: nil, group: group)) + + expect(doc.css('a').last.text).to eq("#{closed_issue.to_reference(other_project)} (closed)") + end + it 'skips cross project references if the user cannot read cross project' do expect(Ability).to receive(:allowed?).with(user, :read_cross_project) { false } link = create_link(closed_issue.to_reference(other_project), issue: closed_issue.id, reference_type: 'issue') diff --git a/spec/lib/gitlab/auth_spec.rb b/spec/lib/gitlab/auth_spec.rb index f969f9e8e38..18cef8ec996 100644 --- a/spec/lib/gitlab/auth_spec.rb +++ b/spec/lib/gitlab/auth_spec.rb @@ -315,13 +315,19 @@ describe Gitlab::Auth do it "tries to autheticate with db before ldap" do expect(Gitlab::Auth::LDAP::Authentication).not_to receive(:login) - gl_auth.find_with_user_password(username, password) + expect(gl_auth.find_with_user_password(username, password)).to eq(user) + end + + it "does not find user by using ldap as fallback to for authentication" do + expect(Gitlab::Auth::LDAP::Authentication).to receive(:login).and_return(nil) + + expect(gl_auth.find_with_user_password('ldap_user', 'password')).to be_nil end - it "uses ldap as fallback to for authentication" do - expect(Gitlab::Auth::LDAP::Authentication).to receive(:login) + it "find new user by using ldap as fallback to for authentication" do + expect(Gitlab::Auth::LDAP::Authentication).to receive(:login).and_return(user) - gl_auth.find_with_user_password('ldap_user', 'password') + expect(gl_auth.find_with_user_password('ldap_user', 'password')).to eq(user) end end diff --git a/spec/lib/gitlab/bare_repository_import/repository_spec.rb b/spec/lib/gitlab/bare_repository_import/repository_spec.rb index 5cb1f4deb5f..0dc3705825d 100644 --- a/spec/lib/gitlab/bare_repository_import/repository_spec.rb +++ b/spec/lib/gitlab/bare_repository_import/repository_spec.rb @@ -54,7 +54,7 @@ describe ::Gitlab::BareRepositoryImport::Repository do context 'hashed storage' do let(:gitlab_shell) { Gitlab::Shell.new } let(:repository_storage) { 'default' } - let(:root_path) { Gitlab.config.repositories.storages[repository_storage]['path'] } + let(:root_path) { Gitlab.config.repositories.storages[repository_storage].legacy_disk_path } let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' } let(:hashed_path) { "@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b" } let(:repo_path) { File.join(root_path, "#{hashed_path}.git") } diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb index 1b03227d67b..dc12ba076bc 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb @@ -5,23 +5,23 @@ describe Gitlab::Ci::Pipeline::Chain::Create do set(:user) { create(:user) } let(:pipeline) do - build(:ci_pipeline_with_one_job, project: project, - ref: 'master') + build(:ci_empty_pipeline, project: project, ref: 'master') end let(:command) do Gitlab::Ci::Pipeline::Chain::Command.new( - project: project, - current_user: user, seeds_block: nil) + project: project, current_user: user) end let(:step) { described_class.new(pipeline, command) } - before do - step.perform! - end - context 'when pipeline is ready to be saved' do + before do + pipeline.stages.build(name: 'test', project: project) + + step.perform! + end + it 'saves a pipeline' do expect(pipeline).to be_persisted end @@ -32,6 +32,7 @@ describe Gitlab::Ci::Pipeline::Chain::Create do it 'creates stages' do expect(pipeline.reload.stages).to be_one + expect(pipeline.stages.first).to be_persisted end end @@ -40,6 +41,10 @@ describe Gitlab::Ci::Pipeline::Chain::Create do build(:ci_pipeline, project: project, ref: nil) end + before do + step.perform! + end + it 'breaks the chain' do expect(step.break?).to be true end @@ -49,18 +54,4 @@ describe Gitlab::Ci::Pipeline::Chain::Create do .to include /Failed to persist the pipeline/ end end - - context 'when there is a seed block present' do - let(:seeds) { spy('pipeline seeds') } - - let(:command) do - double('command', project: project, - current_user: user, - seeds_block: seeds) - end - - it 'executes the block' do - expect(seeds).to have_received(:call).with(pipeline) - end - end end diff --git a/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb new file mode 100644 index 00000000000..2258ae83f38 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb @@ -0,0 +1,153 @@ +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Chain::Populate do + set(:project) { create(:project) } + set(:user) { create(:user) } + + let(:pipeline) do + build(:ci_pipeline_with_one_job, project: project, + ref: 'master') + end + + let(:command) do + Gitlab::Ci::Pipeline::Chain::Command.new( + project: project, + current_user: user, + seeds_block: nil) + end + + let(:step) { described_class.new(pipeline, command) } + + context 'when pipeline doesn not have seeds block' do + before do + step.perform! + end + + it 'does not persist the pipeline' do + expect(pipeline).not_to be_persisted + end + + it 'does not break the chain' do + expect(step.break?).to be false + end + + it 'populates pipeline with stages' do + expect(pipeline.stages).to be_one + expect(pipeline.stages.first).not_to be_persisted + end + + it 'populates pipeline with builds' do + expect(pipeline.builds).to be_one + expect(pipeline.builds.first).not_to be_persisted + expect(pipeline.stages.first.builds).to be_one + expect(pipeline.stages.first.builds.first).not_to be_persisted + end + end + + context 'when pipeline is empty' do + let(:config) do + { rspec: { + script: 'ls', + only: ['something'] + } } + end + + let(:pipeline) do + build(:ci_pipeline, project: project, config: config) + end + + before do + step.perform! + end + + it 'breaks the chain' do + expect(step.break?).to be true + end + + it 'appends an error about missing stages' do + expect(pipeline.errors.to_a) + .to include 'No stages / jobs for this pipeline.' + end + end + + context 'when pipeline has validation errors' do + let(:pipeline) do + build(:ci_pipeline, project: project, ref: nil) + end + + before do + step.perform! + end + + it 'breaks the chain' do + expect(step.break?).to be true + end + + it 'appends validation error' do + expect(pipeline.errors.to_a) + .to include 'Failed to build the pipeline!' + end + end + + context 'when there is a seed blocks present' do + let(:command) do + Gitlab::Ci::Pipeline::Chain::Command.new( + project: project, + current_user: user, + seeds_block: seeds_block) + end + + context 'when seeds block builds some resources' do + let(:seeds_block) do + ->(pipeline) { pipeline.variables.build(key: 'VAR', value: '123') } + end + + it 'populates pipeline with resources described in the seeds block' do + step.perform! + + expect(pipeline).not_to be_persisted + expect(pipeline.variables).not_to be_empty + expect(pipeline.variables.first).not_to be_persisted + expect(pipeline.variables.first.key).to eq 'VAR' + expect(pipeline.variables.first.value).to eq '123' + end + end + + context 'when seeds block tries to persist some resources' do + let(:seeds_block) do + ->(pipeline) { pipeline.variables.create!(key: 'VAR', value: '123') } + end + + it 'raises exception' do + expect { step.perform! }.to raise_error(ActiveRecord::RecordNotSaved) + end + end + end + + context 'when pipeline gets persisted during the process' do + let(:pipeline) { create(:ci_pipeline, project: project) } + + it 'raises error' do + expect { step.perform! }.to raise_error(described_class::PopulateError) + end + end + + context 'when using only/except build policies' do + let(:config) do + { rspec: { script: 'rspec', stage: 'test', only: ['master'] }, + prod: { script: 'cap prod', stage: 'deploy', only: ['tags'] } } + end + + let(:pipeline) do + build(:ci_pipeline, ref: 'master', config: config) + end + + it 'populates pipeline according to used policies' do + step.perform! + + expect(pipeline.stages.size).to eq 1 + expect(pipeline.builds.size).to eq 1 + expect(pipeline.builds.first.name).to eq 'rspec' + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/chain/validate/config_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/validate/config_spec.rb index 5c12c6e6392..c53294d091c 100644 --- a/spec/lib/gitlab/ci/pipeline/chain/validate/config_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/chain/validate/config_spec.rb @@ -76,28 +76,6 @@ describe Gitlab::Ci::Pipeline::Chain::Validate::Config do end end - context 'when pipeline has no stages / jobs' do - let(:config) do - { rspec: { - script: 'ls', - only: ['something'] - } } - end - - let(:pipeline) do - build(:ci_pipeline, project: project, config: config) - end - - it 'appends an error about missing stages' do - expect(pipeline.errors.to_a) - .to include 'No stages / jobs for this pipeline.' - end - - it 'breaks the chain' do - expect(step.break?).to be true - end - end - context 'when pipeline contains configuration validation errors' do let(:config) { { rspec: {} } } diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb new file mode 100644 index 00000000000..116573379e0 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -0,0 +1,242 @@ +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Seed::Build do + let(:pipeline) { create(:ci_empty_pipeline) } + + let(:attributes) do + { name: 'rspec', + ref: 'master', + commands: 'rspec' } + end + + subject do + described_class.new(pipeline, attributes) + end + + describe '#attributes' do + it 'returns hash attributes of a build' do + expect(subject.attributes).to be_a Hash + expect(subject.attributes) + .to include(:name, :project, :ref, :commands) + end + end + + describe '#user=' do + let(:user) { build(:user) } + + it 'assignes user to a build' do + subject.user = user + + expect(subject.attributes).to include(user: user) + end + end + + describe '#to_resource' do + it 'returns a valid build resource' do + expect(subject.to_resource).to be_a(::Ci::Build) + expect(subject.to_resource).to be_valid + end + + it 'memoizes a resource object' do + build = subject.to_resource + + expect(build.object_id).to eq subject.to_resource.object_id + end + + it 'can not be persisted without explicit assignment' do + build = subject.to_resource + + pipeline.save! + + expect(build).not_to be_persisted + end + end + + describe 'applying only/except policies' do + context 'when no branch policy is specified' do + let(:attributes) { { name: 'rspec' } } + + it { is_expected.to be_included } + end + + context 'when branch policy does not match' do + context 'when using only' do + let(:attributes) { { name: 'rspec', only: { refs: ['deploy'] } } } + + it { is_expected.not_to be_included } + end + + context 'when using except' do + let(:attributes) { { name: 'rspec', except: { refs: ['deploy'] } } } + + it { is_expected.to be_included } + end + end + + context 'when branch regexp policy does not match' do + context 'when using only' do + let(:attributes) { { name: 'rspec', only: { refs: ['/^deploy$/'] } } } + + it { is_expected.not_to be_included } + end + + context 'when using except' do + let(:attributes) { { name: 'rspec', except: { refs: ['/^deploy$/'] } } } + + it { is_expected.to be_included } + end + end + + context 'when branch policy matches' do + context 'when using only' do + let(:attributes) { { name: 'rspec', only: { refs: %w[deploy master] } } } + + it { is_expected.to be_included } + end + + context 'when using except' do + let(:attributes) { { name: 'rspec', except: { refs: %w[deploy master] } } } + + it { is_expected.not_to be_included } + end + end + + context 'when keyword policy matches' do + context 'when using only' do + let(:attributes) { { name: 'rspec', only: { refs: ['branches'] } } } + + it { is_expected.to be_included } + end + + context 'when using except' do + let(:attributes) { { name: 'rspec', except: { refs: ['branches'] } } } + + it { is_expected.not_to be_included } + end + end + + context 'when keyword policy does not match' do + context 'when using only' do + let(:attributes) { { name: 'rspec', only: { refs: ['tags'] } } } + + it { is_expected.not_to be_included } + end + + context 'when using except' do + let(:attributes) { { name: 'rspec', except: { refs: ['tags'] } } } + + it { is_expected.to be_included } + end + end + + context 'when keywords and pipeline source policy matches' do + possibilities = [%w[pushes push], + %w[web web], + %w[triggers trigger], + %w[schedules schedule], + %w[api api], + %w[external external]] + + context 'when using only' do + possibilities.each do |keyword, source| + context "when using keyword `#{keyword}` and source `#{source}`" do + let(:pipeline) do + build(:ci_empty_pipeline, ref: 'deploy', tag: false, source: source) + end + + let(:attributes) { { name: 'rspec', only: { refs: [keyword] } } } + + it { is_expected.to be_included } + end + end + end + + context 'when using except' do + possibilities.each do |keyword, source| + context "when using keyword `#{keyword}` and source `#{source}`" do + let(:pipeline) do + build(:ci_empty_pipeline, ref: 'deploy', tag: false, source: source) + end + + let(:attributes) { { name: 'rspec', except: { refs: [keyword] } } } + + it { is_expected.not_to be_included } + end + end + end + end + + context 'when keywords and pipeline source does not match' do + possibilities = [%w[pushes web], + %w[web push], + %w[triggers schedule], + %w[schedules external], + %w[api trigger], + %w[external api]] + + context 'when using only' do + possibilities.each do |keyword, source| + context "when using keyword `#{keyword}` and source `#{source}`" do + let(:pipeline) do + build(:ci_empty_pipeline, ref: 'deploy', tag: false, source: source) + end + + let(:attributes) { { name: 'rspec', only: { refs: [keyword] } } } + + it { is_expected.not_to be_included } + end + end + end + + context 'when using except' do + possibilities.each do |keyword, source| + context "when using keyword `#{keyword}` and source `#{source}`" do + let(:pipeline) do + build(:ci_empty_pipeline, ref: 'deploy', tag: false, source: source) + end + + let(:attributes) { { name: 'rspec', except: { refs: [keyword] } } } + + it { is_expected.to be_included } + end + end + end + end + + context 'when repository path matches' do + context 'when using only' do + let(:attributes) do + { name: 'rspec', only: { refs: ["branches@#{pipeline.project_full_path}"] } } + end + + it { is_expected.to be_included } + end + + context 'when using except' do + let(:attributes) do + { name: 'rspec', except: { refs: ["branches@#{pipeline.project_full_path}"] } } + end + + it { is_expected.not_to be_included } + end + end + + context 'when repository path does not matches' do + context 'when using only' do + let(:attributes) do + { name: 'rspec', only: { refs: ['branches@fork'] } } + end + + it { is_expected.not_to be_included } + end + + context 'when using except' do + let(:attributes) do + { name: 'rspec', except: { refs: ['branches@fork'] } } + end + + it { is_expected.to be_included } + end + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb new file mode 100644 index 00000000000..8f0bf40d624 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb @@ -0,0 +1,133 @@ +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Seed::Stage do + let(:pipeline) { create(:ci_empty_pipeline) } + + let(:attributes) do + { name: 'test', + index: 0, + builds: [{ name: 'rspec' }, + { name: 'spinach' }, + { name: 'deploy', only: { refs: ['feature'] } }] } + end + + subject do + described_class.new(pipeline, attributes) + end + + describe '#size' do + it 'returns a number of jobs in the stage' do + expect(subject.size).to eq 2 + end + end + + describe '#attributes' do + it 'returns hash attributes of a stage' do + expect(subject.attributes).to be_a Hash + expect(subject.attributes).to include(:name, :project) + end + end + + describe '#included?' do + context 'when it contains builds seeds' do + let(:attributes) do + { name: 'test', + index: 0, + builds: [{ name: 'deploy', only: { refs: ['master'] } }] } + end + + it { is_expected.to be_included } + end + + context 'when it does not contain build seeds' do + let(:attributes) do + { name: 'test', + index: 0, + builds: [{ name: 'deploy', only: { refs: ['feature'] } }] } + end + + it { is_expected.not_to be_included } + end + end + + describe '#seeds' do + it 'returns build seeds' do + expect(subject.seeds).to all(be_a Gitlab::Ci::Pipeline::Seed::Build) + end + + it 'returns build seeds including valid attributes' do + expect(subject.seeds.size).to eq 2 + expect(subject.seeds.map(&:attributes)).to all(include(ref: 'master')) + expect(subject.seeds.map(&:attributes)).to all(include(tag: false)) + expect(subject.seeds.map(&:attributes)).to all(include(project: pipeline.project)) + expect(subject.seeds.map(&:attributes)) + .to all(include(trigger_request: pipeline.trigger_requests.first)) + end + + context 'when a ref is protected' do + before do + allow_any_instance_of(Project).to receive(:protected_for?).and_return(true) + end + + it 'returns protected builds' do + expect(subject.seeds.map(&:attributes)).to all(include(protected: true)) + end + end + + context 'when a ref is not protected' do + before do + allow_any_instance_of(Project).to receive(:protected_for?).and_return(false) + end + + it 'returns unprotected builds' do + expect(subject.seeds.map(&:attributes)).to all(include(protected: false)) + end + end + + it 'filters seeds using only/except policies' do + expect(subject.seeds.map(&:attributes)).to satisfy do |seeds| + seeds.any? { |hash| hash.fetch(:name) == 'rspec' } + end + + expect(subject.seeds.map(&:attributes)).not_to satisfy do |seeds| + seeds.any? { |hash| hash.fetch(:name) == 'deploy' } + end + end + end + + describe '#user=' do + let(:user) { build(:user) } + + it 'assignes relevant pipeline attributes' do + subject.user = user + + expect(subject.seeds.map(&:attributes)).to all(include(user: user)) + end + end + + describe '#to_resource' do + it 'builds a valid stage object with all builds' do + subject.to_resource.save! + + expect(pipeline.reload.stages.count).to eq 1 + expect(pipeline.reload.builds.count).to eq 2 + expect(pipeline.builds).to all(satisfy { |job| job.stage_id.present? }) + expect(pipeline.builds).to all(satisfy { |job| job.pipeline.present? }) + expect(pipeline.builds).to all(satisfy { |job| job.project.present? }) + expect(pipeline.stages) + .to all(satisfy { |stage| stage.pipeline.present? }) + expect(pipeline.stages) + .to all(satisfy { |stage| stage.project.present? }) + end + + it 'can not be persisted without explicit pipeline assignment' do + stage = subject.to_resource + + pipeline.save! + + expect(stage).not_to be_persisted + expect(pipeline.reload.stages.count).to eq 0 + expect(pipeline.reload.builds.count).to eq 0 + end + end +end diff --git a/spec/lib/gitlab/ci/stage/seed_spec.rb b/spec/lib/gitlab/ci/stage/seed_spec.rb deleted file mode 100644 index 3fe8d50c49a..00000000000 --- a/spec/lib/gitlab/ci/stage/seed_spec.rb +++ /dev/null @@ -1,83 +0,0 @@ -require 'spec_helper' - -describe Gitlab::Ci::Stage::Seed do - let(:pipeline) { create(:ci_empty_pipeline) } - - let(:builds) do - [{ name: 'rspec' }, { name: 'spinach' }] - end - - subject do - described_class.new(pipeline, 'test', builds) - end - - describe '#size' do - it 'returns a number of jobs in the stage' do - expect(subject.size).to eq 2 - end - end - - describe '#stage' do - it 'returns hash attributes of a stage' do - expect(subject.stage).to be_a Hash - expect(subject.stage).to include(:name, :project) - end - end - - describe '#builds' do - it 'returns hash attributes of all builds' do - expect(subject.builds.size).to eq 2 - expect(subject.builds).to all(include(ref: 'master')) - expect(subject.builds).to all(include(tag: false)) - expect(subject.builds).to all(include(project: pipeline.project)) - expect(subject.builds) - .to all(include(trigger_request: pipeline.trigger_requests.first)) - end - - context 'when a ref is protected' do - before do - allow_any_instance_of(Project).to receive(:protected_for?).and_return(true) - end - - it 'returns protected builds' do - expect(subject.builds).to all(include(protected: true)) - end - end - - context 'when a ref is unprotected' do - before do - allow_any_instance_of(Project).to receive(:protected_for?).and_return(false) - end - - it 'returns unprotected builds' do - expect(subject.builds).to all(include(protected: false)) - end - end - end - - describe '#user=' do - let(:user) { build(:user) } - - it 'assignes relevant pipeline attributes' do - subject.user = user - - expect(subject.builds).to all(include(user: user)) - end - end - - describe '#create!' do - it 'creates all stages and builds' do - subject.create! - - expect(pipeline.reload.stages.count).to eq 1 - expect(pipeline.reload.builds.count).to eq 2 - expect(pipeline.builds).to all(satisfy { |job| job.stage_id.present? }) - expect(pipeline.builds).to all(satisfy { |job| job.pipeline.present? }) - expect(pipeline.builds).to all(satisfy { |job| job.project.present? }) - expect(pipeline.stages) - .to all(satisfy { |stage| stage.pipeline.present? }) - expect(pipeline.stages) - .to all(satisfy { |stage| stage.project.present? }) - end - end -end diff --git a/spec/lib/gitlab/ci/trace/http_io_spec.rb b/spec/lib/gitlab/ci/trace/http_io_spec.rb new file mode 100644 index 00000000000..5474e2f518c --- /dev/null +++ b/spec/lib/gitlab/ci/trace/http_io_spec.rb @@ -0,0 +1,315 @@ +require 'spec_helper' + +describe Gitlab::Ci::Trace::HttpIO do + include HttpIOHelpers + + let(:http_io) { described_class.new(url, size) } + let(:url) { remote_trace_url } + let(:size) { remote_trace_size } + + describe '#close' do + subject { http_io.close } + + it { is_expected.to be_nil } + end + + describe '#binmode' do + subject { http_io.binmode } + + it { is_expected.to be_nil } + end + + describe '#binmode?' do + subject { http_io.binmode? } + + it { is_expected.to be_truthy } + end + + describe '#path' do + subject { http_io.path } + + it { is_expected.to be_nil } + end + + describe '#url' do + subject { http_io.url } + + it { is_expected.to eq(url) } + end + + describe '#seek' do + subject { http_io.seek(pos, where) } + + context 'when moves pos to end of the file' do + let(:pos) { 0 } + let(:where) { IO::SEEK_END } + + it { is_expected.to eq(size) } + end + + context 'when moves pos to middle of the file' do + let(:pos) { size / 2 } + let(:where) { IO::SEEK_SET } + + it { is_expected.to eq(size / 2) } + end + + context 'when moves pos around' do + it 'matches the result' do + expect(http_io.seek(0)).to eq(0) + expect(http_io.seek(100, IO::SEEK_CUR)).to eq(100) + expect { http_io.seek(size + 1, IO::SEEK_CUR) }.to raise_error('new position is outside of file') + end + end + end + + describe '#eof?' do + subject { http_io.eof? } + + context 'when current pos is at end of the file' do + before do + http_io.seek(size, IO::SEEK_SET) + end + + it { is_expected.to be_truthy } + end + + context 'when current pos is not at end of the file' do + before do + http_io.seek(0, IO::SEEK_SET) + end + + it { is_expected.to be_falsey } + end + end + + describe '#each_line' do + subject { http_io.each_line } + + let(:string_io) { StringIO.new(remote_trace_body) } + + before do + stub_remote_trace_206 + end + + it 'yields lines' do + expect { |b| http_io.each_line(&b) }.to yield_successive_args(*string_io.each_line.to_a) + end + + context 'when buckets on GCS' do + context 'when BUFFER_SIZE is larger than file size' do + before do + stub_remote_trace_200 + set_larger_buffer_size_than(size) + end + + it 'calls get_chunk only once' do + expect_any_instance_of(Net::HTTP).to receive(:request).once.and_call_original + + http_io.each_line { |line| } + end + end + end + end + + describe '#read' do + subject { http_io.read(length) } + + context 'when there are no network issue' do + before do + stub_remote_trace_206 + end + + context 'when read whole size' do + let(:length) { nil } + + context 'when BUFFER_SIZE is smaller than file size' do + before do + set_smaller_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body) + end + end + + context 'when BUFFER_SIZE is larger than file size' do + before do + set_larger_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body) + end + end + end + + context 'when read only first 100 bytes' do + let(:length) { 100 } + + context 'when BUFFER_SIZE is smaller than file size' do + before do + set_smaller_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body[0, length]) + end + end + + context 'when BUFFER_SIZE is larger than file size' do + before do + set_larger_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body[0, length]) + end + end + end + + context 'when tries to read oversize' do + let(:length) { size + 1000 } + + context 'when BUFFER_SIZE is smaller than file size' do + before do + set_smaller_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body) + end + end + + context 'when BUFFER_SIZE is larger than file size' do + before do + set_larger_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to eq(remote_trace_body) + end + end + end + + context 'when tries to read 0 bytes' do + let(:length) { 0 } + + context 'when BUFFER_SIZE is smaller than file size' do + before do + set_smaller_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to be_empty + end + end + + context 'when BUFFER_SIZE is larger than file size' do + before do + set_larger_buffer_size_than(size) + end + + it 'reads a trace' do + is_expected.to be_empty + end + end + end + end + + context 'when there is anetwork issue' do + let(:length) { nil } + + before do + stub_remote_trace_500 + end + + it 'reads a trace' do + expect { subject }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError) + end + end + end + + describe '#readline' do + subject { http_io.readline } + + let(:string_io) { StringIO.new(remote_trace_body) } + + before do + stub_remote_trace_206 + end + + shared_examples 'all line matching' do + it 'reads a line' do + (0...remote_trace_body.lines.count).each do + expect(http_io.readline).to eq(string_io.readline) + end + end + end + + context 'when there is anetwork issue' do + let(:length) { nil } + + before do + stub_remote_trace_500 + end + + it 'reads a trace' do + expect { subject }.to raise_error(Gitlab::Ci::Trace::HttpIO::FailedToGetChunkError) + end + end + + context 'when BUFFER_SIZE is smaller than file size' do + before do + set_smaller_buffer_size_than(size) + end + + it_behaves_like 'all line matching' + end + + context 'when BUFFER_SIZE is larger than file size' do + before do + set_larger_buffer_size_than(size) + end + + it_behaves_like 'all line matching' + end + + context 'when pos is at middle of the file' do + before do + set_smaller_buffer_size_than(size) + + http_io.seek(size / 2) + string_io.seek(size / 2) + end + + it 'reads from pos' do + expect(http_io.readline).to eq(string_io.readline) + end + end + end + + describe '#write' do + subject { http_io.write(nil) } + + it { expect { subject }.to raise_error(NotImplementedError) } + end + + describe '#truncate' do + subject { http_io.truncate(nil) } + + it { expect { subject }.to raise_error(NotImplementedError) } + end + + describe '#flush' do + subject { http_io.flush } + + it { expect { subject }.to raise_error(NotImplementedError) } + end + + describe '#present?' do + subject { http_io.present? } + + it { is_expected.to be_truthy } + end +end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index f83f932e61e..fbc2af29b98 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -18,6 +18,34 @@ module Gitlab describe '#build_attributes' do subject { described_class.new(config).build_attributes(:rspec) } + describe 'attributes list' do + let(:config) do + YAML.dump( + before_script: ['pwd'], + rspec: { script: 'rspec' } + ) + end + + it 'returns valid build attributes' do + expect(subject).to eq({ + stage: "test", + stage_idx: 1, + name: "rspec", + commands: "pwd\nrspec", + coverage_regex: nil, + tag_list: [], + options: { + before_script: ["pwd"], + script: ["rspec"] + }, + allow_failure: false, + when: "on_success", + environment: nil, + yaml_variables: [] + }) + end + end + describe 'coverage entry' do describe 'code coverage regexp' do let(:config) do @@ -105,512 +133,118 @@ module Gitlab end end - describe '#stage_seeds' do - context 'when no refs policy is specified' do - let(:config) do - YAML.dump(production: { stage: 'deploy', script: 'cap prod' }, - rspec: { stage: 'test', script: 'rspec' }, - spinach: { stage: 'test', script: 'spinach' }) - end - - let(:pipeline) { create(:ci_empty_pipeline) } - - it 'correctly fabricates a stage seeds object' do - seeds = subject.stage_seeds(pipeline) - - expect(seeds.size).to eq 2 - expect(seeds.first.stage[:name]).to eq 'test' - expect(seeds.second.stage[:name]).to eq 'deploy' - expect(seeds.first.builds.dig(0, :name)).to eq 'rspec' - expect(seeds.first.builds.dig(1, :name)).to eq 'spinach' - expect(seeds.second.builds.dig(0, :name)).to eq 'production' - end - end - - context 'when refs policy is specified' do - let(:config) do - YAML.dump(production: { stage: 'deploy', script: 'cap prod', only: ['master'] }, - spinach: { stage: 'test', script: 'spinach', only: ['tags'] }) - end - - let(:pipeline) do - create(:ci_empty_pipeline, ref: 'feature', tag: true) - end - - it 'returns stage seeds only assigned to master to master' do - seeds = subject.stage_seeds(pipeline) - - expect(seeds.size).to eq 1 - expect(seeds.first.stage[:name]).to eq 'test' - expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' - end - end - - context 'when source policy is specified' do - let(:config) do - YAML.dump(production: { stage: 'deploy', script: 'cap prod', only: ['triggers'] }, - spinach: { stage: 'test', script: 'spinach', only: ['schedules'] }) - end - - let(:pipeline) do - create(:ci_empty_pipeline, source: :schedule) - end - - it 'returns stage seeds only assigned to schedules' do - seeds = subject.stage_seeds(pipeline) - - expect(seeds.size).to eq 1 - expect(seeds.first.stage[:name]).to eq 'test' - expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' - end + describe '#stages_attributes' do + let(:config) do + YAML.dump( + rspec: { script: 'rspec', stage: 'test', only: ['branches'] }, + prod: { script: 'cap prod', stage: 'deploy', only: ['tags'] } + ) end - context 'when kubernetes policy is specified' do - let(:config) do - YAML.dump( - spinach: { stage: 'test', script: 'spinach' }, - production: { - stage: 'deploy', - script: 'cap', - only: { kubernetes: 'active' } - } - ) - end - - context 'when kubernetes is active' do - shared_examples 'same behavior between KubernetesService and Platform::Kubernetes' do - it 'returns seeds for kubernetes dependent job' do - seeds = subject.stage_seeds(pipeline) - - expect(seeds.size).to eq 2 - expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' - expect(seeds.second.builds.dig(0, :name)).to eq 'production' - end - end - - context 'when user configured kubernetes from Integration > Kubernetes' do - let(:project) { create(:kubernetes_project) } - let(:pipeline) { create(:ci_empty_pipeline, project: project) } - - it_behaves_like 'same behavior between KubernetesService and Platform::Kubernetes' - end - - context 'when user configured kubernetes from CI/CD > Clusters' do - let!(:cluster) { create(:cluster, :project, :provided_by_gcp) } - let(:project) { cluster.project } - let(:pipeline) { create(:ci_empty_pipeline, project: project) } - - it_behaves_like 'same behavior between KubernetesService and Platform::Kubernetes' - end - end - - context 'when kubernetes is not active' do - it 'does not return seeds for kubernetes dependent job' do - seeds = subject.stage_seeds(pipeline) - - expect(seeds.size).to eq 1 - expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' - end - end + let(:attributes) do + [{ name: "build", + index: 0, + builds: [] }, + { name: "test", + index: 1, + builds: + [{ stage_idx: 1, + stage: "test", + commands: "rspec", + tag_list: [], + name: "rspec", + allow_failure: false, + when: "on_success", + environment: nil, + coverage_regex: nil, + yaml_variables: [], + options: { script: ["rspec"] }, + only: { refs: ["branches"] }, + except: {} }] }, + { name: "deploy", + index: 2, + builds: + [{ stage_idx: 2, + stage: "deploy", + commands: "cap prod", + tag_list: [], + name: "prod", + allow_failure: false, + when: "on_success", + environment: nil, + coverage_regex: nil, + yaml_variables: [], + options: { script: ["cap prod"] }, + only: { refs: ["tags"] }, + except: {} }] }] + end + + it 'returns stages seed attributes' do + expect(subject.stages_attributes).to eq attributes end end - describe "#pipeline_stage_builds" do - let(:type) { 'test' } + describe 'only / except policies validations' do + context 'when `only` has an invalid value' do + let(:config) { { rspec: { script: "rspec", type: "test", only: only } } } + let(:processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)) } - it "returns builds if no branch specified" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec" } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "master")).size).to eq(1) - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "master")).first).to eq({ - stage: "test", - stage_idx: 1, - name: "rspec", - commands: "pwd\nrspec", - coverage_regex: nil, - tag_list: [], - options: { - before_script: ["pwd"], - script: ["rspec"] - }, - allow_failure: false, - when: "on_success", - environment: nil, - yaml_variables: [] - }) - end - - describe 'only' do - it "does not return builds if only has another branch" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", only: ["deploy"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "master")).size).to eq(0) - end - - it "does not return builds if only has regexp with another branch" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", only: ["/^deploy$/"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "master")).size).to eq(0) - end - - it "returns builds if only has specified this branch" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", only: ["master"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "master")).size).to eq(1) - end - - it "returns builds if only has a list of branches including specified" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, only: %w(master deploy) } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "deploy")).size).to eq(1) - end - - it "returns builds if only has a branches keyword specified" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, only: ["branches"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "deploy")).size).to eq(1) - end - - it "does not return builds if only has a tags keyword" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, only: ["tags"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "deploy")).size).to eq(0) - end + context 'when it is integer' do + let(:only) { 1 } - it "returns builds if only has special keywords specified and source matches" do - possibilities = [{ keyword: 'pushes', source: 'push' }, - { keyword: 'web', source: 'web' }, - { keyword: 'triggers', source: 'trigger' }, - { keyword: 'schedules', source: 'schedule' }, - { keyword: 'api', source: 'api' }, - { keyword: 'external', source: 'external' }] - - possibilities.each do |possibility| - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, only: [possibility[:keyword]] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: 'deploy', tag: false, source: possibility[:source])).size).to eq(1) + it do + expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, + 'jobs:rspec:only has to be either an array of conditions or a hash') end end - it "does not return builds if only has special keywords specified and source doesn't match" do - possibilities = [{ keyword: 'pushes', source: 'web' }, - { keyword: 'web', source: 'push' }, - { keyword: 'triggers', source: 'schedule' }, - { keyword: 'schedules', source: 'external' }, - { keyword: 'api', source: 'trigger' }, - { keyword: 'external', source: 'api' }] - - possibilities.each do |possibility| - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, only: [possibility[:keyword]] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) + context 'when it is an array of integers' do + let(:only) { [1, 1] } - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: 'deploy', tag: false, source: possibility[:source])).size).to eq(0) + it do + expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, + 'jobs:rspec:only config should be an array of strings or regexps') end end - it "returns builds if only has current repository path" do - seed_pipeline = pipeline(ref: 'deploy') - - config = YAML.dump({ - before_script: ["pwd"], - rspec: { - script: "rspec", - type: type, - only: ["branches@#{seed_pipeline.project_full_path}"] - } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) + context 'when it is invalid regex' do + let(:only) { ["/*invalid/"] } - expect(config_processor.pipeline_stage_builds(type, seed_pipeline).size).to eq(1) - end - - it "does not return builds if only has different repository path" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, only: ["branches@fork"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "deploy")).size).to eq(0) - end - - it "returns build only for specified type" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: "test", only: %w(master deploy) }, - staging: { script: "deploy", type: "deploy", only: %w(master deploy) }, - production: { script: "deploy", type: "deploy", only: ["master@path", "deploy"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds("deploy", pipeline(ref: "deploy")).size).to eq(2) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "deploy")).size).to eq(1) - expect(config_processor.pipeline_stage_builds("deploy", pipeline(ref: "master")).size).to eq(1) - end - - context 'for invalid value' do - let(:config) { { rspec: { script: "rspec", type: "test", only: only } } } - let(:processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)) } - - context 'when it is integer' do - let(:only) { 1 } - - it do - expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, - 'jobs:rspec:only has to be either an array of conditions or a hash') - end - end - - context 'when it is an array of integers' do - let(:only) { [1, 1] } - - it do - expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, - 'jobs:rspec:only config should be an array of strings or regexps') - end - end - - context 'when it is invalid regex' do - let(:only) { ["/*invalid/"] } - - it do - expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, - 'jobs:rspec:only config should be an array of strings or regexps') - end + it do + expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, + 'jobs:rspec:only config should be an array of strings or regexps') end end end - describe 'except' do - it "returns builds if except has another branch" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", except: ["deploy"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "master")).size).to eq(1) - end - - it "returns builds if except has regexp with another branch" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", except: ["/^deploy$/"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) + context 'when `except` has an invalid value' do + let(:config) { { rspec: { script: "rspec", except: except } } } + let(:processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)) } - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "master")).size).to eq(1) - end - - it "does not return builds if except has specified this branch" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", except: ["master"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "master")).size).to eq(0) - end - - it "does not return builds if except has a list of branches including specified" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, except: %w(master deploy) } - }) + context 'when it is integer' do + let(:except) { 1 } - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "deploy")).size).to eq(0) - end - - it "does not return builds if except has a branches keyword specified" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, except: ["branches"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "deploy")).size).to eq(0) - end - - it "returns builds if except has a tags keyword" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, except: ["tags"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "deploy")).size).to eq(1) - end - - it "does not return builds if except has special keywords specified and source matches" do - possibilities = [{ keyword: 'pushes', source: 'push' }, - { keyword: 'web', source: 'web' }, - { keyword: 'triggers', source: 'trigger' }, - { keyword: 'schedules', source: 'schedule' }, - { keyword: 'api', source: 'api' }, - { keyword: 'external', source: 'external' }] - - possibilities.each do |possibility| - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, except: [possibility[:keyword]] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: 'deploy', tag: false, source: possibility[:source])).size).to eq(0) + it do + expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, + 'jobs:rspec:except has to be either an array of conditions or a hash') end end - it "returns builds if except has special keywords specified and source doesn't match" do - possibilities = [{ keyword: 'pushes', source: 'web' }, - { keyword: 'web', source: 'push' }, - { keyword: 'triggers', source: 'schedule' }, - { keyword: 'schedules', source: 'external' }, - { keyword: 'api', source: 'trigger' }, - { keyword: 'external', source: 'api' }] - - possibilities.each do |possibility| - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, except: [possibility[:keyword]] } - }) + context 'when it is an array of integers' do + let(:except) { [1, 1] } - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: 'deploy', tag: false, source: possibility[:source])).size).to eq(1) + it do + expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, + 'jobs:rspec:except config should be an array of strings or regexps') end end - it "does not return builds if except has current repository path" do - seed_pipeline = pipeline(ref: 'deploy') - - config = YAML.dump({ - before_script: ["pwd"], - rspec: { - script: "rspec", - type: type, - except: ["branches@#{seed_pipeline.project_full_path}"] - } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, seed_pipeline).size).to eq(0) - end - - it "returns builds if except has different repository path" do - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: type, except: ["branches@fork"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds(type, pipeline(ref: "deploy")).size).to eq(1) - end - - it "returns build except specified type" do - master_pipeline = pipeline(ref: 'master') - test_pipeline = pipeline(ref: 'test') - deploy_pipeline = pipeline(ref: 'deploy') - - config = YAML.dump({ - before_script: ["pwd"], - rspec: { script: "rspec", type: "test", except: ["master", "deploy", "test@#{test_pipeline.project_full_path}"] }, - staging: { script: "deploy", type: "deploy", except: ["master"] }, - production: { script: "deploy", type: "deploy", except: ["master@#{master_pipeline.project_full_path}"] } - }) - - config_processor = Gitlab::Ci::YamlProcessor.new(config) - - expect(config_processor.pipeline_stage_builds("deploy", deploy_pipeline).size).to eq(2) - expect(config_processor.pipeline_stage_builds("test", test_pipeline).size).to eq(0) - expect(config_processor.pipeline_stage_builds("deploy", master_pipeline).size).to eq(0) - end - - context 'for invalid value' do - let(:config) { { rspec: { script: "rspec", except: except } } } - let(:processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)) } + context 'when it is invalid regex' do + let(:except) { ["/*invalid/"] } - context 'when it is integer' do - let(:except) { 1 } - - it do - expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, - 'jobs:rspec:except has to be either an array of conditions or a hash') - end - end - - context 'when it is an array of integers' do - let(:except) { [1, 1] } - - it do - expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, - 'jobs:rspec:except config should be an array of strings or regexps') - end - end - - context 'when it is invalid regex' do - let(:except) { ["/*invalid/"] } - - it do - expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, - 'jobs:rspec:except config should be an array of strings or regexps') - end + it do + expect { processor }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, + 'jobs:rspec:except config should be an array of strings or regexps') end end end @@ -620,7 +254,7 @@ module Gitlab let(:config_data) { YAML.dump(config) } let(:config_processor) { Gitlab::Ci::YamlProcessor.new(config_data) } - subject { config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).first } + subject { config_processor.stage_builds_attributes('test').first } describe "before_script" do context "in global context" do @@ -703,8 +337,8 @@ module Gitlab config_processor = Gitlab::Ci::YamlProcessor.new(config) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).size).to eq(1) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).first).to eq({ + expect(config_processor.stage_builds_attributes("test").size).to eq(1) + expect(config_processor.stage_builds_attributes("test").first).to eq({ stage: "test", stage_idx: 1, name: "rspec", @@ -738,8 +372,8 @@ module Gitlab config_processor = Gitlab::Ci::YamlProcessor.new(config) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).size).to eq(1) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).first).to eq({ + expect(config_processor.stage_builds_attributes("test").size).to eq(1) + expect(config_processor.stage_builds_attributes("test").first).to eq({ stage: "test", stage_idx: 1, name: "rspec", @@ -771,8 +405,8 @@ module Gitlab config_processor = Gitlab::Ci::YamlProcessor.new(config) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).size).to eq(1) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).first).to eq({ + expect(config_processor.stage_builds_attributes("test").size).to eq(1) + expect(config_processor.stage_builds_attributes("test").first).to eq({ stage: "test", stage_idx: 1, name: "rspec", @@ -800,8 +434,8 @@ module Gitlab config_processor = Gitlab::Ci::YamlProcessor.new(config) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).size).to eq(1) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).first).to eq({ + expect(config_processor.stage_builds_attributes("test").size).to eq(1) + expect(config_processor.stage_builds_attributes("test").first).to eq({ stage: "test", stage_idx: 1, name: "rspec", @@ -946,8 +580,8 @@ module Gitlab }) config_processor = Gitlab::Ci::YamlProcessor.new(config) + builds = config_processor.stage_builds_attributes("test") - builds = config_processor.pipeline_stage_builds("test", pipeline(ref: "master")) expect(builds.size).to eq(1) expect(builds.first[:when]).to eq(when_state) end @@ -978,8 +612,8 @@ module Gitlab config_processor = Gitlab::Ci::YamlProcessor.new(config) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).size).to eq(1) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).first[:options][:cache]).to eq( + expect(config_processor.stage_builds_attributes("test").size).to eq(1) + expect(config_processor.stage_builds_attributes("test").first[:options][:cache]).to eq( paths: ["logs/", "binaries/"], untracked: true, key: 'key', @@ -997,8 +631,8 @@ module Gitlab config_processor = Gitlab::Ci::YamlProcessor.new(config) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).size).to eq(1) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).first[:options][:cache]).to eq( + expect(config_processor.stage_builds_attributes("test").size).to eq(1) + expect(config_processor.stage_builds_attributes("test").first[:options][:cache]).to eq( paths: ["logs/", "binaries/"], untracked: true, key: 'key', @@ -1017,8 +651,8 @@ module Gitlab config_processor = Gitlab::Ci::YamlProcessor.new(config) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).size).to eq(1) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).first[:options][:cache]).to eq( + expect(config_processor.stage_builds_attributes("test").size).to eq(1) + expect(config_processor.stage_builds_attributes("test").first[:options][:cache]).to eq( paths: ["test/"], untracked: false, key: 'local', @@ -1046,8 +680,8 @@ module Gitlab config_processor = Gitlab::Ci::YamlProcessor.new(config) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).size).to eq(1) - expect(config_processor.pipeline_stage_builds("test", pipeline(ref: "master")).first).to eq({ + expect(config_processor.stage_builds_attributes("test").size).to eq(1) + expect(config_processor.stage_builds_attributes("test").first).to eq({ stage: "test", stage_idx: 1, name: "rspec", @@ -1083,8 +717,8 @@ module Gitlab }) config_processor = Gitlab::Ci::YamlProcessor.new(config) + builds = config_processor.stage_builds_attributes("test") - builds = config_processor.pipeline_stage_builds("test", pipeline(ref: "master")) expect(builds.size).to eq(1) expect(builds.first[:options][:artifacts][:when]).to eq(when_state) end @@ -1099,7 +733,7 @@ module Gitlab end let(:processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)) } - let(:builds) { processor.pipeline_stage_builds('deploy', pipeline(ref: 'master')) } + let(:builds) { processor.stage_builds_attributes('deploy') } context 'when a production environment is specified' do let(:environment) { 'production' } @@ -1256,7 +890,7 @@ module Gitlab describe "Hidden jobs" do let(:config_processor) { Gitlab::Ci::YamlProcessor.new(config) } - subject { config_processor.pipeline_stage_builds("test", pipeline(ref: "master")) } + subject { config_processor.stage_builds_attributes("test") } shared_examples 'hidden_job_handling' do it "doesn't create jobs that start with dot" do @@ -1304,7 +938,7 @@ module Gitlab describe "YAML Alias/Anchor" do let(:config_processor) { Gitlab::Ci::YamlProcessor.new(config) } - subject { config_processor.pipeline_stage_builds("build", pipeline(ref: "master")) } + subject { config_processor.stage_builds_attributes("build") } shared_examples 'job_templates_handling' do it "is correctly supported for jobs" do @@ -1344,13 +978,13 @@ module Gitlab context 'when template is a job' do let(:config) do - <<EOT -job1: &JOBTMPL - stage: build - script: execute-script-for-job + <<~EOT + job1: &JOBTMPL + stage: build + script: execute-script-for-job -job2: *JOBTMPL -EOT + job2: *JOBTMPL + EOT end it_behaves_like 'job_templates_handling' @@ -1358,15 +992,15 @@ EOT context 'when template is a hidden job' do let(:config) do - <<EOT -.template: &JOBTMPL - stage: build - script: execute-script-for-job + <<~EOT + .template: &JOBTMPL + stage: build + script: execute-script-for-job -job1: *JOBTMPL + job1: *JOBTMPL -job2: *JOBTMPL -EOT + job2: *JOBTMPL + EOT end it_behaves_like 'job_templates_handling' @@ -1374,18 +1008,18 @@ EOT context 'when job adds its own keys to a template definition' do let(:config) do - <<EOT -.template: &JOBTMPL - stage: build - -job1: - <<: *JOBTMPL - script: execute-script-for-job - -job2: - <<: *JOBTMPL - script: execute-script-for-job -EOT + <<~EOT + .template: &JOBTMPL + stage: build + + job1: + <<: *JOBTMPL + script: execute-script-for-job + + job2: + <<: *JOBTMPL + script: execute-script-for-job + EOT end it_behaves_like 'job_templates_handling' @@ -1724,10 +1358,6 @@ EOT it { is_expected.to be_nil } end end - - def pipeline(**attributes) - build_stubbed(:ci_empty_pipeline, **attributes) - end end end end diff --git a/spec/lib/gitlab/encoding_helper_spec.rb b/spec/lib/gitlab/encoding_helper_spec.rb index 83d431a7458..e68c9850f6b 100644 --- a/spec/lib/gitlab/encoding_helper_spec.rb +++ b/spec/lib/gitlab/encoding_helper_spec.rb @@ -161,6 +161,11 @@ describe Gitlab::EncodingHelper do 'removes invalid bytes from ASCII-8bit encoded multibyte string.', "Lorem ipsum\xC3\n dolor sit amet, xy\xC3\xA0y\xC3\xB9abcd\xC3\xB9efg".force_encoding('ASCII-8BIT'), "Lorem ipsum\n dolor sit amet, xyà yùabcdùefg" + ], + [ + 'handles UTF-16BE encoded strings', + "\xFE\xFF\x00\x41".force_encoding('ASCII-8BIT'), # An "A" prepended with UTF-16 BOM + "\xEF\xBB\xBFA" # An "A" prepended with UTF-8 BOM ] ].each do |description, test_string, xpect| it description do diff --git a/spec/lib/gitlab/git/conflict/file_spec.rb b/spec/lib/gitlab/git/conflict/file_spec.rb new file mode 100644 index 00000000000..afed6c32af6 --- /dev/null +++ b/spec/lib/gitlab/git/conflict/file_spec.rb @@ -0,0 +1,50 @@ +# coding: utf-8 +require 'spec_helper' + +describe Gitlab::Git::Conflict::File do + let(:conflict) { { theirs: { path: 'foo', mode: 33188 }, ours: { path: 'foo', mode: 33188 } } } + let(:invalid_content) { described_class.new(nil, nil, conflict, "a\xC4\xFC".force_encoding(Encoding::ASCII_8BIT)) } + let(:valid_content) { described_class.new(nil, nil, conflict, "Espa\xC3\xB1a".force_encoding(Encoding::ASCII_8BIT)) } + + describe '#lines' do + context 'when the content contains non-UTF-8 characters' do + it 'raises UnsupportedEncoding' do + expect { invalid_content.lines } + .to raise_error(described_class::UnsupportedEncoding) + end + end + + context 'when the content can be converted to UTF-8' do + it 'sets lines to the lines' do + expect(valid_content.lines).to eq([{ + full_line: 'España', + type: nil, + line_obj_index: 0, + line_old: 1, + line_new: 1 + }]) + end + + it 'sets the type to text' do + expect(valid_content.type).to eq('text') + end + end + end + + describe '#content' do + context 'when the content contains non-UTF-8 characters' do + it 'raises UnsupportedEncoding' do + expect { invalid_content.content } + .to raise_error(described_class::UnsupportedEncoding) + end + end + + context 'when the content can be converted to UTF-8' do + it 'returns a valid UTF-8 string' do + expect(valid_content.content).to eq('España') + expect(valid_content.content).to be_valid_encoding + expect(valid_content.content.encoding).to eq(Encoding::UTF_8) + end + end + end +end diff --git a/spec/lib/gitlab/git/conflict/parser_spec.rb b/spec/lib/gitlab/git/conflict/parser_spec.rb index 7b035a381f1..29a1702a1c6 100644 --- a/spec/lib/gitlab/git/conflict/parser_spec.rb +++ b/spec/lib/gitlab/git/conflict/parser_spec.rb @@ -212,13 +212,6 @@ CONFLICT .not_to raise_error end end - - context 'when the file contains non-UTF-8 characters' do - it 'raises UnsupportedEncoding' do - expect { parse_text("a\xC4\xFC".force_encoding(Encoding::ASCII_8BIT)) } - .to raise_error(Gitlab::Git::Conflict::Parser::UnsupportedEncoding) - end - end end end end diff --git a/spec/lib/gitlab/health_checks/fs_shards_check_spec.rb b/spec/lib/gitlab/health_checks/fs_shards_check_spec.rb index 4c1ca4349ea..9dcf272d25e 100644 --- a/spec/lib/gitlab/health_checks/fs_shards_check_spec.rb +++ b/spec/lib/gitlab/health_checks/fs_shards_check_spec.rb @@ -26,7 +26,7 @@ describe Gitlab::HealthChecks::FsShardsCheck do let(:storages_paths) do { - default: { path: tmp_dir } + default: Gitlab::GitalyClient::StorageSettings.new('path' => tmp_dir) }.with_indifferent_access end @@ -56,7 +56,7 @@ describe Gitlab::HealthChecks::FsShardsCheck do context 'storage points to not existing folder' do let(:storages_paths) do { - default: { path: 'tmp/this/path/doesnt/exist' } + default: Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/this/path/doesnt/exist') }.with_indifferent_access end @@ -102,7 +102,7 @@ describe Gitlab::HealthChecks::FsShardsCheck do context 'storage points to not existing folder' do let(:storages_paths) do { - default: { path: 'tmp/this/path/doesnt/exist' } + default: Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/this/path/doesnt/exist') }.with_indifferent_access end diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 0b938892da5..0716852f57f 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -265,7 +265,9 @@ CommitStatus: - target_url - description - artifacts_file +- artifacts_file_store - artifacts_metadata +- artifacts_metadata_store - erased_by_id - erased_at - artifacts_expire_at @@ -458,6 +460,7 @@ Project: - merge_requests_ff_only_enabled - merge_requests_rebase_enabled - jobs_cache_index +- pages_https_only Author: - name ProjectFeature: diff --git a/spec/lib/gitlab/repo_path_spec.rb b/spec/lib/gitlab/repo_path_spec.rb index b67bcc77bd4..f030f371372 100644 --- a/spec/lib/gitlab/repo_path_spec.rb +++ b/spec/lib/gitlab/repo_path_spec.rb @@ -48,8 +48,8 @@ describe ::Gitlab::RepoPath do describe '.strip_storage_path' do before do allow(Gitlab.config.repositories).to receive(:storages).and_return({ - 'storage1' => { 'path' => '/foo' }, - 'storage2' => { 'path' => '/bar' } + 'storage1' => Gitlab::GitalyClient::StorageSettings.new('path' => '/foo'), + 'storage2' => Gitlab::GitalyClient::StorageSettings.new('path' => '/bar') }) end diff --git a/spec/lib/gitlab/shell_spec.rb b/spec/lib/gitlab/shell_spec.rb index 14b59c5e945..ea5ce58e34b 100644 --- a/spec/lib/gitlab/shell_spec.rb +++ b/spec/lib/gitlab/shell_spec.rb @@ -405,7 +405,7 @@ describe Gitlab::Shell do describe '#create_repository' do shared_examples '#create_repository' do let(:repository_storage) { 'default' } - let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] } + let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage].legacy_disk_path } let(:repo_name) { 'project/path' } let(:created_path) { File.join(repository_storage_path, repo_name + '.git') } @@ -679,7 +679,7 @@ describe Gitlab::Shell do describe 'namespace actions' do subject { described_class.new } - let(:storage_path) { Gitlab.config.repositories.storages.default.path } + let(:storage_path) { Gitlab.config.repositories.storages.default.legacy_disk_path } describe '#add_namespace' do it 'creates a namespace' do diff --git a/spec/lib/gitlab/verify/lfs_objects_spec.rb b/spec/lib/gitlab/verify/lfs_objects_spec.rb index 64f3a9660e0..0f890e2c7ce 100644 --- a/spec/lib/gitlab/verify/lfs_objects_spec.rb +++ b/spec/lib/gitlab/verify/lfs_objects_spec.rb @@ -31,5 +31,21 @@ describe Gitlab::Verify::LfsObjects do expect(failures.keys).to contain_exactly(lfs_object) expect(failure.to_s).to include('Checksum mismatch') end + + context 'with remote files' do + before do + stub_lfs_object_storage + end + + it 'skips LFS objects in object storage' do + local_failure = create(:lfs_object) + create(:lfs_object, :object_storage) + + failures = {} + described_class.new(batch_size: 10).run_batches { |_, failed| failures.merge!(failed) } + + expect(failures.keys).to contain_exactly(local_failure) + end + end end end diff --git a/spec/lib/gitlab/verify/uploads_spec.rb b/spec/lib/gitlab/verify/uploads_spec.rb index 6146ce61226..85768308edc 100644 --- a/spec/lib/gitlab/verify/uploads_spec.rb +++ b/spec/lib/gitlab/verify/uploads_spec.rb @@ -40,5 +40,21 @@ describe Gitlab::Verify::Uploads do expect(failures.keys).to contain_exactly(upload) expect(failure.to_s).to include('Checksum missing') end + + context 'with remote files' do + before do + stub_uploads_object_storage(AvatarUploader) + end + + it 'skips uploads in object storage' do + local_failure = create(:upload) + create(:upload, :object_storage) + + failures = {} + described_class.new(batch_size: 10).run_batches { |_, failed| failures.merge!(failed) } + + expect(failures.keys).to contain_exactly(local_failure) + end + end end end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index 37a0bf1ad36..95b63fc91fc 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -55,7 +55,7 @@ describe Gitlab::Workhorse do end end - context 'when Gitaly workhorse_archive feature is disabled', :skip_gitaly_mock do + context 'when Gitaly workhorse_archive feature is disabled', :disable_gitaly do it 'sets the header correctly' do key, command, params = decode_workhorse_header(subject) @@ -100,7 +100,7 @@ describe Gitlab::Workhorse do end end - context 'when Gitaly workhorse_send_git_patch feature is disabled', :skip_gitaly_mock do + context 'when Gitaly workhorse_send_git_patch feature is disabled', :disable_gitaly do it 'sets the header correctly' do key, command, params = decode_workhorse_header(subject) @@ -173,7 +173,7 @@ describe Gitlab::Workhorse do end end - context 'when Gitaly workhorse_send_git_diff feature is disabled', :skip_gitaly_mock do + context 'when Gitaly workhorse_send_git_diff feature is disabled', :disable_gitaly do it 'sets the header correctly' do key, command, params = decode_workhorse_header(subject) @@ -455,7 +455,7 @@ describe Gitlab::Workhorse do end end - context 'when Gitaly workhorse_raw_show feature is disabled', :skip_gitaly_mock do + context 'when Gitaly workhorse_raw_show feature is disabled', :disable_gitaly do it 'sets the header correctly' do key, command, params = decode_workhorse_header(subject) diff --git a/spec/migrations/remove_dot_git_from_usernames_spec.rb b/spec/migrations/remove_dot_git_from_usernames_spec.rb index 129374cb38c..3a88a66a476 100644 --- a/spec/migrations/remove_dot_git_from_usernames_spec.rb +++ b/spec/migrations/remove_dot_git_from_usernames_spec.rb @@ -29,7 +29,9 @@ describe RemoveDotGitFromUsernames do update_namespace(user, 'test.git') update_namespace(user2, 'test_git') - storages = { 'default' => 'tmp/tests/custom_repositories' } + default_hash = Gitlab.config.repositories.storages.default.to_h + default_hash['path'] = 'tmp/tests/custom_repositories' + storages = { 'default' => Gitlab::GitalyClient::StorageSettings.new(default_hash) } allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) allow(migration).to receive(:route_exists?).with('test_git').and_return(true) diff --git a/spec/migrations/remove_empty_fork_networks_spec.rb b/spec/migrations/remove_empty_fork_networks_spec.rb index 7f7ce91378b..f6d030ab25c 100644 --- a/spec/migrations/remove_empty_fork_networks_spec.rb +++ b/spec/migrations/remove_empty_fork_networks_spec.rb @@ -19,6 +19,10 @@ describe RemoveEmptyForkNetworks, :migration do deleted_project.destroy! end + after do + Upload.reset_column_information + end + it 'deletes only the fork network without members' do expect(fork_networks.count).to eq(2) diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 30a352fd090..7d935cf8d76 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -198,6 +198,16 @@ describe Ci::Build do end context 'when legacy artifacts are used' do + let(:build) { create(:ci_build, :legacy_artifacts) } + + subject { build.artifacts? } + + context 'is expired' do + let(:build) { create(:ci_build, :legacy_artifacts, :expired) } + + it { is_expected.to be_falsy } + end + context 'artifacts archive does not exist' do let(:build) { create(:ci_build) } @@ -208,13 +218,25 @@ describe Ci::Build do let(:build) { create(:ci_build, :legacy_artifacts) } it { is_expected.to be_truthy } + end + end + end - context 'is expired' do - let(:build) { create(:ci_build, :legacy_artifacts, :expired) } + describe '#browsable_artifacts?' do + subject { build.browsable_artifacts? } - it { is_expected.to be_falsy } - end + context 'artifacts metadata does not exist' do + before do + build.update_attributes(legacy_artifacts_metadata: nil) end + + it { is_expected.to be_falsy } + end + + context 'artifacts metadata does exists' do + let(:build) { create(:ci_build, :artifacts) } + + it { is_expected.to be_truthy } end end diff --git a/spec/models/ci/job_artifact_spec.rb b/spec/models/ci/job_artifact_spec.rb index a2bd36537e6..1aa28434879 100644 --- a/spec/models/ci/job_artifact_spec.rb +++ b/spec/models/ci/job_artifact_spec.rb @@ -15,6 +15,50 @@ describe Ci::JobArtifact do it { is_expected.to delegate_method(:open).to(:file) } it { is_expected.to delegate_method(:exists?).to(:file) } + describe 'callbacks' do + subject { create(:ci_job_artifact, :archive) } + + describe '#schedule_background_upload' do + context 'when object storage is disabled' do + before do + stub_artifacts_object_storage(enabled: false) + end + + it 'does not schedule the migration' do + expect(ObjectStorageUploadWorker).not_to receive(:perform_async) + + subject + end + end + + context 'when object storage is enabled' do + context 'when background upload is enabled' do + before do + stub_artifacts_object_storage(background_upload: true) + end + + it 'schedules the model for migration' do + expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with('JobArtifactUploader', described_class.name, :file, kind_of(Numeric)) + + subject + end + end + + context 'when background upload is disabled' do + before do + stub_artifacts_object_storage(background_upload: false) + end + + it 'schedules the model for migration' do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + subject + end + end + end + end + end + describe '#set_size' do it 'sets the size' do expect(artifact.size).to eq(106365) diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 4635f8cfe9d..92f00cfbc19 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -177,6 +177,24 @@ describe Ci::Pipeline, :mailer do end end + describe '#protected_ref?' do + it 'delegates method to project' do + expect(pipeline).not_to be_protected_ref + end + end + + describe '#legacy_trigger' do + let(:trigger_request) { create(:ci_trigger_request) } + + before do + pipeline.trigger_requests << trigger_request + end + + it 'returns first trigger request' do + expect(pipeline.legacy_trigger).to eq trigger_request + end + end + describe '#auto_canceled?' do subject { pipeline.auto_canceled? } @@ -215,142 +233,257 @@ describe Ci::Pipeline, :mailer do end describe 'pipeline stages' do - before do - create(:commit_status, pipeline: pipeline, - stage: 'build', - name: 'linux', - stage_idx: 0, - status: 'success') - - create(:commit_status, pipeline: pipeline, - stage: 'build', - name: 'mac', - stage_idx: 0, - status: 'failed') - - create(:commit_status, pipeline: pipeline, - stage: 'deploy', - name: 'staging', - stage_idx: 2, - status: 'running') - - create(:commit_status, pipeline: pipeline, - stage: 'test', - name: 'rspec', - stage_idx: 1, - status: 'success') - end - describe '#stage_seeds' do - let(:pipeline) do - build(:ci_pipeline, config: { rspec: { script: 'rake' } }) - end + let(:pipeline) { build(:ci_pipeline, config: config) } + let(:config) { { rspec: { script: 'rake' } } } it 'returns preseeded stage seeds object' do - expect(pipeline.stage_seeds).to all(be_a Gitlab::Ci::Stage::Seed) + expect(pipeline.stage_seeds) + .to all(be_a Gitlab::Ci::Pipeline::Seed::Base) expect(pipeline.stage_seeds.count).to eq 1 end - end - describe '#seeds_size' do - let(:pipeline) { build(:ci_pipeline_with_one_job) } + context 'when no refs policy is specified' do + let(:config) do + { production: { stage: 'deploy', script: 'cap prod' }, + rspec: { stage: 'test', script: 'rspec' }, + spinach: { stage: 'test', script: 'spinach' } } + end - it 'returns number of jobs in stage seeds' do - expect(pipeline.seeds_size).to eq 1 + it 'correctly fabricates a stage seeds object' do + seeds = pipeline.stage_seeds + + expect(seeds.size).to eq 2 + expect(seeds.first.attributes[:name]).to eq 'test' + expect(seeds.second.attributes[:name]).to eq 'deploy' + expect(seeds.dig(0, 0, :name)).to eq 'rspec' + expect(seeds.dig(0, 1, :name)).to eq 'spinach' + expect(seeds.dig(1, 0, :name)).to eq 'production' + end end - end - describe '#legacy_stages' do - subject { pipeline.legacy_stages } + context 'when refs policy is specified' do + let(:pipeline) do + build(:ci_pipeline, ref: 'feature', tag: true, config: config) + end - context 'stages list' do - it 'returns ordered list of stages' do - expect(subject.map(&:name)).to eq(%w[build test deploy]) + let(:config) do + { production: { stage: 'deploy', script: 'cap prod', only: ['master'] }, + spinach: { stage: 'test', script: 'spinach', only: ['tags'] } } + end + + it 'returns stage seeds only assigned to master to master' do + seeds = pipeline.stage_seeds + + expect(seeds.size).to eq 1 + expect(seeds.first.attributes[:name]).to eq 'test' + expect(seeds.dig(0, 0, :name)).to eq 'spinach' end end - context 'stages with statuses' do - let(:statuses) do - subject.map { |stage| [stage.name, stage.status] } + context 'when source policy is specified' do + let(:pipeline) { build(:ci_pipeline, source: :schedule, config: config) } + + let(:config) do + { production: { stage: 'deploy', script: 'cap prod', only: ['triggers'] }, + spinach: { stage: 'test', script: 'spinach', only: ['schedules'] } } end - it 'returns list of stages with correct statuses' do - expect(statuses).to eq([%w(build failed), - %w(test success), - %w(deploy running)]) + it 'returns stage seeds only assigned to schedules' do + seeds = pipeline.stage_seeds + + expect(seeds.size).to eq 1 + expect(seeds.first.attributes[:name]).to eq 'test' + expect(seeds.dig(0, 0, :name)).to eq 'spinach' end + end - context 'when commit status is retried' do - before do - create(:commit_status, pipeline: pipeline, - stage: 'build', - name: 'mac', - stage_idx: 0, - status: 'success') + context 'when kubernetes policy is specified' do + let(:config) do + { + spinach: { stage: 'test', script: 'spinach' }, + production: { + stage: 'deploy', + script: 'cap', + only: { kubernetes: 'active' } + } + } + end + + context 'when kubernetes is active' do + shared_examples 'same behavior between KubernetesService and Platform::Kubernetes' do + it 'returns seeds for kubernetes dependent job' do + seeds = pipeline.stage_seeds - pipeline.process! + expect(seeds.size).to eq 2 + expect(seeds.dig(0, 0, :name)).to eq 'spinach' + expect(seeds.dig(1, 0, :name)).to eq 'production' + end end - it 'ignores the previous state' do - expect(statuses).to eq([%w(build success), - %w(test success), - %w(deploy running)]) + context 'when user configured kubernetes from Integration > Kubernetes' do + let(:project) { create(:kubernetes_project) } + let(:pipeline) { build(:ci_pipeline, project: project, config: config) } + + it_behaves_like 'same behavior between KubernetesService and Platform::Kubernetes' end - end - end - context 'when there is a stage with warnings' do - before do - create(:commit_status, pipeline: pipeline, - stage: 'deploy', - name: 'prod:2', - stage_idx: 2, - status: 'failed', - allow_failure: true) + context 'when user configured kubernetes from CI/CD > Clusters' do + let!(:cluster) { create(:cluster, :project, :provided_by_gcp) } + let(:project) { cluster.project } + let(:pipeline) { build(:ci_pipeline, project: project, config: config) } + + it_behaves_like 'same behavior between KubernetesService and Platform::Kubernetes' + end end - it 'populates stage with correct number of warnings' do - deploy_stage = pipeline.legacy_stages.third + context 'when kubernetes is not active' do + it 'does not return seeds for kubernetes dependent job' do + seeds = pipeline.stage_seeds - expect(deploy_stage).not_to receive(:statuses) - expect(deploy_stage).to have_warnings + expect(seeds.size).to eq 1 + expect(seeds.dig(0, 0, :name)).to eq 'spinach' + end end end end - describe '#stages_count' do - it 'returns a valid number of stages' do - expect(pipeline.stages_count).to eq(3) - end - end + describe '#seeds_size' do + context 'when refs policy is specified' do + let(:config) do + { production: { stage: 'deploy', script: 'cap prod', only: ['master'] }, + spinach: { stage: 'test', script: 'spinach', only: ['tags'] } } + end - describe '#stages_names' do - it 'returns a valid names of stages' do - expect(pipeline.stages_names).to eq(%w(build test deploy)) + let(:pipeline) do + build(:ci_pipeline, ref: 'feature', tag: true, config: config) + end + + it 'returns real seeds size' do + expect(pipeline.seeds_size).to eq 1 + end end end - end - - describe '#legacy_stage' do - subject { pipeline.legacy_stage('test') } - context 'with status in stage' do + describe 'legacy stages' do before do - create(:commit_status, pipeline: pipeline, stage: 'test') + create(:commit_status, pipeline: pipeline, + stage: 'build', + name: 'linux', + stage_idx: 0, + status: 'success') + + create(:commit_status, pipeline: pipeline, + stage: 'build', + name: 'mac', + stage_idx: 0, + status: 'failed') + + create(:commit_status, pipeline: pipeline, + stage: 'deploy', + name: 'staging', + stage_idx: 2, + status: 'running') + + create(:commit_status, pipeline: pipeline, + stage: 'test', + name: 'rspec', + stage_idx: 1, + status: 'success') + end + + describe '#legacy_stages' do + subject { pipeline.legacy_stages } + + context 'stages list' do + it 'returns ordered list of stages' do + expect(subject.map(&:name)).to eq(%w[build test deploy]) + end + end + + context 'stages with statuses' do + let(:statuses) do + subject.map { |stage| [stage.name, stage.status] } + end + + it 'returns list of stages with correct statuses' do + expect(statuses).to eq([%w(build failed), + %w(test success), + %w(deploy running)]) + end + + context 'when commit status is retried' do + before do + create(:commit_status, pipeline: pipeline, + stage: 'build', + name: 'mac', + stage_idx: 0, + status: 'success') + + pipeline.process! + end + + it 'ignores the previous state' do + expect(statuses).to eq([%w(build success), + %w(test success), + %w(deploy running)]) + end + end + end + + context 'when there is a stage with warnings' do + before do + create(:commit_status, pipeline: pipeline, + stage: 'deploy', + name: 'prod:2', + stage_idx: 2, + status: 'failed', + allow_failure: true) + end + + it 'populates stage with correct number of warnings' do + deploy_stage = pipeline.legacy_stages.third + + expect(deploy_stage).not_to receive(:statuses) + expect(deploy_stage).to have_warnings + end + end + end + + describe '#stages_count' do + it 'returns a valid number of stages' do + expect(pipeline.stages_count).to eq(3) + end end - it { expect(subject).to be_a Ci::LegacyStage } - it { expect(subject.name).to eq 'test' } - it { expect(subject.statuses).not_to be_empty } + describe '#stages_names' do + it 'returns a valid names of stages' do + expect(pipeline.stages_names).to eq(%w(build test deploy)) + end + end end - context 'without status in stage' do - before do - create(:commit_status, pipeline: pipeline, stage: 'build') + describe '#legacy_stage' do + subject { pipeline.legacy_stage('test') } + + context 'with status in stage' do + before do + create(:commit_status, pipeline: pipeline, stage: 'test') + end + + it { expect(subject).to be_a Ci::LegacyStage } + it { expect(subject.name).to eq 'test' } + it { expect(subject.statuses).not_to be_empty } end - it 'return stage object' do - is_expected.to be_nil + context 'without status in stage' do + before do + create(:commit_status, pipeline: pipeline, stage: 'build') + end + + it 'return stage object' do + is_expected.to be_nil + end end end end @@ -589,20 +722,6 @@ describe Ci::Pipeline, :mailer do end end - describe '#has_stage_seeds?' do - context 'when pipeline has stage seeds' do - subject { build(:ci_pipeline_with_one_job) } - - it { is_expected.to have_stage_seeds } - end - - context 'when pipeline does not have stage seeds' do - subject { create(:ci_pipeline_without_jobs) } - - it { is_expected.not_to have_stage_seeds } - end - end - describe '#has_warnings?' do subject { pipeline.has_warnings? } diff --git a/spec/models/group_spec.rb b/spec/models/group_spec.rb index abfc0896a41..d620943693c 100644 --- a/spec/models/group_spec.rb +++ b/spec/models/group_spec.rb @@ -240,7 +240,7 @@ describe Group do it "is false if avatar is html page" do group.update_attribute(:avatar, 'uploads/avatar.html') - expect(group.avatar_type).to eq(["only images allowed"]) + expect(group.avatar_type).to eq(["file format is not supported. Please try one of the following supported formats: png, jpg, jpeg, gif, bmp, tiff"]) end end diff --git a/spec/models/lfs_object_spec.rb b/spec/models/lfs_object_spec.rb new file mode 100644 index 00000000000..a182116d637 --- /dev/null +++ b/spec/models/lfs_object_spec.rb @@ -0,0 +1,85 @@ +require 'spec_helper' + +describe LfsObject do + describe '#local_store?' do + it 'returns true when file_store is nil' do + subject.file_store = nil + + expect(subject.local_store?).to eq true + end + + it 'returns true when file_store is equal to LfsObjectUploader::Store::LOCAL' do + subject.file_store = LfsObjectUploader::Store::LOCAL + + expect(subject.local_store?).to eq true + end + + it 'returns false whe file_store is equal to LfsObjectUploader::Store::REMOTE' do + subject.file_store = LfsObjectUploader::Store::REMOTE + + expect(subject.local_store?).to eq false + end + end + + describe '#schedule_background_upload' do + before do + stub_lfs_setting(enabled: true) + end + + subject { create(:lfs_object, :with_file) } + + context 'when object storage is disabled' do + before do + stub_lfs_object_storage(enabled: false) + end + + it 'does not schedule the migration' do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + subject + end + end + + context 'when object storage is enabled' do + context 'when background upload is enabled' do + context 'when is licensed' do + before do + stub_lfs_object_storage(background_upload: true) + end + + it 'schedules the model for migration' do + expect(ObjectStorage::BackgroundMoveWorker) + .to receive(:perform_async) + .with('LfsObjectUploader', described_class.name, :file, kind_of(Numeric)) + .once + + subject + end + + it 'schedules the model for migration once' do + expect(ObjectStorage::BackgroundMoveWorker) + .to receive(:perform_async) + .with('LfsObjectUploader', described_class.name, :file, kind_of(Numeric)) + .once + + lfs_object = create(:lfs_object) + lfs_object.file = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png") + lfs_object.save! + end + end + end + + context 'when background upload is disabled' do + before do + stub_lfs_object_storage(background_upload: false) + end + + it 'schedules the model for migration' do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + subject + end + end + end + end +end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index ee142718f7e..62e95a622eb 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -305,7 +305,7 @@ describe Namespace do end describe '#rm_dir', 'callback' do - let(:repository_storage_path) { Gitlab.config.repositories.storages.default['path'] } + let(:repository_storage_path) { Gitlab.config.repositories.storages.default.legacy_disk_path } let(:path_in_dir) { File.join(repository_storage_path, namespace.full_path) } let(:deleted_path) { namespace.full_path.gsub(namespace.path, "#{namespace.full_path}+#{namespace.id}+deleted") } let(:deleted_path_in_dir) { File.join(repository_storage_path, deleted_path) } diff --git a/spec/models/pages_domain_spec.rb b/spec/models/pages_domain_spec.rb index 95713d8b85b..4b85c5e8720 100644 --- a/spec/models/pages_domain_spec.rb +++ b/spec/models/pages_domain_spec.rb @@ -18,24 +18,63 @@ describe PagesDomain do it { is_expected.to validate_uniqueness_of(:domain).case_insensitive } end - { - 'my.domain.com' => true, - '123.456.789' => true, - '0x12345.com' => true, - '0123123' => true, - '_foo.com' => false, - 'reserved.com' => false, - 'a.reserved.com' => false, - nil => false - }.each do |value, validity| - context "domain #{value.inspect} validity" do - before do - allow(Settings.pages).to receive(:host).and_return('reserved.com') + describe "hostname" do + { + 'my.domain.com' => true, + '123.456.789' => true, + '0x12345.com' => true, + '0123123' => true, + '_foo.com' => false, + 'reserved.com' => false, + 'a.reserved.com' => false, + nil => false + }.each do |value, validity| + context "domain #{value.inspect} validity" do + before do + allow(Settings.pages).to receive(:host).and_return('reserved.com') + end + + let(:domain) { value } + + it { expect(pages_domain.valid?).to eq(validity) } + end + end + end + + describe "HTTPS-only" do + using RSpec::Parameterized::TableSyntax + + let(:domain) { 'my.domain.com' } + + let(:project) do + instance_double(Project, pages_https_only?: pages_https_only) + end + + let(:pages_domain) do + build(:pages_domain, certificate: certificate, key: key).tap do |pd| + allow(pd).to receive(:project).and_return(project) + pd.valid? end + end - let(:domain) { value } + where(:pages_https_only, :certificate, :key, :errors_on) do + attributes = attributes_for(:pages_domain) + cert, key = attributes.fetch_values(:certificate, :key) + + true | nil | nil | %i(certificate key) + true | cert | nil | %i(key) + true | nil | key | %i(certificate key) + true | cert | key | [] + false | nil | nil | [] + false | cert | nil | %i(key) + false | nil | key | %i(key) + false | cert | key | [] + end - it { expect(pages_domain.valid?).to eq(validity) } + with_them do + it "is adds the expected errors" do + expect(pages_domain.errors.keys).to eq errors_on + end end end end @@ -43,26 +82,26 @@ describe PagesDomain do describe 'validate certificate' do subject { domain } - context 'when only certificate is specified' do - let(:domain) { build(:pages_domain, :with_certificate) } + context 'with matching key' do + let(:domain) { build(:pages_domain) } - it { is_expected.not_to be_valid } + it { is_expected.to be_valid } end - context 'when only key is specified' do - let(:domain) { build(:pages_domain, :with_key) } + context 'when no certificate is specified' do + let(:domain) { build(:pages_domain, :without_certificate) } it { is_expected.not_to be_valid } end - context 'with matching key' do - let(:domain) { build(:pages_domain, :with_certificate, :with_key) } + context 'when no key is specified' do + let(:domain) { build(:pages_domain, :without_key) } - it { is_expected.to be_valid } + it { is_expected.not_to be_valid } end context 'for not matching key' do - let(:domain) { build(:pages_domain, :with_missing_chain, :with_key) } + let(:domain) { build(:pages_domain, :with_missing_chain) } it { is_expected.not_to be_valid } end @@ -103,30 +142,26 @@ describe PagesDomain do describe '#url' do subject { domain.url } - context 'without the certificate' do - let(:domain) { build(:pages_domain, certificate: '') } + let(:domain) { build(:pages_domain) } - it { is_expected.to eq("http://#{domain.domain}") } - end + it { is_expected.to eq("https://#{domain.domain}") } - context 'with a certificate' do - let(:domain) { build(:pages_domain, :with_certificate) } + context 'without the certificate' do + let(:domain) { build(:pages_domain, :without_certificate) } - it { is_expected.to eq("https://#{domain.domain}") } + it { is_expected.to eq("http://#{domain.domain}") } end end describe '#has_matching_key?' do subject { domain.has_matching_key? } - context 'for matching key' do - let(:domain) { build(:pages_domain, :with_certificate, :with_key) } + let(:domain) { build(:pages_domain) } - it { is_expected.to be_truthy } - end + it { is_expected.to be_truthy } context 'for invalid key' do - let(:domain) { build(:pages_domain, :with_missing_chain, :with_key) } + let(:domain) { build(:pages_domain, :with_missing_chain) } it { is_expected.to be_falsey } end @@ -136,7 +171,7 @@ describe PagesDomain do subject { domain.has_intermediates? } context 'for self signed' do - let(:domain) { build(:pages_domain, :with_certificate) } + let(:domain) { build(:pages_domain) } it { is_expected.to be_truthy } end @@ -162,7 +197,7 @@ describe PagesDomain do subject { domain.expired? } context 'for valid' do - let(:domain) { build(:pages_domain, :with_certificate) } + let(:domain) { build(:pages_domain) } it { is_expected.to be_falsey } end @@ -175,7 +210,7 @@ describe PagesDomain do end describe '#subject' do - let(:domain) { build(:pages_domain, :with_certificate) } + let(:domain) { build(:pages_domain) } subject { domain.subject } @@ -183,7 +218,7 @@ describe PagesDomain do end describe '#certificate_text' do - let(:domain) { build(:pages_domain, :with_certificate) } + let(:domain) { build(:pages_domain) } subject { domain.certificate_text } @@ -191,6 +226,18 @@ describe PagesDomain do it { is_expected.not_to be_empty } end + describe "#https?" do + context "when a certificate is present" do + subject { build(:pages_domain) } + it { is_expected.to be_https } + end + + context "when no certificate is present" do + subject { build(:pages_domain, :without_certificate) } + it { is_expected.not_to be_https } + end + end + describe '#update_daemon' do it 'runs when the domain is created' do domain = build(:pages_domain) @@ -267,29 +314,30 @@ describe PagesDomain do end context 'TLS configuration' do - set(:domain_with_tls) { create(:pages_domain, :with_key, :with_certificate) } + set(:domain_without_tls) { create(:pages_domain, :without_certificate, :without_key) } + set(:domain) { create(:pages_domain) } - let(:cert1) { domain_with_tls.certificate } + let(:cert1) { domain.certificate } let(:cert2) { cert1 + ' ' } - let(:key1) { domain_with_tls.key } + let(:key1) { domain.key } let(:key2) { key1 + ' ' } it 'updates when added' do - expect(domain).to receive(:update_daemon) + expect(domain_without_tls).to receive(:update_daemon) - domain.update!(key: key1, certificate: cert1) + domain_without_tls.update!(key: key1, certificate: cert1) end it 'updates when changed' do - expect(domain_with_tls).to receive(:update_daemon) + expect(domain).to receive(:update_daemon) - domain_with_tls.update!(key: key2, certificate: cert2) + domain.update!(key: key2, certificate: cert2) end it 'updates when removed' do - expect(domain_with_tls).to receive(:update_daemon) + expect(domain).to receive(:update_daemon) - domain_with_tls.update!(key: nil, certificate: nil) + domain.update!(key: nil, certificate: nil) end end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 4cf8d861595..1a7a6e035ea 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -922,7 +922,7 @@ describe Project do it 'is false if avatar is html page' do project.update_attribute(:avatar, 'uploads/avatar.html') - expect(project.avatar_type).to eq(['only images allowed']) + expect(project.avatar_type).to eq(['file format is not supported. Please try one of the following supported formats: png, jpg, jpeg, gif, bmp, tiff']) end end @@ -1101,8 +1101,8 @@ describe Project do before do storages = { - 'default' => { 'path' => 'tmp/tests/repositories' }, - 'picked' => { 'path' => 'tmp/tests/repositories' } + 'default' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/repositories'), + 'picked' => Gitlab::GitalyClient::StorageSettings.new('path' => 'tmp/tests/repositories') } allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) end @@ -3479,4 +3479,49 @@ describe Project do end end end + + describe "#pages_https_only?" do + subject { build(:project) } + + context "when HTTPS pages are disabled" do + it { is_expected.not_to be_pages_https_only } + end + + context "when HTTPS pages are enabled", :https_pages_enabled do + it { is_expected.to be_pages_https_only } + end + end + + describe "#pages_https_only? validation", :https_pages_enabled do + subject(:project) do + # set-up dirty object: + create(:project, pages_https_only: false).tap do |p| + p.pages_https_only = true + end + end + + context "when no domains are associated" do + it { is_expected.to be_valid } + end + + context "when domains including keys and certificates are associated" do + before do + allow(project) + .to receive(:pages_domains) + .and_return([instance_double(PagesDomain, https?: true)]) + end + + it { is_expected.to be_valid } + end + + context "when domains including no keys or certificates are associated" do + before do + allow(project) + .to receive(:pages_domains) + .and_return([instance_double(PagesDomain, https?: false)]) + end + + it { is_expected.not_to be_valid } + end + end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 5680eb24985..c61674fff13 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -1222,7 +1222,7 @@ describe User do it 'is false if avatar is html page' do user.update_attribute(:avatar, 'uploads/avatar.html') - expect(user.avatar_type).to eq(['only images allowed']) + expect(user.avatar_type).to eq(['file format is not supported. Please try one of the following supported formats: png, jpg, jpeg, gif, bmp, tiff']) end end diff --git a/spec/presenters/project_presenter_spec.rb b/spec/presenters/project_presenter_spec.rb index f8c93d91ec5..55962f345d4 100644 --- a/spec/presenters/project_presenter_spec.rb +++ b/spec/presenters/project_presenter_spec.rb @@ -339,7 +339,7 @@ describe ProjectPresenter do it 'returns link to clusters page if more than one exists' do project.add_master(user) - create(:cluster, projects: [project]) + create(:cluster, :production_environment, projects: [project]) create(:cluster, projects: [project]) expect(presenter.kubernetes_cluster_anchor_data).to eq(OpenStruct.new(enabled: true, diff --git a/spec/requests/api/jobs_spec.rb b/spec/requests/api/jobs_spec.rb index 6192bbd4abb..3ffdfdc0e9a 100644 --- a/spec/requests/api/jobs_spec.rb +++ b/spec/requests/api/jobs_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' describe API::Jobs do + include HttpIOHelpers + set(:project) do create(:project, :repository, public_builds: false) end @@ -112,6 +114,7 @@ describe API::Jobs do let(:query) { Hash.new } before do + job get api("/projects/#{project.id}/pipelines/#{pipeline.id}/jobs", api_user), query end @@ -335,10 +338,55 @@ describe API::Jobs do end end + context 'when artifacts are stored remotely' do + let(:proxy_download) { false } + + before do + stub_artifacts_object_storage(proxy_download: proxy_download) + end + + let(:job) { create(:ci_build, pipeline: pipeline) } + let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) } + + before do + job.reload + + get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user) + end + + context 'when proxy download is enabled' do + let(:proxy_download) { true } + + it 'responds with the workhorse send-url' do + expect(response.headers[Gitlab::Workhorse::SEND_DATA_HEADER]).to start_with("send-url:") + end + end + + context 'when proxy download is disabled' do + it 'returns location redirect' do + expect(response).to have_gitlab_http_status(302) + end + end + + context 'authorized user' do + it 'returns the file remote URL' do + expect(response).to redirect_to(artifact.file.url) + end + end + + context 'unauthorized user' do + let(:api_user) { nil } + + it 'does not return specific job artifacts' do + expect(response).to have_gitlab_http_status(404) + end + end + end + it 'does not return job artifacts if not uploaded' do get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user) - expect(response).to have_gitlab_http_status(404) + expect(response).to have_gitlab_http_status(:not_found) end end end @@ -349,6 +397,7 @@ describe API::Jobs do let(:job) { create(:ci_build, :artifacts, pipeline: pipeline, user: api_user) } before do + stub_artifacts_object_storage job.success end @@ -412,9 +461,24 @@ describe API::Jobs do "attachment; filename=#{job.artifacts_file.filename}" } end - it { expect(response).to have_gitlab_http_status(200) } + it { expect(response).to have_http_status(:ok) } it { expect(response.headers).to include(download_headers) } end + + context 'when artifacts are stored remotely' do + let(:job) { create(:ci_build, pipeline: pipeline, user: api_user) } + let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) } + + before do + job.reload + + get api("/projects/#{project.id}/jobs/#{job.id}/artifacts", api_user) + end + + it 'returns location redirect' do + expect(response).to have_http_status(:found) + end + end end context 'with regular branch' do @@ -451,6 +515,22 @@ describe API::Jobs do end context 'authorized user' do + context 'when trace is in ObjectStorage' do + let!(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) } + + before do + stub_remote_trace_206 + allow_any_instance_of(JobArtifactUploader).to receive(:file_storage?) { false } + allow_any_instance_of(JobArtifactUploader).to receive(:url) { remote_trace_url } + allow_any_instance_of(JobArtifactUploader).to receive(:size) { remote_trace_size } + end + + it 'returns specific job trace' do + expect(response).to have_gitlab_http_status(200) + expect(response.body).to eq(job.trace.raw) + end + end + context 'when trace is artifact' do let(:job) { create(:ci_build, :trace_artifact, pipeline: pipeline) } diff --git a/spec/requests/api/pages_domains_spec.rb b/spec/requests/api/pages_domains_spec.rb index dc3a116c060..a9ccbb32666 100644 --- a/spec/requests/api/pages_domains_spec.rb +++ b/spec/requests/api/pages_domains_spec.rb @@ -1,17 +1,17 @@ require 'rails_helper' describe API::PagesDomains do - set(:project) { create(:project, path: 'my.project') } + set(:project) { create(:project, path: 'my.project', pages_https_only: false) } set(:user) { create(:user) } set(:admin) { create(:admin) } - set(:pages_domain) { create(:pages_domain, domain: 'www.domain.test', project: project) } - set(:pages_domain_secure) { create(:pages_domain, :with_certificate, :with_key, domain: 'ssl.domain.test', project: project) } - set(:pages_domain_expired) { create(:pages_domain, :with_expired_certificate, :with_key, domain: 'expired.domain.test', project: project) } + set(:pages_domain) { create(:pages_domain, :without_key, :without_certificate, domain: 'www.domain.test', project: project) } + set(:pages_domain_secure) { create(:pages_domain, domain: 'ssl.domain.test', project: project) } + set(:pages_domain_expired) { create(:pages_domain, :with_expired_certificate, domain: 'expired.domain.test', project: project) } - let(:pages_domain_params) { build(:pages_domain, domain: 'www.other-domain.test').slice(:domain) } - let(:pages_domain_secure_params) { build(:pages_domain, :with_certificate, :with_key, domain: 'ssl.other-domain.test', project: project).slice(:domain, :certificate, :key) } - let(:pages_domain_secure_key_missmatch_params) {build(:pages_domain, :with_trusted_chain, :with_key, project: project).slice(:domain, :certificate, :key) } + let(:pages_domain_params) { build(:pages_domain, :without_key, :without_certificate, domain: 'www.other-domain.test').slice(:domain) } + let(:pages_domain_secure_params) { build(:pages_domain, domain: 'ssl.other-domain.test', project: project).slice(:domain, :certificate, :key) } + let(:pages_domain_secure_key_missmatch_params) {build(:pages_domain, :with_trusted_chain, project: project).slice(:domain, :certificate, :key) } let(:pages_domain_secure_missing_chain_params) {build(:pages_domain, :with_missing_chain, project: project).slice(:certificate) } let(:route) { "/projects/#{project.id}/pages/domains" } diff --git a/spec/requests/api/projects_spec.rb b/spec/requests/api/projects_spec.rb index cee93f6ed14..d73a42f48ad 100644 --- a/spec/requests/api/projects_spec.rb +++ b/spec/requests/api/projects_spec.rb @@ -1718,6 +1718,12 @@ describe API::Projects do group end + let(:group3) do + group = create(:group, name: 'group3_name', parent: group2) + group.add_owner(user2) + group + end + before do project.add_reporter(user2) end @@ -1813,6 +1819,15 @@ describe API::Projects do expect(json_response['namespace']['name']).to eq(group2.name) end + it 'forks to owned subgroup' do + full_path = "#{group2.path}/#{group3.path}" + post api("/projects/#{project.id}/fork", user2), namespace: full_path + + expect(response).to have_gitlab_http_status(201) + expect(json_response['namespace']['name']).to eq(group3.name) + expect(json_response['namespace']['full_path']).to eq(full_path) + end + it 'fails to fork to not owned group' do post api("/projects/#{project.id}/fork", user2), namespace: group.name diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index 95c23726a79..f3dd121faa9 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -200,7 +200,7 @@ describe API::Runner do let(:project) { create(:project, shared_runners_enabled: false) } let(:pipeline) { create(:ci_pipeline_without_jobs, project: project, ref: 'master') } let(:runner) { create(:ci_runner) } - let!(:job) do + let(:job) do create(:ci_build, :artifacts, :extended_options, pipeline: pipeline, name: 'spinach', stage: 'test', stage_idx: 0, commands: "ls\ndate") end @@ -215,6 +215,7 @@ describe API::Runner do let(:user_agent) { 'gitlab-runner 9.0.0 (9-0-stable; go1.7.4; linux/amd64)' } before do + job stub_container_registry_config(enabled: false) end @@ -888,6 +889,7 @@ describe API::Runner do let(:file_upload2) { fixture_file_upload(Rails.root + 'spec/fixtures/dk.png', 'image/gif') } before do + stub_artifacts_object_storage job.run! end @@ -1179,27 +1181,67 @@ describe API::Runner do describe 'GET /api/v4/jobs/:id/artifacts' do let(:token) { job.token } - before do - download_artifact - end - context 'when job has artifacts' do - let(:job) { create(:ci_build, :artifacts) } - let(:download_headers) do - { 'Content-Transfer-Encoding' => 'binary', - 'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' } + let(:job) { create(:ci_build) } + let(:store) { JobArtifactUploader::Store::LOCAL } + + before do + create(:ci_job_artifact, :archive, file_store: store, job: job) end context 'when using job token' do - it 'download artifacts' do - expect(response).to have_gitlab_http_status(200) - expect(response.headers).to include download_headers + context 'when artifacts are stored locally' do + let(:download_headers) do + { 'Content-Transfer-Encoding' => 'binary', + 'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' } + end + + before do + download_artifact + end + + it 'download artifacts' do + expect(response).to have_http_status(200) + expect(response.headers).to include download_headers + end + end + + context 'when artifacts are stored remotely' do + let(:store) { JobArtifactUploader::Store::REMOTE } + let!(:job) { create(:ci_build) } + + context 'when proxy download is being used' do + before do + download_artifact(direct_download: false) + end + + it 'uses workhorse send-url' do + expect(response).to have_gitlab_http_status(200) + expect(response.headers).to include( + 'Gitlab-Workhorse-Send-Data' => /send-url:/) + end + end + + context 'when direct download is being used' do + before do + download_artifact(direct_download: true) + end + + it 'receive redirect for downloading artifacts' do + expect(response).to have_gitlab_http_status(302) + expect(response.headers).to include('Location') + end + end end end context 'when using runnners token' do let(:token) { job.project.runners_token } + before do + download_artifact + end + it 'responds with forbidden' do expect(response).to have_gitlab_http_status(403) end @@ -1208,12 +1250,16 @@ describe API::Runner do context 'when job does not has artifacts' do it 'responds with not found' do + download_artifact + expect(response).to have_gitlab_http_status(404) end end def download_artifact(params = {}, request_headers = headers) params = params.merge(token: token) + job.reload + get api("/jobs/#{job.id}/artifacts"), params, request_headers end end diff --git a/spec/requests/api/v3/builds_spec.rb b/spec/requests/api/v3/builds_spec.rb index 79041c6a792..00f067889a0 100644 --- a/spec/requests/api/v3/builds_spec.rb +++ b/spec/requests/api/v3/builds_spec.rb @@ -216,6 +216,7 @@ describe API::V3::Builds do describe 'GET /projects/:id/builds/:build_id/artifacts' do before do + stub_artifacts_object_storage get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user) end @@ -230,13 +231,24 @@ describe API::V3::Builds do end it 'returns specific job artifacts' do - expect(response).to have_gitlab_http_status(200) + expect(response).to have_http_status(200) expect(response.headers).to include(download_headers) expect(response.body).to match_file(build.artifacts_file.file.file) end end end + context 'when artifacts are stored remotely' do + let(:build) { create(:ci_build, pipeline: pipeline) } + let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: build) } + + it 'returns location redirect' do + get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user) + + expect(response).to have_gitlab_http_status(302) + end + end + context 'unauthorized user' do let(:api_user) { nil } @@ -256,6 +268,7 @@ describe API::V3::Builds do let(:build) { create(:ci_build, :artifacts, pipeline: pipeline) } before do + stub_artifacts_object_storage build.success end @@ -318,9 +331,24 @@ describe API::V3::Builds do "attachment; filename=#{build.artifacts_file.filename}" } end - it { expect(response).to have_gitlab_http_status(200) } + it { expect(response).to have_http_status(200) } it { expect(response.headers).to include(download_headers) } end + + context 'when artifacts are stored remotely' do + let(:build) { create(:ci_build, pipeline: pipeline) } + let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: build) } + + before do + build.reload + + get v3_api("/projects/#{project.id}/builds/#{build.id}/artifacts", api_user) + end + + it 'returns location redirect' do + expect(response).to have_http_status(302) + end + end end context 'with regular branch' do diff --git a/spec/requests/lfs_http_spec.rb b/spec/requests/lfs_http_spec.rb index 971b45c411d..f7c04c19903 100644 --- a/spec/requests/lfs_http_spec.rb +++ b/spec/requests/lfs_http_spec.rb @@ -191,10 +191,12 @@ describe 'Git LFS API and storage' do describe 'when fetching lfs object' do let(:project) { create(:project) } let(:update_permissions) { } + let(:before_get) { } before do enable_lfs update_permissions + before_get get "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}", nil, headers end @@ -239,6 +241,21 @@ describe 'Git LFS API and storage' do end it_behaves_like 'responds with a file' + + context 'when LFS uses object storage' do + let(:before_get) do + stub_lfs_object_storage + lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) + end + + it 'responds with redirect' do + expect(response).to have_gitlab_http_status(302) + end + + it 'responds with the file location' do + expect(response.location).to include(lfs_object.reload.file.path) + end + end end end @@ -978,6 +995,32 @@ describe 'Git LFS API and storage' do end end + context 'and workhorse requests upload finalize for a new lfs object' do + before do + lfs_object.destroy + end + + context 'with object storage disabled' do + it "doesn't attempt to migrate file to object storage" do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + put_finalize(with_tempfile: true) + end + end + + context 'with object storage enabled' do + before do + stub_lfs_object_storage(background_upload: true) + end + + it 'schedules migration of file to object storage' do + expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with('LfsObjectUploader', 'LfsObject', :file, kind_of(Numeric)) + + put_finalize(with_tempfile: true) + end + end + end + context 'invalid tempfiles' do it 'rejects slashes in the tempfile name (path traversal' do put_finalize('foo/bar') @@ -1177,7 +1220,9 @@ describe 'Git LFS API and storage' do put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}/authorize", nil, authorize_headers end - def put_finalize(lfs_tmp = lfs_tmp_file) + def put_finalize(lfs_tmp = lfs_tmp_file, with_tempfile: false) + setup_tempfile(lfs_tmp) if with_tempfile + put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}", nil, headers.merge('X-Gitlab-Lfs-Tmp' => lfs_tmp).compact end @@ -1185,6 +1230,13 @@ describe 'Git LFS API and storage' do def lfs_tmp_file "#{sample_oid}012345678" end + + def setup_tempfile(lfs_tmp) + upload_path = LfsObjectUploader.workhorse_upload_path + + FileUtils.mkdir_p(upload_path) + FileUtils.touch(File.join(upload_path, lfs_tmp)) + end end def enable_lfs diff --git a/spec/serializers/pipeline_serializer_spec.rb b/spec/serializers/pipeline_serializer_spec.rb index c38795ad1a1..f51c11b141f 100644 --- a/spec/serializers/pipeline_serializer_spec.rb +++ b/spec/serializers/pipeline_serializer_spec.rb @@ -117,6 +117,7 @@ describe PipelineSerializer do shared_examples 'no N+1 queries' do it 'verifies number of queries', :request_store do recorded = ActiveRecord::QueryRecorder.new { subject } + expect(recorded.count).to be_within(1).of(36) expect(recorded.cached_count).to eq(0) end diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index db9c216d3f4..b86a3d72bb4 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -28,7 +28,8 @@ describe Ci::RetryBuildService do %i[type lock_version target_url base_tags trace_sections commit_id deployments erased_by_id last_deployment project_id runner_id tag_taggings taggings tags trigger_request_id - user_id auto_canceled_by_id retried failure_reason].freeze + user_id auto_canceled_by_id retried failure_reason + artifacts_file_store artifacts_metadata_store].freeze shared_examples 'build duplication' do let(:another_pipeline) { create(:ci_empty_pipeline, project: project) } diff --git a/spec/services/clusters/create_service_spec.rb b/spec/services/clusters/create_service_spec.rb index e2e64659dfa..1c2f9c5cf43 100644 --- a/spec/services/clusters/create_service_spec.rb +++ b/spec/services/clusters/create_service_spec.rb @@ -82,7 +82,7 @@ describe Clusters::CreateService do context 'when project has a cluster' do include_context 'valid params' - let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } + let!(:cluster) { create(:cluster, :provided_by_gcp, :production_environment, projects: [project]) } it 'does not create a cluster' do expect(ClusterProvisionWorker).not_to receive(:perform_async) diff --git a/spec/services/issues/move_service_spec.rb b/spec/services/issues/move_service_spec.rb index c148a98569b..a9aee9e100f 100644 --- a/spec/services/issues/move_service_spec.rb +++ b/spec/services/issues/move_service_spec.rb @@ -6,7 +6,7 @@ describe Issues::MoveService do let(:title) { 'Some issue' } let(:description) { 'Some issue description' } let(:old_project) { create(:project) } - let(:new_project) { create(:project, group: create(:group)) } + let(:new_project) { create(:project) } let(:milestone1) { create(:milestone, project_id: old_project.id, title: 'v9.0') } let(:old_issue) do diff --git a/spec/services/merge_requests/refresh_service_spec.rb b/spec/services/merge_requests/refresh_service_spec.rb index 903aa0a5078..2536c6e2514 100644 --- a/spec/services/merge_requests/refresh_service_spec.rb +++ b/spec/services/merge_requests/refresh_service_spec.rb @@ -24,6 +24,14 @@ describe MergeRequests::RefreshService do merge_when_pipeline_succeeds: true, merge_user: @user) + @another_merge_request = create(:merge_request, + source_project: @project, + source_branch: 'master', + target_branch: 'test', + target_project: @project, + merge_when_pipeline_succeeds: true, + merge_user: @user) + @fork_merge_request = create(:merge_request, source_project: @fork_project, source_branch: 'master', @@ -52,9 +60,11 @@ describe MergeRequests::RefreshService do context 'push to origin repo source branch' do let(:refresh_service) { service.new(@project, @user) } + let(:notification_service) { spy('notification_service') } before do allow(refresh_service).to receive(:execute_hooks) + allow(NotificationService).to receive(:new) { notification_service } end it 'executes hooks with update action' do @@ -64,6 +74,11 @@ describe MergeRequests::RefreshService do expect(refresh_service).to have_received(:execute_hooks) .with(@merge_request, 'update', old_rev: @oldrev) + expect(notification_service).to have_received(:push_to_merge_request) + .with(@merge_request, @user, new_commits: anything, existing_commits: anything) + expect(notification_service).to have_received(:push_to_merge_request) + .with(@another_merge_request, @user, new_commits: anything, existing_commits: anything) + expect(@merge_request.notes).not_to be_empty expect(@merge_request).to be_open expect(@merge_request.merge_when_pipeline_succeeds).to be_falsey @@ -119,11 +134,13 @@ describe MergeRequests::RefreshService do context 'push to origin repo source branch when an MR was reopened' do let(:refresh_service) { service.new(@project, @user) } + let(:notification_service) { spy('notification_service') } before do @merge_request.update(state: :reopened) allow(refresh_service).to receive(:execute_hooks) + allow(NotificationService).to receive(:new) { notification_service } refresh_service.execute(@oldrev, @newrev, 'refs/heads/master') reload_mrs end @@ -131,6 +148,10 @@ describe MergeRequests::RefreshService do it 'executes hooks with update action' do expect(refresh_service).to have_received(:execute_hooks) .with(@merge_request, 'update', old_rev: @oldrev) + expect(notification_service).to have_received(:push_to_merge_request) + .with(@merge_request, @user, new_commits: anything, existing_commits: anything) + expect(notification_service).to have_received(:push_to_merge_request) + .with(@another_merge_request, @user, new_commits: anything, existing_commits: anything) expect(@merge_request.notes).not_to be_empty expect(@merge_request).to be_open diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb index 3943148f0db..f8fa2540804 100644 --- a/spec/services/notification_service_spec.rb +++ b/spec/services/notification_service_spec.rb @@ -1090,6 +1090,36 @@ describe NotificationService, :mailer do end end + describe '#push_to_merge_request' do + before do + update_custom_notification(:push_to_merge_request, @u_guest_custom, resource: project) + update_custom_notification(:push_to_merge_request, @u_custom_global) + end + + it do + notification.push_to_merge_request(merge_request, @u_disabled) + + should_email(merge_request.assignee) + should_email(@u_guest_custom) + should_email(@u_custom_global) + should_email(@u_participant_mentioned) + should_email(@subscriber) + should_email(@watcher_and_subscriber) + should_not_email(@u_watcher) + should_not_email(@u_guest_watcher) + should_not_email(@unsubscriber) + should_not_email(@u_participating) + should_not_email(@u_disabled) + should_not_email(@u_lazy_participant) + end + + it_behaves_like 'participating notifications' do + let(:participant) { create(:user, username: 'user-participant') } + let(:issuable) { merge_request } + let(:notification_trigger) { notification.push_to_merge_request(merge_request, @u_disabled) } + end + end + describe '#relabel_merge_request' do let(:group_label_1) { create(:group_label, group: group, title: 'Group Label 1', merge_requests: [merge_request]) } let(:group_label_2) { create(:group_label, group: group, title: 'Group Label 2') } diff --git a/spec/services/projects/create_service_spec.rb b/spec/services/projects/create_service_spec.rb index 8471467d2fa..4413c6ef83e 100644 --- a/spec/services/projects/create_service_spec.rb +++ b/spec/services/projects/create_service_spec.rb @@ -153,7 +153,7 @@ describe Projects::CreateService, '#execute' do context 'when another repository already exists on disk' do let(:repository_storage) { 'default' } - let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] } + let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage].legacy_disk_path } let(:opts) do { diff --git a/spec/services/projects/fork_service_spec.rb b/spec/services/projects/fork_service_spec.rb index d1011b07db6..0f7c46367d0 100644 --- a/spec/services/projects/fork_service_spec.rb +++ b/spec/services/projects/fork_service_spec.rb @@ -105,7 +105,7 @@ describe Projects::ForkService do context 'repository already exists' do let(:repository_storage) { 'default' } - let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] } + let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage].legacy_disk_path } before do gitlab_shell.create_repository(repository_storage, "#{@to_user.namespace.full_path}/#{@from_project.path}") diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index ce567fe3879..95a6771c59d 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -146,7 +146,7 @@ describe Projects::TransferService do context 'namespace which contains orphan repository with same projects path name' do let(:repository_storage) { 'default' } - let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] } + let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage].legacy_disk_path } before do group.add_owner(user) diff --git a/spec/services/projects/update_service_spec.rb b/spec/services/projects/update_service_spec.rb index f3f97b6b921..f48d466d263 100644 --- a/spec/services/projects/update_service_spec.rb +++ b/spec/services/projects/update_service_spec.rb @@ -190,7 +190,7 @@ describe Projects::UpdateService do context 'when renaming a project' do let(:repository_storage) { 'default' } - let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage]['path'] } + let(:repository_storage_path) { Gitlab.config.repositories.storages[repository_storage].legacy_disk_path } context 'with legacy storage' do let(:project) { create(:project, :legacy_storage, :repository, creator: user, namespace: user.namespace) } @@ -241,6 +241,27 @@ describe Projects::UpdateService do }) end end + + context 'when updating #pages_https_only', :https_pages_enabled do + subject(:call_service) do + update_project(project, admin, pages_https_only: false) + end + + it 'updates the attribute' do + expect { call_service } + .to change { project.pages_https_only? } + .to(false) + end + + it 'calls Projects::UpdatePagesConfigurationService' do + expect(Projects::UpdatePagesConfigurationService) + .to receive(:new) + .with(project) + .and_call_original + + call_service + end + end end describe '#run_auto_devops_pipeline?' do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 9f6f0204a16..5051cd34564 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -197,6 +197,22 @@ RSpec.configure do |config| Ability.allowed?(*args) end end + + config.before(:each, :http_pages_enabled) do |_| + allow(Gitlab.config.pages).to receive(:external_http).and_return(['1.1.1.1:80']) + end + + config.before(:each, :https_pages_enabled) do |_| + allow(Gitlab.config.pages).to receive(:external_https).and_return(['1.1.1.1:443']) + end + + config.before(:each, :http_pages_disabled) do |_| + allow(Gitlab.config.pages).to receive(:external_http).and_return(false) + end + + config.before(:each, :https_pages_disabled) do |_| + allow(Gitlab.config.pages).to receive(:external_https).and_return(false) + end end # add simpler way to match asset paths containing digest strings diff --git a/spec/support/http_io/http_io_helpers.rb b/spec/support/http_io/http_io_helpers.rb new file mode 100644 index 00000000000..31e07e720cd --- /dev/null +++ b/spec/support/http_io/http_io_helpers.rb @@ -0,0 +1,64 @@ +module HttpIOHelpers + def stub_remote_trace_206 + WebMock.stub_request(:get, remote_trace_url) + .to_return { |request| remote_trace_response(request, 206) } + end + + def stub_remote_trace_200 + WebMock.stub_request(:get, remote_trace_url) + .to_return { |request| remote_trace_response(request, 200) } + end + + def stub_remote_trace_500 + WebMock.stub_request(:get, remote_trace_url) + .to_return(status: [500, "Internal Server Error"]) + end + + def remote_trace_url + "http://trace.com/trace" + end + + def remote_trace_response(request, responce_status) + range = request.headers['Range'].match(/bytes=(\d+)-(\d+)/) + + { + status: responce_status, + headers: remote_trace_response_headers(responce_status, range[1].to_i, range[2].to_i), + body: range_trace_body(range[1].to_i, range[2].to_i) + } + end + + def remote_trace_response_headers(responce_status, from, to) + headers = { 'Content-Type' => 'text/plain' } + + if responce_status == 206 + headers.merge('Content-Range' => "bytes #{from}-#{to}/#{remote_trace_size}") + end + + headers + end + + def range_trace_body(from, to) + remote_trace_body[from..to] + end + + def remote_trace_body + @remote_trace_body ||= File.read(expand_fixture_path('trace/sample_trace')) + end + + def remote_trace_size + remote_trace_body.length + end + + def set_smaller_buffer_size_than(file_size) + blocks = (file_size / 128) + new_size = (blocks / 2) * 128 + stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size) + end + + def set_larger_buffer_size_than(file_size) + blocks = (file_size / 128) + new_size = (blocks * 2) * 128 + stub_const("Gitlab::Ci::Trace::HttpIO::BUFFER_SIZE", new_size) + end +end diff --git a/spec/support/shared_examples/controllers/variables_shared_examples.rb b/spec/support/shared_examples/controllers/variables_shared_examples.rb index d7acf8c0032..b615a8f54cf 100644 --- a/spec/support/shared_examples/controllers/variables_shared_examples.rb +++ b/spec/support/shared_examples/controllers/variables_shared_examples.rb @@ -16,19 +16,19 @@ shared_examples 'PATCH #update updates variables' do let(:variable_attributes) do { id: variable.id, key: variable.key, - value: variable.value, + secret_value: variable.value, protected: variable.protected?.to_s } end let(:new_variable_attributes) do { key: 'new_key', - value: 'dummy_value', + secret_value: 'dummy_value', protected: 'false' } end context 'with invalid new variable parameters' do let(:variables_attributes) do [ - variable_attributes.merge(value: 'other_value'), + variable_attributes.merge(secret_value: 'other_value'), new_variable_attributes.merge(key: '...?') ] end @@ -52,7 +52,7 @@ shared_examples 'PATCH #update updates variables' do let(:variables_attributes) do [ new_variable_attributes, - new_variable_attributes.merge(value: 'other_value') + new_variable_attributes.merge(secret_value: 'other_value') ] end @@ -74,7 +74,7 @@ shared_examples 'PATCH #update updates variables' do context 'with valid new variable parameters' do let(:variables_attributes) do [ - variable_attributes.merge(value: 'other_value'), + variable_attributes.merge(secret_value: 'other_value'), new_variable_attributes ] end diff --git a/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb b/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb new file mode 100644 index 00000000000..cd9974cd6e2 --- /dev/null +++ b/spec/support/shared_examples/uploaders/object_storage_shared_examples.rb @@ -0,0 +1,126 @@ +shared_context 'with storage' do |store, **stub_params| + before do + subject.object_store = store + end +end + +shared_examples "migrates" do |to_store:, from_store: nil| + let(:to) { to_store } + let(:from) { from_store || subject.object_store } + + def migrate(to) + subject.migrate!(to) + end + + def checksum + Digest::SHA256.hexdigest(subject.read) + end + + before do + migrate(from) + end + + it 'returns corresponding file type' do + expect(subject).to be_an(CarrierWave::Uploader::Base) + expect(subject).to be_a(ObjectStorage::Concern) + + if from == described_class::Store::REMOTE + expect(subject.file).to be_a(CarrierWave::Storage::Fog::File) + elsif from == described_class::Store::LOCAL + expect(subject.file).to be_a(CarrierWave::SanitizedFile) + else + raise 'Unexpected file type' + end + end + + it 'does nothing when migrating to the current store' do + expect { migrate(from) }.not_to change { subject.object_store }.from(from) + end + + it 'migrate to the specified store' do + from_checksum = checksum + + expect { migrate(to) }.to change { subject.object_store }.from(from).to(to) + expect(checksum).to eq(from_checksum) + end + + it 'removes the original file after the migration' do + original_file = subject.file.path + migrate(to) + + expect(File.exist?(original_file)).to be_falsey + end + + it 'can access to the original file during migration' do + file = subject.file + + allow(subject).to receive(:delete_migrated_file) { } # Remove as a callback of :migrate + allow(subject).to receive(:record_upload) { } # Remove as a callback of :store (:record_upload) + + expect(file.exists?).to be_truthy + expect { migrate(to) }.not_to change { file.exists? } + end + + context 'when migrate! is not oqqupied by another process' do + it 'executes migrate!' do + expect(subject).to receive(:object_store=).at_least(1) + + migrate(to) + end + end + + context 'when migrate! is occupied by another process' do + let(:exclusive_lease_key) { "object_storage_migrate:#{subject.model.class}:#{subject.model.id}" } + + before do + @uuid = Gitlab::ExclusiveLease.new(exclusive_lease_key, timeout: 1.hour.to_i).try_obtain + end + + it 'does not execute migrate!' do + expect(subject).not_to receive(:unsafe_migrate!) + + expect { migrate(to) }.to raise_error('Already running') + end + + after do + Gitlab::ExclusiveLease.cancel(exclusive_lease_key, @uuid) + end + end + + context 'migration is unsuccessful' do + shared_examples "handles gracefully" do |error:| + it 'does not update the object_store' do + expect { migrate(to) }.to raise_error(error) + expect(subject.object_store).to eq(from) + end + + it 'does not delete the original file' do + expect { migrate(to) }.to raise_error(error) + expect(subject.exists?).to be_truthy + end + end + + context 'when the store is not supported' do + let(:to) { -1 } # not a valid store + + include_examples "handles gracefully", error: ObjectStorage::UnknownStoreError + end + + context 'upon a fog failure' do + before do + storage_class = subject.send(:storage_for, to).class + expect_any_instance_of(storage_class).to receive(:store!).and_raise("Store failure.") + end + + include_examples "handles gracefully", error: "Store failure." + end + + context 'upon a database failure' do + before do + expect(uploader).to receive(:persist_object_store!).and_raise("ActiveRecord failure.") + end + + include_examples "handles gracefully", error: "ActiveRecord failure." + end + end +end diff --git a/spec/support/stored_repositories.rb b/spec/support/stored_repositories.rb index 52e47ae2d34..21995c89a6e 100644 --- a/spec/support/stored_repositories.rb +++ b/spec/support/stored_repositories.rb @@ -4,7 +4,7 @@ RSpec.configure do |config| end config.before(:all, :broken_storage) do - FileUtils.rm_rf Gitlab.config.repositories.storages.broken['path'] + FileUtils.rm_rf Gitlab.config.repositories.storages.broken.legacy_disk_path end config.before(:each, :broken_storage) do diff --git a/spec/support/stub_configuration.rb b/spec/support/stub_configuration.rb index 9f08c139322..bad1d34df3a 100644 --- a/spec/support/stub_configuration.rb +++ b/spec/support/stub_configuration.rb @@ -50,8 +50,12 @@ module StubConfiguration # Default storage is always required messages['default'] ||= Gitlab.config.repositories.storages.default - messages.each do |storage_name, storage_settings| - storage_settings['path'] = TestEnv.repos_path unless storage_settings.key?('path') + messages.each do |storage_name, storage_hash| + if !storage_hash.key?('path') || storage_hash['path'] == Gitlab::GitalyClient::StorageSettings::Deprecated + storage_hash['path'] = TestEnv.repos_path + end + + messages[storage_name] = Gitlab::GitalyClient::StorageSettings.new(storage_hash.to_h) end allow(Gitlab.config.repositories).to receive(:storages).and_return(Settingslogic.new(messages)) diff --git a/spec/support/stub_object_storage.rb b/spec/support/stub_object_storage.rb new file mode 100644 index 00000000000..1a0a2feb27d --- /dev/null +++ b/spec/support/stub_object_storage.rb @@ -0,0 +1,43 @@ +module StubConfiguration + def stub_object_storage_uploader( + config:, uploader:, remote_directory:, + enabled: true, + proxy_download: false, + background_upload: false) + Fog.mock! + + allow(config).to receive(:enabled) { enabled } + allow(config).to receive(:proxy_download) { proxy_download } + allow(config).to receive(:background_upload) { background_upload } + + return unless enabled + + ::Fog::Storage.new(uploader.object_store_credentials).tap do |connection| + begin + connection.directories.create(key: remote_directory) + rescue Excon::Error::Conflict + end + end + end + + def stub_artifacts_object_storage(**params) + stub_object_storage_uploader(config: Gitlab.config.artifacts.object_store, + uploader: JobArtifactUploader, + remote_directory: 'artifacts', + **params) + end + + def stub_lfs_object_storage(**params) + stub_object_storage_uploader(config: Gitlab.config.lfs.object_store, + uploader: LfsObjectUploader, + remote_directory: 'lfs-objects', + **params) + end + + def stub_uploads_object_storage(uploader = described_class, **params) + stub_object_storage_uploader(config: Gitlab.config.uploads.object_store, + uploader: uploader, + remote_directory: 'uploads', + **params) + end +end diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb index 01321989f01..f14e69b1041 100644 --- a/spec/support/test_env.rb +++ b/spec/support/test_env.rb @@ -225,7 +225,7 @@ module TestEnv end def repos_path - Gitlab.config.repositories.storages.default['path'] + Gitlab.config.repositories.storages.default.legacy_disk_path end def backup_path diff --git a/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb new file mode 100644 index 00000000000..8544fb62b5a --- /dev/null +++ b/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb @@ -0,0 +1,118 @@ +require 'rake_helper' + +describe 'gitlab:artifacts namespace rake task' do + before(:context) do + Rake.application.rake_require 'tasks/gitlab/artifacts/migrate' + end + + let(:object_storage_enabled) { false } + + before do + stub_artifacts_object_storage(enabled: object_storage_enabled) + end + + subject { run_rake_task('gitlab:artifacts:migrate') } + + context 'legacy artifacts' do + describe 'migrate' do + let!(:build) { create(:ci_build, :legacy_artifacts, artifacts_file_store: store, artifacts_metadata_store: store) } + + context 'when local storage is used' do + let(:store) { ObjectStorage::Store::LOCAL } + + context 'and job does not have file store defined' do + let(:object_storage_enabled) { true } + let(:store) { nil } + + it "migrates file to remote storage" do + subject + + expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE) + expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is defined' do + let(:object_storage_enabled) { true } + + it "migrates file to remote storage" do + subject + + expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE) + expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is not defined' do + it "fails to migrate to remote storage" do + subject + + expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::LOCAL) + expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::LOCAL) + end + end + end + + context 'when remote storage is used' do + let(:object_storage_enabled) { true } + + let(:store) { ObjectStorage::Store::REMOTE } + + it "file stays on remote storage" do + subject + + expect(build.reload.artifacts_file_store).to eq(ObjectStorage::Store::REMOTE) + expect(build.reload.artifacts_metadata_store).to eq(ObjectStorage::Store::REMOTE) + end + end + end + end + + context 'job artifacts' do + let!(:artifact) { create(:ci_job_artifact, :archive, file_store: store) } + + context 'when local storage is used' do + let(:store) { ObjectStorage::Store::LOCAL } + + context 'and job does not have file store defined' do + let(:object_storage_enabled) { true } + let(:store) { nil } + + it "migrates file to remote storage" do + subject + + expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is defined' do + let(:object_storage_enabled) { true } + + it "migrates file to remote storage" do + subject + + expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end + + context 'and remote storage is not defined' do + it "fails to migrate to remote storage" do + subject + + expect(artifact.reload.file_store).to eq(ObjectStorage::Store::LOCAL) + end + end + end + + context 'when remote storage is used' do + let(:object_storage_enabled) { true } + let(:store) { ObjectStorage::Store::REMOTE } + + it "file stays on remote storage" do + subject + + expect(artifact.reload.file_store).to eq(ObjectStorage::Store::REMOTE) + end + end + end +end diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 168facd51a6..0d24782f317 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -195,14 +195,23 @@ describe 'gitlab:app namespace rake task' do end context 'multiple repository storages' do - let(:gitaly_address) { Gitlab.config.repositories.storages.default.gitaly_address } + let(:storage_default) do + Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage')) + end + let(:test_second_storage) do + Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/custom_storage')) + end let(:storages) do { - 'default' => { 'path' => Settings.absolute('tmp/tests/default_storage'), 'gitaly_address' => gitaly_address }, - 'test_second_storage' => { 'path' => Settings.absolute('tmp/tests/custom_storage'), 'gitaly_address' => gitaly_address } + 'default' => storage_default, + 'test_second_storage' => test_second_storage } end + before(:all) do + @default_storage_hash = Gitlab.config.repositories.storages.default.to_h + end + before do # We only need a backup of the repositories for this test stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry') diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb index 9e746ceddd6..2bf873c923f 100644 --- a/spec/tasks/gitlab/cleanup_rake_spec.rb +++ b/spec/tasks/gitlab/cleanup_rake_spec.rb @@ -6,13 +6,16 @@ describe 'gitlab:cleanup rake tasks' do end describe 'cleanup' do - let(:gitaly_address) { Gitlab.config.repositories.storages.default.gitaly_address } let(:storages) do { - 'default' => { 'path' => Settings.absolute('tmp/tests/default_storage'), 'gitaly_address' => gitaly_address } + 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage')) } end + before(:all) do + @default_storage_hash = Gitlab.config.repositories.storages.default.to_h + end + before do FileUtils.mkdir(Settings.absolute('tmp/tests/default_storage')) allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) diff --git a/spec/tasks/gitlab/git_rake_spec.rb b/spec/tasks/gitlab/git_rake_spec.rb index 9aebf7b0b4a..1efaecc63a5 100644 --- a/spec/tasks/gitlab/git_rake_spec.rb +++ b/spec/tasks/gitlab/git_rake_spec.rb @@ -1,10 +1,13 @@ require 'rake_helper' describe 'gitlab:git rake tasks' do + before(:all) do + @default_storage_hash = Gitlab.config.repositories.storages.default.to_h + end + before do Rake.application.rake_require 'tasks/gitlab/git' - - storages = { 'default' => { 'path' => Settings.absolute('tmp/tests/default_storage') } } + storages = { 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage')) } FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git')) allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) diff --git a/spec/tasks/gitlab/gitaly_rake_spec.rb b/spec/tasks/gitlab/gitaly_rake_spec.rb index 1f4053ff9ad..1e507c0236e 100644 --- a/spec/tasks/gitlab/gitaly_rake_spec.rb +++ b/spec/tasks/gitlab/gitaly_rake_spec.rb @@ -99,14 +99,14 @@ describe 'gitlab:gitaly namespace rake task' do describe 'storage_config' do it 'prints storage configuration in a TOML format' do config = { - 'default' => { + 'default' => Gitlab::GitalyClient::StorageSettings.new( 'path' => '/path/to/default', 'gitaly_address' => 'unix:/path/to/my.socket' - }, - 'nfs_01' => { + ), + 'nfs_01' => Gitlab::GitalyClient::StorageSettings.new( 'path' => '/path/to/nfs_01', 'gitaly_address' => 'unix:/path/to/my.socket' - } + ) } allow(Gitlab.config.repositories).to receive(:storages).and_return(config) allow(Rails.env).to receive(:test?).and_return(false) @@ -134,7 +134,7 @@ describe 'gitlab:gitaly namespace rake task' do parsed_output = TomlRB.parse(expected_output) config.each do |name, params| - expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params['path'] }) + expect(parsed_output['storage']).to include({ 'name' => name, 'path' => params.legacy_disk_path }) end end end diff --git a/spec/tasks/gitlab/lfs/migrate_rake_spec.rb b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb new file mode 100644 index 00000000000..66d1a192a96 --- /dev/null +++ b/spec/tasks/gitlab/lfs/migrate_rake_spec.rb @@ -0,0 +1,37 @@ +require 'rake_helper' + +describe 'gitlab:lfs namespace rake task' do + before :all do + Rake.application.rake_require 'tasks/gitlab/lfs/migrate' + end + + describe 'migrate' do + let(:local) { ObjectStorage::Store::LOCAL } + let(:remote) { ObjectStorage::Store::REMOTE } + let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) } + + def lfs_migrate + run_rake_task('gitlab:lfs:migrate') + end + + context 'object storage disabled' do + before do + stub_lfs_object_storage(enabled: false) + end + + it "doesn't migrate files" do + expect { lfs_migrate }.not_to change { lfs_object.reload.file_store } + end + end + + context 'object storage enabled' do + before do + stub_lfs_object_storage + end + + it 'migrates local file to object storage' do + expect { lfs_migrate }.to change { lfs_object.reload.file_store }.from(local).to(remote) + end + end + end +end diff --git a/spec/tasks/gitlab/shell_rake_spec.rb b/spec/tasks/gitlab/shell_rake_spec.rb index 65155cb044d..4a756c5742d 100644 --- a/spec/tasks/gitlab/shell_rake_spec.rb +++ b/spec/tasks/gitlab/shell_rake_spec.rb @@ -11,7 +11,7 @@ describe 'gitlab:shell rake tasks' do it 'invokes create_hooks task' do expect(Rake::Task['gitlab:shell:create_hooks']).to receive(:invoke) - storages = Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } + storages = Gitlab.config.repositories.storages.values.map(&:legacy_disk_path) expect(Kernel).to receive(:system).with('bin/install', *storages).and_call_original expect(Kernel).to receive(:system).with('bin/compile').and_call_original diff --git a/spec/tasks/gitlab/uploads/migrate_rake_spec.rb b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb new file mode 100644 index 00000000000..b778d26060d --- /dev/null +++ b/spec/tasks/gitlab/uploads/migrate_rake_spec.rb @@ -0,0 +1,28 @@ +require 'rake_helper' + +describe 'gitlab:uploads:migrate rake tasks' do + let!(:projects) { create_list(:project, 10, :with_avatar) } + let(:model_class) { Project } + let(:uploader_class) { AvatarUploader } + let(:mounted_as) { :avatar } + let(:batch_size) { 3 } + + before do + stub_env('BATCH', batch_size.to_s) + stub_uploads_object_storage(uploader_class) + Rake.application.rake_require 'tasks/gitlab/uploads/migrate' + + allow(ObjectStorage::MigrateUploadsWorker).to receive(:perform_async) + end + + def run + args = [uploader_class.to_s, model_class.to_s, mounted_as].compact + run_rake_task("gitlab:uploads:migrate", *args) + end + + it 'enqueue jobs in batch' do + expect(ObjectStorage::MigrateUploadsWorker).to receive(:enqueue!).exactly(4).times + + run + end +end diff --git a/spec/uploaders/attachment_uploader_spec.rb b/spec/uploaders/attachment_uploader_spec.rb index 091ba824fc6..d302c14efb9 100644 --- a/spec/uploaders/attachment_uploader_spec.rb +++ b/spec/uploaders/attachment_uploader_spec.rb @@ -11,4 +11,26 @@ describe AttachmentUploader do store_dir: %r[uploads/-/system/note/attachment/], upload_path: %r[uploads/-/system/note/attachment/], absolute_path: %r[#{CarrierWave.root}/uploads/-/system/note/attachment/] + + context "object_store is REMOTE" do + before do + stub_uploads_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like 'builds correct paths', + store_dir: %r[note/attachment/], + upload_path: %r[note/attachment/] + end + + describe "#migrate!" do + before do + uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt'))) + stub_uploads_object_storage + end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL + end end diff --git a/spec/uploaders/avatar_uploader_spec.rb b/spec/uploaders/avatar_uploader_spec.rb index bf9028c9260..b0468bc35ff 100644 --- a/spec/uploaders/avatar_uploader_spec.rb +++ b/spec/uploaders/avatar_uploader_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe AvatarUploader do - let(:model) { create(:user, :with_avatar) } + let(:model) { build_stubbed(:user) } let(:uploader) { described_class.new(model, :avatar) } let(:upload) { create(:upload, model: model) } @@ -12,15 +12,28 @@ describe AvatarUploader do upload_path: %r[uploads/-/system/user/avatar/], absolute_path: %r[#{CarrierWave.root}/uploads/-/system/user/avatar/] - describe '#move_to_cache' do - it 'is false' do - expect(uploader.move_to_cache).to eq(false) + context "object_store is REMOTE" do + before do + stub_uploads_object_storage end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like 'builds correct paths', + store_dir: %r[user/avatar/], + upload_path: %r[user/avatar/] end - describe '#move_to_store' do - it 'is false' do - expect(uploader.move_to_store).to eq(false) + context "with a file" do + let(:project) { create(:project, :with_avatar) } + let(:uploader) { project.avatar } + let(:upload) { uploader.upload } + + before do + stub_uploads_object_storage end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL end end diff --git a/spec/uploaders/file_mover_spec.rb b/spec/uploaders/file_mover_spec.rb index bc024cd307c..68b7e24776d 100644 --- a/spec/uploaders/file_mover_spec.rb +++ b/spec/uploaders/file_mover_spec.rb @@ -36,6 +36,12 @@ describe FileMover do it 'creates a new update record' do expect { subject }.to change { Upload.count }.by(1) end + + it 'schedules a background migration' do + expect_any_instance_of(PersonalFileUploader).to receive(:schedule_background_upload).once + + subject + end end context 'when update_markdown fails' do diff --git a/spec/uploaders/file_uploader_spec.rb b/spec/uploaders/file_uploader_spec.rb index b42ce982b27..db2810bbe1d 100644 --- a/spec/uploaders/file_uploader_spec.rb +++ b/spec/uploaders/file_uploader_spec.rb @@ -11,32 +11,41 @@ describe FileUploader do shared_examples 'builds correct legacy storage paths' do include_examples 'builds correct paths', store_dir: %r{awesome/project/\h+}, + upload_path: %r{\h+/<filename>}, absolute_path: %r{#{described_class.root}/awesome/project/secret/foo.jpg} end - shared_examples 'uses hashed storage' do - context 'when rolled out attachments' do - let(:project) { build_stubbed(:project, namespace: group, name: 'project') } + context 'legacy storage' do + it_behaves_like 'builds correct legacy storage paths' - before do - allow(project).to receive(:disk_path).and_return('ca/fe/fe/ed') - end + context 'uses hashed storage' do + context 'when rolled out attachments' do + let(:project) { build_stubbed(:project, namespace: group, name: 'project') } - it_behaves_like 'builds correct paths', - store_dir: %r{ca/fe/fe/ed/\h+}, - absolute_path: %r{#{described_class.root}/ca/fe/fe/ed/secret/foo.jpg} - end + include_examples 'builds correct paths', + store_dir: %r{@hashed/\h{2}/\h{2}/\h+}, + upload_path: %r{\h+/<filename>} + end - context 'when only repositories are rolled out' do - let(:project) { build_stubbed(:project, namespace: group, name: 'project', storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) } + context 'when only repositories are rolled out' do + let(:project) { build_stubbed(:project, namespace: group, name: 'project', storage_version: Project::HASHED_STORAGE_FEATURES[:repository]) } - it_behaves_like 'builds correct legacy storage paths' + it_behaves_like 'builds correct legacy storage paths' + end end end - context 'legacy storage' do - it_behaves_like 'builds correct legacy storage paths' - include_examples 'uses hashed storage' + context 'object store is remote' do + before do + stub_uploads_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + # always use hashed storage path for remote uploads + it_behaves_like 'builds correct paths', + store_dir: %r{@hashed/\h{2}/\h{2}/\h+}, + upload_path: %r{@hashed/\h{2}/\h{2}/\h+/\h+/<filename>} end describe 'initialize' do @@ -78,6 +87,16 @@ describe FileUploader do end end + describe "#migrate!" do + before do + uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/dk.png'))) + stub_uploads_object_storage + end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL + end + describe '#upload=' do let(:secret) { SecureRandom.hex } let(:upload) { create(:upload, :issuable_upload, secret: secret, filename: 'file.txt') } @@ -93,15 +112,5 @@ describe FileUploader do uploader.upload = upload end - - context 'uploader_context is empty' do - it 'fallbacks to regex based extraction' do - expect(upload).to receive(:uploader_context).and_return({}) - - uploader.upload = upload - expect(uploader.secret).to eq(secret) - expect(uploader.instance_variable_get(:@identifier)).to eq('file.txt') - end - end end end diff --git a/spec/uploaders/job_artifact_uploader_spec.rb b/spec/uploaders/job_artifact_uploader_spec.rb index 5612ec7e661..42036d67f3d 100644 --- a/spec/uploaders/job_artifact_uploader_spec.rb +++ b/spec/uploaders/job_artifact_uploader_spec.rb @@ -1,7 +1,8 @@ require 'spec_helper' describe JobArtifactUploader do - let(:job_artifact) { create(:ci_job_artifact) } + let(:store) { described_class::Store::LOCAL } + let(:job_artifact) { create(:ci_job_artifact, file_store: store) } let(:uploader) { described_class.new(job_artifact, :file) } subject { uploader } @@ -11,6 +12,17 @@ describe JobArtifactUploader do cache_dir: %r[artifacts/tmp/cache], work_dir: %r[artifacts/tmp/work] + context "object store is REMOTE" do + before do + stub_artifacts_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like "builds correct paths", + store_dir: %r[\h{2}/\h{2}/\h{64}/\d{4}_\d{1,2}_\d{1,2}/\d+/\d+\z] + end + describe '#open' do subject { uploader.open } @@ -36,6 +48,17 @@ describe JobArtifactUploader do end end end + + context 'when trace is stored in Object storage' do + before do + allow(uploader).to receive(:file_storage?) { false } + allow(uploader).to receive(:url) { 'http://object_storage.com/trace' } + end + + it 'returns http io stream' do + is_expected.to be_a(Gitlab::Ci::Trace::HttpIO) + end + end end context 'file is stored in valid local_path' do @@ -55,4 +78,14 @@ describe JobArtifactUploader do it { is_expected.to include("/#{job_artifact.job_id}/#{job_artifact.id}/") } it { is_expected.to end_with("ci_build_artifacts.zip") } end + + describe "#migrate!" do + before do + uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/trace/sample_trace'))) + stub_artifacts_object_storage + end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL + end end diff --git a/spec/uploaders/legacy_artifact_uploader_spec.rb b/spec/uploaders/legacy_artifact_uploader_spec.rb index 54c6a8b869b..eeb6fd90c9d 100644 --- a/spec/uploaders/legacy_artifact_uploader_spec.rb +++ b/spec/uploaders/legacy_artifact_uploader_spec.rb @@ -1,7 +1,8 @@ require 'rails_helper' describe LegacyArtifactUploader do - let(:job) { create(:ci_build) } + let(:store) { described_class::Store::LOCAL } + let(:job) { create(:ci_build, artifacts_file_store: store) } let(:uploader) { described_class.new(job, :legacy_artifacts_file) } let(:local_path) { described_class.root } @@ -20,6 +21,17 @@ describe LegacyArtifactUploader do cache_dir: %r[artifacts/tmp/cache], work_dir: %r[artifacts/tmp/work] + context 'object store is remote' do + before do + stub_artifacts_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like "builds correct paths", + store_dir: %r[\d{4}_\d{1,2}/\d+/\d+\z] + end + describe '#filename' do # we need to use uploader, as this makes to use mounter # which initialises uploader.file object diff --git a/spec/uploaders/lfs_object_uploader_spec.rb b/spec/uploaders/lfs_object_uploader_spec.rb index 6ebc885daa8..a2fb3886610 100644 --- a/spec/uploaders/lfs_object_uploader_spec.rb +++ b/spec/uploaders/lfs_object_uploader_spec.rb @@ -11,4 +11,62 @@ describe LfsObjectUploader do store_dir: %r[\h{2}/\h{2}], cache_dir: %r[/lfs-objects/tmp/cache], work_dir: %r[/lfs-objects/tmp/work] + + context "object store is REMOTE" do + before do + stub_lfs_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like "builds correct paths", + store_dir: %r[\h{2}/\h{2}] + end + + describe 'migration to object storage' do + context 'with object storage disabled' do + it "is skipped" do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + lfs_object + end + end + + context 'with object storage enabled' do + before do + stub_lfs_object_storage(background_upload: true) + end + + it 'is scheduled to run after creation' do + expect(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async).with(described_class.name, 'LfsObject', :file, kind_of(Numeric)) + + lfs_object + end + end + end + + describe 'remote file' do + let(:remote) { described_class::Store::REMOTE } + let(:lfs_object) { create(:lfs_object, file_store: remote) } + + context 'with object storage enabled' do + before do + stub_lfs_object_storage + end + + it 'can store file remotely' do + allow(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async) + + store_file(lfs_object) + + expect(lfs_object.file_store).to eq remote + expect(lfs_object.file.path).not_to be_blank + end + end + end + + def store_file(lfs_object) + lfs_object.file = fixture_file_upload(Rails.root.join("spec/fixtures/dk.png"), "`/png") + lfs_object.save! + end end diff --git a/spec/uploaders/namespace_file_uploader_spec.rb b/spec/uploaders/namespace_file_uploader_spec.rb index 24a2fc0f72e..a8ba01d70b8 100644 --- a/spec/uploaders/namespace_file_uploader_spec.rb +++ b/spec/uploaders/namespace_file_uploader_spec.rb @@ -13,4 +13,26 @@ describe NamespaceFileUploader do store_dir: %r[uploads/-/system/namespace/\d+], upload_path: IDENTIFIER, absolute_path: %r[#{CarrierWave.root}/uploads/-/system/namespace/\d+/#{IDENTIFIER}] + + context "object_store is REMOTE" do + before do + stub_uploads_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like 'builds correct paths', + store_dir: %r[namespace/\d+/\h+], + upload_path: IDENTIFIER + end + + describe "#migrate!" do + before do + uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt'))) + stub_uploads_object_storage + end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL + end end diff --git a/spec/uploaders/object_storage_spec.rb b/spec/uploaders/object_storage_spec.rb new file mode 100644 index 00000000000..489b6707c6e --- /dev/null +++ b/spec/uploaders/object_storage_spec.rb @@ -0,0 +1,326 @@ +require 'rails_helper' +require 'carrierwave/storage/fog' + +class Implementation < GitlabUploader + include ObjectStorage::Concern + include ::RecordsUploads::Concern + prepend ::ObjectStorage::Extension::RecordsUploads + + storage_options Gitlab.config.uploads + + private + + # user/:id + def dynamic_segment + File.join(model.class.to_s.underscore, model.id.to_s) + end +end + +describe ObjectStorage do + let(:uploader_class) { Implementation } + let(:object) { build_stubbed(:user) } + let(:uploader) { uploader_class.new(object, :file) } + + before do + allow(uploader_class).to receive(:object_store_enabled?).and_return(true) + end + + describe '#object_store=' do + it "reload the local storage" do + uploader.object_store = described_class::Store::LOCAL + expect(uploader.file_storage?).to be_truthy + end + + it "reload the REMOTE storage" do + uploader.object_store = described_class::Store::REMOTE + expect(uploader.file_storage?).to be_falsey + end + end + + context 'object_store is Store::LOCAL' do + before do + uploader.object_store = described_class::Store::LOCAL + end + + describe '#store_dir' do + it 'is the composition of (base_dir, dynamic_segment)' do + expect(uploader.store_dir).to start_with("uploads/-/system/user/") + end + end + end + + context 'object_store is Store::REMOTE' do + before do + uploader.object_store = described_class::Store::REMOTE + end + + describe '#store_dir' do + it 'is the composition of (dynamic_segment)' do + expect(uploader.store_dir).to start_with("user/") + end + end + end + + describe '#object_store' do + it "delegates to <mount>_store on model" do + expect(object).to receive(:file_store) + + uploader.object_store + end + + context 'when store is null' do + before do + expect(object).to receive(:file_store).and_return(nil) + end + + it "returns Store::LOCAL" do + expect(uploader.object_store).to eq(described_class::Store::LOCAL) + end + end + + context 'when value is set' do + before do + expect(object).to receive(:file_store).and_return(described_class::Store::REMOTE) + end + + it "returns the given value" do + expect(uploader.object_store).to eq(described_class::Store::REMOTE) + end + end + end + + describe '#file_cache_storage?' do + context 'when file storage is used' do + before do + uploader_class.cache_storage(:file) + end + + it { expect(uploader).to be_file_cache_storage } + end + + context 'when is remote storage' do + before do + uploader_class.cache_storage(:fog) + end + + it { expect(uploader).not_to be_file_cache_storage } + end + end + + # this means the model shall include + # include RecordsUpload::Concern + # prepend ObjectStorage::Extension::RecordsUploads + # the object_store persistence is delegated to the `Upload` model. + # + context 'when persist_object_store? is false' do + let(:object) { create(:project, :with_avatar) } + let(:uploader) { object.avatar } + + it { expect(object).to be_a(Avatarable) } + it { expect(uploader.persist_object_store?).to be_falsey } + + describe 'delegates the object_store logic to the `Upload` model' do + it 'sets @upload to the found `upload`' do + expect(uploader.upload).to eq(uploader.upload) + end + + it 'sets @object_store to the `Upload` value' do + expect(uploader.object_store).to eq(uploader.upload.store) + end + end + + describe '#migrate!' do + let(:new_store) { ObjectStorage::Store::REMOTE } + + before do + stub_uploads_object_storage(uploader: AvatarUploader) + end + + subject { uploader.migrate!(new_store) } + + it 'persist @object_store to the recorded upload' do + subject + + expect(uploader.upload.store).to eq(new_store) + end + + describe 'fails' do + it 'is handled gracefully' do + store = uploader.object_store + expect_any_instance_of(Upload).to receive(:save!).and_raise("An error") + + expect { subject }.to raise_error("An error") + expect(uploader.exists?).to be_truthy + expect(uploader.upload.store).to eq(store) + end + end + end + end + + # this means the model holds an <mounted_as>_store attribute directly + # and do not delegate the object_store persistence to the `Upload` model. + # + context 'persist_object_store? is true' do + context 'when using JobArtifactsUploader' do + let(:store) { described_class::Store::LOCAL } + let(:object) { create(:ci_job_artifact, :archive, file_store: store) } + let(:uploader) { object.file } + + context 'checking described_class' do + it "uploader include described_class::Concern" do + expect(uploader).to be_a(described_class::Concern) + end + end + + describe '#use_file' do + context 'when file is stored locally' do + it "calls a regular path" do + expect { |b| uploader.use_file(&b) }.not_to yield_with_args(%r[tmp/cache]) + end + end + + context 'when file is stored remotely' do + let(:store) { described_class::Store::REMOTE } + + before do + stub_artifacts_object_storage + end + + it "calls a cache path" do + expect { |b| uploader.use_file(&b) }.to yield_with_args(%r[tmp/cache]) + end + end + end + + describe '#migrate!' do + subject { uploader.migrate!(new_store) } + + shared_examples "updates the underlying <mounted>_store" do + it do + subject + + expect(object.file_store).to eq(new_store) + end + end + + context 'when using the same storage' do + let(:new_store) { store } + + it "to not migrate the storage" do + subject + + expect(uploader).not_to receive(:store!) + expect(uploader.object_store).to eq(store) + end + end + + context 'when migrating to local storage' do + let(:store) { described_class::Store::REMOTE } + let(:new_store) { described_class::Store::LOCAL } + + before do + stub_artifacts_object_storage + end + + include_examples "updates the underlying <mounted>_store" + + it "local file does not exist" do + expect(File.exist?(uploader.path)).to eq(false) + end + + it "remote file exist" do + expect(uploader.file.exists?).to be_truthy + end + + it "does migrate the file" do + subject + + expect(uploader.object_store).to eq(new_store) + expect(File.exist?(uploader.path)).to eq(true) + end + end + + context 'when migrating to remote storage' do + let(:new_store) { described_class::Store::REMOTE } + let!(:current_path) { uploader.path } + + it "file does exist" do + expect(File.exist?(current_path)).to eq(true) + end + + context 'when storage is disabled' do + before do + stub_artifacts_object_storage(enabled: false) + end + + it "to raise an error" do + expect { subject }.to raise_error(/Object Storage is not enabled/) + end + end + + context 'when credentials are set' do + before do + stub_artifacts_object_storage + end + + include_examples "updates the underlying <mounted>_store" + + it "does migrate the file" do + subject + + expect(uploader.object_store).to eq(new_store) + end + + it "does delete original file" do + subject + + expect(File.exist?(current_path)).to eq(false) + end + + context 'when subject save fails' do + before do + expect(uploader).to receive(:persist_object_store!).and_raise(RuntimeError, "exception") + end + + it "original file is not removed" do + expect { subject }.to raise_error(/exception/) + + expect(File.exist?(current_path)).to eq(true) + end + end + end + end + end + end + end + + describe '#fog_directory' do + let(:remote_directory) { 'directory' } + + before do + uploader_class.storage_options double(object_store: double(remote_directory: remote_directory)) + end + + subject { uploader.fog_directory } + + it { is_expected.to eq(remote_directory) } + end + + describe '#fog_credentials' do + let(:connection) { Settingslogic.new("provider" => "AWS") } + + before do + uploader_class.storage_options double(object_store: double(connection: connection)) + end + + subject { uploader.fog_credentials } + + it { is_expected.to eq(provider: 'AWS') } + end + + describe '#fog_public' do + subject { uploader.fog_public } + + it { is_expected.to eq(false) } + end +end diff --git a/spec/uploaders/personal_file_uploader_spec.rb b/spec/uploaders/personal_file_uploader_spec.rb index ed1fba6edda..c70521d90dc 100644 --- a/spec/uploaders/personal_file_uploader_spec.rb +++ b/spec/uploaders/personal_file_uploader_spec.rb @@ -14,6 +14,18 @@ describe PersonalFileUploader do upload_path: IDENTIFIER, absolute_path: %r[#{CarrierWave.root}/uploads/-/system/personal_snippet/\d+/#{IDENTIFIER}] + context "object_store is REMOTE" do + before do + stub_uploads_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like 'builds correct paths', + store_dir: %r[\d+/\h+], + upload_path: IDENTIFIER + end + describe '#to_h' do before do subject.instance_variable_set(:@secret, 'secret') @@ -30,4 +42,14 @@ describe PersonalFileUploader do ) end end + + describe "#migrate!" do + before do + uploader.store!(fixture_file_upload(Rails.root.join('spec/fixtures/doc_sample.txt'))) + stub_uploads_object_storage + end + + it_behaves_like "migrates", to_store: described_class::Store::REMOTE + it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL + end end diff --git a/spec/views/ci/lints/show.html.haml_spec.rb b/spec/views/ci/lints/show.html.haml_spec.rb index 7724d54c569..ded320793ea 100644 --- a/spec/views/ci/lints/show.html.haml_spec.rb +++ b/spec/views/ci/lints/show.html.haml_spec.rb @@ -5,6 +5,7 @@ describe 'ci/lints/show' do describe 'XSS protection' do let(:config_processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(content)) } + before do assign(:status, true) assign(:builds, config_processor.builds) diff --git a/spec/workers/object_storage_upload_worker_spec.rb b/spec/workers/object_storage_upload_worker_spec.rb new file mode 100644 index 00000000000..32ddcbe9757 --- /dev/null +++ b/spec/workers/object_storage_upload_worker_spec.rb @@ -0,0 +1,108 @@ +require 'spec_helper' + +describe ObjectStorageUploadWorker do + let(:local) { ObjectStorage::Store::LOCAL } + let(:remote) { ObjectStorage::Store::REMOTE } + + def perform + described_class.perform_async(uploader_class.name, subject_class, file_field, subject_id) + end + + context 'for LFS' do + let!(:lfs_object) { create(:lfs_object, :with_file, file_store: local) } + let(:uploader_class) { LfsObjectUploader } + let(:subject_class) { LfsObject } + let(:file_field) { :file } + let(:subject_id) { lfs_object.id } + + context 'when object storage is enabled' do + before do + stub_lfs_object_storage(background_upload: true) + end + + it 'uploads object to storage' do + expect { perform }.to change { lfs_object.reload.file_store }.from(local).to(remote) + end + + context 'when background upload is disabled' do + before do + allow(Gitlab.config.lfs.object_store).to receive(:background_upload) { false } + end + + it 'is skipped' do + expect { perform }.not_to change { lfs_object.reload.file_store } + end + end + end + + context 'when object storage is disabled' do + before do + stub_lfs_object_storage(enabled: false) + end + + it "doesn't migrate files" do + perform + + expect(lfs_object.reload.file_store).to eq(local) + end + end + end + + context 'for legacy artifacts' do + let(:build) { create(:ci_build, :legacy_artifacts) } + let(:uploader_class) { LegacyArtifactUploader } + let(:subject_class) { Ci::Build } + let(:file_field) { :artifacts_file } + let(:subject_id) { build.id } + + context 'when local storage is used' do + let(:store) { local } + + context 'and remote storage is defined' do + before do + stub_artifacts_object_storage(background_upload: true) + end + + it "migrates file to remote storage" do + perform + + expect(build.reload.artifacts_file_store).to eq(remote) + end + + context 'for artifacts_metadata' do + let(:file_field) { :artifacts_metadata } + + it 'migrates metadata to remote storage' do + perform + + expect(build.reload.artifacts_metadata_store).to eq(remote) + end + end + end + end + end + + context 'for job artifacts' do + let(:artifact) { create(:ci_job_artifact, :archive) } + let(:uploader_class) { JobArtifactUploader } + let(:subject_class) { Ci::JobArtifact } + let(:file_field) { :file } + let(:subject_id) { artifact.id } + + context 'when local storage is used' do + let(:store) { local } + + context 'and remote storage is defined' do + before do + stub_artifacts_object_storage(background_upload: true) + end + + it "migrates file to remote storage" do + perform + + expect(artifact.reload.file_store).to eq(remote) + end + end + end + end +end diff --git a/yarn.lock b/yarn.lock index c32b66e0669..584951b5da0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4232,7 +4232,7 @@ ignore@^3.2.0: version "3.3.3" resolved "https://registry.yarnpkg.com/ignore/-/ignore-3.3.3.tgz#432352e57accd87ab3110e82d3fea0e47812156d" -ignore@^3.3.5: +ignore@^3.3.5, ignore@^3.3.7: version "3.3.7" resolved "https://registry.yarnpkg.com/ignore/-/ignore-3.3.7.tgz#612289bfb3c220e186a58118618d5be8c1bab021" |
