diff options
34 files changed, 579 insertions, 102 deletions
diff --git a/app/assets/javascripts/boards/components/new_list_dropdown.js b/app/assets/javascripts/boards/components/new_list_dropdown.js index c19c989680d..cf0bb5f5376 100644 --- a/app/assets/javascripts/boards/components/new_list_dropdown.js +++ b/app/assets/javascripts/boards/components/new_list_dropdown.js @@ -1,5 +1,6 @@ /* eslint-disable func-names, no-new, space-before-function-paren, one-var, promise/catch-or-return */ +import axios from '~/lib/utils/axios_utils'; import _ from 'underscore'; import CreateLabelDropdown from '../../create_label'; @@ -28,9 +29,9 @@ gl.issueBoards.newListDropdownInit = () => { $this.glDropdown({ data(term, callback) { - $.get($this.attr('data-list-labels-path')) - .then((resp) => { - callback(resp); + axios.get($this.attr('data-list-labels-path')) + .then(({ data }) => { + callback(data); }); }, renderRow (label) { diff --git a/app/assets/javascripts/dispatcher.js b/app/assets/javascripts/dispatcher.js index aceaffdfcb9..f8082c74943 100644 --- a/app/assets/javascripts/dispatcher.js +++ b/app/assets/javascripts/dispatcher.js @@ -1,15 +1,9 @@ /* eslint-disable func-names, space-before-function-paren, no-var, prefer-arrow-callback, wrap-iife, no-shadow, consistent-return, one-var, one-var-declaration-per-line, camelcase, default-case, no-new, quotes, no-duplicate-case, no-case-declarations, no-fallthrough, max-len */ -import MergeRequest from './merge_request'; import Flash from './flash'; import GfmAutoComplete from './gfm_auto_complete'; -import ZenMode from './zen_mode'; -import initNotes from './init_notes'; -import initIssuableSidebar from './init_issuable_sidebar'; import { convertPermissionToBoolean } from './lib/utils/common_utils'; import GlFieldErrors from './gl_field_errors'; import Shortcuts from './shortcuts'; -import ShortcutsIssuable from './shortcuts_issuable'; -import Diff from './diff'; import SearchAutocomplete from './search_autocomplete'; var Dispatcher; @@ -262,17 +256,10 @@ var Dispatcher; .catch(fail); break; case 'projects:merge_requests:show': - new Diff(); - new ZenMode(); - - initIssuableSidebar(); - initNotes(); - - const mrShowNode = document.querySelector('.merge-request'); - window.mergeRequest = new MergeRequest({ - action: mrShowNode.dataset.mrAction, - }); - shortcut_handler = new ShortcutsIssuable(true); + import('./pages/projects/merge_requests/show') + .then(callDefault) + .catch(fail); + shortcut_handler = true; break; case 'dashboard:activity': import('./pages/dashboard/activity') diff --git a/app/assets/javascripts/gl_form.js b/app/assets/javascripts/gl_form.js index d0f9e6af0f8..d200044b79f 100644 --- a/app/assets/javascripts/gl_form.js +++ b/app/assets/javascripts/gl_form.js @@ -1,5 +1,4 @@ -/* global autosize */ - +import autosize from 'autosize'; import GfmAutoComplete from './gfm_auto_complete'; import dropzoneInput from './dropzone_input'; import textUtils from './lib/utils/text_markdown'; diff --git a/app/assets/javascripts/pages/projects/merge_requests/show/index.js b/app/assets/javascripts/pages/projects/merge_requests/show/index.js new file mode 100644 index 00000000000..c3463c266e3 --- /dev/null +++ b/app/assets/javascripts/pages/projects/merge_requests/show/index.js @@ -0,0 +1,24 @@ +import MergeRequest from '~/merge_request'; +import ZenMode from '~/zen_mode'; +import initNotes from '~/init_notes'; +import initIssuableSidebar from '~/init_issuable_sidebar'; +import ShortcutsIssuable from '~/shortcuts_issuable'; +import Diff from '~/diff'; +import { handleLocationHash } from '~/lib/utils/common_utils'; + +export default () => { + new Diff(); // eslint-disable-line no-new + new ZenMode(); // eslint-disable-line no-new + + initIssuableSidebar(); // eslint-disable-line no-new + initNotes(); // eslint-disable-line no-new + + const mrShowNode = document.querySelector('.merge-request'); + + window.mergeRequest = new MergeRequest({ + action: mrShowNode.dataset.mrAction, + }); + + new ShortcutsIssuable(true); // eslint-disable-line no-new + handleLocationHash(); +}; diff --git a/app/controllers/root_controller.rb b/app/controllers/root_controller.rb index 8acefd58e77..63e5fdb1da5 100644 --- a/app/controllers/root_controller.rb +++ b/app/controllers/root_controller.rb @@ -13,10 +13,7 @@ class RootController < Dashboard::ProjectsController before_action :redirect_logged_user, if: -> { current_user.present? } def index - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37434 - Gitlab::GitalyClient.allow_n_plus_1_calls do - super - end + super end private diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index d025062f562..5bec68ce4f6 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -158,10 +158,12 @@ class MergeRequest < ActiveRecord::Base end def rebase_in_progress? - # The source project can be deleted - return false unless source_project + strong_memoize(:rebase_in_progress) do + # The source project can be deleted + next false unless source_project - source_project.repository.rebase_in_progress?(id) + source_project.repository.rebase_in_progress?(id) + end end # Use this method whenever you need to make sure the head_pipeline is synced with the diff --git a/app/models/user.rb b/app/models/user.rb index 05c93d3cb17..4097fe2b5dc 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -551,7 +551,7 @@ class User < ActiveRecord::Base gpg_keys.each(&:update_invalid_gpg_signatures) end - # Returns the groups a user has access to + # Returns the groups a user has access to, either through a membership or a project authorization def authorized_groups union = Gitlab::SQL::Union .new([groups.select(:id), authorized_projects.select(:namespace_id)]) @@ -559,6 +559,11 @@ class User < ActiveRecord::Base Group.where("namespaces.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end + # Returns the groups a user is a member of, either directly or through a parent group + def membership_groups + Gitlab::GroupHierarchy.new(groups).base_and_descendants + end + # Returns a relation of groups the user has access to, including their parent # and child groups (recursively). def all_expanded_groups diff --git a/changelogs/unreleased/bvl-fix-concurrent-fork-network-migrations.yml b/changelogs/unreleased/bvl-fix-concurrent-fork-network-migrations.yml new file mode 100644 index 00000000000..b2a77f75e55 --- /dev/null +++ b/changelogs/unreleased/bvl-fix-concurrent-fork-network-migrations.yml @@ -0,0 +1,5 @@ +--- +title: Avoid running `PopulateForkNetworksRange`-migration multiple times +merge_request: 16988 +author: +type: fixed diff --git a/changelogs/unreleased/feature-oidc-groups-claim.yml b/changelogs/unreleased/feature-oidc-groups-claim.yml new file mode 100644 index 00000000000..bde19130114 --- /dev/null +++ b/changelogs/unreleased/feature-oidc-groups-claim.yml @@ -0,0 +1,4 @@ +--- +title: Add groups to OpenID Connect claims +merge_request: 16929 +author: Hassan Zamani diff --git a/config/initializers/doorkeeper_openid_connect.rb b/config/initializers/doorkeeper_openid_connect.rb index af174def047..98e1f6e830f 100644 --- a/config/initializers/doorkeeper_openid_connect.rb +++ b/config/initializers/doorkeeper_openid_connect.rb @@ -31,6 +31,7 @@ Doorkeeper::OpenidConnect.configure do o.claim(:website) { |user| user.full_website_url if user.website_url? } o.claim(:profile) { |user| Gitlab::Routing.url_helpers.user_url user } o.claim(:picture) { |user| user.avatar_url(only_path: false) } + o.claim(:groups) { |user| user.membership_groups.map(&:full_path) } end end end diff --git a/config/locales/doorkeeper.en.yml b/config/locales/doorkeeper.en.yml index b1c71095d4f..889111282ef 100644 --- a/config/locales/doorkeeper.en.yml +++ b/config/locales/doorkeeper.en.yml @@ -68,7 +68,7 @@ en: read_user: Read-only access to the user's profile information, like username, public email and full name openid: - The ability to authenticate using GitLab, and read-only access to the user's profile information + The ability to authenticate using GitLab, and read-only access to the user's profile information and group memberships sudo: Access to the Sudo feature, to perform API actions as any user in the system (only available for admins) flash: diff --git a/db/migrate/20170929131201_populate_fork_networks.rb b/db/migrate/20170929131201_populate_fork_networks.rb index 1214962770f..ddbf27e1852 100644 --- a/db/migrate/20170929131201_populate_fork_networks.rb +++ b/db/migrate/20170929131201_populate_fork_networks.rb @@ -6,22 +6,8 @@ class PopulateForkNetworks < ActiveRecord::Migration DOWNTIME = false - MIGRATION = 'PopulateForkNetworksRange'.freeze - BATCH_SIZE = 100 - DELAY_INTERVAL = 15.seconds - - disable_ddl_transaction! - - class ForkedProjectLink < ActiveRecord::Base - include EachBatch - - self.table_name = 'forked_project_links' - end - def up - say 'Populating the `fork_networks` based on existing `forked_project_links`' - - queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller' end def down diff --git a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb index 05430efe1f6..26f917d5a1e 100644 --- a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb +++ b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb @@ -3,22 +3,8 @@ class RescheduleForkNetworkCreation < ActiveRecord::Migration DOWNTIME = false - MIGRATION = 'PopulateForkNetworksRange'.freeze - BATCH_SIZE = 100 - DELAY_INTERVAL = 15.seconds - - disable_ddl_transaction! - - class ForkedProjectLink < ActiveRecord::Base - include EachBatch - - self.table_name = 'forked_project_links' - end - def up - say 'Populating the `fork_networks` based on existing `forked_project_links`' - - queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller' end def down diff --git a/doc/development/background_migrations.md b/doc/development/background_migrations.md index af2026c483e..fc1b202b5eb 100644 --- a/doc/development/background_migrations.md +++ b/doc/development/background_migrations.md @@ -94,6 +94,18 @@ jobs = [['BackgroundMigrationClassName', [1]], BackgroundMigrationWorker.bulk_perform_in(5.minutes, jobs) ``` +### Rescheduling background migrations + +If one of the background migrations contains a bug that is fixed in a patch +release, the background migration needs to be rescheduled so the migration would +be repeated on systems that already performed the initial migration. + +When you reschedule the background migration, make sure to turn the original +scheduling into a no-op by clearing up the `#up` and `#down` methods of the +migration performing the scheduling. Otherwise the background migration would be +scheduled multiple times on systems that are upgrading multiple patch releases at +once. + ## Cleaning Up >**Note:** diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index f4542932295..c0a325a83e9 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -126,6 +126,9 @@ strings and remove any strings that aren't used anymore. You should check this file in. Once the changes are on master, they will be picked up by [Crowdin](http://translate.gitlab.com) and be presented for translation. +If there are merge conflicts in the `gitlab.pot` file, you can delete the file +and regenerate it using the same command. Confirm that you are not deleting any strings accidentally by looking over the diff. + The command also updates the translation files for each language: `locale/*/gitlab.po` These changes can be discarded, the languange files will be updated by Crowdin automatically. diff --git a/doc/install/installation.md b/doc/install/installation.md index 6eb8890cc4f..4dfc03d0fe0 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -299,9 +299,9 @@ sudo usermod -aG redis git ### Clone the Source # Clone GitLab repository - sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 10-4-stable gitlab + sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 10-5-stable gitlab -**Note:** You can change `10-4-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server! +**Note:** You can change `10-5-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server! ### Configure It diff --git a/doc/integration/openid_connect_provider.md b/doc/integration/openid_connect_provider.md index 56f367d841e..ad41be52045 100644 --- a/doc/integration/openid_connect_provider.md +++ b/doc/integration/openid_connect_provider.md @@ -39,6 +39,7 @@ Currently the following user information is shared with clients: | `website` | `string` | URL for the user's website | `profile` | `string` | URL for the user's GitLab profile | `picture` | `string` | URL for the user's GitLab avatar +| `groups` | `array` | Names of the groups the user is a member of [OpenID Connect]: http://openid.net/connect/ "OpenID Connect website" [doorkeeper-openid_connect]: https://github.com/doorkeeper-gem/doorkeeper-openid_connect "Doorkeeper::OpenidConnect website" diff --git a/doc/update/10.4-to-10.5.md b/doc/update/10.4-to-10.5.md new file mode 100644 index 00000000000..313419ed13d --- /dev/null +++ b/doc/update/10.4-to-10.5.md @@ -0,0 +1,361 @@ +--- +comments: false +--- + +# From 10.4 to 10.5 + +Make sure you view this update guide from the tag (version) of GitLab you would +like to install. In most cases this should be the highest numbered production +tag (without rc in it). You can select the tag in the version dropdown at the +top left corner of GitLab (below the menu bar). + +If the highest number stable branch is unclear please check the +[GitLab Blog](https://about.gitlab.com/blog/archives.html) for installation +guide links by version. + +### 1. Stop server + +```bash +sudo service gitlab stop +``` + +### 2. Backup + +NOTE: If you installed GitLab from source, make sure `rsync` is installed. + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production +``` + +### 3. Update Ruby + +NOTE: GitLab 9.0 and higher only support Ruby 2.3.x and dropped support for Ruby 2.1.x. Be +sure to upgrade your interpreter if necessary. + +You can check which version you are running with `ruby -v`. + +Download and compile Ruby: + +```bash +mkdir /tmp/ruby && cd /tmp/ruby +curl --remote-name --progress https://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.6.tar.gz +echo '4e6a0f828819e15d274ae58485585fc8b7caace0 ruby-2.3.6.tar.gz' | shasum -c - && tar xzf ruby-2.3.6.tar.gz +cd ruby-2.3.6 +./configure --disable-install-rdoc +make +sudo make install +``` + +Install Bundler: + +```bash +sudo gem install bundler --no-ri --no-rdoc +``` + +### 4. Update Node + +GitLab now runs [webpack](http://webpack.js.org) to compile frontend assets. +We require a minimum version of node v6.0.0. + +You can check which version you are running with `node -v`. If you are running +a version older than `v6.0.0` you will need to update to a newer version. You +can find instructions to install from community maintained packages or compile +from source at the nodejs.org website. + +<https://nodejs.org/en/download/> + +Since 8.17, GitLab requires the use of yarn `>= v0.17.0` to manage +JavaScript dependencies. + +```bash +curl --silent --show-error https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - +echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list +sudo apt-get update +sudo apt-get install yarn +``` + +More information can be found on the [yarn website](https://yarnpkg.com/en/docs/install). + +### 5. Update Go + +NOTE: GitLab 9.2 and higher only supports Go 1.8.3 and dropped support for Go +1.5.x through 1.7.x. Be sure to upgrade your installation if necessary. + +You can check which version you are running with `go version`. + +Download and install Go: + +```bash +# Remove former Go installation folder +sudo rm -rf /usr/local/go + +curl --remote-name --progress https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz +echo '1862f4c3d3907e59b04a757cfda0ea7aa9ef39274af99a784f5be843c80c6772 go1.8.3.linux-amd64.tar.gz' | shasum -a256 -c - && \ + sudo tar -C /usr/local -xzf go1.8.3.linux-amd64.tar.gz +sudo ln -sf /usr/local/go/bin/{go,godoc,gofmt} /usr/local/bin/ +rm go1.8.3.linux-amd64.tar.gz +``` + +### 6. Get latest code + +```bash +cd /home/git/gitlab + +sudo -u git -H git fetch --all +sudo -u git -H git checkout -- db/schema.rb # local changes will be restored automatically +sudo -u git -H git checkout -- locale +``` + +For GitLab Community Edition: + +```bash +cd /home/git/gitlab + +sudo -u git -H git checkout 10-5-stable +``` + +OR + +For GitLab Enterprise Edition: + +```bash +cd /home/git/gitlab + +sudo -u git -H git checkout 10-5-stable-ee +``` + +### 7. Update gitlab-shell + +```bash +cd /home/git/gitlab-shell + +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_SHELL_VERSION) +sudo -u git -H bin/compile +``` + +### 8. Update gitlab-workhorse + +Install and compile gitlab-workhorse. GitLab-Workhorse uses +[GNU Make](https://www.gnu.org/software/make/). +If you are not using Linux you may have to run `gmake` instead of +`make` below. + +```bash +cd /home/git/gitlab-workhorse + +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v$(</home/git/gitlab/GITLAB_WORKHORSE_VERSION) +sudo -u git -H make +``` + +### 9. Update Gitaly + +#### New Gitaly configuration options required + +In order to function Gitaly needs some additional configuration information. Below we assume you installed Gitaly in `/home/git/gitaly` and GitLab Shell in `/home/git/gitlab-shell`. + +```shell +echo ' +[gitaly-ruby] +dir = "/home/git/gitaly/ruby" + +[gitlab-shell] +dir = "/home/git/gitlab-shell" +' | sudo -u git tee -a /home/git/gitaly/config.toml +``` + +#### Check Gitaly configuration + +Due to a bug in the `rake gitlab:gitaly:install` script your Gitaly +configuration file may contain syntax errors. The block name +`[[storages]]`, which may occur more than once in your `config.toml` +file, should be `[[storage]]` instead. + +```shell +sudo -u git -H sed -i.pre-10.1 's/\[\[storages\]\]/[[storage]]/' /home/git/gitaly/config.toml +``` + +#### Compile Gitaly + +```shell +cd /home/git/gitaly +sudo -u git -H git fetch --all --tags +sudo -u git -H git checkout v$(</home/git/gitlab/GITALY_SERVER_VERSION) +sudo -u git -H make +``` + +### 10. Update MySQL permissions + +If you are using MySQL you need to grant the GitLab user the necessary +permissions on the database: + +```bash +mysql -u root -p -e "GRANT TRIGGER ON \`gitlabhq_production\`.* TO 'git'@'localhost';" +``` + +If you use MySQL with replication, or just have MySQL configured with binary logging, +you will need to also run the following on all of your MySQL servers: + +```bash +mysql -u root -p -e "SET GLOBAL log_bin_trust_function_creators = 1;" +``` + +You can make this setting permanent by adding it to your `my.cnf`: + +``` +log_bin_trust_function_creators=1 +``` + +### 11. Update configuration files + +#### New configuration options for `gitlab.yml` + +There might be configuration options available for [`gitlab.yml`][yaml]. View them with the command below and apply them manually to your current `gitlab.yml`: + +```sh +cd /home/git/gitlab + +git diff origin/10-4-stable:config/gitlab.yml.example origin/10-5-stable:config/gitlab.yml.example +``` + +#### Nginx configuration + +Ensure you're still up-to-date with the latest NGINX configuration changes: + +```sh +cd /home/git/gitlab + +# For HTTPS configurations +git diff origin/10-4-stable:lib/support/nginx/gitlab-ssl origin/10-5-stable:lib/support/nginx/gitlab-ssl + +# For HTTP configurations +git diff origin/10-4-stable:lib/support/nginx/gitlab origin/10-5-stable:lib/support/nginx/gitlab +``` + +If you are using Strict-Transport-Security in your installation to continue using it you must enable it in your Nginx +configuration as GitLab application no longer handles setting it. + +If you are using Apache instead of NGINX please see the updated [Apache templates]. +Also note that because Apache does not support upstreams behind Unix sockets you +will need to let gitlab-workhorse listen on a TCP port. You can do this +via [/etc/default/gitlab]. + +[Apache templates]: https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache +[/etc/default/gitlab]: https://gitlab.com/gitlab-org/gitlab-ce/blob/10-5-stable/lib/support/init.d/gitlab.default.example#L38 + +#### SMTP configuration + +If you're installing from source and use SMTP to deliver mail, you will need to add the following line +to config/initializers/smtp_settings.rb: + +```ruby +ActionMailer::Base.delivery_method = :smtp +``` + +See [smtp_settings.rb.sample] as an example. + +[smtp_settings.rb.sample]: https://gitlab.com/gitlab-org/gitlab-ce/blob/10-5-stable/config/initializers/smtp_settings.rb.sample#L13 + +#### Init script + +There might be new configuration options available for [`gitlab.default.example`][gl-example]. View them with the command below and apply them manually to your current `/etc/default/gitlab`: + +```sh +cd /home/git/gitlab + +git diff origin/10-4-stable:lib/support/init.d/gitlab.default.example origin/10-5-stable:lib/support/init.d/gitlab.default.example +``` + +Ensure you're still up-to-date with the latest init script changes: + +```bash +cd /home/git/gitlab + +sudo cp lib/support/init.d/gitlab /etc/init.d/gitlab +``` + +For Ubuntu 16.04.1 LTS: + +```bash +sudo systemctl daemon-reload +``` + +### 12. Install libs, migrations, etc. + +```bash +cd /home/git/gitlab + +# MySQL installations (note: the line below states '--without postgres') +sudo -u git -H bundle install --without postgres development test --deployment + +# PostgreSQL installations (note: the line below states '--without mysql') +sudo -u git -H bundle install --without mysql development test --deployment + +# Optional: clean up old gems +sudo -u git -H bundle clean + +# Run database migrations +sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production + +# Compile GetText PO files + +sudo -u git -H bundle exec rake gettext:compile RAILS_ENV=production + +# Update node dependencies and recompile assets +sudo -u git -H bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile RAILS_ENV=production NODE_ENV=production + +# Clean up cache +sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production +``` + +**MySQL installations**: Run through the `MySQL strings limits` and `Tables and data conversion to utf8mb4` [tasks](../install/database_mysql.md). + +### 13. Start application + +```bash +sudo service gitlab start +sudo service nginx restart +``` + +### 14. Check application status + +Check if GitLab and its environment are configured correctly: + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production +``` + +To make sure you didn't miss anything run a more thorough check: + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production +``` + +If all items are green, then congratulations, the upgrade is complete! + +## Things went south? Revert to previous version (10.4) + +### 1. Revert the code to the previous version + +Follow the [upgrade guide from 10.3 to 10.4](10.3-to-10.4.md), except for the +database migration (the backup is already migrated to the previous version). + +### 2. Restore from the backup + +```bash +cd /home/git/gitlab + +sudo -u git -H bundle exec rake gitlab:backup:restore RAILS_ENV=production +``` + +If you have more than one backup `*.tar` file(s) please add `BACKUP=timestamp_of_backup` to the command above. + +[yaml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/10-5-stable/config/gitlab.yml.example +[gl-example]: https://gitlab.com/gitlab-org/gitlab-ce/blob/10-5-stable/lib/support/init.d/gitlab.default.example diff --git a/doc/user/project/merge_requests/work_in_progress_merge_requests.md b/doc/user/project/merge_requests/work_in_progress_merge_requests.md index 546c8bdc5e5..f01da06fa6e 100644 --- a/doc/user/project/merge_requests/work_in_progress_merge_requests.md +++ b/doc/user/project/merge_requests/work_in_progress_merge_requests.md @@ -7,7 +7,8 @@ have been marked a **Work In Progress**.  To mark a merge request a Work In Progress, simply start its title with `[WIP]` -or `WIP:`. +or `WIP:`. As an alternative, you're also able to do it by sending a commit +with its title starting with `wip` or `WIP` to the merge request's source branch.  diff --git a/doc/user/project/pages/getting_started_part_three.md b/doc/user/project/pages/getting_started_part_three.md index 0096f8507d2..a153610c712 100644 --- a/doc/user/project/pages/getting_started_part_three.md +++ b/doc/user/project/pages/getting_started_part_three.md @@ -155,15 +155,40 @@ Certificates are NOT required to add to your custom (sub)domain on your GitLab Pages project, though they are highly recommendable. -The importance of having any website securely served under HTTPS -is explained on the introductory section of the blog post -[Secure GitLab Pages with StartSSL](https://about.gitlab.com/2016/06/24/secure-gitlab-pages-with-startssl/#https-a-quick-overview). +Let's start with an introduction to the importance of HTTPS. +Alternatively, jump ahead to [adding certificates to your project](#adding-certificates-to-your-project). -The reason why certificates are so important is that they encrypt +#### Why should I care about HTTPS? + +This might be your first question. If our sites are hosted by GitLab Pages, +they are static, hence we are not dealing with server-side scripts +nor credit card transactions, then why do we need secure connections? + +Back in the 1990s, where HTTPS came out, [SSL](https://en.wikipedia.org/wiki/Transport_Layer_Security#SSL_1.0.2C_2.0_and_3.0) was considered a "special" +security measure, necessary just for big companies, like banks and shoppings sites +with financial transactions. +Now we have a different picture. [According to Josh Aas](https://letsencrypt.org/2015/10/29/phishing-and-malware.html), Executive Director at [ISRG](https://en.wikipedia.org/wiki/Internet_Security_Research_Group): + +> _We’ve since come to realize that HTTPS is important for almost all websites. It’s important for any website that allows people to log in with a password, any website that [tracks its users](https://www.washingtonpost.com/news/the-switch/wp/2013/12/10/nsa-uses-google-cookies-to-pinpoint-targets-for-hacking/) in any way, any website that [doesn’t want its content altered](http://arstechnica.com/tech-policy/2014/09/why-comcasts-javascript-ad-injections-threaten-security-net-neutrality/), and for any site that offers content people might not want others to know they are consuming. We’ve also learned that any site not secured by HTTPS [can be used to attack other sites](http://krebsonsecurity.com/2015/04/dont-be-fodder-for-chinas-great-cannon/)._ + +Therefore, the reason why certificates are so important is that they encrypt the connection between the **client** (you, me, your visitors) and the **server** (where you site lives), through a keychain of authentications and validations. +How about taking Josh's advice and protecting our sites too? We will be +well supported, and we'll contribute to a safer internet. + +#### Organizations supporting HTTPS + +There is a huge movement in favor of securing all the web. W3C fully +[supports the cause](https://w3ctag.github.io/web-https/) and explains very well +the reasons for that. Richard Barnes, a writer for Mozilla Security Blog, +suggested that [Firefox would deprecate HTTP](https://blog.mozilla.org/security/2015/04/30/deprecating-non-secure-http/), +and would no longer accept unsecured connections. Recently, Mozilla published a +[communication](https://blog.mozilla.org/security/2016/03/29/march-2016-ca-communication/) +reiterating the importance of HTTPS. + ### Issuing Certificates GitLab Pages accepts [PEM](https://support.quovadisglobal.com/kb/a37/what-is-pem-format.aspx) certificates issued by diff --git a/doc/user/project/pages/index.md b/doc/user/project/pages/index.md index 8404d789de6..df245710940 100644 --- a/doc/user/project/pages/index.md +++ b/doc/user/project/pages/index.md @@ -54,7 +54,6 @@ _Blog posts for securing GitLab Pages custom domains with SSL/TLS certificates:_ - [CloudFlare](https://about.gitlab.com/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/) - [Let's Encrypt](https://about.gitlab.com/2016/04/11/tutorial-securing-your-gitlab-pages-with-tls-and-letsencrypt/) (outdated) -- [StartSSL](https://about.gitlab.com/2016/06/24/secure-gitlab-pages-with-startssl/) (deprecated) ## Advanced use diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index 03b17b319fa..1b4a9e8a194 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -14,6 +14,14 @@ module Gitlab def perform(start_id, end_id) log("Creating memberships for forks: #{start_id} - #{end_id}") + insert_members(start_id, end_id) + + if missing_members?(start_id, end_id) + BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + end + + def insert_members(start_id, end_id) ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id) @@ -33,10 +41,9 @@ module Gitlab WHERE existing_members.project_id = forked_project_links.forked_to_project_id ) INSERT_MEMBERS - - if missing_members?(start_id, end_id) - BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) - end + rescue ActiveRecord::RecordNotUnique => e + # `fork_network_member` was created concurrently in another migration + log(e.message) end def missing_members?(start_id, end_id) diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index f4a41dc3eda..4ba44e0feef 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -294,7 +294,8 @@ module Gitlab # add_namespace("/path/to/storage", "gitlab") # def add_namespace(storage, name) - Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:add_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).add(name) else @@ -315,7 +316,8 @@ module Gitlab # rm_namespace("/path/to/storage", "gitlab") # def rm_namespace(storage, name) - Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:remove_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).remove(name) else @@ -333,7 +335,8 @@ module Gitlab # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # def mv_namespace(storage, old_name, new_name) - Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:rename_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).rename(old_name, new_name) else @@ -368,7 +371,8 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) - Gitlab::GitalyClient.migrate(:namespace_exists) do |enabled| + Gitlab::GitalyClient.migrate(:namespace_exists, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).exists?(dir_name) else diff --git a/spec/javascripts/gl_form_spec.js b/spec/javascripts/gl_form_spec.js index 5a8009e57fd..9c1fc0fda9e 100644 --- a/spec/javascripts/gl_form_spec.js +++ b/spec/javascripts/gl_form_spec.js @@ -1,10 +1,8 @@ -import Autosize from 'autosize'; +import autosize from 'autosize'; import GLForm from '~/gl_form'; import '~/lib/utils/text_utility'; import '~/lib/utils/common_utils'; -window.autosize = Autosize; - describe('GLForm', () => { describe('when instantiated', function () { beforeEach((done) => { @@ -13,14 +11,12 @@ describe('GLForm', () => { spyOn($.prototype, 'off').and.returnValue(this.textarea); spyOn($.prototype, 'on').and.returnValue(this.textarea); spyOn($.prototype, 'css'); - spyOn(window, 'autosize'); - this.glForm = new GLForm(this.form); + this.glForm = new GLForm(this.form, false); setTimeout(() => { $.prototype.off.calls.reset(); $.prototype.on.calls.reset(); $.prototype.css.calls.reset(); - window.autosize.calls.reset(); done(); }); }); @@ -43,10 +39,6 @@ describe('GLForm', () => { expect($.prototype.on).toHaveBeenCalledWith('mouseup.autosize', jasmine.any(Function)); }); - it('should autosize the textarea', () => { - expect(window.autosize).toHaveBeenCalledWith(jasmine.any(Object)); - }); - it('should set the resize css property to vertical', () => { expect($.prototype.css).toHaveBeenCalledWith('resize', 'vertical'); }); @@ -74,7 +66,7 @@ describe('GLForm', () => { spyOn($.prototype, 'data'); spyOn($.prototype, 'outerHeight').and.returnValue(200); spyOn(window, 'outerHeight').and.returnValue(400); - spyOn(window.autosize, 'destroy'); + spyOn(autosize, 'destroy'); this.glForm.destroyAutosize(); }); @@ -88,7 +80,7 @@ describe('GLForm', () => { }); it('should call autosize destroy', () => { - expect(window.autosize.destroy).toHaveBeenCalledWith(this.textarea); + expect(autosize.destroy).toHaveBeenCalledWith(this.textarea); }); it('should set the data-height attribute', () => { @@ -107,9 +99,9 @@ describe('GLForm', () => { it('should return undefined if the data-height equals the outerHeight', () => { spyOn($.prototype, 'outerHeight').and.returnValue(200); spyOn($.prototype, 'data').and.returnValue(200); - spyOn(window.autosize, 'destroy'); + spyOn(autosize, 'destroy'); expect(this.glForm.destroyAutosize()).toBeUndefined(); - expect(window.autosize.destroy).not.toHaveBeenCalled(); + expect(autosize.destroy).not.toHaveBeenCalled(); }); }); }); diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index cb02d526a98..76a6aef39cc 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -1586,14 +1586,37 @@ describe User do describe '#authorized_groups' do let!(:user) { create(:user) } let!(:private_group) { create(:group) } + let!(:child_group) { create(:group, parent: private_group) } + + let!(:project_group) { create(:group) } + let!(:project) { create(:project, group: project_group) } before do private_group.add_user(user, Gitlab::Access::MASTER) + project.add_master(user) end subject { user.authorized_groups } - it { is_expected.to eq([private_group]) } + it { is_expected.to contain_exactly private_group, project_group } + end + + describe '#membership_groups' do + let!(:user) { create(:user) } + let!(:parent_group) { create(:group) } + let!(:child_group) { create(:group, parent: parent_group) } + + before do + parent_group.add_user(user, Gitlab::Access::MASTER) + end + + subject { user.membership_groups } + + if Group.supports_nested_groups? + it { is_expected.to contain_exactly parent_group, child_group } + else + it { is_expected.to contain_exactly parent_group } + end end describe '#authorized_projects', :delete do diff --git a/spec/requests/openid_connect_spec.rb b/spec/requests/openid_connect_spec.rb index 1a5ad9b04e4..5d349f45a33 100644 --- a/spec/requests/openid_connect_spec.rb +++ b/spec/requests/openid_connect_spec.rb @@ -65,10 +65,20 @@ describe 'OpenID Connect requests' do ) end - let(:public_email) { build :email, email: 'public@example.com' } - let(:private_email) { build :email, email: 'private@example.com' } + let!(:public_email) { build :email, email: 'public@example.com' } + let!(:private_email) { build :email, email: 'private@example.com' } - it 'includes all user information' do + let!(:group1) { create :group, path: 'group1' } + let!(:group2) { create :group, path: 'group2' } + let!(:group3) { create :group, path: 'group3', parent: group2 } + let!(:group4) { create :group, path: 'group4', parent: group3 } + + before do + group1.add_user(user, GroupMember::OWNER) + group3.add_user(user, Gitlab::Access::DEVELOPER) + end + + it 'includes all user information and group memberships' do request_user_info expect(json_response).to eq({ @@ -79,7 +89,13 @@ describe 'OpenID Connect requests' do 'email_verified' => true, 'website' => 'https://example.com', 'profile' => 'http://localhost/alice', - 'picture' => "http://localhost/uploads/-/system/user/avatar/#{user.id}/dk.png" + 'picture' => "http://localhost/uploads/-/system/user/avatar/#{user.id}/dk.png", + 'groups' => + if Group.supports_nested_groups? + ['group1', 'group2/group3', 'group2/group3/group4'] + else + ['group1', 'group2/group3'] + end }) end end diff --git a/vendor/gitignore/Android.gitignore b/vendor/gitignore/Android.gitignore index addf405e4f5..d57137223ed 100644 --- a/vendor/gitignore/Android.gitignore +++ b/vendor/gitignore/Android.gitignore @@ -54,3 +54,10 @@ google-services.json freeline.py freeline/ freeline_project_description.json + +# fastlane +fastlane/report.xml +fastlane/Preview.html +fastlane/screenshots +fastlane/test_output +fastlane/readme.md diff --git a/vendor/gitignore/Dart.gitignore b/vendor/gitignore/Dart.gitignore index 4d2a4d6db7c..58950beb4fa 100644 --- a/vendor/gitignore/Dart.gitignore +++ b/vendor/gitignore/Dart.gitignore @@ -1,6 +1,7 @@ # See https://www.dartlang.org/tools/private-files.html # Files and directories created by pub +.dart_tool/ .packages .pub/ build/ diff --git a/vendor/gitignore/Global/JetBrains.gitignore b/vendor/gitignore/Global/JetBrains.gitignore index a30eacf1d98..9c01e12b050 100644 --- a/vendor/gitignore/Global/JetBrains.gitignore +++ b/vendor/gitignore/Global/JetBrains.gitignore @@ -1,4 +1,4 @@ -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff: @@ -9,7 +9,6 @@ # Sensitive or high-churn files: .idea/**/dataSources/ .idea/**/dataSources.ids -.idea/**/dataSources.xml .idea/**/dataSources.local.xml .idea/**/sqlDataSources.xml .idea/**/dynamic.xml diff --git a/vendor/gitignore/Python.gitignore b/vendor/gitignore/Python.gitignore index af2f537516d..b989be6ca15 100644 --- a/vendor/gitignore/Python.gitignore +++ b/vendor/gitignore/Python.gitignore @@ -45,6 +45,7 @@ nosetests.xml coverage.xml *.cover .hypothesis/ +.pytest_cache/ # Translations *.mo diff --git a/vendor/gitignore/ROS.gitignore b/vendor/gitignore/ROS.gitignore index 425641f2c3a..35d74bb771f 100644 --- a/vendor/gitignore/ROS.gitignore +++ b/vendor/gitignore/ROS.gitignore @@ -13,6 +13,8 @@ msg/*Feedback.msg msg/*Goal.msg msg/*Result.msg msg/_*.py +build_isolated/ +devel_isolated/ # Generated by dynamic reconfigure *.cfgc diff --git a/vendor/gitignore/TeX.gitignore b/vendor/gitignore/TeX.gitignore index 9bb63365618..5359e544bcf 100644 --- a/vendor/gitignore/TeX.gitignore +++ b/vendor/gitignore/TeX.gitignore @@ -10,6 +10,7 @@ *.fot *.cb *.cb2 +.*.lb ## Intermediate documents: *.dvi diff --git a/vendor/gitignore/VisualStudio.gitignore b/vendor/gitignore/VisualStudio.gitignore index d3d5371b415..c49041ff7d2 100644 --- a/vendor/gitignore/VisualStudio.gitignore +++ b/vendor/gitignore/VisualStudio.gitignore @@ -237,6 +237,7 @@ _UpgradeReport_Files/ Backup*/ UpgradeLog*.XML UpgradeLog*.htm +ServiceFabricBackup/ # SQL Server files *.mdf diff --git a/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml index b2439d96e00..094d6791505 100644 --- a/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml +++ b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml @@ -110,12 +110,15 @@ performance: kubernetes: active sast: - image: registry.gitlab.com/gitlab-org/gl-sast:latest + image: docker:latest variables: - POSTGRES_DB: "false" + DOCKER_DRIVER: overlay2 allow_failure: true + services: + - docker:dind script: - - sast . + - setup_docker + - sast artifacts: paths: [gl-sast-report.json] @@ -285,6 +288,12 @@ production: export TILLER_NAMESPACE=$KUBE_NAMESPACE function sast_container() { + if [[ -n "$CI_REGISTRY_USER" ]]; then + echo "Logging to GitLab Container Registry with CI credentials..." + docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + echo "" + fi + docker run -d --name db arminc/clair-db:latest docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1 apk add -U wget ca-certificates @@ -309,7 +318,12 @@ production: function sast() { case "$CI_SERVER_VERSION" in *-ee) - /app/bin/run "$@" + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" + SAST_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + + docker run --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code ;; *) echo "GitLab EE is required" @@ -346,6 +360,12 @@ production: replicas="$new_replicas" fi + if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then + secret_name='gitlab-registry' + else + secret_name='' + fi + helm upgrade --install \ --wait \ --set service.enabled="$service_enabled" \ @@ -353,6 +373,7 @@ production: --set image.repository="$CI_APPLICATION_REPOSITORY" \ --set image.tag="$CI_APPLICATION_TAG" \ --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ --set application.track="$track" \ --set application.database_url="$DATABASE_URL" \ --set service.url="$CI_ENVIRONMENT_URL" \ @@ -482,6 +503,9 @@ production: function create_secret() { echo "Create secret..." + if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then + return + fi kubectl create secret -n "$KUBE_NAMESPACE" \ docker-registry gitlab-registry \ |