diff options
17 files changed, 2119 insertions, 494 deletions
diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index 3e107b475c9..dd61cb3f035 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -4,6 +4,7 @@ lint-ci-gitlab: extends: - .default-tags - .default-retry + - .except-docs image: sdesbure/yamllint:latest dependencies: [] script: diff --git a/changelogs/unreleased/ab-unconfirmed-email-index.yml b/changelogs/unreleased/ab-unconfirmed-email-index.yml new file mode 100644 index 00000000000..3887cd87e41 --- /dev/null +++ b/changelogs/unreleased/ab-unconfirmed-email-index.yml @@ -0,0 +1,5 @@ +--- +title: Create index for users.unconfirmed_email +merge_request: 32664 +author: +type: performance diff --git a/changelogs/unreleased/fj-66723-add-dns-rebinding-protection-check.yml b/changelogs/unreleased/fj-66723-add-dns-rebinding-protection-check.yml new file mode 100644 index 00000000000..c1372a4a73e --- /dev/null +++ b/changelogs/unreleased/fj-66723-add-dns-rebinding-protection-check.yml @@ -0,0 +1,5 @@ +--- +title: Allow not resolvable urls when dns rebind protection is disabled +merge_request: 32523 +author: +type: fixed diff --git a/db/migrate/20190904173203_add_index_on_users_unconfirmed_email.rb b/db/migrate/20190904173203_add_index_on_users_unconfirmed_email.rb new file mode 100644 index 00000000000..e78d47f023f --- /dev/null +++ b/db/migrate/20190904173203_add_index_on_users_unconfirmed_email.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddIndexOnUsersUnconfirmedEmail < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :users, :unconfirmed_email, where: 'unconfirmed_email IS NOT NULL' + end + + def down + remove_concurrent_index :users, :unconfirmed_email, where: 'unconfirmed_email IS NOT NULL' + end +end diff --git a/db/schema.rb b/db/schema.rb index 79f4551ee26..98c4403efe1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_09_02_160015) do +ActiveRecord::Schema.define(version: 2019_09_04_173203) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -3563,6 +3563,7 @@ ActiveRecord::Schema.define(version: 2019_09_02_160015) do t.index ["state"], name: "index_users_on_state" t.index ["state"], name: "index_users_on_state_and_internal", where: "(ghost IS NOT TRUE)" t.index ["state"], name: "index_users_on_state_and_internal_ee", where: "((ghost IS NOT TRUE) AND (bot_type IS NULL))" + t.index ["unconfirmed_email"], name: "index_users_on_unconfirmed_email", where: "(unconfirmed_email IS NOT NULL)" t.index ["username"], name: "index_users_on_username" t.index ["username"], name: "index_users_on_username_trigram", opclass: :gin_trgm_ops, using: :gin end diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index 407539885a6..7228cc6948e 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -315,7 +315,7 @@ section to resolve the error. Otherwise, the secret is lost and you'll need to [geo-limitations]: ../replication/index.md#current-limitations [planned-failover]: planned_failover.md [setup-geo]: ../replication/index.md#setup-instructions -[updating-geo]: ../replication/updating_the_geo_nodes.md#upgrading-to-gitlab-105 +[updating-geo]: ../replication/version_specific_updates.md#updating-to-gitlab-105 [sec-tfa]: ../../../security/two_factor_authentication.md#disabling-2fa-for-everyone [gitlab-org/omnibus-gitlab#3058]: https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3058 [gitlab-org/gitlab-ee#4284]: https://gitlab.com/gitlab-org/gitlab-ee/issues/4284 diff --git a/doc/administration/geo/replication/updating_the_geo_nodes.md b/doc/administration/geo/replication/updating_the_geo_nodes.md index 8c27c4dac4f..fda0ebbbeac 100644 --- a/doc/administration/geo/replication/updating_the_geo_nodes.md +++ b/doc/administration/geo/replication/updating_the_geo_nodes.md @@ -1,6 +1,26 @@ # Updating the Geo nodes **(PREMIUM ONLY)** -Depending on which version of Geo you are updating to/from, there may be different steps. +Updating Geo nodes involves performing: + +1. [Version-specific update steps](#version-specific-update-steps), depending on the + version being updated to or from. +1. [General update steps](#general-update-steps), for all updates. + +## Version specific update steps + +Depending on which version of Geo you are updating to/from, there may be +different steps. + +- [Updating to GitLab 12.1](version_specific_updates.md#updating-to-gitlab-121) +- [Updating to GitLab 10.8](version_specific_updates.md#updating-to-gitlab-108) +- [Updating to GitLab 10.6](version_specific_updates.md#updating-to-gitlab-106) +- [Updating to GitLab 10.5](version_specific_updates.md#updating-to-gitlab-105) +- [Updating to GitLab 10.3](version_specific_updates.md#updating-to-gitlab-103) +- [Updating to GitLab 10.2](version_specific_updates.md#updating-to-gitlab-102) +- [Updating to GitLab 10.1](version_specific_updates.md#updating-to-gitlab-101) +- [Updating to GitLab 10.0](version_specific_updates.md#updating-to-gitlab-100) +- [Updating from GitLab 9.3 or older](version_specific_updates.md#updating-from-gitlab-93-or-older) +- [Updating to GitLab 9.0](version_specific_updates.md#updating-to-gitlab-90) ## General update steps @@ -31,428 +51,3 @@ everything is working correctly: is received by **secondary** nodes. If you encounter any issues, please consult the [Geo troubleshooting guide](troubleshooting.md). - -## Upgrading to GitLab 12.1 - -By default, GitLab 12.1 will attempt to automatically upgrade the embedded PostgreSQL server to 10.7 from 9.6. Please see [the omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) for the recommended procedure. - -This can be temporarily disabled by running the following before ugprading: - -```sh -sudo touch /etc/gitlab/disable-postgresql-upgrade -``` - -## Upgrading to GitLab 10.8 - -Before 10.8, broadcast messages would not propagate without flushing the cache on the **secondary** nodes. This has been fixed in 10.8, but requires one last cache flush on each **secondary** node: - -```sh -sudo gitlab-rake cache:clear -``` - -## Upgrading to GitLab 10.6 - -In 10.4, we started to recommend that you define a password for database user (`gitlab`). - -We now require this change as we use this password to enable the Foreign Data Wrapper, as a way to optimize -the Geo Tracking Database. We are also improving security by disabling the use of **trust** -authentication method. - -1. **(primary)** Login to your **primary** node and run: - - ```sh - gitlab-ctl pg-password-md5 gitlab - # Enter password: <your_password_here> - # Confirm password: <your_password_here> - # fca0b89a972d69f00eb3ec98a5838484 - ``` - - Copy the generated hash and edit `/etc/gitlab/gitlab.rb`: - - ```ruby - # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab` - postgresql['sql_user_password'] = '<md5_hash_of_your_password>' - - # Every node that runs Unicorn or Sidekiq needs to have the database - # password specified as below. If you have a high-availability setup, this - # must be present in all application nodes. - gitlab_rails['db_password'] = '<your_password_here>' - ``` - - Still in the configuration file, locate and remove the `trust_auth_cidr_address`: - - ```ruby - postgresql['trust_auth_cidr_addresses'] = ['127.0.0.1/32','1.2.3.4/32'] # <- Remove this - ``` - -1. **(primary)** Reconfigure and restart: - - ```sh - sudo gitlab-ctl reconfigure - sudo gitlab-ctl restart - ``` - -1. **(secondary)** Login to all **secondary** nodes and edit `/etc/gitlab/gitlab.rb`: - - ```ruby - # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab` - postgresql['sql_user_password'] = '<md5_hash_of_your_password>' - - # Every node that runs Unicorn or Sidekiq needs to have the database - # password specified as below. If you have a high-availability setup, this - # must be present in all application nodes. - gitlab_rails['db_password'] = '<your_password_here>' - - # Enable Foreign Data Wrapper - geo_secondary['db_fdw'] = true - - # Secondary address in CIDR format, for example '5.6.7.8/32' - postgresql['md5_auth_cidr_addresses'] = ['<secondary_node_ip>/32'] - ``` - - Still in the configuration file, locate and remove the `trust_auth_cidr_address`: - - ```ruby - postgresql['trust_auth_cidr_addresses'] = ['127.0.0.1/32','5.6.7.8/32'] # <- Remove this - ``` - -1. **(secondary)** Reconfigure and restart: - - ```sh - sudo gitlab-ctl reconfigure - sudo gitlab-ctl restart - ``` - -## Upgrading to GitLab 10.5 - -For Geo Disaster Recovery to work with minimum downtime, your **secondary** node -should use the same set of secrets as the **primary** node. However, setup instructions -prior to the 10.5 release only synchronized the `db_key_base` secret. - -To rectify this error on existing installations, you should **overwrite** the -contents of `/etc/gitlab/gitlab-secrets.json` on each **secondary** node with the -contents of `/etc/gitlab/gitlab-secrets.json` on the **primary** node, then run the -following command on each **secondary** node: - -```sh -sudo gitlab-ctl reconfigure -``` - -If you do not perform this step, you may find that two-factor authentication -[is broken following DR](../disaster_recovery/index.html#i-followed-the-disaster-recovery-instructions-and-now-two-factor-auth-is-broken). - -To prevent SSH requests to the newly promoted **primary** node from failing -due to SSH host key mismatch when updating the **primary** node domain's DNS record -you should perform the step to [Manually replicate **primary** SSH host keys](configuration.md#step-2-manually-replicate-the-primary-nodes-ssh-host-keys) in each -**secondary** node. - -## Upgrading to GitLab 10.4 - -There are no Geo-specific steps to take! - -## Upgrading to GitLab 10.3 - -### Support for SSH repository synchronization removed - -In GitLab 10.2, synchronizing secondaries over SSH was deprecated. In 10.3, -support is removed entirely. All installations will switch to the HTTP/HTTPS -cloning method instead. Before upgrading, ensure that all your Geo nodes are -configured to use this method and that it works for your installation. In -particular, ensure that [Git access over HTTP/HTTPS is enabled](configuration.md#step-6-enable-git-access-over-httphttps). - -Synchronizing repositories over the public Internet using HTTP is insecure, so -you should ensure that you have HTTPS configured before upgrading. Note that -file synchronization is **also** insecure in these cases! - -## Upgrading to GitLab 10.2 - -### Secure PostgreSQL replication - -Support for TLS-secured PostgreSQL replication has been added. If you are -currently using PostgreSQL replication across the open internet without an -external means of securing the connection (e.g., a site-to-site VPN), then you -should immediately reconfigure your **primary** and **secondary** PostgreSQL instances -according to the [updated instructions][database]. - -If you *are* securing the connections externally and wish to continue doing so, -ensure you include the new option `--sslmode=prefer` in future invocations of -`gitlab-ctl replicate-geo-database`. - -### HTTPS repository sync - -Support for replicating repositories and wikis over HTTP/HTTPS has been added. -Replicating over SSH has been deprecated, and support for this option will be -removed in a future release. - -To switch to HTTP/HTTPS replication, log into the **primary** node as an admin and visit -**Admin Area > Geo** (`/admin/geo/nodes`). For each **secondary** node listed, -press the "Edit" button, change the "Repository cloning" setting from -"SSH (deprecated)" to "HTTP/HTTPS", and press "Save changes". This should take -effect immediately. - -Any new secondaries should be created using HTTP/HTTPS replication - this is the -default setting. - -After you've verified that HTTP/HTTPS replication is working, you should remove -the now-unused SSH keys from your secondaries, as they may cause problems if the -**secondary** node if ever promoted to a **primary** node: - -1. **(secondary)** Login to **all** your **secondary** nodes and run: - - ```ruby - sudo -u git -H rm ~git/.ssh/id_rsa ~git/.ssh/id_rsa.pub - ``` - -### Hashed Storage - -CAUTION: **Warning:** -Hashed storage is in **Alpha**. It is considered experimental and not -production-ready. See [Hashed Storage] for more detail. - -If you previously enabled Hashed Storage and migrated all your existing -projects to Hashed Storage, disabling hashed storage will not migrate projects -to their previous project based storage path. As such, once enabled and -migrated we recommend leaving Hashed Storage enabled. - -## Upgrading to GitLab 10.1 - -CAUTION: **Warning:** -Hashed storage is in **Alpha**. It is considered experimental and not -production-ready. See [Hashed Storage] for more detail. - -[Hashed storage] was introduced in GitLab 10.0, and a [migration path][hashed-migration] -for existing repositories was added in GitLab 10.1. - -## Upgrading to GitLab 10.0 - -Since GitLab 10.0, we require all **Geo** systems to [use SSH key lookups via -the database][ssh-fast-lookup] to avoid having to maintain consistency of the -`authorized_keys` file for SSH access. Failing to do this will prevent users -from being able to clone via SSH. - -Note that in older versions of Geo, attachments downloaded on the **secondary** -nodes would be saved to the wrong directory. We recommend that you do the -following to clean this up. - -On the **secondary** Geo nodes, run as root: - -```sh -mv /var/opt/gitlab/gitlab-rails/working /var/opt/gitlab/gitlab-rails/working.old -mkdir /var/opt/gitlab/gitlab-rails/working -chmod 700 /var/opt/gitlab/gitlab-rails/working -chown git:git /var/opt/gitlab/gitlab-rails/working -``` - -You may delete `/var/opt/gitlab/gitlab-rails/working.old` any time. - -Once this is done, we advise restarting GitLab on the **secondary** nodes for the -new working directory to be used: - -```sh -sudo gitlab-ctl restart -``` - -## Upgrading from GitLab 9.3 or older - -If you started running Geo on GitLab 9.3 or older, we recommend that you -resync your **secondary** PostgreSQL databases to use replication slots. If you -started using Geo with GitLab 9.4 or 10.x, no further action should be -required because replication slots are used by default. However, if you -started with GitLab 9.3 and upgraded later, you should still follow the -instructions below. - -When in doubt, it does not hurt to do a resync. The easiest way to do this in -Omnibus is the following: - -1. Make sure you have Omnibus GitLab on the **primary** server. -1. Run `gitlab-ctl reconfigure` and `gitlab-ctl restart postgresql`. This will enable replication slots on the **primary** database. -1. Check the steps about defining `postgresql['sql_user_password']`, `gitlab_rails['db_password']`. -1. Make sure `postgresql['max_replication_slots']` matches the number of **secondary** Geo nodes locations. -1. Install GitLab on the **secondary** server. -1. Re-run the [database replication process](database.md#step-3-initiate-the-replication-process). - -## Special update notes for 9.0.x - -> **IMPORTANT**: -With GitLab 9.0, the PostgreSQL version is upgraded to 9.6 and manual steps are -required in order to update the **secondary** nodes and keep the Streaming -Replication working. Downtime is required, so plan ahead. - -The following steps apply only if you upgrade from a 8.17 GitLab version to -9.0+. For previous versions, update to GitLab 8.17 first before attempting to -upgrade to 9.0+. - ---- - -Make sure to follow the steps in the exact order as they appear below and pay -extra attention in what node (either **primary** or **secondary**) you execute them! Each step -is prepended with the relevant node for better clarity: - -1. **(secondary)** Login to **all** your **secondary** nodes and stop all services: - - ```ruby - sudo gitlab-ctl stop - ``` - -1. **(secondary)** Make a backup of the `recovery.conf` file on **all** - **secondary** nodes to preserve PostgreSQL's credentials: - - ```sh - sudo cp /var/opt/gitlab/postgresql/data/recovery.conf /var/opt/gitlab/ - ``` - -1. **(primary)** Update the **primary** node to GitLab 9.0 following the - [regular update docs][update]. At the end of the update, the **primary** node - will be running with PostgreSQL 9.6. - -1. **(primary)** To prevent a de-synchronization of the repository replication, - stop all services except `postgresql` as we will use it to re-initialize the - **secondary** node's database: - - ```sh - sudo gitlab-ctl stop - sudo gitlab-ctl start postgresql - ``` - -1. **(secondary)** Run the following steps on each of the **secondary** nodes: - - 1. **(secondary)** Stop all services: - - ```sh - sudo gitlab-ctl stop - ``` - - 1. **(secondary)** Prevent running database migrations: - - ```sh - sudo touch /etc/gitlab/skip-auto-migrations - ``` - - 1. **(secondary)** Move the old database to another directory: - - ```sh - sudo mv /var/opt/gitlab/postgresql{,.bak} - ``` - - 1. **(secondary)** Update to GitLab 9.0 following the [regular update docs][update]. - At the end of the update, the node will be running with PostgreSQL 9.6. - - 1. **(secondary)** Make sure all services are up: - - ```sh - sudo gitlab-ctl start - ``` - - 1. **(secondary)** Reconfigure GitLab: - - ```sh - sudo gitlab-ctl reconfigure - ``` - - 1. **(secondary)** Run the PostgreSQL upgrade command: - - ```sh - sudo gitlab-ctl pg-upgrade - ``` - - 1. **(secondary)** See the stored credentials for the database that you will - need to re-initialize the replication: - - ```sh - sudo grep -s primary_conninfo /var/opt/gitlab/recovery.conf - ``` - - 1. **(secondary)** Save the snippet below in a file, let's say `/tmp/replica.sh`. Modify the - embedded paths if necessary: - - ``` - #!/bin/bash - - PORT="5432" - USER="gitlab_replicator" - echo --------------------------------------------------------------- - echo WARNING: Make sure this script is run from the secondary server - echo --------------------------------------------------------------- - echo - echo Enter the IP or FQDN of the primary PostgreSQL server - read HOST - echo Enter the password for $USER@$HOST - read -s PASSWORD - echo Enter the required sslmode - read SSLMODE - - echo Stopping PostgreSQL and all GitLab services - sudo service gitlab stop - sudo service postgresql stop - - echo Backing up postgresql.conf - sudo -u postgres mv /var/opt/gitlab/postgresql/data/postgresql.conf /var/opt/gitlab/postgresql/ - - echo Cleaning up old cluster directory - sudo -u postgres rm -rf /var/opt/gitlab/postgresql/data - - echo Starting base backup as the replicator user - echo Enter the password for $USER@$HOST - sudo -u postgres /opt/gitlab/embedded/bin/pg_basebackup -h $HOST -D /var/opt/gitlab/postgresql/data -U gitlab_replicator -v -x -P - - echo Writing recovery.conf file - sudo -u postgres bash -c "cat > /var/opt/gitlab/postgresql/data/recovery.conf <<- _EOF1_ - standby_mode = 'on' - primary_conninfo = 'host=$HOST port=$PORT user=$USER password=$PASSWORD sslmode=$SSLMODE' - _EOF1_ - " - - echo Restoring postgresql.conf - sudo -u postgres mv /var/opt/gitlab/postgresql/postgresql.conf /var/opt/gitlab/postgresql/data/ - - echo Starting PostgreSQL - sudo service postgresql start - ``` - - 1. **(secondary)** Run the recovery script using the credentials from the - previous step: - - ```sh - sudo bash /tmp/replica.sh - ``` - - 1. **(secondary)** Reconfigure GitLab: - - ```sh - sudo gitlab-ctl reconfigure - ``` - - 1. **(secondary)** Start all services: - - ```sh - sudo gitlab-ctl start - ``` - - 1. **(secondary)** Repeat the steps for the remaining **secondary** nodes. - -1. **(primary)** After all **secondary** nodes are updated, start all services in - **primary** node: - - ```sh - sudo gitlab-ctl start - ``` - -### Update tracking database on **secondary** node - -After updating a **secondary** node, you might need to run migrations on -the tracking database. The tracking database was added in GitLab 9.1, -and it is required since 10.0. - -1. Run database migrations on tracking database: - - ```sh - sudo gitlab-rake geo:db:migrate - ``` - -1. Repeat this step for each **secondary** node. - -[update]: ../../../update/README.md -[database]: database.md -[Hashed Storage]: ../../repository_storage_types.md -[hashed-migration]: ../../raketasks/storage.md -[ssh-fast-lookup]: ../../operations/fast_ssh_key_lookup.md diff --git a/doc/administration/geo/replication/version_specific_updates.md b/doc/administration/geo/replication/version_specific_updates.md new file mode 100644 index 00000000000..6d550a49df4 --- /dev/null +++ b/doc/administration/geo/replication/version_specific_updates.md @@ -0,0 +1,426 @@ +# Version specific update instructions + +Check this document if it includes instructions for the version you are updating. +These steps go together with the [general steps](updating_the_geo_nodes.md#general-update-steps) +for updating Geo nodes. + +## Updating to GitLab 12.1 + +By default, GitLab 12.1 will attempt to automatically update the +embedded PostgreSQL server to 10.7 from 9.6. Please see +[the omnibus documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance) +for the recommended procedure. + +This can be temporarily disabled by running the following before updating: + +```sh +sudo touch /etc/gitlab/disable-postgresql-upgrade +``` + +## Updating to GitLab 10.8 + +Before 10.8, broadcast messages would not propagate without flushing +the cache on the **secondary** nodes. This has been fixed in 10.8, but +requires one last cache flush on each **secondary** node: + +```sh +sudo gitlab-rake cache:clear +``` + +## Updating to GitLab 10.6 + +In 10.4, we started to recommend that you define a password for database user (`gitlab`). + +We now require this change as we use this password to enable the Foreign Data Wrapper, as a way to optimize +the Geo Tracking Database. We are also improving security by disabling the use of **trust** +authentication method. + +1. **(primary)** Login to your **primary** node and run: + + ```sh + gitlab-ctl pg-password-md5 gitlab + # Enter password: <your_password_here> + # Confirm password: <your_password_here> + # fca0b89a972d69f00eb3ec98a5838484 + ``` + + Copy the generated hash and edit `/etc/gitlab/gitlab.rb`: + + ```ruby + # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab` + postgresql['sql_user_password'] = '<md5_hash_of_your_password>' + + # Every node that runs Unicorn or Sidekiq needs to have the database + # password specified as below. If you have a high-availability setup, this + # must be present in all application nodes. + gitlab_rails['db_password'] = '<your_password_here>' + ``` + + Still in the configuration file, locate and remove the `trust_auth_cidr_address`: + + ```ruby + postgresql['trust_auth_cidr_addresses'] = ['127.0.0.1/32','1.2.3.4/32'] # <- Remove this + ``` + +1. **(primary)** Reconfigure and restart: + + ```sh + sudo gitlab-ctl reconfigure + sudo gitlab-ctl restart + ``` + +1. **(secondary)** Login to all **secondary** nodes and edit `/etc/gitlab/gitlab.rb`: + + ```ruby + # Fill with the hash generated by `gitlab-ctl pg-password-md5 gitlab` + postgresql['sql_user_password'] = '<md5_hash_of_your_password>' + + # Every node that runs Unicorn or Sidekiq needs to have the database + # password specified as below. If you have a high-availability setup, this + # must be present in all application nodes. + gitlab_rails['db_password'] = '<your_password_here>' + + # Enable Foreign Data Wrapper + geo_secondary['db_fdw'] = true + + # Secondary address in CIDR format, for example '5.6.7.8/32' + postgresql['md5_auth_cidr_addresses'] = ['<secondary_node_ip>/32'] + ``` + + Still in the configuration file, locate and remove the `trust_auth_cidr_address`: + + ```ruby + postgresql['trust_auth_cidr_addresses'] = ['127.0.0.1/32','5.6.7.8/32'] # <- Remove this + ``` + +1. **(secondary)** Reconfigure and restart: + + ```sh + sudo gitlab-ctl reconfigure + sudo gitlab-ctl restart + ``` + +## Updating to GitLab 10.5 + +For Geo Disaster Recovery to work with minimum downtime, your **secondary** node +should use the same set of secrets as the **primary** node. However, setup instructions +prior to the 10.5 release only synchronized the `db_key_base` secret. + +To rectify this error on existing installations, you should **overwrite** the +contents of `/etc/gitlab/gitlab-secrets.json` on each **secondary** node with the +contents of `/etc/gitlab/gitlab-secrets.json` on the **primary** node, then run the +following command on each **secondary** node: + +```sh +sudo gitlab-ctl reconfigure +``` + +If you do not perform this step, you may find that two-factor authentication +[is broken following DR](../disaster_recovery/index.html#i-followed-the-disaster-recovery-instructions-and-now-two-factor-auth-is-broken). + +To prevent SSH requests to the newly promoted **primary** node from failing +due to SSH host key mismatch when updating the **primary** node domain's DNS record +you should perform the step to [Manually replicate **primary** SSH host keys](configuration.md#step-2-manually-replicate-the-primary-nodes-ssh-host-keys) in each +**secondary** node. + +## Updating to GitLab 10.3 + +### Support for SSH repository synchronization removed + +In GitLab 10.2, synchronizing secondaries over SSH was deprecated. In 10.3, +support is removed entirely. All installations will switch to the HTTP/HTTPS +cloning method instead. Before updating, ensure that all your Geo nodes are +configured to use this method and that it works for your installation. In +particular, ensure that [Git access over HTTP/HTTPS is enabled](configuration.md#step-6-enable-git-access-over-httphttps). + +Synchronizing repositories over the public Internet using HTTP is insecure, so +you should ensure that you have HTTPS configured before updating. Note that +file synchronization is **also** insecure in these cases! + +## Updating to GitLab 10.2 + +### Secure PostgreSQL replication + +Support for TLS-secured PostgreSQL replication has been added. If you are +currently using PostgreSQL replication across the open internet without an +external means of securing the connection (e.g., a site-to-site VPN), then you +should immediately reconfigure your **primary** and **secondary** PostgreSQL instances +according to the [updated instructions](database.md). + +If you *are* securing the connections externally and wish to continue doing so, +ensure you include the new option `--sslmode=prefer` in future invocations of +`gitlab-ctl replicate-geo-database`. + +### HTTPS repository sync + +Support for replicating repositories and wikis over HTTP/HTTPS has been added. +Replicating over SSH has been deprecated, and support for this option will be +removed in a future release. + +To switch to HTTP/HTTPS replication, log into the **primary** node as an admin and visit +**Admin Area > Geo** (`/admin/geo/nodes`). For each **secondary** node listed, +press the "Edit" button, change the "Repository cloning" setting from +"SSH (deprecated)" to "HTTP/HTTPS", and press "Save changes". This should take +effect immediately. + +Any new secondaries should be created using HTTP/HTTPS replication - this is the +default setting. + +After you've verified that HTTP/HTTPS replication is working, you should remove +the now-unused SSH keys from your secondaries, as they may cause problems if the +**secondary** node if ever promoted to a **primary** node: + +1. **(secondary)** Login to **all** your **secondary** nodes and run: + + ```ruby + sudo -u git -H rm ~git/.ssh/id_rsa ~git/.ssh/id_rsa.pub + ``` + +### Hashed Storage + +CAUTION: **Warning:** +Hashed storage is in **Alpha**. It is considered experimental and not +production-ready. See [Hashed Storage](../../repository_storage_types.md) for more detail. + +If you previously enabled Hashed Storage and migrated all your existing +projects to Hashed Storage, disabling hashed storage will not migrate projects +to their previous project based storage path. As such, once enabled and +migrated we recommend leaving Hashed Storage enabled. + +## Updating to GitLab 10.1 + +CAUTION: **Warning:** +Hashed storage is in **Alpha**. It is considered experimental and not +production-ready. See [Hashed Storage](../../repository_storage_types.md) for more detail. + +[Hashed storage](../../repository_storage_types.md) was introduced in +GitLab 10.0, and a [migration path](../../raketasks/storage.md) for +existing repositories was added in GitLab 10.1. + +## Updating to GitLab 10.0 + +Since GitLab 10.0, we require all **Geo** systems to [use SSH key lookups via +the database](../../operations/fast_ssh_key_lookup.md) to avoid having to maintain consistency of the +`authorized_keys` file for SSH access. Failing to do this will prevent users +from being able to clone via SSH. + +Note that in older versions of Geo, attachments downloaded on the **secondary** +nodes would be saved to the wrong directory. We recommend that you do the +following to clean this up. + +On the **secondary** Geo nodes, run as root: + +```sh +mv /var/opt/gitlab/gitlab-rails/working /var/opt/gitlab/gitlab-rails/working.old +mkdir /var/opt/gitlab/gitlab-rails/working +chmod 700 /var/opt/gitlab/gitlab-rails/working +chown git:git /var/opt/gitlab/gitlab-rails/working +``` + +You may delete `/var/opt/gitlab/gitlab-rails/working.old` any time. + +Once this is done, we advise restarting GitLab on the **secondary** nodes for the +new working directory to be used: + +```sh +sudo gitlab-ctl restart +``` + +## Updating from GitLab 9.3 or older + +If you started running Geo on GitLab 9.3 or older, we recommend that you +resync your **secondary** PostgreSQL databases to use replication slots. If you +started using Geo with GitLab 9.4 or 10.x, no further action should be +required because replication slots are used by default. However, if you +started with GitLab 9.3 and updated later, you should still follow the +instructions below. + +When in doubt, it does not hurt to do a resync. The easiest way to do this in +Omnibus is the following: + +1. Make sure you have Omnibus GitLab on the **primary** server. +1. Run `gitlab-ctl reconfigure` and `gitlab-ctl restart postgresql`. This will enable replication slots on the **primary** database. +1. Check the steps about defining `postgresql['sql_user_password']`, `gitlab_rails['db_password']`. +1. Make sure `postgresql['max_replication_slots']` matches the number of **secondary** Geo nodes locations. +1. Install GitLab on the **secondary** server. +1. Re-run the [database replication process](database.md#step-3-initiate-the-replication-process). + +## Updating to GitLab 9.0 + +> **IMPORTANT**: +With GitLab 9.0, the PostgreSQL version is updated to 9.6 and manual steps are +required in order to update the **secondary** nodes and keep the Streaming +Replication working. Downtime is required, so plan ahead. + +The following steps apply only if you update from a 8.17 GitLab version to +9.0+. For previous versions, update to GitLab 8.17 first before attempting to +update to 9.0+. + +--- + +Make sure to follow the steps in the exact order as they appear below and pay +extra attention in what node (either **primary** or **secondary**) you execute them! Each step +is prepended with the relevant node for better clarity: + +1. **(secondary)** Log in to **all** your **secondary** nodes and stop all services: + + ```ruby + sudo gitlab-ctl stop + ``` + +1. **(secondary)** Make a backup of the `recovery.conf` file on **all** + **secondary** nodes to preserve PostgreSQL's credentials: + + ```sh + sudo cp /var/opt/gitlab/postgresql/data/recovery.conf /var/opt/gitlab/ + ``` + +1. **(primary)** Update the **primary** node to GitLab 9.0 following the + [regular update docs](../../../update/README.md). At the end of the + update, the **primary** node will be running with PostgreSQL 9.6. + +1. **(primary)** To prevent a de-synchronization of the repository replication, + stop all services except `postgresql` as we will use it to re-initialize the + **secondary** node's database: + + ```sh + sudo gitlab-ctl stop + sudo gitlab-ctl start postgresql + ``` + +1. **(secondary)** Run the following steps on each of the **secondary** nodes: + + 1. **(secondary)** Stop all services: + + ```sh + sudo gitlab-ctl stop + ``` + + 1. **(secondary)** Prevent running database migrations: + + ```sh + sudo touch /etc/gitlab/skip-auto-migrations + ``` + + 1. **(secondary)** Move the old database to another directory: + + ```sh + sudo mv /var/opt/gitlab/postgresql{,.bak} + ``` + + 1. **(secondary)** Update to GitLab 9.0 following the [regular update docs](../../../update/README.md). + At the end of the update, the node will be running with PostgreSQL 9.6. + + 1. **(secondary)** Make sure all services are up: + + ```sh + sudo gitlab-ctl start + ``` + + 1. **(secondary)** Reconfigure GitLab: + + ```sh + sudo gitlab-ctl reconfigure + ``` + + 1. **(secondary)** Run the PostgreSQL upgrade command: + + ```sh + sudo gitlab-ctl pg-upgrade + ``` + + 1. **(secondary)** See the stored credentials for the database that you will + need to re-initialize the replication: + + ```sh + sudo grep -s primary_conninfo /var/opt/gitlab/recovery.conf + ``` + + 1. **(secondary)** Save the snippet below in a file, let's say `/tmp/replica.sh`. Modify the + embedded paths if necessary: + + ``` + #!/bin/bash + + PORT="5432" + USER="gitlab_replicator" + echo --------------------------------------------------------------- + echo WARNING: Make sure this script is run from the secondary server + echo --------------------------------------------------------------- + echo + echo Enter the IP or FQDN of the primary PostgreSQL server + read HOST + echo Enter the password for $USER@$HOST + read -s PASSWORD + echo Enter the required sslmode + read SSLMODE + + echo Stopping PostgreSQL and all GitLab services + sudo service gitlab stop + sudo service postgresql stop + + echo Backing up postgresql.conf + sudo -u postgres mv /var/opt/gitlab/postgresql/data/postgresql.conf /var/opt/gitlab/postgresql/ + + echo Cleaning up old cluster directory + sudo -u postgres rm -rf /var/opt/gitlab/postgresql/data + + echo Starting base backup as the replicator user + echo Enter the password for $USER@$HOST + sudo -u postgres /opt/gitlab/embedded/bin/pg_basebackup -h $HOST -D /var/opt/gitlab/postgresql/data -U gitlab_replicator -v -x -P + + echo Writing recovery.conf file + sudo -u postgres bash -c "cat > /var/opt/gitlab/postgresql/data/recovery.conf <<- _EOF1_ + standby_mode = 'on' + primary_conninfo = 'host=$HOST port=$PORT user=$USER password=$PASSWORD sslmode=$SSLMODE' + _EOF1_ + " + + echo Restoring postgresql.conf + sudo -u postgres mv /var/opt/gitlab/postgresql/postgresql.conf /var/opt/gitlab/postgresql/data/ + + echo Starting PostgreSQL + sudo service postgresql start + ``` + + 1. **(secondary)** Run the recovery script using the credentials from the + previous step: + + ```sh + sudo bash /tmp/replica.sh + ``` + + 1. **(secondary)** Reconfigure GitLab: + + ```sh + sudo gitlab-ctl reconfigure + ``` + + 1. **(secondary)** Start all services: + + ```sh + sudo gitlab-ctl start + ``` + + 1. **(secondary)** Repeat the steps for the remaining **secondary** nodes. + +1. **(primary)** After all **secondary** nodes are updated, start all services in + **primary** node: + + ```sh + sudo gitlab-ctl start + ``` + +### Update tracking database on **secondary** node + +After updating a **secondary** node, you might need to run migrations on +the tracking database. The tracking database was added in GitLab 9.1, +and it is required since 10.0. + +1. Run database migrations on tracking database: + + ```sh + sudo gitlab-rake geo:db:migrate + ``` + +1. Repeat this step for each **secondary** node. diff --git a/doc/administration/index.md b/doc/administration/index.md index 711eebcd61e..d557068e6c8 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -187,13 +187,29 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Debugging tips](troubleshooting/debug.md): Tips to debug problems when things go wrong - [Log system](logs.md): Where to look for logs. - [Sidekiq Troubleshooting](troubleshooting/sidekiq.md): Debug when Sidekiq appears hung and is not processing jobs. -- Useful [diagnostics tools](troubleshooting/diagnostics_tools.md) that are sometimes used by the GitLab - Support team. -- [Troubleshooting ElasticSearch](troubleshooting/elasticsearch.md): Tips to troubleshoot ElasticSearch. -- [Kubernetes troubleshooting](troubleshooting/kubernetes_cheat_sheet.md): Commands and tips useful - for troubleshooting Kubernetes-related issues. -- Useful links from the Support Team: - - [GitLab Developer Docs](https://docs.gitlab.com/ee/development/README.html). - - [Repairing and recovering broken Git repositories](https://git.seveas.net/repairing-and-recovering-broken-git-repositories.html). - - [Testing with OpenSSL](https://www.feistyduck.com/library/openssl-cookbook/online/ch-testing-with-openssl.html). - - [Strace zine](https://wizardzines.com/zines/strace/). +- [Troubleshooting ElasticSearch](troubleshooting/elasticsearch.md) + +### Support Team Docs + +The GitLab Support Team has collected a lot of information about troubleshooting GitLab +instances. These documents are normally used by the Support Team itself, or by customers +with direct guidance from a Support Team member. GitLab administrators may find the +information useful for troubleshooting, but if you are experiencing trouble with your +GitLab instance, you should check your [support options](https://about.gitlab.com/support/) +before referring to these documents. + +CAUTION: **Warning:** +Using the commands listed in the documentation below could result in data loss or +other damage to a GitLab instance, and should only be used by experienced administrators +who are aware of the risks. + +- [Useful diagnostics tools](troubleshooting/diagnostics_tools.md) +- [Useful Linux commands](troubleshooting/linux_cheat_sheet.md) +- [Troubleshooting Kubernetes](troubleshooting/kubernetes_cheat_sheet.md) +- [Guide to test environments](troubleshooting/test_environments.md) (for Support Engineers) +- [GitLab rails console commands](troubleshooting/gitlab_rails_cheat_sheet.md) (for Support Engineers) +- Useful links: + - [GitLab Developer Docs](../development/README.md) + - [Repairing and recovering broken Git repositories](https://git.seveas.net/repairing-and-recovering-broken-git-repositories.html) + - [Testing with OpenSSL](https://www.feistyduck.com/library/openssl-cookbook/online/ch-testing-with-openssl.html) + - [Strace zine](https://wizardzines.com/zines/strace/) diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md new file mode 100644 index 00000000000..0c5611aa6cd --- /dev/null +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -0,0 +1,1040 @@ +--- +type: reference +--- + +# GitLab Rails Console Cheat Sheet + +This is the GitLab Support Team's collection of information regarding the GitLab rails +console, for use while troubleshooting. It is listed here for transparency, +and it may be useful for users with experience with these tools. If you are currently +having an issue with GitLab, it is highly recommended that you check your +[support options](https://about.gitlab.com/support/) first, before attempting to use +this information. + +CAUTION: **CAUTION:** +Please note that some of these scripts could be damaging if not run correctly, +or under the right conditions. We highly recommend running them under the +guidance of a Support Engineer, or running them in a test environment with a +backup of the instance ready to be restored, just in case. + +CAUTION: **CAUTION:** +Please also note that as GitLab changes, changes to the code are inevitable, +and so some scripts may not work as they once used to. These are not kept +up-to-date as these scripts/commands were added as they were found/needed. As +mentioned above, we recommend running these scripts under the supervision of a +Support Engineer, who can also verify that they will continue to work as they +should and, if needed, update the script for the latest version of GitLab. + +## Use the Rails Runner + +If the script you want to run is short, you can use the Rails Runner to avoid +entering the rails console in the first place. Here's an example of its use: + +```bash +gitlab-rails runner "RAILS_COMMAND" + +# Example with a 2-line script +gitlab-rails runner "user = User.first; puts user.username" +``` + +## Enable debug logging on rails console + +```ruby +Rails.logger.level = 0 +``` + +## Enable debug logging for ActiveRecord (db issues) + +```ruby +ActiveRecord::Base.logger = Logger.new(STDOUT) +``` + +## Temporarily Disable Timeout + +```ruby +ActiveRecord::Base.connection.execute('SET statement_timeout TO 0') +``` + +## Find specific methods for an object + +```ruby +Array.methods.select { |m| m.to_s.include? "sing" } +Array.methods.grep(/sing/) +``` + +## Find method source + +Works for [non-instrumented methods](https://docs.gitlab.com/ce/development/instrumentation.html#checking-instrumented-methods): + +```ruby +instance_of_object.method(:foo).source_location + +# Example for when we would call project.private? +project.method(:private?).source_location +``` + +## Query an object + +```ruby +o = Object.where('attribute like ?', 'ex') +``` + +## View all keys in cache + +```ruby +Rails.cache.instance_variable_get(:@data).keys +``` + +## Rails console history + +```ruby +puts Readline::HISTORY.to_a +``` + +## Profile a page + +```ruby +# Before 11.6.0 +logger = Logger.new(STDOUT) +admin_token = User.find_by_username('ADMIN_USERNAME').personal_access_tokens.first.token +app.get("URL/?private_token=#{admin_token}") + +# From 11.6.0 +admin = User.find_by_username('ADMIN_USERNAME') +url = "/url/goes/here" +Gitlab::Profiler.with_user(admin) { app.get(url) } +``` + +## Using the GitLab profiler inside console (used as of 10.5) + +```ruby +logger = Logger.new(STDOUT) +admin = User.find_by_username('ADMIN_USERNAME') +Gitlab::Profiler.profile('URL', logger: logger, user: admin) +``` + +## Time an operation + +```ruby +# A single operation +Benchmark.measure { <operation> } + +# A breakdown of multiple operations +Benchmark.bm do |x| + x.report(:label1) { <operation_1> } + x.report(:label2) { <operation_2> } +end +``` + +## Command Line + +### Check the GitLab version fast + +```bash +grep -m 1 gitlab /opt/gitlab/version-manifest.txt +``` + +### Debugging SSH + +```bash +GIT_SSH_COMMAND="ssh -vvv" git clone <repository> +``` + +### Debugging over HTTPS + +```bash +GIT_CURL_VERBOSE=1 GIT_TRACE=1 git clone <repository> +``` + +## Projects + +### Find projects + +```ruby +# A single project +project = Project.find_by_full_path('PROJECT_PATH') + +# All projects in a particular namespace. Can be a username, a group +# ('gitlab-org'), or even include subgroups ('gitlab-org/distribution') +namespace = Namespace.find_by_full_path('NAMESPACE_PATH') +projects = namespace.all_projects +``` + +### Clear a project's cache + +```ruby +ProjectCacheWorker.perform_async(project.id) +``` + +### Expire the .exists? cache + +```ruby +project.repository.expire_exists_cache +``` + +### Make all projects private + +```ruby +Project.update_all(visibility_level: 0) +``` + +### Find & remove projects that are pending deletion + +```ruby +# +# This section will list all the projects which are pending deletion +# +projects = Project.where(pending_delete: true) +projects.each do |p| + puts "Project name: #{p.id}" + puts "Project name: #{p.name}" + puts "Repository path: #{p.repository.storage_path}" +end + +# +# Assign a user (the root user will do) +# +user = User.find_by_username('root') + +# +# For each project listed repeat these two commands +# + +# Find the project, update the xxx-changeme values from above +project = Project.find_by_full_path('group-changeme/project-changeme') + +# Delete the project +::Projects::DestroyService.new(project, user, {}).execute +``` + +Next, run `sudo gitlab-rake gitlab:cleanup:repos` on the command line to finish. + +### Destroy a project + +```ruby +project = Project.find_by_full_path('') +user = User.find_by_username('') +ProjectDestroyWorker.perform_async(project.id, user.id, {}) +# or ProjectDestroyWorker.new.perform(project.id, user.id, {}) +# or Projects::DestroyService.new(project, user).execute +``` + +### Remove fork relationship manually + +```ruby +p = Project.find_by_full_path('') +u = User.find_by_username('') +::Projects::UnlinkForkService.new(p, u).execute +``` + +### Make a project read-only (can only be done in the console) + +```ruby +# Make a project read-only +project.repository_read_only = true; project.save + +# OR +project.update!(repository_read_only: true) +``` + +### Bulk update service integration password for _all_ projects + +For example, change the Jira user's password for all projects that have the Jira +integration active: + +```ruby +p = Project.find_by_sql("SELECT p.id FROM projects p LEFT JOIN services s ON p.id = s.project_id WHERE s.type = 'JiraService' AND s.active = true") + +p.each do |project| + project.jira_service.update_attribute(:password, '<your-new-password>') +end +``` + +### Identify un-indexed projects + +```ruby +Project.find_each do |project| + puts "id #{project.id}: #{project.namespace.name.to_s}/#{project.name.to_s}" if project.index_status.nil? +end +``` + +## Imports / Exports + +```ruby +# Find the project and get the error +p = Project.find_by_full_path('<username-or-group>/<project-name>') + +p.import_error + +# To finish the import on GitLab running version before 11.6 +p.import_finish + +# To finish the import on GitLab running version 11.6 or after +p.import_state.mark_as_failed("Failed manually through console.") +``` + +### Rename imported repository + +In a specific situation, an imported repository needed to be renamed. The Support +Team was informed of a backup restore that failed on a single repository, which created +the project with an empty repository. The project was successfully restored to a dev +instance, then exported, and imported into a new project under a different name. + +The Support Team was able to transfer the incorrectly named imported project into the +correctly named empty project using the steps below. + +Move the new repository to the empty repository: + +```bash +mv /var/opt/gitlab/git-data/repositories/<group>/<new-project> /var/opt/gitlab/git-data/repositories/<group>/<empty-project> +``` + +Make sure the permissions are correct: + +```bash +chown -R git:git <path-to-directory>.git +``` + +Clear the cache: + +```bash +sudo gitlab-rake cache:clear +``` + +## Repository + +### Search sequence of pushes to a repository + +If it seems that a commit has gone "missing", search the sequence of pushes to a repository. +[This StackOverflow article](https://stackoverflow.com/questions/13468027/the-mystery-of-the-missing-commit-across-merges) +describes how you can end up in this state without a force push. + +If you look at the output from the sample code below for the target branch, you will +see a discontinuity in the from/to commits as you step through the output. Each new +push should be "from" the "to" SHA of the previous push. When this discontinuity happens, +you will see two pushes with the same "from" SHA: + +```ruby +p = Project.find_with_namespace('u/p') +p.events.code_push.last(100).each do |e| + printf "%-20.20s %8s...%8s (%s)\n", e.data[:ref], e.data[:before], e.data[:after], e.author.try(:username) +end +``` + +GitLab 9.5 and above: + +```ruby +p = Project.find_by_full_path('u/p') +p.events.code_push.last(100).each do |e| + printf "%-20.20s %8s...%8s (%s)\n", e.push_event_payload[:ref], e.push_event_payload[:commit_from], e.push_event_payload[:commit_to], e.author.try(:username) +end +``` + +## Mirrors + +### Find mirrors with "bad decrypt" errors + +```ruby +total = 0 +bad = [] +ProjectImportData.find_each do |data| + begin + total += 1 + data.credentials + rescue => e + bad << data + end +end + +puts "Bad count: #{bad.count} / #{total}" +bad.each do |repo| + puts Project.find(repo.project_id).full_path +end; bad.count +``` + +### Transfer mirror users and tokens to a single service account + +Use case: If you have multiple users using their own GitHub credentials to set up +repository mirroring, mirroring breaks when people leave the company. Use this +script to migrate disparate mirroring users and tokens into a single service account: + +```ruby +svc_user = User.find_by(username: 'ourServiceUser') +token = 'githubAccessToken' + +Project.where(mirror: true).each do |project| + import_url = project.import_url + + # The url we want is https://token@project/path.git + repo_url = if import_url.include?('@') + # Case 1: The url is something like https://23423432@project/path.git + import_url.split('@').last + elsif import_url.include?('//') + # Case 2: The url is something like https://project/path.git + import_url.split('//').last + end + + next unless repo_url + + final_url = "https://#{token}@#{repo_url}" + + project.mirror_user = svc_user + project.import_url = final_url + project.username_only_import_url = final_url + project.save +end +``` + +## Users + +### Finding users + +```ruby +# By username +user = User.find_by(username: '') + +# By primary email +user = User.find_by(email: '') + +# By any email (primary or secondary) +user = User.find_by_any_email('') + +# Admins +User.admins +admin = User.admins.first +``` + +### Block + +```ruby +User.find_by_username().block! +``` + +### Unblock + +```ruby +User.find_by_username().active +``` + +### Skip reconfirmation + +```ruby +user = User.find_by_username '' +user.skip_reconfirmation! +``` + +### Get an admin token + +```ruby +# Get the first admin's first access token (no longer works on 11.9+. see: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22743) +User.where(admin:true).first.personal_access_tokens.first.token + +# Get the first admin's private token (no longer works on 10.2+) +User.where(admin:true).private_token +``` + +### Create personal access token + +```ruby +personal_access_token = User.find(123).personal_access_tokens.create( + name: 'apitoken', + impersonation: false, + scopes: [:api] +) + +personal_access_token.token +``` + +You might also want to manually set the token string: + +```ruby +User.find(123).personal_access_tokens.create( + name: 'apitoken', + token_digest: Gitlab::CryptoHelper.sha256('some-token-string-here'), + impersonation: false, + scopes: [:api] +) +``` + +### Disable 2FA on a user + +```ruby +user = User.find_by_username('username') +user.disable_two_factor! +``` + +### Active users & Historical users + +```ruby +# Active users on the instance, now +User.active.count + +# The historical max on the instance as of the past year +::HistoricalData.max_historical_user_count +``` + +```bash +# Using curl and jq (up to a max 100, see [pagination](https://docs.gitlab.com/ee/api/#pagination) +curl --silent --header "Private-Token: ********************" "https://gitlab.example.com/api/v4/users?per_page=100&active" | jq --compact-output '.[] | [.id,.name,.username]' +``` + +### Block or Delete Users that have no projects or groups + +```ruby +users = User.where('id NOT IN (select distinct(user_id) from project_authorizations)') + +# How many users will be removed? +users.count + +# If that count looks sane: + +# You can either block the users: +users.each { |user| user.block! } + +# Or you can delete them: + # need 'current user' (your user) for auditing purposes +current_user = User.find_by(username: '<your username>') + +users.each do |user| + DeleteUserWorker.perform_async(current_user.id, user.id) +end +``` + +### Block Users that have no recent activity + +```ruby +days_inactive = 60 +inactive_users = User.active.where("last_activity_on <= ?", days_inactive.days.ago) + +inactive_users.each do |user| + puts "user '#{user.username}': #{user.last_activity_on}" + user.block! +end +``` + +### Find Max permissions for project/group + +```ruby +user = User.find_by_username 'username' +project = Project.find_by_full_path 'group/project' +user.max_member_access_for_project project.id +``` + +```ruby +user = User.find_by_username 'username' +group = Group.find_by_full_path 'group' +user.max_member_access_for_group group.id +``` + +## Groups + +### Count unique users in a group and sub-groups + +```ruby +group = Group.find_by_path_or_name("groupname") +members = [] +for member in group.members_with_descendants + members.push(member.user_name) +end + +members.uniq.length +``` + +```ruby +group = Group.find_by_path_or_name("groupname") + +# Count users from subgroup and up (inherited) +group.members_with_parents.count + +# Count users from parent group and down (specific grants) +parent.members_with_descendants.count +``` + +### Delete a group + +```ruby +GroupDestroyWorker.perform_async(group_id, user_id) +``` + +## LDAP + +### LDAP commands in the rails console + +TIP: **TIP:** +Use the rails runner to avoid entering the rails console in the first place. +This is great when only a single command (such as a UserSync or GroupSync) +is needed. + +```ruby +# Get debug output +Rails.logger.level = Logger::DEBUG + +# Run a UserSync (normally performed once a day) +LdapSyncWorker.new.perform + +# Run a GroupSync for all groups (9.3-) +LdapGroupSyncWorker.new.perform + +# Run a GroupSync for all groups (9.3+) +LdapAllGroupsSyncWorker.new.perform + +# Run a GroupSync for a single group (10.6-) +group = Group.find_by(name: 'my_gitlab_group') +EE::Gitlab::LDAP::Sync::Group.execute_all_providers(group) + +# Run a GroupSync for a single group (10.6+) +group = Group.find_by(name: 'my_gitlab_group') +EE::Gitlab::Auth::LDAP::Sync::Group.execute_all_providers(group) + +# Query an LDAP group directly (10.6-) +adapter = Gitlab::LDAP::Adapter.new('ldapmain') # If `main` is the LDAP provider +ldap_group = EE::Gitlab::LDAP::Group.find_by_cn('group_cn_here', adapter) +ldap_group.member_dns +ldap_group.member_uids + +# Query an LDAP group directly (10.6+) +adapter = Gitlab::Auth::LDAP::Adapter.new('ldapmain') # If `main` is the LDAP provider +ldap_group = EE::Gitlab::Auth::LDAP::Group.find_by_cn('group_cn_here', adapter) +ldap_group.member_dns +ldap_group.member_uids + +# Lookup a particular user (10.6+) +# This could expose potential errors connecting to and/or querying LDAP that may seem to +# fail silently in the GitLab UI +adapter = Gitlab::Auth::LDAP::Adapter.new('ldapmain') # If `main` is the LDAP provider +user = Gitlab::Auth::LDAP::Person.find_by_uid('<username>',adapter) + +# Query the LDAP server directly (10.6+) +## For an example, see https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/ee/gitlab/auth/ldap/adapter.rb +adapter = Gitlab::Auth::LDAP::Adapter.new('ldapmain') +options = { + # the :base is required + # use adapter.config.base for the base or .group_base for the group_base + base: adapter.config.group_base, + + # :filter is optional + # 'cn' looks for all "cn"s under :base + # '*' is the search string - here, it's a wildcard + filter: Net::LDAP::Filter.eq('cn', '*'), + + # :attributes is optional + # the attributes we want to get returned + attributes: %w(dn cn memberuid member submember uniquemember memberof) +} +adapter.ldap_search(options) +``` + +### Update user accounts when the `dn` and email change + +The following will require that any accounts with the new email address are removed. +Emails have to be unique in GitLab. This is expected to work but unverified as of yet: + +```ruby +# Here's an example with a couple users. +# Each entry will have to include the old username and the new email +emails = { + 'ORIGINAL_USERNAME' => 'NEW_EMAIL_ADDRESS', + ... +} + +emails.each do |username, email| + user = User.find_by_username(username) + user.email = email + user.skip_reconfirmation! + user.save! +end + +# Run the UserSync to update the above users' data +LdapSyncWorker.new.perform +``` + +## Routes + +### Remove redirecting routes + +See <https://gitlab.com/gitlab-org/gitlab-ce/issues/41758#note_54828133>. + +```ruby +path = 'foo' +conflicting_permanent_redirects = RedirectRoute.matching_path_and_descendants(path) + +# Check that conflicting_permanent_redirects is as expected +conflicting_permanent_redirects.destroy_all +``` + +## Merge Requests + +### Find Merge Request + +```ruby +m = project.merge_requests.find_by(iid: <IID>) +m = MergeRequest.find_by_title('NEEDS UNIQUE TITLE!!!') +``` + +### Close a merge request properly (if merged but still marked as open) + +```ruby +p = Project.find_by_full_path('') +m = project.merge_requests.find_by(iid: ) +u = User.find_by_username('') +MergeRequests::PostMergeService.new(p, u).execute(m) +``` + +### Rebase manually + +```ruby +p = Project.find_by_full_path('') +m = project.merge_requests.find_by(iid: ) +u = User.find_by_username('') +MergeRequests::RebaseService.new(m.target_project, u).execute(m) +``` + +## CI + +### Cancel stuck pending pipelines + +See <https://gitlab.com/gitlab-com/support-forum/issues/2449#note_41929707>. + +```ruby +Ci::Pipeline.where(project_id: p.id).where(status: 'pending').count +Ci::Pipeline.where(project_id: p.id).where(status: 'pending').each {|p| p.cancel} +Ci::Pipeline.where(project_id: p.id).where(status: 'pending').count +``` + +### Manually modify runner minutes + +```ruby +Namespace.find_by_full_path("user/proj").namespace_statistics.update(shared_runners_seconds: 27360) +``` + +### Remove artifacts more than a week old + +```ruby +### SELECTING THE BUILDS TO CLEAR +# For a single project: +project = Project.find_by_full_path('') +builds_with_artifacts = project.builds.with_artifacts_archive + +# Prior to 10.6 the above line would be: +# builds_with_artifacts = project.builds.with_artifacts + +# Instance-wide: +builds_with_artifacts = Ci::Build.with_artifacts + +### CLEAR THEM OUT +builds_to_clear = builds_with_artifacts.where("finished_at < ?", 1.week.ago) +builds_to_clear.each do |build| + build.artifacts_expire_at = Time.now + build.erase_erasable_artifacts! +end +``` + +### Find reason failure (for when build trace is empty) (Introduced in 10.3.0) + +See <https://gitlab.com/gitlab-org/gitlab-ce/issues/41111>. + +```ruby +build = Ci::Build.find(78420) + +build.failure_reason + +build.dependencies.each do |d| { puts "status: #{d.status}, finished at: #{d.finished_at}, + completed: #{d.complete?}, artifacts_expired: #{d.artifacts_expired?}, erased: #{d.erased?}" } +``` + +### Disable strict artifact checking (Introduced in GitLab 10.3.0) + +See <https://docs.gitlab.com/ee/administration/job_artifacts.html#validation-for-dependencies>. + +```ruby +Feature.enable('ci_disable_validates_dependencies') +``` + +### Remove CI traces older than 6 months + +```ruby +current_user = User.find_by_email('cindy@gitlap.com') +Ci::Build.where("finished_at < ?", 6.months.ago.to_date).each {|b| puts b.id; b.erase(erased_by: current_user) if b.erasable?};nil +``` + +### Try CI service + +```ruby +p = Project.find_by_full_path('') +m = project.merge_requests.find_by(iid: ) +m.project.try(:ci_service) +``` + +### Disable AutoDevOps on Existing Projects + +```ruby +Project.all.each do |p| + p.auto_devops_attributes={"enabled"=>"0"} + p.save +end +``` + +## License + +### See license plan name (since v9.3.0-ee) + +```ruby +License.current.plan +``` + +### Check if a project feature is available on the instance + +Features listed in <https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/app/models/license.rb>. + +```ruby +License.current.feature_available?(:jira_dev_panel_integration) +``` + +### Check if a project feature is available in a project + +Features listed in <https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/app/models/license.rb>. + +```ruby +p = Project.find_by_full_path('<group>/<project>') +p.feature_available?(:jira_dev_panel_integration) +``` + +### Add a license through the console + +```ruby +key = "<key>" +license = License.new(data: key) +license.save +License.current # check to make sure it applied +``` + +## Unicorn + +From [Zendesk ticket #91083](https://gitlab.zendesk.com/agent/tickets/91083) (internal) + +### Poll unicorn requests by seconds + +```ruby +require 'rubygems' +require 'unicorn' + +# Usage for this program +def usage + puts "ruby unicorn_status.rb <path to unix socket> <poll interval in seconds>" + puts "Polls the given Unix socket every interval in seconds. Will not allow you to drop below 3 second poll intervals." + puts "Example: /opt/gitlab/embedded/bin/ruby poll_unicorn.rb /var/opt/gitlab/gitlab-rails/sockets/gitlab.socket 10" +end + +# Look for required args. Throw usage and exit if they don't exist. +if ARGV.count < 2 + usage + exit 1 +end + +# Get the socket and threshold values. +socket = ARGV[0] +threshold = (ARGV[1]).to_i + +# Check threshold - is it less than 3? If so, set to 3 seconds. Safety first! +if threshold.to_i < 3 + threshold = 3 +end + +# Check - does that socket exist? +unless File.exist?(socket) + puts "Socket file not found: #{socket}" + exit 1 +end + +# Poll the given socket every THRESHOLD seconds as specified above. +puts "Running infinite loop. Use CTRL+C to exit." +puts "------------------------------------------" +loop do + Raindrops::Linux.unix_listener_stats([socket]).each do |addr, stats| + puts DateTime.now.to_s + " Active: " + stats.active.to_s + " Queued: " + stats.queued.to_s + end + sleep threshold +end +``` + +## Sidekiq + +### Size of a queue + +```ruby +Sidekiq::Queue.new('background_migration').size +``` + +### Kill a worker's Sidekiq jobs + +```ruby +queue = Sidekiq::Queue.new('repository_import') +queue.each { |job| job.delete if <condition>} +``` + +### Enable debug logging of Sidekiq + +```ruby +gitlab_rails['env'] = { + 'SIDEKIQ_LOG_ARGUMENTS' => "1" +} +``` + +Then `gitlab-ctl reconfigure; gitlab-ctl restart sidekiq`. The Sidekiq logs will now include additional data for troubleshooting. + +### Sidekiq kill signals + +See <https://github.com/mperham/sidekiq/wiki/Signals#ttin>. + +## Redis + +### Connect to redis (omnibus) + +```sh +/opt/gitlab/embedded/bin/redis-cli -s /var/opt/gitlab/redis/redis.socket +``` + +### Connect to redis (HA) + +```sh +/opt/gitlab/embedded/bin/redis-cli -h <host ip> -a <password> +``` + +## LFS + +### Get info about LFS objects and associated project + +```ruby +o=LfsObject.find_by(oid: "<oid>") +p=Project.find(LfsObjectsProject.find_by_lfs_object_id(o.id).project_id) +``` + +You can then delete these records from the database with: + +```ruby +LfsObjectsProject.find_by_lfs_object_id(o.id).destroy +o.destroy +``` + +You would also want to combine this with deleting the LFS file in the LFS storage +area on disk. It remains to be seen exactly how or whether the deletion is useful, however. + +## Decryption Problems + +### Bad Decrypt Script (for encrypted variables) + +See <https://gitlab.com/snippets/1730735/raw>. + +This script will go through all the encrypted variables and count how many are not able +to be decrypted. Might be helpful to run on multiple nodes to see which `gitlab-secrets.json` +file is most up to date: + +```bash +wget -O /tmp/bad-decrypt.rb https://gitlab.com/snippets/1730735/raw +gitlab-rails runner /tmp/bad-decrypt.rb +``` + +If `ProjectImportData Bad count:` is detected and the decision is made to delete the +encrypted credentials to allow manual reentry: + +```ruby + # Find the ids of the corrupt ProjectImportData objects + total = 0 + bad = [] + ProjectImportData.find_each do |data| + begin + total += 1 + data.credentials + rescue => e + bad << data.id + end + end + + puts "Bad count: #{bad.count} / #{total}" + + # See the bad ProjectImportData ids + bad + + # Remove the corrupted credentials + import_data = ProjectImportData.where(id: bad) + import_data.each do |data| + data.update_columns({ encrypted_credentials: nil, encrypted_credentials_iv: nil, encrypted_credentials_salt: nil}) + end +``` + +If `User OTP Secret Bad count:` is detected. For each user listed disable/enable +two-factor authentication. + +### Decrypt Script for encrypted tokens + +This script will search for all encrypted tokens that are causing decryption errors, +and update or reset as needed: + +```bash +wget -O /tmp/encrypted-tokens.rb https://gitlab.com/snippets/1876342/raw +gitlab-rails runner /tmp/encrypted-tokens.rb +``` + +## Geo + +### Artifacts + +#### Find failed artifacts + +```ruby +Geo::JobArtifactRegistry.failed +``` + +#### Download artifact + +```ruby +Gitlab::Geo::JobArtifactDownloader.new(:job_artifact, <artifact_id>).execute +``` + +#### Get a count of the synced artifacts + +```ruby +Geo::JobArtifactRegistry.synced.count +``` + +#### Find `ID` of synced artifacts that are missing on primary + +```ruby +Geo::JobArtifactRegistry.synced.missing_on_primary.pluck(:artifact_id) +``` + +### Repository verification failures + +#### Get the number of verification failed repositories + +```ruby +Geo::ProjectRegistryFinder.new.count_verification_failed_repositories +``` + +#### Find the verification failed repositories + +```ruby +Geo::ProjectRegistry.verification_failed_repos +``` + +### Find repositories that failed to sync + +```ruby +Geo::ProjectRegistryFinder.new.find_failed_project_registries('repository') +``` + +### Resync repositories + +#### Queue up all repositories for resync. Sidekiq will handle each sync + +```ruby +Geo::ProjectRegistry.update_all(resync_repository: true, resync_wiki: true) +``` + +#### Sync individual repository now + +```ruby +project = Project.find_by_full_path('<group/project>') + +Geo::RepositorySyncService.new(project).execute +``` diff --git a/doc/administration/troubleshooting/linux_cheat_sheet.md b/doc/administration/troubleshooting/linux_cheat_sheet.md new file mode 100644 index 00000000000..2bbb498f020 --- /dev/null +++ b/doc/administration/troubleshooting/linux_cheat_sheet.md @@ -0,0 +1,339 @@ +--- +type: reference +--- + +# Linux Cheat Sheet + +This is the GitLab Support Team's collection of information regarding Linux, that they +sometimes use while troubleshooting. It is listed here for transparency, +and it may be useful for users with experience with Linux. If you are currently +having an issue with GitLab, you may want to check your [support options](https://about.gitlab.com/support/) +first, before attempting to use this information. + +CAUTION: **CAUTION:** +If you are administering GitLab you are expected to know these commands for your distribution +of choice. If you are a GitLab Support Engineer, consider this a cross-reference to +translate `yum` -> `apt-get` and the like. + +Note: **Note:** +Most of the commands below have not been labeled as to which distribution they work +on. Contributions are welcome to help add them. + +## System Commands + +### Distro Information + +```bash +# Debian/Ubuntu +uname -a +lsb_release -a + +# CentOS/RedHat +cat /etc/centos-release +cat /etc/redhat-release + +# This will provide a lot more information +cat /etc/os-release +``` + +### Shut down or Reboot + +```bash +shutdown -h now +reboot +``` + +### Permissions + +```bash +# change the user:group ownership of a file/dir +chown root:git <file_or_dir> + +# make a file executable +chmod u+x <file> +``` + +### Files & Dirs + +```bash +# create a new directory and all subdirectories +mkdir -p dir/dir2/dir3 + +# Send a command's output to file.txt, no STDOUT +ls > file.txt + +# Send a command's output to file.txt AND see it in STDOUT +ls | tee /tmp/file.txt + +# Search and Replace within a file +sed -i 's/original-text/new-text/g' <filename> +``` + +### See all set environment variables + +```bash +env +``` + +## Searching + +### File names + +```bash +# search for a file in a filesystem +find . -name 'filename.rb' -print + +# locate a file +locate <filename> + +# see command history +history + +# search CLI history +<ctrl>-R +``` + +### File contents + +```bash +# -B/A = show 2 lines before/after search_term +grep -B 2 -A 2 search_term <filename> + +# -<number> shows both before and after +grep -2 search_term <filename> + +# Search on all files in directory (recursively) +grep -r search_term <directory> + +# search through *.gz files is the same except with zgrep +zgrep search_term <filename> + +# Fast grep printing lines containing a string pattern +fgrep -R string_pattern <filename or directory> +``` + +### CLI + +```bash +# View command history +history + +# Run last command that started with 'his' (3 letters min) +!his + +# Search through command history +<ctrl>-R + +# Execute last command with sudo +sudo !! +``` + +## Managing resources + +### Memory, Disk, & CPU usage + +```bash +# disk space info. The '-h' gives the data in human-readable values +df -h + +# size of each file/dir and its contents in the current dir +du -hd 1 + +# or alternative +du -h --max-depth=1 + +# find files greater than certain size(k, M, G) and list them in order +# get rid of the + for exact, - for less than +find / -type f -size +100M -print0 | xargs -0 du -hs | sort -h + +# Find free memory on a system +free -m + +# Find what processes are using memory/CPU and organize by it +# Load average is 1/CPU for 1, 5, and 15 minutes +top -o %MEM +top -o %CPU +``` + +### Strace + +```bash +# strace a process +strace -tt -T -f -y -s 1024 -p <pid> + +# -tt print timestamps with microsecond accuracy + +# -T print the time spent in each syscall + +# -f also trace any child processes that forked + +# -y print the path associated with file handles + +# -s max string length to print for an event + +# -o output file + +# run strace on all unicorn processes +ps auwx | grep unicorn | awk '{ print " -p " $2}' | xargs strace -tt -T -f -y -s 1024 -o /tmp/unicorn.txt +``` + +See the [strace zine](https://wizardzines.com/zines/strace/) for a quick walkthrough. + +Brendan Gregg has a more detailed explanation of [how to use strace](http://www.brendangregg.com/blog/2014-05-11/strace-wow-much-syscall.html). + +Be aware that strace can have major impacts to system performance when it is running. + +### The Strace Parser tool + +Our [strace-parser tool](https://gitlab.com/wchandler/strace-parser) can be used to +provide a high level summary of the `strace` output. It is similar to `strace -C`, +but provides much more detailed statistics. + +MacOS and Linux binaries [are available](https://gitlab.com/gitlab-com/support/toolbox/strace-parser/-/tags), +or you can build it from source if you have the Rust compiler. + +#### How to use the tool + +First run the tool with no arguments other than the strace output file name to get +a summary of the top processes sorted by time spent actively performing tasks. You +can also sort based on total time, # of syscalls made, PID #, and # of child processes +using the `-S` or `--sort` flag. The number of results defaults to 25 processes, but +can be changed using the `-c`/`--count` option. See `--help` for full details. + +```sh +$ ./strace-parser strace.txt + +Top 25 PIDs +----------- + + pid active (ms) wait (ms) total (ms) % active syscalls + ---------- ---------- --------- --------- --------- --------- + 8795 689.072 45773.832 46462.902 16.89% 23018 + 13408 679.432 55910.891 56590.320 16.65% 28593 + 6423 554.822 13175.485 13730.308 13.60% 13735 +... +``` + +Based on the summary, you can then view the details of syscalls made by one or more +procsses using the `-p`/`--pid` for a specific process, or `-s`/`--stats` flags for +a sorted list. `--stats` takes the same sorting and count options as summary. + +```sh +$ ./strace-parse strace.text -p 6423 + +PID 6423 +13735 syscalls, active time: 554.822ms, total time: 13730.308ms + + syscall count total max avg min errors + (ms) (ms) (ms) (ms) + --------------- -------- ---------- ---------- ---------- ---------- -------- + epoll_wait 628 13175.485 21.259 20.980 0.020 + clock_gettime 7326 199.500 0.249 0.027 0.013 + stat 2101 110.768 19.056 0.053 0.017 ENOENT: 2076 + ... + --------------- + + Parent PID: 495 + Child PIDs: 8383, 8418, 8419, 8420, 8421 + + Slowest file access times for PID 6423: + + open (ms) timestamp error file name + ----------- --------------- --------------- ---------- + 29.818 10:53:11.528954 /srv/gitlab-data/builds/2018_08/6174/954448.log + 12.309 10:53:46.708274 /srv/gitlab-data/builds/2018_08/5342/954186.log + 0.039 10:53:49.222110 /opt/gitlab/embedded/service/gitlab-rails/app/views/events/event/_note.html.haml + 0.035 10:53:49.125115 /opt/gitlab/embedded/service/gitlab-rails/app/views/events/event/_push.html.haml + ... +``` + +In the example above, we can see that file opening times on `/srv/gitlab-data` are +extremely slow, about 100X slower than `/opt/gitlab`. + +When nothing stands out in the results, a good way to get more context is to run `strace` +on your own GitLab instance while performing the action performed by the customer, +then compare summaries of both results and dive into the differences. + +#### Stats for the open syscall + +Rough numbers for calls to `open` and `openat` (used to access files) on various configurations. +Slow storage can cause the dreaded `DeadlineExceeded` error in Gitaly. + +Also [see this entry](https://docs.gitlab.com/ee/administration/operations/filesystem_benchmarking.html) +in the handbook for quick tests customers can perform to check their filesystem performance. + +Keep in mind that timing information from `strace` is often somewhat inaccurate, so +small differences should not be considered significant. + +|Setup | access times | +|:--------------|:--------------| +| EFS | 10 - 30ms | +| Local Storage | 0.01 - 1ms | + +## Networking + +### Ports + +```bash +# Find the programs that are listening on ports +netstat -plnt +ss -plnt +lsof -i -P | grep <port> +``` + +### Internet/DNS + +```bash +# Show domain IP address +dig +short example.com +nslookup example.com + +# Check DNS using specific nameserver +# 8.8.8.8 = google, 1.1.1.1 = cloudflare, 208.67.222.222 = opendns +dig @8.8.8.8 example.com +nslookup example.com 1.1.1.1 + +# Find host provider +whois <ip_address> | grep -i "orgname\|netname" + +# Curl headers with redirect +curl --head --location https://example.com +``` + +## Package Management + +```bash +# Debian/Ubuntu + +# List packages +dpkg -l +apt list --installed + +# Find an installed package +dpkg -l | grep <package> +apt list --installed | grep <package> + +# Install a package +dpkg -i <package_name>.deb +apt-get install <package> +apt install <package> + +# CentOS/RedHat + +# Install a package +yum install <package> +dnf install <package> # RHEL/CentOS 8+ + +rpm -ivh <package_name>.rpm + +# Find an installed package +rpm -qa | grep <package> +``` + +## Logs + +```bash +# Print last lines in log file where 'n' +# is the number of lines to print +tail -n /path/to/log/file +``` diff --git a/doc/administration/troubleshooting/test_environments.md b/doc/administration/troubleshooting/test_environments.md new file mode 100644 index 00000000000..075effc5dc3 --- /dev/null +++ b/doc/administration/troubleshooting/test_environments.md @@ -0,0 +1,126 @@ +--- +type: reference +--- + +# Apps for a Testing Environment + +This is the GitLab Support Team's collection of information regarding testing environments, +for use while troubleshooting. It is listed here for transparency, and it may be useful +for users with experience with these tools. If you are currently having an issue with +GitLab, you may want to check your [support options](https://about.gitlab.com/support/) +first, before attempting to use this information. + +NOTE: **Note:** +This page was initially written for Support Engineers, so some of the links +are only available internally at GitLab. + +## Docker + +The following were tested on docker containers running in the cloud. Support Engineers, +please see [these docs](https://gitlab.com/gitlab-com/dev-resources/tree/master/dev-resources#running-docker-containers) +on how to run Docker containers on `dev-resources`. Other setups haven't been tested, +but contributions are welcome. + +### GitLab + +Please see [our Docker test environment docs](https://docs.gitlab.com/ee/install/digitaloceandocker.html#create-new-gitlab-container) +for how to run GitLab on Docker. When spinning this up with `docker-machine`, ensure +you change a few things: + +1. Update the name of the `docker-machine` host. You can see a list of hosts + with `docker-machine ls`. +1. Expose the necessary ports using the `-p` flag. Docker normally doesn't + allow access to any ports it uses outside of the container, so they must be + explicitly exposed. +1. Add any necessary `gitlab.rb` configuration to the + `GITLAB_OMNIBUS_CONFIG` variable. + +For example, when the `docker-machine` host we want to use is `do-docker`: + +```sh +docker run --detach --name gitlab \ +--env GITLAB_OMNIBUS_CONFIG="external_url 'http://$(docker-machine ip do-docker)'; gitlab_rails['gitlab_shell_ssh_port'] = 2222;" \ +--hostname $(docker-machine ip do-docker) \ +-p 80:80 -p 2222:22 \ +gitlab/gitlab-ee:11.5.3-ee.0 +``` + +### SAML + +#### SAML for Authentication + +We can use the [test-saml-idp Docker image](https://hub.docker.com/r/jamedjo/test-saml-idp) +to do the work for us: + +```sh +docker run --name gitlab_saml -p 8080:8080 -p 8443:8443 \ +-e SIMPLESAMLPHP_SP_ENTITY_ID=<GITLAB_IP_OR_DOMAIN> \ +-e SIMPLESAMLPHP_SP_ASSERTION_CONSUMER_SERVICE=<GITLAB_IP_OR_DOMAIN>/users/auth/saml/callback \ +-d jamedjo/test-saml-idp +``` + +The following will also need to go in your `/etc/gitlab/gitlab.rb`. See [our SAML docs](https://docs.gitlab.com/ee/integration/saml.html) +for more, as well as the list of [default usernames, passwords, and emails](https://hub.docker.com/r/jamedjo/test-saml-idp/#usage). + +```ruby +gitlab_rails['omniauth_enabled'] = true +gitlab_rails['omniauth_allow_single_sign_on'] = ['saml'] +gitlab_rails['omniauth_sync_email_from_provider'] = 'saml' +gitlab_rails['omniauth_sync_profile_from_provider'] = ['saml'] +gitlab_rails['omniauth_sync_profile_attributes'] = ['email'] +gitlab_rails['omniauth_auto_sign_in_with_provider'] = 'saml' +gitlab_rails['omniauth_block_auto_created_users'] = false +gitlab_rails['omniauth_auto_link_ldap_user'] = false +gitlab_rails['omniauth_auto_link_saml_user'] = true +gitlab_rails['omniauth_providers'] = [ + { + "name" => "saml", + "label" => "SAML", + "args" => { + assertion_consumer_service_url: '<GITLAB_IP_OR_DOMAIN>/users/auth/saml/callback', + idp_cert_fingerprint: '119b9e027959cdb7c662cfd075d9e2ef384e445f', + idp_sso_target_url: '<SAML_IP_OR_DOMAIN>:8080/simplesaml/saml2/idp/SSOService.php', + issuer: '<GITLAB_IP_OR_DOMAIN>', + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent' + } + } +] +``` + +#### GroupSAML for GitLab.com + +See [the GDK SAML documentation](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/saml.md). + +### ElasticSearch + +```sh +docker run -d --name elasticsearch \ +-p 9200:9200 -p 9300:9300 \ +-e "discovery.type=single-node" \ +docker.elastic.co/elasticsearch/elasticsearch:5.5.1 +``` + +Then confirm it works in the browser at `curl http://<IP_ADDRESS>:9200/_cat/health`. +ElasticSearch's default username is `elastic` and password is `changeme`. + +### PlantUML + +See [our PlantUML docs](../integration/plantuml.md#docker) +on running PlantUML in Docker. + +### Jira + +```sh +docker run -d -p 8081:8080 cptactionhank/atlassian-jira:latest +``` + +Then go to `<IP_ADDRESS>:8081` in the browser to set it up. This requires a +Jira license. + +### Grafana + +```sh +docker run -d --name grafana -e "GF_SECURITY_ADMIN_PASSWORD=gitlab" -p 3000:3000 grafana/grafana +``` + +Access it at `<IP_ADDRESS>:3000`. diff --git a/doc/development/testing_guide/end_to_end/best_practices.md b/doc/development/testing_guide/end_to_end/best_practices.md index 89500ef9a90..2200069ecfd 100644 --- a/doc/development/testing_guide/end_to_end/best_practices.md +++ b/doc/development/testing_guide/end_to_end/best_practices.md @@ -1,5 +1,7 @@ # Best practices when writing end-to-end tests +## Avoid using a GUI when it's not required + The majority of the end-to-end tests require some state to be built in the application for the tests to happen. A good example is a user being logged in as a pre-condition for testing the feature. @@ -36,3 +38,18 @@ Finally, interacting with the application only by its GUI generates a higher rat - When depending only on the GUI to create the application's state and tests fail due to front-end issues, we can't rely on the test failures rate, and we generate a higher rate of test flakiness. Now that we are aware of all of it, [let's go create some tests](quick_start_guide.md). + +## Prefer to split tests across multiple files + +Our framework includes a couple of parallelization mechanisms that work by executing spec files in parallel. + +However, because tests are parallelized by spec *file* and not by test/example, we can't achieve greater parallelization if a new test is added to an existing file. + +Nonetheless, there could be other reasons to add a new test to an existing file. + +For example, if tests share state that is expensive to set up it might be more efficient to perform that setup once even if it means the tests that use the setup can't be parallelized. + +In summary: + +- **Do**: Split tests across separate files, unless the tests share expensive setup. +- **Don't**: Put new tests in an existing file without considering the impact on parallelization. diff --git a/doc/user/permissions.md b/doc/user/permissions.md index c28a5e49ec4..02b5f7437ee 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -37,6 +37,8 @@ usernames. A GitLab administrator can configure the GitLab instance to NOTE: **Note:** In GitLab 11.0, the Master role was renamed to Maintainer. +While Maintainer is the highest project-level role, some actions can only be performed by a personal namespace or group owner. + The following table depicts the various user permission levels in a project. | Action | Guest | Reporter | Developer |Maintainer| Owner | @@ -74,7 +76,7 @@ The following table depicts the various user permission levels in a project. | View project statistics | | ✓ | ✓ | ✓ | ✓ | | View Error Tracking list | | ✓ | ✓ | ✓ | ✓ | | Pull from [Maven repository](project/packages/maven_repository.md) or [NPM registry](project/packages/npm_registry.md) **(PREMIUM)** | | ✓ | ✓ | ✓ | ✓ | -| Publish to [Maven repository](project/packages/maven_repository.md) or [NPM registry](project/packages/npm_registry.md) **(PREMIUM)** | | | ✓ | ✓ | ✓ || +| Publish to [Maven repository](project/packages/maven_repository.md) or [NPM registry](project/packages/npm_registry.md) **(PREMIUM)** | | | ✓ | ✓ | ✓ | | Upload [Design Management](project/issues/design_management.md) files **(PREMIUM)** | | | ✓ | ✓ | ✓ | | Create new branches | | | ✓ | ✓ | ✓ | | Push to non-protected branches | | | ✓ | ✓ | ✓ | diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 5c1c0c142e5..f704266b73d 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -54,7 +54,7 @@ variables: ROLLOUT_RESOURCE_TYPE: deployment - DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 + DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 stages: - build @@ -73,16 +73,16 @@ stages: - cleanup include: - - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml - - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml - - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml - - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml - - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml - - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml + - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml + - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml + - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml + - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml + - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml + - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml + - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml # Override DAST job to exclude master branch dast: diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 9c35d200dcb..fab504aa603 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -49,7 +49,7 @@ module Gitlab hostname = uri.hostname port = get_port(uri) - address_info = get_address_info(hostname, port) + address_info = get_address_info(hostname, port, dns_rebind_protection) return [uri, nil] unless address_info ip_address = ip_address(address_info) @@ -110,11 +110,15 @@ module Gitlab validate_unicode_restriction(uri) if ascii_only end - def get_address_info(hostname, port) + def get_address_info(hostname, port, dns_rebind_protection) Addrinfo.getaddrinfo(hostname, port, nil, :STREAM).map do |addr| addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr end rescue SocketError + # If the dns rebinding protection is not enabled, we allow + # urls that can't be resolved at this point. + return unless dns_rebind_protection + # In the test suite we use a lot of mocked urls that are either invalid or # don't exist. In order to avoid modifying a ton of tests and factories # we allow invalid urls unless the environment variable RSPEC_ALLOW_INVALID_URLS diff --git a/spec/lib/gitlab/url_blocker_spec.rb b/spec/lib/gitlab/url_blocker_spec.rb index df8a1f82f81..6d1d7e48326 100644 --- a/spec/lib/gitlab/url_blocker_spec.rb +++ b/spec/lib/gitlab/url_blocker_spec.rb @@ -4,80 +4,114 @@ require 'spec_helper' describe Gitlab::UrlBlocker do + include StubRequests + describe '#validate!' do + subject { described_class.validate!(import_url) } + + shared_examples 'validates URI and hostname' do + it 'runs the url validations' do + uri, hostname = subject + + expect(uri).to eq(Addressable::URI.parse(expected_uri)) + expect(hostname).to eq(expected_hostname) + end + end + context 'when URI is nil' do let(:import_url) { nil } - it 'returns no URI and hostname' do - uri, hostname = described_class.validate!(import_url) - - expect(uri).to be(nil) - expect(hostname).to be(nil) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { nil } + let(:expected_hostname) { nil } end end context 'when URI is internal' do let(:import_url) { 'http://localhost' } - it 'returns URI and no hostname' do - uri, hostname = described_class.validate!(import_url) - - expect(uri).to eq(Addressable::URI.parse('http://[::1]')) - expect(hostname).to eq('localhost') + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'http://[::1]' } + let(:expected_hostname) { 'localhost' } end end context 'when the URL hostname is a domain' do - let(:import_url) { 'https://example.org' } + context 'when domain can be resolved' do + let(:import_url) { 'https://example.org' } - it 'returns URI and hostname' do - uri, hostname = described_class.validate!(import_url) + before do + stub_dns(import_url, ip_address: '93.184.216.34') + end - expect(uri).to eq(Addressable::URI.parse('https://93.184.216.34')) - expect(hostname).to eq('example.org') + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'https://93.184.216.34' } + let(:expected_hostname) { 'example.org' } + end + end + + context 'when domain cannot be resolved' do + let(:import_url) { 'http://foobar.x' } + + it 'raises an error' do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + + expect { subject }.to raise_error(described_class::BlockedUrlError) + end end end context 'when the URL hostname is an IP address' do let(:import_url) { 'https://93.184.216.34' } - it 'returns URI and no hostname' do - uri, hostname = described_class.validate!(import_url) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + end + + context 'when the address is invalid' do + let(:import_url) { 'http://1.1.1.1.1' } - expect(uri).to eq(Addressable::URI.parse('https://93.184.216.34')) - expect(hostname).to be(nil) + it 'raises an error' do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + + expect { subject }.to raise_error(described_class::BlockedUrlError) + end end end context 'disabled DNS rebinding protection' do + subject { described_class.validate!(import_url, dns_rebind_protection: false) } + context 'when URI is internal' do let(:import_url) { 'http://localhost' } - it 'returns URI and no hostname' do - uri, hostname = described_class.validate!(import_url, dns_rebind_protection: false) - - expect(uri).to eq(Addressable::URI.parse('http://localhost')) - expect(hostname).to be(nil) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } end end context 'when the URL hostname is a domain' do let(:import_url) { 'https://example.org' } - it 'returns URI and no hostname' do - uri, hostname = described_class.validate!(import_url, dns_rebind_protection: false) + before do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + end - expect(uri).to eq(Addressable::URI.parse('https://example.org')) - expect(hostname).to eq(nil) + context 'when domain can be resolved' do + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + end end - context 'when it cannot be resolved' do + context 'when domain cannot be resolved' do let(:import_url) { 'http://foobar.x' } - it 'raises error' do - stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') - - expect { described_class.validate!(import_url) }.to raise_error(described_class::BlockedUrlError) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } end end end @@ -85,20 +119,17 @@ describe Gitlab::UrlBlocker do context 'when the URL hostname is an IP address' do let(:import_url) { 'https://93.184.216.34' } - it 'returns URI and no hostname' do - uri, hostname = described_class.validate!(import_url, dns_rebind_protection: false) - - expect(uri).to eq(Addressable::URI.parse('https://93.184.216.34')) - expect(hostname).to be(nil) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } end context 'when it is invalid' do let(:import_url) { 'http://1.1.1.1.1' } - it 'raises an error' do - stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') - - expect { described_class.validate!(import_url) }.to raise_error(described_class::BlockedUrlError) + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } end end end |