From 7dd545b130cb0cc2196ac15d7968736f679e8072 Mon Sep 17 00:00:00 2001 From: Elliot Rushton Date: Thu, 8 Aug 2019 01:00:23 +0000 Subject: Fix required runner permissions --- doc/ci/runners/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/ci/runners/README.md b/doc/ci/runners/README.md index 03a219e03ca..f5f8e04755e 100644 --- a/doc/ci/runners/README.md +++ b/doc/ci/runners/README.md @@ -88,7 +88,7 @@ visit the project you want to make the Runner work for in GitLab: ## Registering a group Runner -Creating a group Runner requires Maintainer permissions for the group. To create a +Creating a group Runner requires Owner permissions for the group. To create a group Runner visit the group you want to make the Runner work for in GitLab: 1. Go to **Settings > CI/CD** to obtain the token @@ -124,9 +124,9 @@ To lock/unlock a Runner: ## Assigning a Runner to another project -If you are Maintainer on a project where a specific Runner is assigned to, and the +If you are an Owner on a project where a specific Runner is assigned to, and the Runner is not [locked only to that project](#locking-a-specific-runner-from-being-enabled-for-other-projects), -you can enable the Runner also on any other project where you have Maintainer permissions. +you can enable the Runner also on any other project where you have Owner permissions. To enable/disable a Runner in your project: @@ -250,7 +250,7 @@ When you [register a Runner][register], its default behavior is to **only pick** [tagged jobs](../yaml/README.md#tags). NOTE: **Note:** -Maintainer [permissions](../../user/permissions.md) are required to change the +Owner [permissions](../../user/permissions.md) are required to change the Runner settings. To make a Runner pick untagged jobs: -- cgit v1.2.1 From 880c9a25a8738a4f4be26b147655b4cb6ac89d4e Mon Sep 17 00:00:00 2001 From: Steve Azzopardi Date: Thu, 25 Jul 2019 14:14:31 +0200 Subject: Update srmX configuration --- doc/user/gitlab_com/index.md | 83 ++++++++++---------------------------------- 1 file changed, 19 insertions(+), 64 deletions(-) diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index c9fbd7effa0..d21a325d401 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -112,57 +112,6 @@ Below are the shared Runners settings. The full contents of our `config.toml` are: -**DigitalOcean** - -```toml -concurrent = X -check_interval = 1 -metrics_server = "X" -sentry_dsn = "X" - -[[runners]] - name = "docker-auto-scale" - request_concurrency = X - url = "https://gitlab.com/" - token = "SHARED_RUNNER_TOKEN" - executor = "docker+machine" - environment = [ - "DOCKER_DRIVER=overlay2" - ] - limit = X - [runners.docker] - image = "ruby:2.5" - privileged = true - [runners.machine] - IdleCount = 20 - IdleTime = 1800 - OffPeakPeriods = ["* * * * * sat,sun *"] - OffPeakTimezone = "UTC" - OffPeakIdleCount = 5 - OffPeakIdleTime = 1800 - MaxBuilds = 1 - MachineName = "srm-%s" - MachineDriver = "digitalocean" - MachineOptions = [ - "digitalocean-image=X", - "digitalocean-ssh-user=core", - "digitalocean-region=nyc1", - "digitalocean-size=s-2vcpu-2gb", - "digitalocean-private-networking", - "digitalocean-tags=shared_runners,gitlab_com", - "engine-registry-mirror=http://INTERNAL_IP_OF_OUR_REGISTRY_MIRROR", - "digitalocean-access-token=DIGITAL_OCEAN_ACCESS_TOKEN", - ] - [runners.cache] - Type = "s3" - BucketName = "runner" - Insecure = true - Shared = true - ServerAddress = "INTERNAL_IP_OF_OUR_CACHE_SERVER" - AccessKey = "ACCESS_KEY" - SecretKey = "ACCESS_SECRET_KEY" -``` - **Google Cloud Platform** ```toml @@ -178,20 +127,25 @@ sentry_dsn = "X" token = "SHARED_RUNNER_TOKEN" executor = "docker+machine" environment = [ - "DOCKER_DRIVER=overlay2" + "DOCKER_DRIVER=overlay2", + "DOCKER_TLS_CERTDIR=" ] limit = X [runners.docker] image = "ruby:2.5" privileged = true + volumes = [ + "/certs/client", + "/dummy-sys-class-dmi-id:/sys/class/dmi/id:ro" # Make kaniko builds work on GCP. + ] [runners.machine] - IdleCount = 20 - IdleTime = 1800 + IdleCount = 50 + IdleTime = 3600 OffPeakPeriods = ["* * * * * sat,sun *"] OffPeakTimezone = "UTC" - OffPeakIdleCount = 5 - OffPeakIdleTime = 1800 - MaxBuilds = 1 + OffPeakIdleCount = 15 + OffPeakIdleTime = 3600 + MaxBuilds = 1 # For security reasons we delete the VM after job has finished so it's not reused. MachineName = "srm-%s" MachineDriver = "google" MachineOptions = [ @@ -202,17 +156,18 @@ sentry_dsn = "X" "google-tags=gitlab-com,srm", "google-use-internal-ip", "google-zone=us-east1-d", + "engine-opt=mtu=1460", # Set MTU for container interface, for more information check https://gitlab.com/gitlab-org/gitlab-runner/issues/3214#note_82892928 "google-machine-image=PROJECT/global/images/IMAGE", - "engine-registry-mirror=http://INTERNAL_IP_OF_OUR_REGISTRY_MIRROR" + "engine-opt=ipv6", # This will create IPv6 interfaces in the containers. + "engine-opt=fixed-cidr-v6=fc00::/7", + "google-operation-backoff-initial-interval=2" # Custom flag from forked docker-machine, for more information check https://github.com/docker/machine/pull/4600 ] [runners.cache] - Type = "s3" - BucketName = "runner" - Insecure = true + Type = "gcs" Shared = true - ServerAddress = "INTERNAL_IP_OF_OUR_CACHE_SERVER" - AccessKey = "ACCESS_KEY" - SecretKey = "ACCESS_SECRET_KEY" + [runners.cache.gcs] + CredentialsFile = "/path/to/file" + BucketName = "bucket-name" ``` ## Sidekiq -- cgit v1.2.1 From 5a574883f95373e13f663568eb4710c9d69d00d6 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Mon, 12 Aug 2019 11:29:10 +0100 Subject: Remove MySQL references from development docs I noticed the doc/development/testing_guide/best_practices.md still referenced the `[run mysql]` tags, etc. They no longer work, so I removed them, then realised I had better clean up the rest of doc/development ! --- doc/development/architecture.md | 22 ++++++++-------- doc/development/hash_indexes.md | 2 +- doc/development/rake_tasks.md | 2 +- doc/development/sha1_as_binary.md | 2 +- doc/development/sql.md | 30 +++++++--------------- doc/development/testing_guide/best_practices.md | 10 -------- doc/development/testing_guide/ci.md | 1 - doc/development/testing_guide/flaky_tests.md | 4 +-- doc/development/verifying_database_capabilities.md | 8 +++--- doc/development/what_requires_downtime.md | 25 ++++-------------- 10 files changed, 34 insertions(+), 72 deletions(-) diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 87405bc2fec..5cb2ddf6e52 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -20,7 +20,7 @@ A typical install of GitLab will be on GNU/Linux. It uses Nginx or Apache as a w We also support deploying GitLab on Kubernetes using our [gitlab Helm chart](https://docs.gitlab.com/charts/). -The GitLab web app uses MySQL or PostgreSQL for persistent database information (e.g. users, permissions, issues, other meta data). GitLab stores the bare git repositories it serves in `/home/git/repositories` by default. It also keeps default branch and hook information with the bare repository. +The GitLab web app uses PostgreSQL for persistent database information (e.g. users, permissions, issues, other meta data). GitLab stores the bare git repositories it serves in `/home/git/repositories` by default. It also keeps default branch and hook information with the bare repository. When serving repositories over HTTP/HTTPS GitLab utilizes the GitLab API to resolve authorization and access as well as serving git objects. @@ -511,7 +511,15 @@ To summarize here's the [directory structure of the `git` user home directory](. ps aux | grep '^git' ``` -GitLab has several components to operate. As a system user (i.e. any user that is not the `git` user) it requires a persistent database (MySQL/PostreSQL) and redis database. It also uses Apache httpd or Nginx to proxypass Unicorn. As the `git` user it starts Sidekiq and Unicorn (a simple ruby HTTP server running on port `8080` by default). Under the GitLab user there are normally 4 processes: `unicorn_rails master` (1 process), `unicorn_rails worker` (2 processes), `sidekiq` (1 process). +GitLab has several components to operate. It requires a persistent database +(PostgreSQL) and redis database, and uses Apache httpd or Nginx to proxypass +Unicorn. All these components should run as different system users to GitLab +(e.g., `postgres`, `redis` and `www-data`, instead of `git`). + +As the `git` user it starts Sidekiq and Unicorn (a simple ruby HTTP server +running on port `8080` by default). Under the GitLab user there are normally 4 +processes: `unicorn_rails master` (1 process), `unicorn_rails worker` +(2 processes), `sidekiq` (1 process). ### Repository access @@ -554,12 +562,9 @@ $ /etc/init.d/nginx Usage: nginx {start|stop|restart|reload|force-reload|status|configtest} ``` -Persistent database (one of the following) +Persistent database ``` -/etc/init.d/mysqld -Usage: /etc/init.d/mysqld {start|stop|status|restart|condrestart|try-restart|reload|force-reload} - $ /etc/init.d/postgresql Usage: /etc/init.d/postgresql {start|stop|restart|reload|force-reload|status} [version ..] ``` @@ -597,11 +602,6 @@ PostgreSQL - `/var/log/postgresql/*` -MySQL - -- `/var/log/mysql/*` -- `/var/log/mysql.*` - ### GitLab specific config files GitLab has configuration files located in `/home/git/gitlab/config/*`. Commonly referenced config files include: diff --git a/doc/development/hash_indexes.md b/doc/development/hash_indexes.md index e6c1b3590b1..417ea18e22f 100644 --- a/doc/development/hash_indexes.md +++ b/doc/development/hash_indexes.md @@ -1,6 +1,6 @@ # Hash Indexes -Both PostgreSQL and MySQL support hash indexes besides the regular btree +PostgreSQL supports hash indexes besides the regular btree indexes. Hash indexes however are to be avoided at all costs. While they may _sometimes_ provide better performance the cost of rehashing can be very high. More importantly: at least until PostgreSQL 10.0 hash indexes are not diff --git a/doc/development/rake_tasks.md b/doc/development/rake_tasks.md index c97e179910b..e9d6cfe00b2 100644 --- a/doc/development/rake_tasks.md +++ b/doc/development/rake_tasks.md @@ -9,7 +9,7 @@ bundle exec rake setup ``` The `setup` task is an alias for `gitlab:setup`. -This tasks calls `db:reset` to create the database, calls `add_limits_mysql` that adds limits to the database schema in case of a MySQL database and finally it calls `db:seed_fu` to seed the database. +This tasks calls `db:reset` to create the database, and calls `db:seed_fu` to seed the database. Note: `db:setup` calls `db:seed` but this does nothing. ### Seeding issues for all or a given project diff --git a/doc/development/sha1_as_binary.md b/doc/development/sha1_as_binary.md index 3151cc29bbc..6c4252ec634 100644 --- a/doc/development/sha1_as_binary.md +++ b/doc/development/sha1_as_binary.md @@ -2,7 +2,7 @@ Storing SHA1 hashes as strings is not very space efficient. A SHA1 as a string requires at least 40 bytes, an additional byte to store the encoding, and -perhaps more space depending on the internals of PostgreSQL and MySQL. +perhaps more space depending on the internals of PostgreSQL. On the other hand, if one were to store a SHA1 as binary one would only need 20 bytes for the actual SHA1, and 1 or 4 bytes of additional space (again depending diff --git a/doc/development/sql.md b/doc/development/sql.md index a256fd46c09..2584dcfb4ca 100644 --- a/doc/development/sql.md +++ b/doc/development/sql.md @@ -15,14 +15,11 @@ FROM issues WHERE title LIKE 'WIP:%'; ``` -On PostgreSQL the `LIKE` statement is case-sensitive. On MySQL this depends on -the case-sensitivity of the collation, which is usually case-insensitive. To -perform a case-insensitive `LIKE` on PostgreSQL you have to use `ILIKE` instead. -This statement in turn isn't supported on MySQL. +On PostgreSQL the `LIKE` statement is case-sensitive. To perform a case-insensitive +`LIKE` you have to use `ILIKE` instead. -To work around this problem you should write `LIKE` queries using Arel instead -of raw SQL fragments as Arel automatically uses `ILIKE` on PostgreSQL and `LIKE` -on MySQL. This means that instead of this: +To handle this automatically you should use `LIKE` queries using Arel instead +of raw SQL fragments, as Arel automatically uses `ILIKE` on PostgreSQL. ```ruby Issue.where('title LIKE ?', 'WIP:%') @@ -45,7 +42,7 @@ table = Issue.arel_table Issue.where(table[:title].matches('WIP:%').or(table[:foo].matches('WIP:%'))) ``` -For PostgreSQL this produces: +On PostgreSQL, this produces: ```sql SELECT * @@ -53,18 +50,10 @@ FROM issues WHERE (title ILIKE 'WIP:%' OR foo ILIKE 'WIP:%') ``` -In turn for MySQL this produces: - -```sql -SELECT * -FROM issues -WHERE (title LIKE 'WIP:%' OR foo LIKE 'WIP:%') -``` - ## LIKE & Indexes -Neither PostgreSQL nor MySQL use any indexes when using `LIKE` / `ILIKE` with a -wildcard at the start. For example, this will not use any indexes: +PostgreSQL won't use any indexes when using `LIKE` / `ILIKE` with a wildcard at +the start. For example, this will not use any indexes: ```sql SELECT * @@ -75,9 +64,8 @@ WHERE title ILIKE '%WIP:%'; Because the value for `ILIKE` starts with a wildcard the database is not able to use an index as it doesn't know where to start scanning the indexes. -MySQL provides no known solution to this problem. Luckily PostgreSQL _does_ -provide a solution: trigram GIN indexes. These indexes can be created as -follows: +Luckily, PostgreSQL _does_ provide a solution: trigram GIN indexes. These +indexes can be created as follows: ```sql CREATE INDEX [CONCURRENTLY] index_name_here diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index 448d9fd01c4..a505d3c79a2 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -15,16 +15,6 @@ manifest themselves within our code. When designing our tests, take time to revi our test design. We can find some helpful heuristics documented in the Handbook in the [Test Design](https://about.gitlab.com/handbook/engineering/quality/guidelines/test-engineering/test-design/) section. -## Run tests against MySQL - -By default, tests are only run against PostgreSQL, but you can run them on -demand against MySQL by following one of the following conventions: - -| Convention | Valid example | -|:----------------------|:-----------------------------| -| Include `mysql` in your branch name | `enhance-mysql-support` | -| Include `[run mysql]` in your commit message | `Fix MySQL support

[run mysql]` | - ## Test speed GitLab has a massive test suite that, without [parallelization], can take hours diff --git a/doc/development/testing_guide/ci.md b/doc/development/testing_guide/ci.md index 87d48726268..d9f66a827de 100644 --- a/doc/development/testing_guide/ci.md +++ b/doc/development/testing_guide/ci.md @@ -39,7 +39,6 @@ slowest test files and try to improve them. ## CI setup -- On CE and EE, the test suite runs both PostgreSQL and MySQL. - Rails logging to `log/test.log` is disabled by default in CI [for performance reasons][logging]. To override this setting, provide the `RAILS_ENABLE_TEST_LOG` environment variable. diff --git a/doc/development/testing_guide/flaky_tests.md b/doc/development/testing_guide/flaky_tests.md index 931cbc51cae..eb0bf6fc563 100644 --- a/doc/development/testing_guide/flaky_tests.md +++ b/doc/development/testing_guide/flaky_tests.md @@ -35,8 +35,8 @@ Once a test is in quarantine, there are 3 choices: Quarantined tests are run on the CI in dedicated jobs that are allowed to fail: -- `rspec-pg-quarantine` and `rspec-mysql-quarantine` (CE & EE) -- `rspec-pg-quarantine-ee` and `rspec-mysql-quarantine-ee` (EE only) +- `rspec-pg-quarantine` (CE & EE) +- `rspec-pg-quarantine-ee` (EE only) ## Automatic retries and flaky tests detection diff --git a/doc/development/verifying_database_capabilities.md b/doc/development/verifying_database_capabilities.md index ccec6f7d719..6b4995aebe2 100644 --- a/doc/development/verifying_database_capabilities.md +++ b/doc/development/verifying_database_capabilities.md @@ -1,15 +1,15 @@ # Verifying Database Capabilities -Sometimes certain bits of code may only work on a certain database and/or +Sometimes certain bits of code may only work on a certain database version. While we try to avoid such code as much as possible sometimes it is necessary to add database (version) specific behaviour. To facilitate this we have the following methods that you can use: -- `Gitlab::Database.postgresql?`: returns `true` if PostgreSQL is being used -- `Gitlab::Database.mysql?`: returns `true` if MySQL is being used +- `Gitlab::Database.postgresql?`: returns `true` if PostgreSQL is being used. + You can normally just assume this is the case. - `Gitlab::Database.version`: returns the PostgreSQL version number as a string - in the format `X.Y.Z`. This method does not work for MySQL + in the format `X.Y.Z`. This allows you to write code such as: diff --git a/doc/development/what_requires_downtime.md b/doc/development/what_requires_downtime.md index f0da1cc2ddc..944bf5900c5 100644 --- a/doc/development/what_requires_downtime.md +++ b/doc/development/what_requires_downtime.md @@ -7,9 +7,8 @@ downtime. ## Adding Columns -On PostgreSQL you can safely add a new column to an existing table as long as it -does **not** have a default value. For example, this query would not require -downtime: +You can safely add a new column to an existing table as long as it does **not** +have a default value. For example, this query would not require downtime: ```sql ALTER TABLE projects ADD COLUMN random_value int; @@ -27,11 +26,6 @@ This requires updating every single row in the `projects` table so that indexes in a table. This in turn acquires enough locks on the table for it to effectively block any other queries. -As of MySQL 5.6 adding a column to a table is still quite an expensive -operation, even when using `ALGORITHM=INPLACE` and `LOCK=NONE`. This means -downtime _may_ be required when modifying large tables as otherwise the -operation could potentially take hours to complete. - Adding a column with a default value _can_ be done without requiring downtime when using the migration helper method `Gitlab::Database::MigrationHelpers#add_column_with_default`. This method works @@ -311,8 +305,7 @@ migrations](background_migrations.md#cleaning-up). ## Adding Indexes Adding indexes is an expensive process that blocks INSERT and UPDATE queries for -the duration. When using PostgreSQL one can work around this by using the -`CONCURRENTLY` option: +the duration. You can work around this by using the `CONCURRENTLY` option: ```sql CREATE INDEX CONCURRENTLY index_name ON projects (column_name); @@ -336,17 +329,9 @@ end Note that `add_concurrent_index` can not be reversed automatically, thus you need to manually define `up` and `down`. -When running this on PostgreSQL the `CONCURRENTLY` option mentioned above is -used. On MySQL this method produces a regular `CREATE INDEX` query. - -MySQL doesn't really have a workaround for this. Supposedly it _can_ create -indexes without the need for downtime but only for variable width columns. The -details on this are a bit sketchy. Since it's better to be safe than sorry one -should assume that adding indexes requires downtime on MySQL. - ## Dropping Indexes -Dropping an index does not require downtime on both PostgreSQL and MySQL. +Dropping an index does not require downtime. ## Adding Tables @@ -370,7 +355,7 @@ transaction this means this approach would require downtime. GitLab allows you to work around this by using `Gitlab::Database::MigrationHelpers#add_concurrent_foreign_key`. This method -ensures that when PostgreSQL is used no downtime is needed. +ensures that no downtime is needed. ## Removing Foreign Keys -- cgit v1.2.1 From b46b9d5e89d54cf1e374a014f0d523735c82ab8c Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 17 Aug 2019 00:42:23 -0700 Subject: Fix pipelines not always being created after a push https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/31741 introduced a regression where not all the right parameters would be passed into `Ci::CreatePipelineService`. We fix this by breaking out the pipeline parameters and reusing a method from `Gitlab::DataBuilder::Push`. Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/66196 --- app/services/git/base_hooks_service.rb | 19 ++++++++++++------- .../unreleased/sh-fix-pipelines-not-being-created.yml | 5 +++++ lib/gitlab/data_builder/push.rb | 2 -- spec/services/git/branch_push_service_spec.rb | 16 ++++++++++++++++ 4 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 changelogs/unreleased/sh-fix-pipelines-not-being-created.yml diff --git a/app/services/git/base_hooks_service.rb b/app/services/git/base_hooks_service.rb index 3fd38444196..47c308c8280 100644 --- a/app/services/git/base_hooks_service.rb +++ b/app/services/git/base_hooks_service.rb @@ -56,7 +56,7 @@ module Git return unless params.fetch(:create_pipelines, true) Ci::CreatePipelineService - .new(project, current_user, base_params) + .new(project, current_user, pipeline_params) .execute(:push, pipeline_options) end @@ -75,24 +75,29 @@ module Git ProjectCacheWorker.perform_async(project.id, file_types, [], false) end - def base_params + def pipeline_params { - oldrev: params[:oldrev], - newrev: params[:newrev], + before: params[:oldrev], + after: params[:newrev], ref: params[:ref], - push_options: params[:push_options] || {} + push_options: params[:push_options] || {}, + checkout_sha: Gitlab::DataBuilder::Push.checkout_sha( + project.repository, params[:newrev], params[:ref]) } end def push_data_params(commits:, with_changed_files: true) - base_params.merge( + { + oldrev: params[:oldrev], + newrev: params[:newrev], + ref: params[:ref], project: project, user: current_user, commits: commits, message: event_message, commits_count: commits_count, with_changed_files: with_changed_files - ) + } end def event_push_data diff --git a/changelogs/unreleased/sh-fix-pipelines-not-being-created.yml b/changelogs/unreleased/sh-fix-pipelines-not-being-created.yml new file mode 100644 index 00000000000..a6937eae588 --- /dev/null +++ b/changelogs/unreleased/sh-fix-pipelines-not-being-created.yml @@ -0,0 +1,5 @@ +--- +title: Fix pipelines not always being created after a push +merge_request: 31927 +author: +type: fixed diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 37fadb47736..75d9a2d55b9 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -129,8 +129,6 @@ module Gitlab SAMPLE_DATA end - private - def checkout_sha(repository, newrev, ref) # Checkout sha is nil when we remove branch or tag return if Gitlab::Git.blank_ref?(newrev) diff --git a/spec/services/git/branch_push_service_spec.rb b/spec/services/git/branch_push_service_spec.rb index ad5d296f5c1..d9e607cd251 100644 --- a/spec/services/git/branch_push_service_spec.rb +++ b/spec/services/git/branch_push_service_spec.rb @@ -76,6 +76,22 @@ describe Git::BranchPushService, services: true do stub_ci_pipeline_to_return_yaml_file end + it 'creates a pipeline with the right parameters' do + expect(Ci::CreatePipelineService) + .to receive(:new) + .with(project, + user, + { + before: oldrev, + after: newrev, + ref: ref, + checkout_sha: SeedRepo::Commit::ID, + push_options: {} + }).and_call_original + + subject + end + it "creates a new pipeline" do expect { subject }.to change { Ci::Pipeline.count } -- cgit v1.2.1 From 56dcd45a766107b7b3069edd4a3993badf8bb2e0 Mon Sep 17 00:00:00 2001 From: Mark Lapierre Date: Mon, 19 Aug 2019 08:07:39 +1000 Subject: Quarantine failing test --- .../features/browser_ui/3_create/repository/add_file_template_spec.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb index 567c6a83ddf..458072b1507 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true module QA - context 'Create' do + # Failure issue: https://gitlab.com/gitlab-org/quality/nightly/issues/127 + context 'Create', :quarantine do describe 'File templates' do include Runtime::Fixtures -- cgit v1.2.1 From 5ba8716dda99a8c60af906f6e0a4540bec27395c Mon Sep 17 00:00:00 2001 From: Ben Bodenmiller Date: Sun, 18 Aug 2019 23:15:22 +0000 Subject: add GitLab CI job token API authentication section --- doc/api/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/api/README.md b/doc/api/README.md index b7ee710b87a..9156d719e11 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -77,11 +77,12 @@ authentication is not provided. For those cases where it is not required, this will be mentioned in the documentation for each individual endpoint. For example, the [`/projects/:id` endpoint](projects.md). -There are three ways to authenticate with the GitLab API: +There are four ways to authenticate with the GitLab API: 1. [OAuth2 tokens](#oauth2-tokens) 1. [Personal access tokens](#personal-access-tokens) 1. [Session cookie](#session-cookie) +1. [GitLab CI job token](#gitlab-ci-job-token-premium) **(PREMIUM)** For admins who want to authenticate with the API as a specific user, or who want to build applications or scripts that do so, two options are available: @@ -151,6 +152,14 @@ The primary user of this authentication method is the web frontend of GitLab its which can use the API as the authenticated user to get a list of their projects, for example, without needing to explicitly pass an access token. +### GitLab CI job token **(PREMIUM)** + +With a few API endpoints you can use a [GitLab CI job token](../user/project/new_ci_build_permissions_model.md#job-token) +to authenticate with the API: + +* [Get job artifacts](jobs.md#get-job-artifacts) +* [Pipeline triggers](pipeline_triggers.md) + ### Impersonation tokens > [Introduced][ce-9099] in GitLab 9.0. Needs admin permissions. -- cgit v1.2.1 From b57e99295a2758174eca2c204faa52d4f8da99f7 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Sun, 18 Aug 2019 23:56:50 +0000 Subject: Mentions related log on Rate limit docs Requests over the rate limit are logged into `auth.log` --- doc/administration/logs.md | 8 ++++++-- doc/user/admin_area/settings/rate_limits_on_raw_endpoints.md | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 306d611f6bf..a57ef8ddc7d 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -284,13 +284,16 @@ Introduced in GitLab 11.3. This file lives in `/var/log/gitlab/gitlab-rails/impo Omnibus GitLab packages or in `/home/git/gitlab/log/importer.log` for installations from source. -## `auth.log` +## `auth.log` Introduced in GitLab 12.0. This file lives in `/var/log/gitlab/gitlab-rails/auth.log` for Omnibus GitLab packages or in `/home/git/gitlab/log/auth.log` for installations from source. -It logs information whenever [Rack Attack] registers an abusive request. +This log records: + +- Information whenever [Rack Attack] registers an abusive request. +- Requests over the [Rate Limit] on raw endpoints. NOTE: **Note:** From [%12.1](https://gitlab.com/gitlab-org/gitlab-ce/issues/62756), user id and username are available on this log. @@ -334,3 +337,4 @@ installations from source. [repocheck]: repository_checks.md [Rack Attack]: ../security/rack_attack.md +[Rate Limit]: ../user/admin_area/settings/rate_limits_on_raw_endpoints.md diff --git a/doc/user/admin_area/settings/rate_limits_on_raw_endpoints.md b/doc/user/admin_area/settings/rate_limits_on_raw_endpoints.md index b2d56be154b..8e53a6995fb 100644 --- a/doc/user/admin_area/settings/rate_limits_on_raw_endpoints.md +++ b/doc/user/admin_area/settings/rate_limits_on_raw_endpoints.md @@ -9,7 +9,7 @@ type: reference This setting allows you to rate limit the requests to raw endpoints, defaults to `300` requests per minute. It can be modified in **Admin Area > Network > Performance Optimization**. -For example, requests over `300` per minute to `https://gitlab.com/gitlab-org/gitlab-ce/raw/master/app/controllers/application_controller.rb` will be blocked. +For example, requests over `300` per minute to `https://gitlab.com/gitlab-org/gitlab-ce/raw/master/app/controllers/application_controller.rb` will be blocked. Access to the raw file will be released after 1 minute. ![Rate limits on raw endpoints](img/rate_limits_on_raw_endpoints.png) @@ -18,3 +18,5 @@ This limit is: - Applied independently per project, per commit and per file path. - Not applied per IP address. - Active by default. To disable, set the option to `0`. + +Requests over the rate limit are logged into `auth.log`. -- cgit v1.2.1 From 1e0cd1b389fe017941756abaa4544f1859211e11 Mon Sep 17 00:00:00 2001 From: Tristan Williams <2390023-tristan@users.noreply.gitlab.com> Date: Mon, 19 Aug 2019 00:26:04 +0000 Subject: Docs: Refresh CI quick start screenshots --- doc/ci/quick_start/img/build_log.png | Bin 35256 -> 138388 bytes doc/ci/quick_start/img/builds_status.png | Bin 19107 -> 47887 bytes doc/ci/quick_start/img/pipelines_status.png | Bin 22872 -> 64605 bytes doc/ci/quick_start/img/runners_activated.png | Bin 18215 -> 104545 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/ci/quick_start/img/build_log.png b/doc/ci/quick_start/img/build_log.png index 2bf0992c50e..16698629edc 100644 Binary files a/doc/ci/quick_start/img/build_log.png and b/doc/ci/quick_start/img/build_log.png differ diff --git a/doc/ci/quick_start/img/builds_status.png b/doc/ci/quick_start/img/builds_status.png index 58978e23978..b4aeeb988d2 100644 Binary files a/doc/ci/quick_start/img/builds_status.png and b/doc/ci/quick_start/img/builds_status.png differ diff --git a/doc/ci/quick_start/img/pipelines_status.png b/doc/ci/quick_start/img/pipelines_status.png index 06d1559f5d2..39a77a26b25 100644 Binary files a/doc/ci/quick_start/img/pipelines_status.png and b/doc/ci/quick_start/img/pipelines_status.png differ diff --git a/doc/ci/quick_start/img/runners_activated.png b/doc/ci/quick_start/img/runners_activated.png index cd83c1a7e4c..ac09e1d0137 100644 Binary files a/doc/ci/quick_start/img/runners_activated.png and b/doc/ci/quick_start/img/runners_activated.png differ -- cgit v1.2.1 From 6eec567310237865bbe461fd9642addc004522f2 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Mon, 19 Aug 2019 00:39:11 +0000 Subject: Improve docs for cross-project MR dependencies Addressing a documentation review by Marcia! --- ...ross-project-dependencies-edit-inaccessible.png | Bin 19461 -> 0 bytes .../img/cross-project-dependencies-edit.png | Bin 19302 -> 0 bytes .../img/cross-project-dependencies-view.png | Bin 37528 -> 0 bytes ...roject_dependencies_edit_inaccessible_v12_2.png | Bin 0 -> 19461 bytes .../img/cross_project_dependencies_edit_v12_2.png | Bin 0 -> 19302 bytes .../img/cross_project_dependencies_view_v12_2.png | Bin 0 -> 37528 bytes .../merge_requests/merge_request_dependencies.md | 30 ++++++++++----------- 7 files changed, 15 insertions(+), 15 deletions(-) delete mode 100644 doc/user/project/merge_requests/img/cross-project-dependencies-edit-inaccessible.png delete mode 100644 doc/user/project/merge_requests/img/cross-project-dependencies-edit.png delete mode 100644 doc/user/project/merge_requests/img/cross-project-dependencies-view.png create mode 100644 doc/user/project/merge_requests/img/cross_project_dependencies_edit_inaccessible_v12_2.png create mode 100644 doc/user/project/merge_requests/img/cross_project_dependencies_edit_v12_2.png create mode 100644 doc/user/project/merge_requests/img/cross_project_dependencies_view_v12_2.png diff --git a/doc/user/project/merge_requests/img/cross-project-dependencies-edit-inaccessible.png b/doc/user/project/merge_requests/img/cross-project-dependencies-edit-inaccessible.png deleted file mode 100644 index 2dc02634fd8..00000000000 Binary files a/doc/user/project/merge_requests/img/cross-project-dependencies-edit-inaccessible.png and /dev/null differ diff --git a/doc/user/project/merge_requests/img/cross-project-dependencies-edit.png b/doc/user/project/merge_requests/img/cross-project-dependencies-edit.png deleted file mode 100644 index 362e7e0ead2..00000000000 Binary files a/doc/user/project/merge_requests/img/cross-project-dependencies-edit.png and /dev/null differ diff --git a/doc/user/project/merge_requests/img/cross-project-dependencies-view.png b/doc/user/project/merge_requests/img/cross-project-dependencies-view.png deleted file mode 100644 index e00231c839b..00000000000 Binary files a/doc/user/project/merge_requests/img/cross-project-dependencies-view.png and /dev/null differ diff --git a/doc/user/project/merge_requests/img/cross_project_dependencies_edit_inaccessible_v12_2.png b/doc/user/project/merge_requests/img/cross_project_dependencies_edit_inaccessible_v12_2.png new file mode 100644 index 00000000000..2dc02634fd8 Binary files /dev/null and b/doc/user/project/merge_requests/img/cross_project_dependencies_edit_inaccessible_v12_2.png differ diff --git a/doc/user/project/merge_requests/img/cross_project_dependencies_edit_v12_2.png b/doc/user/project/merge_requests/img/cross_project_dependencies_edit_v12_2.png new file mode 100644 index 00000000000..362e7e0ead2 Binary files /dev/null and b/doc/user/project/merge_requests/img/cross_project_dependencies_edit_v12_2.png differ diff --git a/doc/user/project/merge_requests/img/cross_project_dependencies_view_v12_2.png b/doc/user/project/merge_requests/img/cross_project_dependencies_view_v12_2.png new file mode 100644 index 00000000000..e00231c839b Binary files /dev/null and b/doc/user/project/merge_requests/img/cross_project_dependencies_view_v12_2.png differ diff --git a/doc/user/project/merge_requests/merge_request_dependencies.md b/doc/user/project/merge_requests/merge_request_dependencies.md index e046b3466c4..b30e24b2386 100644 --- a/doc/user/project/merge_requests/merge_request_dependencies.md +++ b/doc/user/project/merge_requests/merge_request_dependencies.md @@ -2,9 +2,9 @@ type: reference, concepts --- -# Cross-project merge request dependencies **(PREMIUM)** +# Cross-project Merge Request dependencies **(PREMIUM)** -> Introduced in GitLab Premium 12.2 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9688) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. Cross-project merge request dependencies allows a required order of merging between merge requests in different projects to be expressed. If a @@ -24,11 +24,11 @@ merge requests in the same project cannot depend on each other. ## Use cases - Ensure changes to a library are merged before changes to a project that - imports the library + imports the library. - Prevent a documentation-only merge request from being merged before the merge request - implementing the feature to be documented + implementing the feature to be documented. - Require an merge request updating a permissions matrix to be merged before merging an - merge request from someone who hasn't yet been granted permissions + merge request from someone who hasn't yet been granted permissions. It is common for a single logical change to span several merge requests, spread out across multiple projects, and the order in which they are merged can be @@ -60,33 +60,33 @@ new merge request in `awesome-project` (or by editing it, if it already exists). The dependency needs to be configured on the **dependent** merge request. There is a "Cross-project dependencies" section in the form: -![Cross-project dependencies form control](img/cross-project-dependencies-edit.png) +![Cross-project dependencies form control](img/cross_project_dependencies_edit_v12_2.png) Anyone who can edit a merge request can change the list of dependencies. New dependencies can be added by reference, or by URL. To remove a dependency, -press the "X" by its reference. +press the **X** by its reference. As dependencies are specified across projects, it's possible that someone else has added a dependency for a merge request in a project you don't have access to. These are shown as a simple count: -![Cross-project dependencies form control with inaccessible merge requests](img/cross-project-dependencies-edit-inaccessible.png) +![Cross-project dependencies form control with inaccessible merge requests](img/cross_project_dependencies_edit_inaccessible_v12_2.png) -If necessary, you can remove all the dependencies like this by pressing the "X", -just as you would for a single, visible dependency. +If necessary, you can remove all the dependencies like this by pressing the +**X**, just as you would for a single, visible dependency. -Once you're finished, press the "Save changes" button to submit the request, or -"Cancel" to return without making any changes. +Once you're finished, press the **Save changes** button to submit the request, +or **Cancel** to return without making any changes. The list of configured dependencies, and the status of each one, is shown in the merge request widget: -![Cross-project dependencies in merge request widget](img/cross-project-dependencies-view.png) +![Cross-project dependencies in merge request widget](img/cross_project_dependencies_view_v12_2.png) -Until all dependencies have, themselves, been merged, the "Merge" +Until all dependencies have, themselves, been merged, the **Merge** button will be disabled for the dependent merge request. In -particular, note that **closed** merge request still prevent their +particular, note that **closed merge requests** still prevent their dependents from being merged - it is impossible to automatically determine whether the dependency expressed by a closed merge request has been satisfied in some other way or not. -- cgit v1.2.1 From 365cc7f638eb870860bc6a9d2038a1c852c1566f Mon Sep 17 00:00:00 2001 From: Tristan Williams <2390023-tristan@users.noreply.gitlab.com> Date: Mon, 19 Aug 2019 01:05:11 +0000 Subject: Docs: Improve clarity of SCIM workflow --- doc/user/group/saml_sso/scim_setup.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index f8bef8b8a6a..5d136ad62da 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -59,15 +59,14 @@ Once [Single sign-on](index.md) has been configured, we can: ### Azure -First, double check the [Single sign-on](index.md) configuration for your group and ensure that **Name identifier value** (NameID) points to `user.objectid` or another unique identifier. This will match the `extern_uid` used on GitLab. +The SAML application that was created during [Single sign-on](index.md) setup now needs to be set up for SCIM. -![Name identifier value mapping](img/scim_name_identifier_mapping.png) +1. Check the configuration for your GitLab SAML app and ensure that **Name identifier value** (NameID) points to `user.objectid` or another unique identifier. This will match the `extern_uid` used on GitLab. -#### Set up admin credentials + ![Name identifier value mapping](img/scim_name_identifier_mapping.png) -Next, configure your GitLab application in Azure by following the -[Provisioning users and groups to applications that support SCIM](https://docs.microsoft.com/en-us/azure/active-directory/manage-apps/use-scim-to-provision-users-and-groups#provisioning-users-and-groups-to-applications-that-support-scim) -section in Azure's SCIM setup documentation. +1. Set up automatic provisioning and administrative credentials by following the + [Provisioning users and groups to applications that support SCIM](https://docs.microsoft.com/en-us/azure/active-directory/manage-apps/use-scim-to-provision-users-and-groups#provisioning-users-and-groups-to-applications-that-support-scim) section in Azure's SCIM setup documentation. During this configuration, note the following: @@ -97,6 +96,7 @@ You can then test the connection by clicking on **Test Connection**. If the conn NOTE: **Note:** If you used a unique identifier **other than** `objectId`, be sure to map it instead to both `id` and `externalId`. 1. Below the mapping list click on **Show advanced options > Edit attribute list for AppName**. + 1. Leave the `id` as the primary and only required field. NOTE: **Note:** @@ -129,8 +129,7 @@ When testing the connection, you may encounter an error: **You appear to have en When checking the Audit Logs for the Provisioning, you can sometimes see the error `Namespace can't be blank, Name can't be blank, and User can't be blank.` -This is likely caused because not all required fields (such as first name and -last name) are present for all users being mapped. +This is likely caused because not all required fields (such as first name and last name) are present for all users being mapped. As a workaround, try an alternate mapping: -- cgit v1.2.1 From 84f753bc10cccbf6b32cfd0228c41a5842cb2d2d Mon Sep 17 00:00:00 2001 From: Marcel Amirault Date: Mon, 19 Aug 2019 01:34:23 +0000 Subject: Update screenshot to fix bad html --- doc/user/discussions/img/make_suggestion.png | Bin 28447 -> 115084 bytes doc/user/discussions/img/suggestion.png | Bin 39775 -> 149758 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/user/discussions/img/make_suggestion.png b/doc/user/discussions/img/make_suggestion.png index 20acc1417da..a24e29770aa 100644 Binary files a/doc/user/discussions/img/make_suggestion.png and b/doc/user/discussions/img/make_suggestion.png differ diff --git a/doc/user/discussions/img/suggestion.png b/doc/user/discussions/img/suggestion.png index 68a67e6ae5e..f7962305a15 100644 Binary files a/doc/user/discussions/img/suggestion.png and b/doc/user/discussions/img/suggestion.png differ -- cgit v1.2.1 From 24e13c86535796b01a7569bd4149d9a65754f0e5 Mon Sep 17 00:00:00 2001 From: Jonathan Love <4422532-jondlove@users.noreply.gitlab.com> Date: Mon, 19 Aug 2019 02:56:45 +0000 Subject: Escape vertical bars inside code blocks in quick_actions.md - Markdown parsing in previews and Gitlab.com appear to parse these as table delimiters even when inside a code block. --- doc/user/project/quick_actions.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index 6758adf2b43..647250bd02a 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -40,18 +40,20 @@ discussions, and descriptions: | `/label ~label1 ~label2` | Add label(s). Label names can also start without ~ but mixed syntax is not supported. | ✓ | ✓ | | `/unlabel ~label1 ~label2` | Remove all or specific label(s)| ✓ | ✓ | | `/relabel ~label1 ~label2` | Replace existing label(s) with those specified | ✓ | ✓ | -| `/copy_metadata <#issue | !merge_request>` | Copy labels and milestone from other issue or merge request in the project | ✓ | ✓ | +| `/copy_metadata <#issue>` | Copy labels and milestone from another issue in the project | ✓ | ✓ | +| `/copy_metadata ` | Copy labels and milestone from another merge request in the project | ✓ | ✓ | | `/estimate <1w 3d 2h 14m>` | Set time estimate | ✓ | ✓ | | `/remove_estimate` | Remove time estimate | ✓ | ✓ | -| `/spend ` | Add or subtract spent time; optionally, specify the date that time was spent on | ✓ | ✓ | +| `/spend ` | Add spent time; optionally, specify the date that time was spent on | ✓ | ✓ | +| `/spend ` | Subtract spent time; optionally, specify the date that time was spent on | ✓ | ✓ | | `/remove_time_spent` | Remove time spent | ✓ | ✓ | | `/lock` | Lock the thread | ✓ | ✓ | | `/unlock` | Unlock the thread | ✓ | ✓ | -| `/due `| Set due date | ✓ | | +| `/due ` | Set due date. Examples of valid `` include `in 2 days`, `this Friday` and `December 31st`. | ✓ | | | `/remove_due_date` | Remove due date | ✓ | | -| `/weight <0 | 1 | 2 | ...>` | Set weight **(STARTER)** | ✓ | | +| `/weight ` | Set weight. Valid options for `` include `0`, `1`, `2`, etc. **(STARTER)** | ✓ | | | `/clear_weight` | Clears weight **(STARTER)** | ✓ | | -| `/epic <&epic | group&epic | Epic URL>` | Add to epic **(ULTIMATE)** | ✓ | | +| `/epic ` | Add to epic ``. The `` value should be in the format of `&epic`, `group&epic` or `epic-URL`. **(ULTIMATE)** | ✓ | | | `/remove_epic` | Removes from epic **(ULTIMATE)** | ✓ | | | `/promote` | Promote issue to epic **(ULTIMATE)** | ✓ | | | `/confidential` | Make confidential | ✓ | | @@ -110,9 +112,9 @@ The following quick actions are applicable for epics threads and description: | `/label ~label1 ~label2` | Add label(s) | | `/unlabel ~label1 ~label2` | Remove all or specific label(s) | | `/relabel ~label1 ~label2` | Replace existing label(s) with those specified | -| `/child_epic <&epic | group&epic | Epic URL>` | Adds child epic to epic ([introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab-ee/issues/7330)) | -| `/remove_child_epic <&epic | group&epic | Epic URL>` | Removes child epic from epic ([introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab-ee/issues/7330)) | -| `/parent_epic <&epic | group&epic | Epic URL>` | Sets parent epic to epic ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-ee/issues/10556)) | +| `/child_epic ` | Adds child epic to ``. The `` value should be in the format of `&epic`, `group&epic` or `epic-URL`. ([Introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab-ee/issues/7330)) **(ULTIMATE)**| +| `/remove_child_epic ` | Removes child epic from ``. The `` value should be in the format of `&epic`, `group&epic` or `epic-URL`. ([Introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab-ee/issues/7330)) **(ULTIMATE)** | +| `/parent_epic ` | Sets parent epic to ``. The `` value should be in the format of `&epic`, `group&epic` or `epic-URL`. ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-ee/issues/10556)) **(ULTIMATE)** | | `/remove_parent_epic` | Removes parent epic from epic ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-ee/issues/10556)) |