diff options
34 files changed, 436 insertions, 356 deletions
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 17a8bbbb21d..d10b2b59c35 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -42,6 +42,23 @@ review-cleanup: script: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb +review-gcp-cleanup: + extends: + - .review:rules:review-gcp-cleanup + stage: prepare + image: gcr.io/google.com/cloudsdktool/cloud-sdk:latest + allow_failure: true + environment: + name: review/auto-gcp-cleanup + action: stop + before_script: + - gcloud auth activate-service-account --key-file=$REVIEW_APPS_GCP_CREDENTIALS + - gcloud config set project $REVIEW_APPS_GCP_PROJECT + - apt-get install -y jq + - source scripts/review_apps/gcp_cleanup.sh + script: + - gcp_cleanup + review-build-cng: extends: - .default-retry diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 3dc09b9988e..c083116cc27 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -496,6 +496,13 @@ - <<: *if-dot-com-gitlab-org-schedule when: on_success +.review:rules:review-gcp-cleanup: + rules: + - <<: *if-dot-com-gitlab-org-merge-request + when: manual + - <<: *if-dot-com-gitlab-org-schedule + when: on_success + .review:rules:danger: rules: - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID' diff --git a/app/workers/concerns/waitable_worker.rb b/app/workers/concerns/waitable_worker.rb index b9ea585d49b..c3abcdafcf2 100644 --- a/app/workers/concerns/waitable_worker.rb +++ b/app/workers/concerns/waitable_worker.rb @@ -14,9 +14,7 @@ module WaitableWorker # are not likely to finish within the timeout. This assumes we can process # 10 jobs per second: # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/205 - if ::Feature.enabled?(:skip_job_waiter_for_large_batches) - return bulk_perform_async(args_list) if args_list.length >= 10 * timeout - end + return bulk_perform_async(args_list) if args_list.length >= 10 * timeout waiter = Gitlab::JobWaiter.new(args_list.size, worker_label: self.to_s) diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index 3008cea365c..606e041c956 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -355,6 +355,30 @@ sudo gitlab-ctl reconfigure To help us resolve this problem, consider commenting on [the issue](https://gitlab.com/gitlab-org/gitlab/issues/4489). +### Message: `LOG: invalid CIDR mask in address` + +This happens on wrongly-formatted addresses in `postgresql['md5_auth_cidr_addresses']`. + +```plaintext +2020-03-20_23:59:57.60499 LOG: invalid CIDR mask in address "***" +2020-03-20_23:59:57.60501 CONTEXT: line 74 of configuration file "/var/opt/gitlab/postgresql/data/pg_hba.conf" +``` + +To fix this, update the IP addresses in `/etc/gitlab/gitlab.rb` under `postgresql['md5_auth_cidr_addresses']` +to respect the CIDR format (i.e. `1.2.3.4/32`). + +### Message: `LOG: invalid IP mask "md5": Name or service not known` + +This happens when you have added IP addresses without a subnet mask in `postgresql['md5_auth_cidr_addresses']`. + +```plaintext +2020-03-21_00:23:01.97353 LOG: invalid IP mask "md5": Name or service not known +2020-03-21_00:23:01.97354 CONTEXT: line 75 of configuration file "/var/opt/gitlab/postgresql/data/pg_hba.conf" +``` + +To fix this, add the subnet mask in `/etc/gitlab/gitlab.rb` under `postgresql['md5_auth_cidr_addresses']` +to respect the CIDR format (i.e. `1.2.3.4/32`). + ### Very large repositories never successfully synchronize on the **secondary** node GitLab places a timeout on all repository clones, including project imports diff --git a/doc/administration/high_availability/pgbouncer.md b/doc/administration/high_availability/pgbouncer.md index 1195c736079..f0e76172a00 100644 --- a/doc/administration/high_availability/pgbouncer.md +++ b/doc/administration/high_availability/pgbouncer.md @@ -273,3 +273,11 @@ In case you are experiencing any issues connecting through PgBouncer, the first ``` Additionally, you can check the output from `show databases` in the [Administrative console](#administrative-console). In the output, you would expect to see values in the `host` field for the `gitlabhq_production` database. Additionally, `current_connections` should be greater than 1. + +### Message: `LOG: invalid CIDR mask in address` + +See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-cidr-mask-in-address). + +### Message: `LOG: invalid IP mask "md5": Name or service not known` + +See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-ip-mask-md5-name-or-service-not-known). diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 6f927d8f920..cc1d744d73b 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -21,7 +21,7 @@ To disable artifacts site-wide, follow the steps below. gitlab_rails['artifacts_enabled'] = false ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. **In installations from source:** @@ -32,7 +32,7 @@ To disable artifacts site-wide, follow the steps below. enabled: false ``` -1. Save the file and [restart GitLab][] for the changes to take effect. +1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. ## Storing job artifacts @@ -57,7 +57,7 @@ _The artifacts are stored by default in gitlab_rails['artifacts_path'] = "/mnt/storage/artifacts" ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. **In installations from source:** @@ -73,7 +73,7 @@ _The artifacts are stored by default in path: /mnt/storage/artifacts ``` -1. Save the file and [restart GitLab][] for the changes to take effect. +1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. ### Using object storage @@ -153,7 +153,7 @@ _The artifacts are stored by default in } ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Migrate any existing local artifacts to the object storage: ```shell @@ -186,7 +186,7 @@ _The artifacts are stored by default in region: eu-central-1 ``` -1. Save the file and [restart GitLab][] for the changes to take effect. +1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. 1. Migrate any existing local artifacts to the object storage: ```shell @@ -205,13 +205,13 @@ by the `gitlab:artifacts:migrate` script. In order to migrate back to local storage: 1. Set both `direct_upload` and `background_upload` to false in `gitlab.rb`, under the artifacts object storage settings. -1. [reconfigure GitLab][]. +1. [Reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Run `gitlab-rake gitlab:artifacts:migrate_to_local`. 1. Disable object_storage for artifacts in `gitlab.rb`: - Set `gitlab_rails['artifacts_object_store_enabled'] = false`. - Comment out all other `artifacts_object_store` settings, including the entire `artifacts_object_store_connection` section, including the closing `}`. -1. [reconfigure GitLab][]. +1. [Reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure). ## Expiring artifacts @@ -231,7 +231,7 @@ steps below. gitlab_rails['expire_build_artifacts_worker_cron'] = "50 * * * *" ``` -1. Save the file and [reconfigure GitLab][] for the changes to take effect. +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. **In installations from source:** @@ -243,7 +243,7 @@ steps below. cron: "50 * * * *" ``` -1. Save the file and [restart GitLab][] for the changes to take effect. +1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. ## Validation for dependencies @@ -295,7 +295,7 @@ and [projects APIs](../api/projects.md). ## Implementation details When GitLab receives an artifacts archive, an archive metadata file is also -generated by [GitLab Workhorse]. This metadata file describes all the entries +generated by [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse). This metadata file describes all the entries that are located in the artifacts archive itself. The metadata file is in a binary format, with additional GZIP compression. @@ -304,14 +304,10 @@ and disk I/O. It instead inspects the metadata file which contains all the relevant information. This is especially important when there is a lot of artifacts, or an archive is a very large file. -When clicking on a specific file, [GitLab Workhorse] extracts it +When clicking on a specific file, [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) extracts it from the archive and the download begins. This implementation saves space, memory and disk I/O. -[reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab" -[restart gitlab]: restart_gitlab.md#installations-from-source "How to restart GitLab" -[gitlab workhorse]: https://gitlab.com/gitlab-org/gitlab-workhorse "GitLab Workhorse repository" - ## Troubleshooting ### Job artifacts using too much disk space diff --git a/doc/administration/monitoring/github_imports.md b/doc/administration/monitoring/github_imports.md index c732730e869..6d5915930b2 100644 --- a/doc/administration/monitoring/github_imports.md +++ b/doc/administration/monitoring/github_imports.md @@ -1,7 +1,7 @@ # Monitoring GitHub imports >**Note:** -Available since [GitLab 10.2][14731]. +Available since [GitLab 10.2](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14731). The GitHub importer exposes various Prometheus metrics that you can use to monitor the health and progress of the importer. @@ -97,5 +97,3 @@ The name of the project is stored in the `project` label in the format This metric tracks the number of imported repositories across all projects. This metric does not expose any labels. - -[14731]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14731 diff --git a/doc/administration/monitoring/ip_whitelist.md b/doc/administration/monitoring/ip_whitelist.md index 700bcc978e0..1b0ea1744d9 100644 --- a/doc/administration/monitoring/ip_whitelist.md +++ b/doc/administration/monitoring/ip_whitelist.md @@ -5,8 +5,8 @@ NOTE: **Note:** We intend to [rename IP whitelist as `IP allowlist`](https://gitlab.com/gitlab-org/gitlab/-/issues/7554). -GitLab provides some [monitoring endpoints] that provide health check information -when probed. +GitLab provides some [monitoring endpoints](../../user/admin_area/monitoring/health_check.md) +that provide health check information when probed. To control access to those endpoints via IP whitelisting, you can add single hosts or use IP ranges: @@ -19,7 +19,7 @@ hosts or use IP ranges: gitlab_rails['monitoring_whitelist'] = ['127.0.0.0/8', '192.168.0.1'] ``` -1. Save the file and [reconfigure] GitLab for the changes to take effect. +1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. --- @@ -35,8 +35,4 @@ hosts or use IP ranges: - 192.168.0.1 ``` -1. Save the file and [restart] GitLab for the changes to take effect. - -[reconfigure]: ../restart_gitlab.md#omnibus-gitlab-reconfigure -[restart]: ../restart_gitlab.md#installations-from-source -[monitoring endpoints]: ../../user/admin_area/monitoring/health_check.md +1. Save the file and [restart](../restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. diff --git a/doc/administration/monitoring/performance/grafana_configuration.md b/doc/administration/monitoring/performance/grafana_configuration.md index 3b7cd8950ec..3c26e399c7f 100644 --- a/doc/administration/monitoring/performance/grafana_configuration.md +++ b/doc/administration/monitoring/performance/grafana_configuration.md @@ -80,7 +80,7 @@ For more information see the [InfluxDB Management README](https://gitlab.com/git You can now import a set of default dashboards that will give you a good start on displaying useful information. GitLab has published a set of default -[Grafana dashboards][grafana-dashboards] to get you started. Clone the +[Grafana dashboards](https://gitlab.com/gitlab-org/grafana-dashboards) to get you started. Clone the repository or download a zip/tarball, then follow these steps to import each JSON file. @@ -102,11 +102,9 @@ navigate away. Repeat this process for each dashboard you wish to import. Alternatively you can automatically import all the dashboards into your Grafana -instance. See the README of the [Grafana dashboards][grafana-dashboards] +instance. See the README of the [Grafana dashboards](https://gitlab.com/gitlab-org/grafana-dashboards) repository for more information on this process. -[grafana-dashboards]: https://gitlab.com/gitlab-org/grafana-dashboards - ## Integration with GitLab UI > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/61005) in GitLab 12.1. diff --git a/doc/administration/monitoring/performance/index.md b/doc/administration/monitoring/performance/index.md index 6569f6a8c6d..3305ea33f2e 100644 --- a/doc/administration/monitoring/performance/index.md +++ b/doc/administration/monitoring/performance/index.md @@ -36,7 +36,8 @@ including (but not limited to): - Ruby garbage collection statistics. Metrics data is written to [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) -over [UDP][influxdb-udp]. Stored data can be visualized using [Grafana](https://grafana.com) or any other application that +over [UDP](https://docs.influxdata.com/influxdb/v0.9/write_protocols/udp/). +Stored data can be visualized using [Grafana](https://grafana.com) or any other application that supports reading data from InfluxDB. Alternatively data can be queried using the InfluxDB CLI. @@ -70,5 +71,3 @@ half above the interval. For example, for a user defined interval of 15 seconds the actual interval can be anywhere between 7.5 and 22.5. The interval is re-generated for every sampling run instead of being generated once and re-used for the duration of the process' lifetime. - -[influxdb-udp]: https://docs.influxdata.com/influxdb/v0.9/write_protocols/udp/ diff --git a/doc/administration/monitoring/performance/influxdb_configuration.md b/doc/administration/monitoring/performance/influxdb_configuration.md index b81f0ce1506..8478272897b 100644 --- a/doc/administration/monitoring/performance/influxdb_configuration.md +++ b/doc/administration/monitoring/performance/influxdb_configuration.md @@ -4,10 +4,10 @@ CAUTION: **InfluxDB is deprecated in favor of Prometheus:** InfluxDB support is scheduled to be removed in GitLab 13.0. You are advised to use [Prometheus](../prometheus/index.md) instead. -The default settings provided by [InfluxDB] are not sufficient for a high traffic -GitLab environment. The settings discussed in this document are based on the -settings GitLab uses for GitLab.com, depending on your own needs you may need to -further adjust them. +The default settings provided by [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) +are not sufficient for a high traffic GitLab environment. The settings discussed in +this document are based on the settings GitLab uses for GitLab.com. Depending on +your own needs, you may need to further adjust them. If you are intending to run InfluxDB on the same server as GitLab, make sure you have plenty of RAM since InfluxDB can use quite a bit depending on traffic. @@ -23,12 +23,13 @@ Unless you are going with a budget setup, it's advised to run it separately. Note that the RAM and storage requirements can differ greatly depending on the amount of data received/stored. To limit the amount of stored data users can -look into [InfluxDB Retention Policies][influxdb-retention]. +look into +[InfluxDB Retention Policies](https://docs.influxdata.com/influxdb/v0.9/query_language/database_management/#retention-policy-management). ## Installation Installing InfluxDB is out of the scope of this document. Please refer to the -[InfluxDB documentation]. +[InfluxDB documentation](https://docs.influxdata.com/influxdb/v0.9/). ## InfluxDB Server Settings @@ -43,7 +44,8 @@ InfluxDB needs to be restarted. InfluxDB comes with different storage engines and as of InfluxDB 0.9.5 a new storage engine is available, called [TSM Tree](https://www.influxdata.com/blog/new-storage-engine-time-structured-merge-tree/). -All users **must** use the new `tsm1` storage engine as this [will be the default engine][tsm1-commit] in +All users **must** use the new `tsm1` storage engine as this +[will be the default engine](https://github.com/influxdata/influxdb/commit/15d723dc77651bac83e09e2b1c94be480966cb0d) in upcoming InfluxDB releases. Make sure you have the following in your configuration file: @@ -67,9 +69,9 @@ file: ### HTTP -HTTP is required when using the [InfluxDB CLI] or other tools such as Grafana, -thus it should be enabled. When enabling make sure to _also_ enable -authentication: +HTTP is required when using the [InfluxDB CLI](https://docs.influxdata.com/influxdb/v0.9/tools/shell/) +or other tools such as Grafana, thus it should be enabled. When enabling +make sure to _also_ enable authentication: ```toml [http] @@ -129,14 +131,15 @@ using _at least_ 100 MB. When enabling UDP, users should take care to not expose the port to the public, as doing so will allow anybody to write data into your InfluxDB database (as -[InfluxDB's UDP protocol][udp] doesn't support authentication). We recommend either +[InfluxDB's UDP protocol](https://docs.influxdata.com/influxdb/v0.9/write_protocols/udp/) +doesn't support authentication). We recommend either whitelisting the allowed IP addresses/ranges, or setting up a VLAN and only allowing traffic from members of said VLAN. ## Create a new admin user If you want to [enable authentication](#http), you might want to [create an -admin user][influx-admin]: +admin user](https://docs.influxdata.com/influxdb/v0.9/administration/authentication_and_authorization/#create-a-new-admin-user): ```shell influx -execute "CREATE USER jeff WITH PASSWORD '1234' WITH ALL PRIVILEGES" @@ -186,11 +189,3 @@ Read more on: - [GitLab Configuration](gitlab_configuration.md) - [InfluxDB Schema](influxdb_schema.md) - [Grafana Install/Configuration](grafana_configuration.md) - -[influxdb-retention]: https://docs.influxdata.com/influxdb/v0.9/query_language/database_management/#retention-policy-management -[influxdb documentation]: https://docs.influxdata.com/influxdb/v0.9/ -[influxdb cli]: https://docs.influxdata.com/influxdb/v0.9/tools/shell/ -[udp]: https://docs.influxdata.com/influxdb/v0.9/write_protocols/udp/ -[influxdb]: https://www.influxdata.com/products/influxdb-overview/ -[tsm1-commit]: https://github.com/influxdata/influxdb/commit/15d723dc77651bac83e09e2b1c94be480966cb0d -[influx-admin]: https://docs.influxdata.com/influxdb/v0.9/administration/authentication_and_authorization/#create-a-new-admin-user diff --git a/doc/administration/monitoring/performance/performance_bar.md b/doc/administration/monitoring/performance/performance_bar.md index 13474e2a183..89246bc9782 100644 --- a/doc/administration/monitoring/performance/performance_bar.md +++ b/doc/administration/monitoring/performance/performance_bar.md @@ -10,9 +10,9 @@ It allows you to see (from left to right): - the current host serving the page - time taken and number of DB queries; click through for details of these queries  -- time taken and number of [Gitaly] calls; click through for details of these calls +- time taken and number of [Gitaly](../../gitaly/index.md) calls; click through for details of these calls  -- time taken and number of [Rugged] calls; click through for details of these calls +- time taken and number of [Rugged](../../high_availability/nfs.md#improving-nfs-performance-with-gitlab) calls; click through for details of these calls  - time taken and number of Redis calls; click through for details of these calls  @@ -68,6 +68,3 @@ display it. You can toggle the Bar using the same shortcut.  - -[Gitaly]: ../../gitaly/index.md -[Rugged]: ../../high_availability/nfs.md#improving-nfs-performance-with-gitlab diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 897f9592d84..11fda995955 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -170,7 +170,7 @@ Some basic Ruby runtime metrics are available: | Metric | Type | Since | Description | |:------------------------------------ |:--------- |:----- |:----------- | | `ruby_gc_duration_seconds` | Counter | 11.1 | Time spent by Ruby in GC | -| `ruby_gc_stat_...` | Gauge | 11.1 | Various metrics from [GC.stat] | +| `ruby_gc_stat_...` | Gauge | 11.1 | Various metrics from [GC.stat](https://ruby-doc.org/core-2.6.5/GC.html#method-c-stat) | | `ruby_file_descriptors` | Gauge | 11.1 | File descriptors per process | | `ruby_memory_bytes` | Gauge | 11.1 | Memory usage by process | | `ruby_sampler_duration_seconds` | Counter | 11.1 | Time spent collecting stats | @@ -179,8 +179,6 @@ Some basic Ruby runtime metrics are available: | `ruby_process_resident_memory_bytes` | Gauge | 12.0 | Memory usage by process | | `ruby_process_start_time_seconds` | Gauge | 12.0 | UNIX timestamp of process start time | -[GC.stat]: https://ruby-doc.org/core-2.6.5/GC.html#method-c-stat - ## Unicorn Metrics Unicorn specific metrics, when Unicorn is used. diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index da368327c5e..03d86012cc7 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -10,7 +10,7 @@ > - Prometheus and its exporters don't authenticate users, and will be available > to anyone who can access them. -[Prometheus] is a powerful time-series monitoring service, providing a flexible +[Prometheus](https://prometheus.io) is a powerful time-series monitoring service, providing a flexible platform for monitoring GitLab and other software products. GitLab provides out of the box monitoring with Prometheus, providing easy access to high quality time-series monitoring of GitLab services. @@ -43,13 +43,13 @@ To disable Prometheus and all of its exporters, as well as any added in the futu prometheus_monitoring['enable'] = false ``` -1. Save the file and [reconfigure GitLab][reconfigure] for the changes to +1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. ### Changing the port and address Prometheus listens on NOTE: **Note:** -The following change was added in [GitLab Omnibus 8.17][1261]. Although possible, +The following change was added in [GitLab Omnibus 8.17](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1261). Although possible, it's not recommended to change the port Prometheus listens on, as this might affect or conflict with other services running on the GitLab server. Proceed at your own risk. @@ -75,7 +75,7 @@ To change the address/port that Prometheus listens on: prometheus['listen_address'] = '0.0.0.0:9090' ``` -1. Save the file and [reconfigure GitLab][reconfigure] for the changes to +1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect ### Adding custom scrape configs @@ -151,7 +151,7 @@ To use an external Prometheus server: } ``` -1. [Reconfigure GitLab][reconfigure] to apply the changes. +1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) to apply the changes. 1. Edit the Prometheus server's configuration file. 1. Add each node's exporters to the Prometheus server's [scrape target configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cscrape_config%3E). @@ -217,17 +217,17 @@ You can visit `http://localhost:9090` for the dashboard that Prometheus offers b >**Note:** If SSL has been enabled on your GitLab instance, you may not be able to access -Prometheus on the same browser as GitLab if using the same FQDN due to [HSTS][hsts]. We plan to -[provide access via GitLab][multi-user-prometheus], but in the interim there are +Prometheus on the same browser as GitLab if using the same FQDN due to [HSTS](https://en.wikipedia.org/wiki/HTTP_Strict_Transport_Security). We plan to +[provide access via GitLab](https://gitlab.com/gitlab-org/multi-user-prometheus), but in the interim there are some workarounds: using a separate FQDN, using server IP, using a separate browser for Prometheus, resetting HSTS, or -having [NGINX proxy it][nginx-custom-config]. +having [NGINX proxy it](https://docs.gitlab.com/omnibus/settings/nginx.html#inserting-custom-nginx-settings-into-the-gitlab-server-block). The performance data collected by Prometheus can be viewed directly in the Prometheus console, or through a compatible dashboard tool. The Prometheus interface provides a [flexible query language](https://prometheus.io/docs/prometheus/latest/querying/basics/) to work with the collected data where you can visualize the output. For a more fully featured dashboard, Grafana can be used and has -[official support for Prometheus][prom-grafana]. +[official support for Prometheus](https://prometheus.io/docs/visualization/grafana/). Sample Prometheus queries: @@ -305,7 +305,7 @@ The GitLab exporter allows you to measure various GitLab metrics, pulled from Re > - Introduced in GitLab 9.0. > - Pod monitoring introduced in GitLab 9.4. -If your GitLab server is running within Kubernetes, Prometheus will collect metrics from the Nodes and [annotated Pods](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) in the cluster, including performance data on each container. This is particularly helpful if your CI/CD environments run in the same cluster, as you can use the [Prometheus project integration][prometheus integration] to monitor them. +If your GitLab server is running within Kubernetes, Prometheus will collect metrics from the Nodes and [annotated Pods](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) in the cluster, including performance data on each container. This is particularly helpful if your CI/CD environments run in the same cluster, as you can use the [Prometheus project integration](../../../user/project/integrations/prometheus.md) to monitor them. To disable the monitoring of Kubernetes: @@ -316,14 +316,5 @@ To disable the monitoring of Kubernetes: prometheus['monitor_kubernetes'] = false ``` -1. Save the file and [reconfigure GitLab][reconfigure] for the changes to +1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -[hsts]: https://en.wikipedia.org/wiki/HTTP_Strict_Transport_Security -[multi-user-prometheus]: https://gitlab.com/gitlab-org/multi-user-prometheus -[nginx-custom-config]: https://docs.gitlab.com/omnibus/settings/nginx.html#inserting-custom-nginx-settings-into-the-gitlab-server-block -[prometheus]: https://prometheus.io -[prom-grafana]: https://prometheus.io/docs/visualization/grafana/ -[reconfigure]: ../../restart_gitlab.md#omnibus-gitlab-reconfigure -[1261]: https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1261 -[prometheus integration]: ../../../user/project/integrations/prometheus.md diff --git a/doc/administration/monitoring/prometheus/node_exporter.md b/doc/administration/monitoring/prometheus/node_exporter.md index bcacfaa3be5..d75b04f1ccd 100644 --- a/doc/administration/monitoring/prometheus/node_exporter.md +++ b/doc/administration/monitoring/prometheus/node_exporter.md @@ -4,8 +4,8 @@ Available since Omnibus GitLab 8.16. For installations from source you'll have to install and configure it yourself. -The [node exporter] allows you to measure various machine resources such as -memory, disk and CPU utilization. +The [node exporter](https://github.com/prometheus/node_exporter) allows you to measure +various machine resources such as memory, disk and CPU utilization. To enable the node exporter: @@ -17,14 +17,10 @@ To enable the node exporter: node_exporter['enable'] = true ``` -1. Save the file and [reconfigure GitLab][reconfigure] for the changes to - take effect +1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect Prometheus will now automatically begin collecting performance data from the node exporter exposed under `localhost:9100`. [← Back to the main Prometheus page](index.md) - -[node exporter]: https://github.com/prometheus/node_exporter -[prometheus]: https://prometheus.io -[reconfigure]: ../../restart_gitlab.md#omnibus-gitlab-reconfigure diff --git a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md index ba7fc74f269..ba8e464efcb 100644 --- a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md +++ b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md @@ -1,10 +1,10 @@ # PgBouncer exporter >**Note:** -Available since [Omnibus GitLab 11.0][2493]. For installations from source -you'll have to install and configure it yourself. +Available since [Omnibus GitLab 11.0](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/2493). +For installations from source you'll have to install and configure it yourself. -The [PgBouncer exporter] allows you to measure various PgBouncer metrics. +The [PgBouncer exporter](https://github.com/stanhu/pgbouncer_exporter) allows you to measure various PgBouncer metrics. To enable the PgBouncer exporter: @@ -16,19 +16,13 @@ To enable the PgBouncer exporter: pgbouncer_exporter['enable'] = true ``` -1. Save the file and [reconfigure GitLab][reconfigure] for the changes to +1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. Prometheus will now automatically begin collecting performance data from the PgBouncer exporter exposed under `localhost:9188`. -The PgBouncer exporter will also be enabled by default if the [`pgbouncer_role`][postgres roles] +The PgBouncer exporter will also be enabled by default if the [`pgbouncer_role`](https://docs.gitlab.com/omnibus/roles/#postgres-roles) role is enabled. [← Back to the main Prometheus page](index.md) - -[2493]: https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/2493 -[PgBouncer exporter]: https://github.com/stanhu/pgbouncer_exporter -[postgres roles]: https://docs.gitlab.com/omnibus/roles/#postgres-roles -[prometheus]: https://prometheus.io -[reconfigure]: ../../restart_gitlab.md#omnibus-gitlab-reconfigure diff --git a/doc/administration/monitoring/prometheus/redis_exporter.md b/doc/administration/monitoring/prometheus/redis_exporter.md index 177ecc329ba..76f4add0c1b 100644 --- a/doc/administration/monitoring/prometheus/redis_exporter.md +++ b/doc/administration/monitoring/prometheus/redis_exporter.md @@ -1,11 +1,12 @@ # Redis exporter >**Note:** -Available since [Omnibus GitLab 8.17][1118]. For installations from source -you'll have to install and configure it yourself. +Available since [Omnibus GitLab 8.17](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1118). +For installations from source you'll have to install and configure it yourself. -The [Redis exporter] allows you to measure various [Redis] metrics. For more -information on what's exported [read the upstream documentation][redis-exp]. +The [Redis exporter](https://github.com/oliver006/redis_exporter) allows you to measure +various [Redis](https://redis.io) metrics. For more information on what's exported, +[read the upstream documentation](https://github.com/oliver006/redis_exporter/blob/master/README.md#whats-exported). To enable the Redis exporter: @@ -17,17 +18,10 @@ To enable the Redis exporter: redis_exporter['enable'] = true ``` -1. Save the file and [reconfigure GitLab][reconfigure] for the changes to - take effect +1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. Prometheus will now automatically begin collecting performance data from the Redis exporter exposed under `localhost:9121`. [← Back to the main Prometheus page](index.md) - -[1118]: https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/1118 -[redis]: https://redis.io -[redis exporter]: https://github.com/oliver006/redis_exporter -[redis-exp]: https://github.com/oliver006/redis_exporter/blob/master/README.md#whats-exported -[prometheus]: https://prometheus.io -[reconfigure]: ../../restart_gitlab.md#omnibus-gitlab-reconfigure diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index ad0a828afa0..84133205bd3 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -4,14 +4,14 @@ description: 'Learn how to administer GitLab Pages.' # GitLab Pages administration -> - [Introduced][ee-80] in GitLab EE 8.3. -> - Custom CNAMEs with TLS support were [introduced][ee-173] in GitLab EE 8.5. -> - GitLab Pages [was ported][ce-14605] to Community Edition in GitLab 8.17. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80) in GitLab EE 8.3. +> - Custom CNAMEs with TLS support were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/173) in GitLab EE 8.5. +> - GitLab Pages [was ported](https://gitlab.com/gitlab-org/gitlab-foss/issues/14605) to Community Edition in GitLab 8.17. > - Support for subgroup project's websites was > [introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/30548) in GitLab 11.8. GitLab Pages allows for hosting of static sites. It must be configured by an -administrator. Separate [user documentation][pages-userguide] is available. +administrator. Separate [user documentation](../../user/project/pages/index.md) is available. NOTE: **Note:** This guide is for Omnibus GitLab installations. If you have installed @@ -20,11 +20,11 @@ GitLab from source, see ## Overview -GitLab Pages makes use of the [GitLab Pages daemon], a simple HTTP server +GitLab Pages makes use of the [GitLab Pages daemon](https://gitlab.com/gitlab-org/gitlab-pages), a simple HTTP server written in Go that can listen on an external IP address and provide support for custom domains and custom certificates. It supports dynamic certificates through SNI and exposes pages using HTTP2 by default. -You are encouraged to read its [README][pages-readme] to fully understand how +You are encouraged to read its [README](https://gitlab.com/gitlab-org/gitlab-pages/blob/master/README.md) to fully understand how it works. In the case of [custom domains](#custom-domains) (but not @@ -124,9 +124,9 @@ The Pages daemon doesn't listen to the outside world. pages_external_url 'http://example.io' ``` -1. [Reconfigure GitLab][reconfigure]. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -Watch the [video tutorial][video-admin] for this configuration. +Watch the [video tutorial](https://youtu.be/dD8c7WNcc6s) for this configuration. ### Wildcard domains with TLS support @@ -156,7 +156,7 @@ outside world. where `pages-nginx.crt` and `pages-nginx.key` are the SSL cert and key, respectively. -1. [Reconfigure GitLab][reconfigure]. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ### Additional configuration for Docker container @@ -171,7 +171,7 @@ behavior: gitlab_pages['inplace_chroot'] = true ``` -1. [Reconfigure GitLab][reconfigure]. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). NOTE: **Note:** `inplace_chroot` option might not work with the other features, such as [Pages Access Control](#access-control). @@ -213,7 +213,7 @@ world. Custom domains are supported, but no TLS. `192.0.2.2` and `2001::2` are the secondary IPs the GitLab Pages daemon listens on. If you don't have IPv6, you can omit the IPv6 address. -1. [Reconfigure GitLab][reconfigure]. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ### Custom domains with TLS support @@ -247,7 +247,7 @@ world. Custom domains and TLS are supported. `192.0.2.2` and `2001::2` are the secondary IPs where the GitLab Pages daemon listens on. If you don't have IPv6, you can omit the IPv6 address. -1. [Reconfigure GitLab][reconfigure]. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ### Custom domain verification @@ -304,7 +304,7 @@ Pages access control is disabled by default. To enable it: gitlab_pages['access_control'] = true ``` -1. [Reconfigure GitLab][reconfigure]. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Users can now configure it in their [projects' settings](../../user/project/pages/pages_access_control.md). #### Disabling public access to all Pages websites @@ -340,7 +340,7 @@ pages: gitlab_pages['http_proxy'] = 'http://example:8080' ``` -1. [Reconfigure GitLab][reconfigure] for the changes to take effect. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. ### Using a custom Certificate Authority (CA) @@ -389,7 +389,7 @@ Follow the steps below to configure verbose logging of GitLab Pages daemon. gitlab_pages['log_verbose'] = true ``` -1. [Reconfigure GitLab][reconfigure]. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ## Change storage path @@ -404,7 +404,7 @@ are stored. gitlab_rails['pages_path'] = "/mnt/storage/pages" ``` -1. [Reconfigure GitLab][reconfigure]. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ## Configure listener for reverse proxy requests @@ -427,7 +427,7 @@ Omnibus GitLab 11.1. gitlab_pages['listen_proxy'] = "localhost:10080" ``` -1. [Reconfigure GitLab][reconfigure]. +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ## Set maximum pages size @@ -559,7 +559,7 @@ then you must use the following procedure to configure [access control](#access- ## Backup -GitLab Pages are part of the [regular backup][backup], so there is no separate backup to configure. +GitLab Pages are part of the [regular backup](../../raketasks/backup_restore.md), so there is no separate backup to configure. ## Security @@ -610,16 +610,3 @@ The fix is to correct the source file permissions and restart Pages: sudo chmod 644 /opt/gitlab/embedded/ssl/certs/cacert.pem sudo gitlab-ctl restart gitlab-pages ``` - -[backup]: ../../raketasks/backup_restore.md -[ce-14605]: https://gitlab.com/gitlab-org/gitlab-foss/issues/14605 -[ee-80]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80 -[ee-173]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/173 -[gitlab pages daemon]: https://gitlab.com/gitlab-org/gitlab-pages -[NGINX configs]: https://gitlab.com/gitlab-org/gitlab/tree/8-5-stable-ee/lib/support/nginx -[pages-readme]: https://gitlab.com/gitlab-org/gitlab-pages/blob/master/README.md -[pages-userguide]: ../../user/project/pages/index.md -[reconfigure]: ../restart_gitlab.md#omnibus-gitlab-reconfigure -[restart]: ../restart_gitlab.md#installations-from-source -[gitlab-pages]: https://gitlab.com/gitlab-org/gitlab-pages/tree/v0.2.4 -[video-admin]: https://youtu.be/dD8c7WNcc6s diff --git a/doc/administration/pages/source.md b/doc/administration/pages/source.md index 87f0afeca12..94d2c5420aa 100644 --- a/doc/administration/pages/source.md +++ b/doc/administration/pages/source.md @@ -17,12 +17,12 @@ Pages to the latest supported version. ## Overview -GitLab Pages makes use of the [GitLab Pages daemon], a simple HTTP server +GitLab Pages makes use of the [GitLab Pages daemon](https://gitlab.com/gitlab-org/gitlab-pages), a simple HTTP server written in Go that can listen on an external IP address and provide support for custom domains and custom certificates. It supports dynamic certificates through SNI and exposes pages using HTTP2 by default. -You are encouraged to read its [README][pages-readme] to fully understand how -it works. +You are encouraged to read its [README](https://gitlab.com/gitlab-org/gitlab-pages/blob/master/README.md) +to fully understand how it works. In the case of [custom domains](#custom-domains) (but not [wildcard domains](#wildcard-domains)), the Pages daemon needs to listen on @@ -55,7 +55,7 @@ Before proceeding with the Pages configuration, make sure that: Pages artifacts. 1. (Optional) You have a **wildcard certificate** for the Pages domain if you decide to serve Pages (`*.example.io`) under HTTPS. -1. (Optional but recommended) You have configured and enabled the [Shared Runners][] +1. (Optional but recommended) You have configured and enabled the [Shared Runners](../../ci/runners/README.md) so that your users don't have to bring their own. ### DNS configuration @@ -144,7 +144,7 @@ The Pages daemon doesn't listen to the outside world. ``` 1. Restart NGINX -1. [Restart GitLab][restart] +1. [Restart GitLab](../restart_gitlab.md#installations-from-source) ### Wildcard domains with TLS support @@ -201,7 +201,7 @@ outside world. ``` 1. Restart NGINX -1. [Restart GitLab][restart] +1. [Restart GitLab](../restart_gitlab.md#installations-from-source) ## Advanced configuration @@ -272,7 +272,7 @@ world. Custom domains are supported, but no TLS. `0.0.0.0` with `192.0.2.1`, where `192.0.2.1` the primary IP where GitLab listens to. 1. Restart NGINX -1. [Restart GitLab][restart] +1. [Restart GitLab](../restart_gitlab.md#installations-from-source) ### Custom domains with TLS support @@ -341,7 +341,7 @@ world. Custom domains and TLS are supported. `0.0.0.0` with `192.0.2.1`, where `192.0.2.1` the primary IP where GitLab listens to. 1. Restart NGINX -1. [Restart GitLab][restart] +1. [Restart GitLab](../restart_gitlab.md#installations-from-source) ## NGINX caveats @@ -402,7 +402,7 @@ Pages access control is disabled by default. To enable it: access_control: true ``` -1. [Restart GitLab][restart]. +1. [Restart GitLab](../restart_gitlab.md#installations-from-source). 1. Create a new [system OAuth application](../../integration/oauth_provider.md#adding-an-application-through-the-profile). This should be called `GitLab Pages` and have a `Redirect URL` of `https://projects.example.io/auth`. It does not need to be a "trusted" @@ -435,7 +435,7 @@ are stored. path: /mnt/storage/pages ``` -1. [Restart GitLab][restart] +1. [Restart GitLab](../restart_gitlab.md#installations-from-source) ## Set maximum Pages size @@ -445,21 +445,9 @@ The default is 100MB. ## Backup -Pages are part of the [regular backup][backup] so there is nothing to configure. +Pages are part of the [regular backup](../../raketasks/backup_restore.md) so there is nothing to configure. ## Security You should strongly consider running GitLab Pages under a different hostname than GitLab to prevent XSS attacks. - -[backup]: ../../raketasks/backup_restore.md -[ee-80]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80 -[ee-173]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/173 -[gitlab pages daemon]: https://gitlab.com/gitlab-org/gitlab-pages -[NGINX configs]: https://gitlab.com/gitlab-org/gitlab/tree/8-5-stable-ee/lib/support/nginx -[pages-readme]: https://gitlab.com/gitlab-org/gitlab-pages/blob/master/README.md -[pages-userguide]: ../../user/project/pages/index.md -[restart]: ../restart_gitlab.md#installations-from-source -[gitlab-pages]: https://gitlab.com/gitlab-org/gitlab-pages/tree/v0.4.0 -[gl-example]: https://gitlab.com/gitlab-org/gitlab/blob/master/lib/support/init.d/gitlab.default.example -[shared runners]: ../../ci/runners/README.md diff --git a/doc/api/jobs.md b/doc/api/jobs.md index 6cd2d0a04f5..4827eafd790 100644 --- a/doc/api/jobs.md +++ b/doc/api/jobs.md @@ -346,8 +346,8 @@ Example of response > **Notes**: > > - [Introduced][ce-2893] in GitLab 8.5. -> - The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced][ee-2346] -> in [GitLab Premium][ee] 9.5. +> - The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2346) +> in [GitLab Premium](https://about.gitlab.com/pricing/) 9.5. Get the job's artifacts zipped archive of a project. @@ -359,7 +359,7 @@ GET /projects/:id/jobs/:job_id/artifacts |-------------|----------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------| | `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | | `job_id` | integer | yes | ID of a job. | -| `job_token` **(PREMIUM)** | string | no | To be used with [triggers] for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | +| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/triggers/README.md#when-a-pipeline-depends-on-the-artifacts-of-another-pipeline-premium) for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | Example request using the `PRIVATE-TOKEN` header: @@ -406,8 +406,8 @@ Possible response status codes: > **Notes**: > > - [Introduced][ce-5347] in GitLab 8.10. -> - The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced][ee-2346] -> in [GitLab Premium][ee] 9.5. +> - The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2346) +> in [GitLab Premium](https://about.gitlab.com/pricing/) 9.5. Download the artifacts zipped archive from the latest successful pipeline for the given reference name and job, provided the job finished successfully. This @@ -425,7 +425,7 @@ Parameters | `id` | integer/string | yes | ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | | `ref_name` | string | yes | Branch or tag name in repository. HEAD or SHA references are not supported. | | `job` | string | yes | The name of the job. | -| `job_token` **(PREMIUM)** | string | no | To be used with [triggers] for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | +| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/triggers/README.md#when-a-pipeline-depends-on-the-artifacts-of-another-pipeline-premium) for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | Example request using the `PRIVATE-TOKEN` header: @@ -841,7 +841,3 @@ Example of response "user": null } ``` - -[ee]: https://about.gitlab.com/pricing/ -[ee-2346]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2346 -[triggers]: ../ci/triggers/README.md#when-a-pipeline-depends-on-the-artifacts-of-another-pipeline-premium diff --git a/doc/api/pipelines.md b/doc/api/pipelines.md index 9db7c1528ff..490f1304130 100644 --- a/doc/api/pipelines.md +++ b/doc/api/pipelines.md @@ -2,7 +2,7 @@ ## List project pipelines -> [Introduced][ce-5837] in GitLab 8.11 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5837) in GitLab 8.11 ```plaintext GET /projects/:id/pipelines @@ -54,7 +54,7 @@ Example of response ## Get a single pipeline -> [Introduced][ce-5837] in GitLab 8.11 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5837) in GitLab 8.11 ```plaintext GET /projects/:id/pipelines/:pipeline_id @@ -132,7 +132,7 @@ Example of response ## Create a new pipeline -> [Introduced][ce-7209] in GitLab 8.14 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7209) in GitLab 8.14 ```plaintext POST /projects/:id/pipeline @@ -180,7 +180,7 @@ Example of response ## Retry jobs in a pipeline -> [Introduced][ce-5837] in GitLab 8.11 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5837) in GitLab 8.11 ```plaintext POST /projects/:id/pipelines/:pipeline_id/retry @@ -227,7 +227,7 @@ Response: ## Cancel a pipelines jobs -> [Introduced][ce-5837] in GitLab 8.11 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5837) in GitLab 8.11 ```plaintext POST /projects/:id/pipelines/:pipeline_id/cancel @@ -288,6 +288,3 @@ DELETE /projects/:id/pipelines/:pipeline_id ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" --request "DELETE" "https://gitlab.example.com/api/v4/projects/1/pipelines/46" ``` - -[ce-5837]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/5837 -[ce-7209]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/7209 diff --git a/doc/api/runners.md b/doc/api/runners.md index fa4631c1bd2..c17069b4474 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -1,8 +1,6 @@ # Runners API -> [Introduced][ce-2640] in GitLab 8.5 - -[ce-2640]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/2640 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/2640) in GitLab 8.5 ## Registration and authentication tokens diff --git a/doc/development/integrations/jenkins.md b/doc/development/integrations/jenkins.md index 31c479bc898..001d1c21fd3 100644 --- a/doc/development/integrations/jenkins.md +++ b/doc/development/integrations/jenkins.md @@ -1,6 +1,6 @@ -# How to run Jenkins in development environment (on OSX) **(STARTER)** +# How to run Jenkins in development environment (on macOS) **(STARTER)** -This is a step by step guide on how to set up [Jenkins](https://jenkins.io/) on your local machine and connect to it from your GitLab instance. GitLab triggers webhooks on Jenkins, and Jenkins is connects to GitLab using the API. By running both applications on the same machine, we can make sure they are able to access each other. +This is a step by step guide on how to set up [Jenkins](https://jenkins.io/) on your local machine and connect to it from your GitLab instance. GitLab triggers webhooks on Jenkins, and Jenkins connects to GitLab using the API. By running both applications on the same machine, we can make sure they are able to access each other. ## Install Jenkins @@ -17,12 +17,12 @@ GitLab does not allow requests to localhost or the local network by default. Whe 1. Log into your GitLab instance as an admin. 1. Go to **{admin}** **Admin Area > Settings > Network**. -1. Expand `Outbound requests` and check the following checkboxes: +1. Expand **Outbound requests** and check the following checkboxes: -- **Allow requests to the local network from web hooks and services** -- **Allow requests to the local network from system hooks** + - **Allow requests to the local network from web hooks and services** + - **Allow requests to the local network from system hooks** -For more details about GitLab webhooks, see [Webhooks and insecure internal web services](../../security/webhooks.md). + For more details about GitLab webhooks, see [Webhooks and insecure internal web services](../../security/webhooks.md). Jenkins uses the GitLab API and needs an access token. @@ -36,7 +36,9 @@ Jenkins uses the GitLab API and needs an access token. Configure your GitLab API connection in Jenkins. 1. Make sure the GitLab plugin is installed on Jenkins. You can manage plugins in **Manage Jenkins > Manage Plugins**. -1. Set up the GitLab connection: Go to **Manage Jenkins > Configure System**, find the **GitLab** section and check the `Enable authentication for '/project' end-point` checkbox. +1. Set up the GitLab connection: + 1. Go to **Manage Jenkins > Configure System**. + 1. Find the **GitLab** section and check the **Enable authentication for '/project' end-point** checkbox. 1. To add your credentials, click **Add** then choose **Jenkins Credential Provider**. 1. Choose **GitLab API token** as the type of token. 1. Paste your GitLab access token and click **Add**. @@ -48,64 +50,45 @@ For more details, see [GitLab documentation about Jenkins CI](../../integration/ ## Configure Jenkins Project -Set up the Jenkins project you are going to run your build on. +Set up the Jenkins project you're going to run your build on. A **Freestyle** project is the easiest option because the Jenkins plugin will update the build status on GitLab. In a **Pipeline** project, updating the status on GitLab needs to be configured in a script. 1. On your Jenkins instance, go to **New Item**. -1. Pick a name, choose **Pipeline** and click **ok**. +1. Pick a name, choose **Freestyle** or **Pipeline** and click **ok**. 1. Choose your GitLab connection from the dropdown. 1. Check the **Build when a change is pushed to GitLab** checkbox. 1. Check the following checkboxes: -- **Accepted Merge Request Events** -- **Closed Merge Request Events** + - **Accepted Merge Request Events** + - **Closed Merge Request Events** -1. Updating the status on GitLab must be done by a pipeline script. Add GitLab update steps, using the following as an example: +1. If you created a **Freestyle** project, choose **Publish build status to GitLab** in the **Post-build Actions** section. -**Example Pipeline Script:** + If you created a **Pipeline** project, updating the status on GitLab has to be done by the pipeline script. Add GitLab update steps as in this example: -```groovy -pipeline { - agent any + ```groovy + pipeline { + agent any - stages { - stage('gitlab') { - steps { - echo 'Notify GitLab' - updateGitlabCommitStatus name: 'build', state: 'pending' - updateGitlabCommitStatus name: 'build', state: 'success' + stages { + stage('gitlab') { + steps { + echo 'Notify GitLab' + updateGitlabCommitStatus name: 'build', state: 'pending' + updateGitlabCommitStatus name: 'build', state: 'success' + } } } } -} -``` + ``` ## Configure your GitLab project To activate the Jenkins service you must have a Starter subscription or higher. 1. Go to your project's page, then **Settings > Integrations > Jenkins CI**. -1. Check the `Active` checkbox and the triggers for `Push` and `Merge request`. +1. Check the **Active** checkbox and the triggers for **Push** and **Merge request**. 1. Fill in your Jenkins host, project name, username and password and click **Test settings and save changes**. ## Test your setup -Make a change in your repository and open an MR. In your Jenkins project it should have triggered a new build and on your MR, there should be a widget saying "Pipeline #NUMBER passed". It will also include a link to your Jenkins build. - -### Run QA test - -The [jenkins_build_status_spec](https://gitlab.com/gitlab-org/gitlab/-/blob/v12.9.0-ee/qa/qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb) performs an end-to-end test of the Jenkins setup using [GitLab QA](https://gitlab.com/gitlab-org/gitlab-qa). - -To run the test against your GDK, follow the [run QA tests against your GDK setup](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/run_qa_against_gdk.md#run-qa-tests-against-your-gdk-setup) instructions. - -The following environment variables are required to run the test: - -- `GITLAB_USERNAME` -- `GITLAB_PASSWORD` -- `GITLAB_ADMIN_USERNAME` -- `GITLAB_ADMIN_PASSWORD` - -Run the test in the **qa** directory using: - -```shell -bundle exec bin/qa Test::Instance::All http://<your_network_ip_address>:3000 -- qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb --tag quarantine -``` +Make a change in your repository and open an MR. In your Jenkins project it should have triggered a new build and on your MR, there should be a widget saying **Pipeline #NUMBER passed**. It will also include a link to your Jenkins build. diff --git a/doc/development/testing_guide/review_apps.md b/doc/development/testing_guide/review_apps.md index 77a94de98c4..c84a5466e03 100644 --- a/doc/development/testing_guide/review_apps.md +++ b/doc/development/testing_guide/review_apps.md @@ -113,6 +113,10 @@ pipelines (and is manual in merge request) stops stale Review Apps after 5 days, deletes their environment after 6 days, and cleans up any dangling Helm releases and Kubernetes resources after 7 days. +The `review-gcp-cleanup` job that automatically runs in scheduled pipelines +(and is manual in merge request) removes any dangling GCP network resources +that were not removed along with the Kubernetes resources. + ## QA runs On every [pipeline][gitlab-pipeline] in the `qa` stage (which comes after the diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index d49de056def..5eb23326e43 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -1,6 +1,6 @@ # Auto DevOps -> - [Introduced][ce-37115] in GitLab 10.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/37115) in GitLab 10.0. > - Generally available on GitLab 11.0. Auto DevOps provides pre-defined CI/CD configuration which allows you to automatically detect, build, test, @@ -317,7 +317,7 @@ When using Auto DevOps, you may want to deploy different environments to different Kubernetes clusters. This is possible due to the 1:1 connection that [exists between them](../../user/project/clusters/index.md#multiple-kubernetes-clusters-premium). -In the [Auto DevOps template] (used behind the scenes by Auto DevOps), there +In the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) (used behind the scenes by Auto DevOps), there are currently 3 defined environment names that you need to know: - `review/` (every environment starting with `review/`) @@ -369,7 +369,7 @@ Auto Build creates a build of the application using an existing `Dockerfile` or Heroku buildpacks. Either way, the resulting Docker image is automatically pushed to the -[Container Registry][container-registry] and tagged with the commit SHA or tag. +[Container Registry](../../user/packages/container_registry/index.md) and tagged with the commit SHA or tag. #### Auto Build using a Dockerfile @@ -430,7 +430,7 @@ Any differences between the source and target branches are also ### Auto SAST **(ULTIMATE)** -> Introduced in [GitLab Ultimate][ee] 10.3. +> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.3. Static Application Security Testing (SAST) uses the [SAST Docker image](https://gitlab.com/gitlab-org/security-products/sast) to run static @@ -445,7 +445,7 @@ Any security warnings are also shown in the merge request widget. Read more how ### Auto Dependency Scanning **(ULTIMATE)** -> Introduced in [GitLab Ultimate][ee] 10.7. +> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.7. Dependency Scanning uses the [Dependency Scanning Docker image](https://gitlab.com/gitlab-org/security-products/dependency-scanning) @@ -462,7 +462,7 @@ Any security warnings are also shown in the merge request widget. Read more abou ### Auto License Compliance **(ULTIMATE)** -> Introduced in [GitLab Ultimate][ee] 11.0. +> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0. License Compliance uses the [License Compliance Docker image](https://gitlab.com/gitlab-org/security-products/license-management) @@ -498,7 +498,7 @@ This is an optional step, since many projects do not have a Kubernetes cluster available. If the [requirements](#requirements) are not met, the job will silently be skipped. -[Review Apps][review-app] are temporary application environments based on the +[Review Apps](../../ci/review_apps/index.md) are temporary application environments based on the branch's code so developers, designers, QA, product managers, and other reviewers can actually see and interact with code changes as part of the review process. Auto Review Apps create a Review App for each branch. @@ -533,7 +533,7 @@ in the first place, and thus not realize that it needs to re-apply the old confi ### Auto DAST **(ULTIMATE)** -> Introduced in [GitLab Ultimate][ee] 10.4. +> Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 10.4. Dynamic Application Security Testing (DAST) uses the popular open source tool [OWASP ZAProxy](https://github.com/zaproxy/zaproxy) @@ -570,7 +570,7 @@ DAST can be disabled: ### Auto Browser Performance Testing **(PREMIUM)** -> Introduced in [GitLab Premium][ee] 10.4. +> Introduced in [GitLab Premium](https://about.gitlab.com/pricing/) 10.4. Auto Browser Performance Testing utilizes the [Sitespeed.io container](https://hub.docker.com/r/sitespeedio/sitespeed.io/) to measure the performance of a web page. A JSON report is created and uploaded as an artifact, which includes the overall performance score for each page. By default, the root page of Review and Production environments will be tested. If you would like to add additional URL's to test, simply add the paths to a file named `.gitlab-urls.txt` in the root directory, one per line. For example: @@ -595,7 +595,7 @@ the Kubernetes cluster, with a namespace based on the project name and unique project ID, for example `project-4321`. Auto Deploy doesn't include deployments to staging or canary by default, but the -[Auto DevOps template] contains job definitions for these tasks if you want to +[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) contains job definitions for these tasks if you want to enable them. You can make use of [environment variables](#environment-variables) to automatically @@ -619,7 +619,7 @@ deploys with Auto DevOps can undo your changes. Also, if you change something and want to undo it by deploying again, Helm may not detect that anything changed in the first place, and thus not realize that it needs to re-apply the old config. -> [Introduced][ce-19507] in GitLab 11.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19507) in GitLab 11.0. For internal and private projects a [GitLab Deploy Token](../../user/project/deploy_tokens/index.md#gitlab-deploy-token) will be automatically created, when Auto DevOps is enabled and the Auto DevOps settings are saved. This Deploy Token @@ -669,7 +669,7 @@ restore your database before opting into version `2`. #### Migrations -> [Introduced][ce-21955] in GitLab 11.4 +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/21955) in GitLab 11.4 Database initialization and migrations for PostgreSQL can be configured to run within the application pod by setting the project variables `DB_INITIALIZE` and @@ -1017,7 +1017,7 @@ to the desired environment. See [Limiting environment scopes of variables](../.. ### Customizing `.gitlab-ci.yml` -Auto DevOps is completely customizable because the [Auto DevOps template]: +Auto DevOps is completely customizable because the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml): - Is just an implementation of a [`.gitlab-ci.yml`](../../ci/yaml/README.md) file. - Uses only features available to any implementation of `.gitlab-ci.yml`. @@ -1033,7 +1033,7 @@ include: ``` Then add any extra changes you want. Your additions will be merged with the -[Auto DevOps template] using the behaviour described for +[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) using the behaviour described for [`include`](../../ci/yaml/README.md#include). It is also possible to copy and paste the contents of the [Auto DevOps @@ -1083,12 +1083,12 @@ include: - template: Jobs/Build.gitlab-ci.yml ``` -Consult the [Auto DevOps template] for information on available jobs. +Consult the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) for information on available jobs. ### PostgreSQL database support In order to support applications that require a database, -[PostgreSQL][postgresql] is provisioned by default. The credentials to access +[PostgreSQL](https://www.postgresql.org/) is provisioned by default. The credentials to access the database are preconfigured, but can be customized by setting the associated [variables](#environment-variables). These credentials can be used for defining a `DATABASE_URL` of the format: @@ -1550,16 +1550,6 @@ spec: service account for your project. For help debugging this issue, see [Troubleshooting failed deployment jobs](../../user/project/clusters/index.md#troubleshooting). -[ce-37115]: https://gitlab.com/gitlab-org/gitlab-foss/issues/37115 -[docker-in-docker]: ../../docker/using_docker_build.md#use-docker-in-docker-executor -[review-app]: ../../ci/review_apps/index.md -[container-registry]: ../../user/packages/container_registry/index.md -[postgresql]: https://www.postgresql.org/ -[Auto DevOps template]: https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml -[ee]: https://about.gitlab.com/pricing/ -[ce-21955]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/21955 -[ce-19507]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19507 - ## Development guides [Development guide for Auto DevOps](../../development/auto_devops.md) diff --git a/doc/user/project/canary_deployments.md b/doc/user/project/canary_deployments.md index 3a19c29b241..26cf58654f2 100644 --- a/doc/user/project/canary_deployments.md +++ b/doc/user/project/canary_deployments.md @@ -1,6 +1,6 @@ # Canary Deployments **(PREMIUM)** -> [Introduced][ee-1659] in [GitLab Premium][eep] 9.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/1659) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.1. A popular [Continuous Deployment](https://en.wikipedia.org/wiki/Continuous_deployment) strategy, where a small portion of the fleet is updated to the new version of @@ -8,7 +8,7 @@ your application. ## Overview -When embracing [Continuous Delivery][cd-blog], a company needs to decide what +When embracing [Continuous Delivery](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/), a company needs to decide what type of deployment strategy to use. One of the most popular strategies is canary deployments, where a small portion of the fleet is updated to the new version first. This subset, the canaries, then serve as the proverbial @@ -18,8 +18,8 @@ If there is a problem with the new version of the application, only a small percentage of users are affected and the change can either be fixed or quickly reverted. -Leveraging [Kubernetes' Canary deployments][kube-canary], visualize your canary -deployments right inside the [Deploy Board], without the need to leave GitLab. +Leveraging [Kubernetes' Canary deployments](https://kubernetes.io/docs/concepts/cluster-administration/manage-deployment/#canary-deployments), visualize your canary +deployments right inside the [Deploy Board](deploy_boards.md), without the need to leave GitLab. ## Use cases @@ -35,7 +35,7 @@ to be careful when using canaries with user-facing changes, because by default, requests from the same user will be randomly distributed between canary and non-canary pods, which could result in confusion or even errors. If needed, you may want to consider [setting `service.spec.sessionAffinity` to `ClientIP` in -your Kubernetes service definitions][kube-net], but that is beyond the scope of +your Kubernetes service definitions][kube-net](https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies), but that is beyond the scope of this document. ## Enabling Canary Deployments @@ -61,10 +61,3 @@ Canary deployments are marked with a yellow dot in the Deploy Board so that you can easily notice them.  - -[eep]: https://about.gitlab.com/pricing/ -[ee-1659]: https://gitlab.com/gitlab-org/gitlab/issues/1659 -[kube-canary]: https://kubernetes.io/docs/concepts/cluster-administration/manage-deployment/#canary-deployments -[deploy board]: deploy_boards.md -[cd-blog]: https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/ -[kube-net]: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies diff --git a/doc/user/project/deploy_boards.md b/doc/user/project/deploy_boards.md index d8d1db4fbcb..019ba083584 100644 --- a/doc/user/project/deploy_boards.md +++ b/doc/user/project/deploy_boards.md @@ -1,9 +1,9 @@ # Deploy Boards **(PREMIUM)** -> [Introduced][ee-1589] in [GitLab Premium][ee] 9.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/1589) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.0. GitLab's Deploy Boards offer a consolidated view of the current health and -status of each CI [environment] running on [Kubernetes], displaying the status +status of each CI [environment](../../ci/environments.md) running on [Kubernetes](https://kubernetes.io), displaying the status of the pods in the deployment. Developers and other teammates can view the progress and status of a rollout, pod by pod, in the workflow they already use without any need to access Kubernetes. @@ -52,12 +52,12 @@ specific environment, there are a lot of use cases. To name a few: stuck or failed. - You've got an MR that looks good, but you want to run it on staging because staging is set up in some way closer to production. You go to the environment - list, find the [Review App][review apps] you're interested in, and click the + list, find the [Review App](../../ci/review_apps/index.md) you're interested in, and click the manual action to deploy it to staging. ## Enabling Deploy Boards -To display the Deploy Boards for a specific [environment] you should: +To display the Deploy Boards for a specific [environment](../../ci/environments.md) you should: 1. Have [defined an environment](../../ci/environments.md#defining-environments) with a deploy stage. @@ -70,9 +70,9 @@ To display the Deploy Boards for a specific [environment] you should: [OpenShift docs](https://docs.openshift.com/container-platform/3.7/dev_guide/deployments/kubernetes_deployments.html#kubernetes-deployments-vs-deployment-configurations) and [GitLab issue #4584](https://gitlab.com/gitlab-org/gitlab/issues/4584). -1. [Configure GitLab Runner][runners] with the [Docker][docker-exec] or - [Kubernetes][kube-exec] executor. -1. Configure the [Kubernetes integration][kube-integration] in your project for the +1. [Configure GitLab Runner](../../ci/runners/README.md) with the [Docker](https://docs.gitlab.com/runner/executors/docker.html) or + [Kubernetes](https://docs.gitlab.com/runner/executors/kubernetes.html) executor. +1. Configure the [Kubernetes integration](clusters/index.md) in your project for the cluster. The Kubernetes namespace is of particular note as you will need it for your deployment scripts (exposed by the `KUBE_NAMESPACE` env variable). 1. Ensure Kubernetes annotations of `app.gitlab.com/env: $CI_ENVIRONMENT_SLUG` @@ -81,7 +81,7 @@ To display the Deploy Boards for a specific [environment] you should: `$CI_PROJECT_PATH_SLUG` are the values of the CI variables. This is so we can lookup the proper environment in a cluster/namespace which may have more than one. These resources should be contained in the namespace defined in - the Kubernetes service setting. You can use an [Autodeploy] `.gitlab-ci.yml` + the Kubernetes service setting. You can use an [Autodeploy](../../topics/autodevops/index.md#auto-deploy) `.gitlab-ci.yml` template which has predefined stages and commands to use, and automatically applies the annotations. Each project will need to have a unique namespace in Kubernetes as well. The image below demonstrates how this is shown inside @@ -139,21 +139,7 @@ version of your application. ## Further reading -- [GitLab Autodeploy][autodeploy] -- [GitLab CI/CD environment variables][variables] -- [Environments and deployments][environment] -- [Kubernetes deploy example][kube-deploy] - -[ee-1589]: https://gitlab.com/gitlab-org/gitlab/issues/1589 "Deploy Boards initial issue" -[ee]: https://about.gitlab.com/pricing/ "GitLab Enterprise Edition landing page" -[kube-deploy]: https://gitlab.com/gitlab-examples/kubernetes-deploy "Kubernetes deploy example project" -[kubernetes]: https://kubernetes.io "Kubernetes website" -[environment]: ../../ci/environments.md "Environments and deployments documentation" -[docker-exec]: https://docs.gitlab.com/runner/executors/docker.html "GitLab Runner Docker executor" -[kube-exec]: https://docs.gitlab.com/runner/executors/kubernetes.html "GitLab Runner Kubernetes executor" -[kube-integration]: clusters/index.md "Kubernetes integration" -[review apps]: ../../ci/review_apps/index.md "Review Apps documentation" -[variables]: ../../ci/variables/README.md "GitLab CI/CD variables" -[autodeploy]: ../../topics/autodevops/index.md#auto-deploy "GitLab Autodeploy" -[kube-image]: https://gitlab.com/gitlab-examples/kubernetes-deploy/container_registry "Kubernetes deploy Container Registry" -[runners]: ../../ci/runners/README.md +- [GitLab Autodeploy](../../topics/autodevops/index.md#auto-deploy) +- [GitLab CI/CD environment variables](../../ci/variables/README.md) +- [Environments and deployments](../../ci/environments.md) +- [Kubernetes deploy example](https://gitlab.com/gitlab-examples/kubernetes-deploy) diff --git a/doc/user/project/deploy_tokens/index.md b/doc/user/project/deploy_tokens/index.md index 03b580c320a..0bd511cf837 100644 --- a/doc/user/project/deploy_tokens/index.md +++ b/doc/user/project/deploy_tokens/index.md @@ -1,6 +1,6 @@ # Deploy Tokens -> - [Introduced][ce-17894] in GitLab 10.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17894) in GitLab 10.7. > - [Moved](https://gitlab.com/gitlab-org/gitlab/issues/199370) from **Settings > Repository** in GitLab 12.9. Deploy tokens allow you to download (`git clone`) or read the container registry images of a project without having a user and a password. @@ -43,7 +43,7 @@ the following table. | Scope | Description | | ----- | ----------- | | `read_repository` | Allows read-access to the repository through `git clone` | -| `read_registry` | Allows read-access to [container registry] images if a project is private and authorization is required. | +| `read_registry` | Allows read-access to [container registry](../../packages/container_registry/index.md) images if a project is private and authorization is required. | ## Deploy token custom username @@ -101,7 +101,7 @@ apply consistently when cloning the repository of related projects. ### GitLab Deploy Token -> [Introduced][ce-18414] in GitLab 10.8. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18414) in GitLab 10.8. There's a special case when it comes to Deploy Tokens. If a user creates one named `gitlab-deploy-token`, the username and token of the Deploy Token will be @@ -115,8 +115,3 @@ those variables: ```shell docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY ``` - -[ce-17894]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/17894 -[ce-11845]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/11845 -[ce-18414]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18414 -[container registry]: ../../packages/container_registry/index.md diff --git a/doc/user/project/new_ci_build_permissions_model.md b/doc/user/project/new_ci_build_permissions_model.md index 9ecfcd12b80..c5ab8d66852 100644 --- a/doc/user/project/new_ci_build_permissions_model.md +++ b/doc/user/project/new_ci_build_permissions_model.md @@ -2,7 +2,7 @@ > Introduced in GitLab 8.12. -GitLab 8.12 has a completely redesigned [job permissions] system. You can find +GitLab 8.12 has a completely redesigned [job permissions](../permissions.md#job-permissions) system. You can find all discussion and all our concerns when choosing the current approach in issue [#18994](https://gitlab.com/gitlab-org/gitlab-foss/issues/18994). @@ -32,7 +32,7 @@ as created by the pusher (local push or via the UI) and any job created in this pipeline will have the read permissions of the pusher but not write permissions. This allows us to make it really easy to evaluate the access for all projects -that have [Git submodules][gitsub] or are using container images that the pusher +that have [Git submodules](../../ci/git_submodules.md) or are using container images that the pusher would have access too. **The permission is granted only for the time that the job is running. The access is revoked after the job is finished.** @@ -132,7 +132,7 @@ With the new job permissions model, there is now an easy way to access all dependent source code in a project. That way, we can: 1. Access a project's dependent repositories -1. Access a project's [Git submodules][gitsub] +1. Access a project's [Git submodules](../../ci/git_submodules.md) 1. Access private container images 1. Access project's and submodule LFS objects @@ -151,15 +151,15 @@ In short here's what you need to do should you encounter any issues. As an administrator: -- **500 errors**: You will need to update [GitLab Workhorse][workhorse] to at +- **500 errors**: You will need to update [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) to at least 0.8.2. This is done automatically for Omnibus installations, you need to - [check manually][update-docs] for installations from source. + [check manually](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/doc/update) for installations from source. - **500 errors**: Check if you have another web proxy sitting in front of NGINX (HAProxy, Apache, etc.). It might be a good idea to let GitLab use the internal NGINX - web server and not disable it completely. See [this comment][comment] for an + web server and not disable it completely. See [this comment](https://gitlab.com/gitlab-org/gitlab-foss/issues/22484#note_16648302) for an example. - **403 errors**: You need to make sure that your installation has [HTTP(S) - cloning enabled][https]. HTTP(S) support is now a **requirement** by GitLab CI + cloning enabled](../admin_area/settings/visibility_and_access_controls.md#enabled-git-access-protocols). HTTP(S) support is now a **requirement** by GitLab CI to clone all sources. As a user: @@ -170,7 +170,7 @@ As a user: ### Dependent repositories -The [Job environment variable][jobenv] `CI_JOB_TOKEN` can be used to +The [Job environment variable](../../ci/variables/README.md#predefined-environment-variables) `CI_JOB_TOKEN` can be used to authenticate any clones of dependent repositories. For example: ```shell @@ -187,7 +187,7 @@ echo -e "machine gitlab.com\nlogin gitlab-ci-token\npassword ${CI_JOB_TOKEN}" > ### Git submodules To properly configure submodules with GitLab CI/CD, read the -[Git submodules documentation][gitsub]. +[Git submodules documentation](../../ci/git_submodules.md). ### Container Registry @@ -203,8 +203,8 @@ Container Registries for private projects. > access token created explicitly for this purpose). This issue is resolved with > latest changes in GitLab Runner 1.8 which receives GitLab credentials with > build data. -> - Starting from GitLab 8.12, if you have [2FA] enabled in your account, you need -> to pass a [personal access token][pat] instead of your password in order to +> - Starting from GitLab 8.12, if you have [2FA](../profile/account/two_factor_authentication.md) enabled in your account, you need +> to pass a [personal access token](../profile/personal_access_tokens.md) instead of your password in order to > login to GitLab's Container Registry. Your jobs can access all container images that you would normally have access @@ -223,7 +223,7 @@ test: ### Pipeline triggers -Since 9.0 [pipeline triggers][triggers] do support the new permission model. +Since 9.0 [pipeline triggers](../../ci/triggers/README.md#ci-job-token) do support the new permission model. The new triggers do impersonate their associated user including their access to projects and their project permissions. @@ -231,14 +231,3 @@ to projects and their project permissions. GitLab API cannot be used via `CI_JOB_TOKEN` but there is a [proposal](https://gitlab.com/gitlab-org/gitlab-foss/issues/29566) to support it. - -[job permissions]: ../permissions.md#job-permissions -[comment]: https://gitlab.com/gitlab-org/gitlab-foss/issues/22484#note_16648302 -[gitsub]: ../../ci/git_submodules.md -[https]: ../admin_area/settings/visibility_and_access_controls.md#enabled-git-access-protocols -[triggers]: ../../ci/triggers/README.md#ci-job-token -[update-docs]: https://gitlab.com/gitlab-org/gitlab-foss/tree/master/doc/update -[workhorse]: https://gitlab.com/gitlab-org/gitlab-workhorse -[jobenv]: ../../ci/variables/README.md#predefined-environment-variables -[2fa]: ../profile/account/two_factor_authentication.md -[pat]: ../profile/personal_access_tokens.md diff --git a/doc/user/project/pages/introduction.md b/doc/user/project/pages/introduction.md index 798efe6fa1d..f77c572664a 100644 --- a/doc/user/project/pages/introduction.md +++ b/doc/user/project/pages/introduction.md @@ -21,7 +21,7 @@ In brief, this is what you need to upload your website in GitLab Pages: 1. Domain of the instance: domain name that is used for GitLab Pages (ask your administrator). -1. GitLab CI/CD: a `.gitlab-ci.yml` file with a specific job named [`pages`][pages] in the root directory of your repository. +1. GitLab CI/CD: a `.gitlab-ci.yml` file with a specific job named [`pages`](../../../ci/yaml/README.md#pages) in the root directory of your repository. 1. A directory called `public` in your site's repo containing the content to be published. 1. GitLab Runner enabled for the project. @@ -61,7 +61,7 @@ If the case of `404.html`, there are different scenarios. For example: Since you cannot use any custom server configuration files, like `.htaccess` or any `.conf` file, if you want to redirect a page to another -location, you can use the [HTTP meta refresh tag][metarefresh]. +location, you can use the [HTTP meta refresh tag](https://en.wikipedia.org/wiki/Meta_refresh). Some static site generators provide plugins for that functionality so that you don't have to create and edit HTML files manually. For example, Jekyll has the @@ -286,8 +286,4 @@ No, you don't. You can create your project first and it will be accessed under ## Known issues -For a list of known issues, visit GitLab's [public issue tracker]. - -[pages]: ../../../ci/yaml/README.md#pages -[metarefresh]: https://en.wikipedia.org/wiki/Meta_refresh -[public issue tracker]: https://gitlab.com/gitlab-org/gitlab/-/issues?label_name[]=Category%3APages +For a list of known issues, visit GitLab's [public issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues?label_name[]=Category%3APages). diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index ce1dfb0753c..305bcada2f5 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -45,7 +45,7 @@ module Gitlab end # CSS sprite fallback takes precedence over image fallback - def gl_emoji_tag(name) + def gl_emoji_tag(name, options = {}) emoji_name = emojis_aliases[name] || name emoji_info = emojis[emoji_name] return unless emoji_info @@ -54,8 +54,9 @@ module Gitlab name: emoji_name, unicode_version: emoji_unicode_version(emoji_name) } + options = { title: emoji_info['description'], data: data }.merge(options) - ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], title: emoji_info['description'], data: data) + ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], options) end private diff --git a/scripts/review_apps/gcp_cleanup.sh b/scripts/review_apps/gcp_cleanup.sh new file mode 100755 index 00000000000..ea6b60ed5ff --- /dev/null +++ b/scripts/review_apps/gcp_cleanup.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +source scripts/utils.sh + +# These scripts require the following environment variables: +# - REVIEW_APPS_GCP_REGION - e.g `us-central1` +# - KUBE_NAMESPACE - e.g `review-apps-ee` + +function delete_firewall_rules() { + if [[ ${#@} -eq 0 ]]; then + echoinfo "No firewall rules to be deleted" true + return + fi + + echoinfo "Deleting firewall rules:" true + echo "${@}" + + if [[ ${DRY_RUN} = 1 ]]; then + echo "[DRY RUN] gcloud compute firewall-rules delete -q" "${@}" + else + gcloud compute firewall-rules delete -q "${@}" + fi +} + +function delete_forwarding_rules() { + if [[ ${#@} -eq 0 ]]; then + echoinfo "No forwarding rules to be deleted" true + return + fi + + echoinfo "Deleting forwarding rules:" true + echo "${@}" + + if [[ ${DRY_RUN} = 1 ]]; then + echo "[DRY RUN] gcloud compute forwarding-rules delete -q" "${@}" "--region ${REVIEW_APPS_GCP_REGION}" + else + gcloud compute forwarding-rules delete -q "${@}" --region "${REVIEW_APPS_GCP_REGION}" + fi +} + +function delete_target_pools() { + if [[ ${#@} -eq 0 ]]; then + echoinfo "No target pools to be deleted" true + return + fi + + echoinfo "Deleting target pools:" true + echo "${@}" + + if [[ ${DRY_RUN} = 1 ]]; then + echo "[DRY RUN] gcloud compute target-pools delete -q" "${@}" "--region ${REVIEW_APPS_GCP_REGION}" + else + gcloud compute target-pools delete -q "${@}" --region "${REVIEW_APPS_GCP_REGION}" + fi +} + +function delete_http_health_checks() { + if [[ ${#@} -eq 0 ]]; then + echoinfo "No http health checks to be deleted" true + return + fi + + echoinfo "Deleting http health checks:" true + echo "${@}" + + if [[ ${DRY_RUN} = 1 ]]; then + echo "[DRY RUN] gcloud compute http-health-checks delete -q" "${@}" + else + gcloud compute http-health-checks delete -q "${@}" + fi +} + +function get_related_firewall_rules() { + local forwarding_rule=${1} + + gcloud compute firewall-rules list --filter "name~${forwarding_rule}" --format "value(name)" +} + +function get_service_name_in_forwarding_rule() { + local forwarding_rule=${1} + + gcloud compute forwarding-rules describe "${forwarding_rule}" --region "${REVIEW_APPS_GCP_REGION}" --format "value(description)" | jq -r '.["kubernetes.io/service-name"]' +} + +function forwarding_rule_k8s_service_exists() { + local namespace="${KUBE_NAMESPACE}" + local namespaced_service_name=$(get_service_name_in_forwarding_rule "$forwarding_rule") + + if [[ ! $namespaced_service_name =~ ^"${namespace}" ]]; then + return 0 # this prevents `review-apps-ee` pipeline from deleting `review-apps-ce` resources and vice versa + fi + + local service_name=$(echo "${namespaced_service_name}" | sed -e "s/${namespace}\///g") + + kubectl get svc "${service_name}" -n "${namespace}" >/dev/null 2>&1 + local status=$? + + return $status +} + +function gcp_cleanup() { + if [[ ! $(command -v kubectl) ]]; then + echoerr "kubectl executable not found" + return 1 + fi + + if [[ -z "${REVIEW_APPS_GCP_REGION}" ]]; then + echoerr "REVIEW_APPS_GCP_REGION is not set." + return 1 + fi + + if [[ -z "${KUBE_NAMESPACE}" ]]; then + echoerr "KUBE_NAMESPACE is not set." + return 1 + fi + + if [[ -n "${DRY_RUN}" ]]; then + echoinfo "Running in DRY_RUN" + fi + + local target_pools_to_delete=() + local firewall_rules_to_delete=() + local forwarding_rules_to_delete=() + local http_health_checks_to_delete=() + + for forwarding_rule in $(gcloud compute forwarding-rules list --filter="region:(${REVIEW_APPS_GCP_REGION})" --format "value(name)"); do + echoinfo "Inspecting forwarding rule ${forwarding_rule}" true + + # We perform clean up when there is no more kubernetes service that require the resources. + # To identify the kubernetes service using the resources, + # we find the service name indicated in the forwarding rule description, e.g: + # + # $ gcloud compute forwarding-rules describe aff68b997da1211e984a042010af0019 + # # ... + # description: '{"kubernetes.io/service-name":"review-apps-ee/review-winh-eslin-809vqz-nginx-ingress-controller"}' + # # ... + if forwarding_rule_k8s_service_exists "${forwarding_rule}"; then + echoinfo "Skip clean up for ${forwarding_rule}" + else + echoinfo "Queuing forwarding rule, firewall rule, target pool and health check for ${forwarding_rule} to be cleaned up" + + firewall_rules_to_delete+=($(get_related_firewall_rules "${forwarding_rule}")) + forwarding_rules_to_delete+=(${forwarding_rule}) + target_pools_to_delete+=(${forwarding_rule}) + http_health_checks_to_delete+=(${forwarding_rule}) + fi + done + + delete_firewall_rules "${firewall_rules_to_delete[@]}" + delete_forwarding_rules "${forwarding_rules_to_delete[@]}" + delete_target_pools "${target_pools_to_delete[@]}" + delete_http_health_checks "${http_health_checks_to_delete[@]}" +} diff --git a/spec/helpers/emoji_helper_spec.rb b/spec/helpers/emoji_helper_spec.rb new file mode 100644 index 00000000000..1b73e956f7a --- /dev/null +++ b/spec/helpers/emoji_helper_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe EmojiHelper do + describe '#emoji_icon' do + let(:options) { {} } + let(:emoji_text) { 'rocket' } + let(:aria_hidden_option) { "aria-hidden=\"true\"" } + + subject { helper.emoji_icon(emoji_text, options) } + + it 'has no options' do + is_expected.to include('<gl-emoji', + "title=\"#{emoji_text}\"", + "data-name=\"#{emoji_text}\"", + "data-unicode-version=\"#{::Gitlab::Emoji.emoji_unicode_version(emoji_text)}\"") + is_expected.not_to include(aria_hidden_option) + end + + context 'with aria-hidden option' do + let(:options) { { 'aria-hidden': true } } + + it 'applies aria-hidden' do + is_expected.to include('<gl-emoji', + "title=\"#{emoji_text}\"", + "data-name=\"#{emoji_text}\"", + "data-unicode-version=\"#{::Gitlab::Emoji.emoji_unicode_version(emoji_text)}\"", + aria_hidden_option) + end + end + end +end diff --git a/spec/workers/concerns/waitable_worker_spec.rb b/spec/workers/concerns/waitable_worker_spec.rb index 1d76480ccb0..e7ef96c685d 100644 --- a/spec/workers/concerns/waitable_worker_spec.rb +++ b/spec/workers/concerns/waitable_worker_spec.rb @@ -58,21 +58,6 @@ describe WaitableWorker do worker.bulk_perform_and_wait(arguments, timeout: 2) end - - context 'when the skip_job_waiter_for_large_batches flag is disabled' do - before do - stub_feature_flags(skip_job_waiter_for_large_batches: false) - end - - it 'runs jobs over 10 * the timeout using a waiter key' do - arguments = 1.upto(21).map { |i| [i] } - arguments_with_waiter = arguments.map { |arg| arg + [anything] } - - expect(worker).to receive(:bulk_perform_async).with(arguments_with_waiter) - - worker.bulk_perform_and_wait(arguments, timeout: 2) - end - end end describe '.bulk_perform_inline' do |