diff options
34 files changed, 338 insertions, 198 deletions
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index a269e15611f..f35d3ce19a4 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -34,8 +34,7 @@ variables: DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 - LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly" - QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}" + GITLAB_EDITION: "ce" build-qa-image: extends: @@ -47,7 +46,9 @@ build-qa-image: - branches@gitlab-org/gitlab stage: test script: - - time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} --file ./qa/Dockerfile ./ + - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' + - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}" + - time docker build --cache-from gitlab/gitlab-${GITLAB_EDITION}-qa:nightly --tag ${QA_IMAGE} --file ./qa/Dockerfile ./ - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY} - time docker push ${QA_IMAGE} @@ -82,11 +83,13 @@ schedule:review-build-cng: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" GITLAB_HELM_CHART_REF: "master" + GITLAB_EDITION: "ce" environment: name: review/${CI_COMMIT_REF_NAME} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} on_stop: review-stop before_script: + - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) @@ -169,6 +172,8 @@ review-cleanup-failed-deployment: expire_in: 7 days when: always before_script: + - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' + - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}" - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)" - echo "${CI_ENVIRONMENT_URL}" - echo "${QA_IMAGE}" @@ -148,7 +148,7 @@ gem 'wikicloth', '0.8.1' gem 'asciidoctor', '~> 2.0.10' gem 'asciidoctor-include-ext', '~> 0.3.1', require: false gem 'asciidoctor-plantuml', '0.0.9' -gem 'rouge', '~> 3.10' +gem 'rouge', '~> 3.7' gem 'truncato', '~> 0.7.11' gem 'bootstrap_form', '~> 4.2.0' gem 'nokogiri', '~> 1.10.4' diff --git a/Gemfile.lock b/Gemfile.lock index 581730b208c..6fd74c1da78 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -834,7 +834,7 @@ GEM retriable (3.1.2) rinku (2.0.0) rotp (2.1.2) - rouge (3.10.0) + rouge (3.7.0) rqrcode (0.7.0) chunky_png rqrcode-rails3 (0.1.7) @@ -1276,7 +1276,7 @@ DEPENDENCIES redis-rails (~> 5.0.2) request_store (~> 1.3) responders (~> 2.0) - rouge (~> 3.10) + rouge (~> 3.7) rqrcode-rails3 (~> 0.1.7) rspec-parameterized rspec-rails (~> 3.8.0) diff --git a/app/graphql/resolvers/issues_resolver.rb b/app/graphql/resolvers/issues_resolver.rb index b50186c5a82..4e71a7a9ead 100644 --- a/app/graphql/resolvers/issues_resolver.rb +++ b/app/graphql/resolvers/issues_resolver.rb @@ -11,31 +11,32 @@ module Resolvers description: 'The list of IIDs of issues, e.g., [1, 2]' argument :state, Types::IssuableStateEnum, required: false, - description: "Current state of Issue" + description: 'Current state of Issue' argument :label_name, GraphQL::STRING_TYPE.to_list_type, required: false, - description: "Labels applied to the Issue" + description: 'Labels applied to the Issue' argument :created_before, Types::TimeType, required: false, - description: "Issues created before this date" + description: 'Issues created before this date' argument :created_after, Types::TimeType, required: false, - description: "Issues created after this date" + description: 'Issues created after this date' argument :updated_before, Types::TimeType, required: false, - description: "Issues updated before this date" + description: 'Issues updated before this date' argument :updated_after, Types::TimeType, required: false, - description: "Issues updated after this date" + description: 'Issues updated after this date' argument :closed_before, Types::TimeType, required: false, - description: "Issues closed before this date" + description: 'Issues closed before this date' argument :closed_after, Types::TimeType, required: false, - description: "Issues closed after this date" + description: 'Issues closed after this date' argument :search, GraphQL::STRING_TYPE, # rubocop:disable Graphql/Descriptions required: false - argument :sort, Types::Sort, # rubocop:disable Graphql/Descriptions + argument :sort, Types::SortEnum, + description: 'Sort issues by this criteria', required: false, default_value: 'created_desc' diff --git a/app/graphql/types/order.rb b/app/graphql/types/order.rb deleted file mode 100644 index c5e1cc406b4..00000000000 --- a/app/graphql/types/order.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -module Types - class Types::Order < Types::BaseEnum - value "id", "Created at date" - value "updated_at", "Updated at date" - end -end diff --git a/app/graphql/types/sort.rb b/app/graphql/types/sort.rb deleted file mode 100644 index 1f756fdab69..00000000000 --- a/app/graphql/types/sort.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Types - class Types::Sort < Types::BaseEnum - value "updated_desc", "Updated at descending order" - value "updated_asc", "Updated at ascending order" - value "created_desc", "Created at descending order" - value "created_asc", "Created at ascending order" - end -end diff --git a/app/graphql/types/sort_enum.rb b/app/graphql/types/sort_enum.rb new file mode 100644 index 00000000000..3245cb33e0d --- /dev/null +++ b/app/graphql/types/sort_enum.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Types + class SortEnum < BaseEnum + graphql_name 'Sort' + description 'Common sort values' + + value 'updated_desc', 'Updated at descending order' + value 'updated_asc', 'Updated at ascending order' + value 'created_desc', 'Created at descending order' + value 'created_asc', 'Created at ascending order' + end +end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index f281cbd1d6f..89cdd8c64f8 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -4,9 +4,12 @@ module Ci # The purpose of this class is to store Build related data that can be disposed. # Data that should be persisted forever, should be stored with Ci::Build model. class BuildMetadata < ApplicationRecord + BuildTimeout = Struct.new(:value, :source) + extend Gitlab::Ci::Model include Presentable include ChronicDurationAttribute + include Gitlab::Utils::StrongMemoize self.table_name = 'ci_builds_metadata' @@ -25,17 +28,16 @@ module Ci enum timeout_source: { unknown_timeout_source: 1, project_timeout_source: 2, - runner_timeout_source: 3 + runner_timeout_source: 3, + job_timeout_source: 4 } def update_timeout_state - return unless build.runner.present? + timeout = timeout_with_highest_precedence - project_timeout = project&.build_timeout - timeout = [project_timeout, build.runner.maximum_timeout].compact.min - timeout_source = timeout < project_timeout ? :runner_timeout_source : :project_timeout_source + return unless timeout - update(timeout: timeout, timeout_source: timeout_source) + update(timeout: timeout.value, timeout_source: timeout.source) end private @@ -43,5 +45,37 @@ module Ci def set_build_project self.project_id ||= self.build.project_id end + + def timeout_with_highest_precedence + [(job_timeout || project_timeout), runner_timeout].compact.min_by { |timeout| timeout.value } + end + + def project_timeout + strong_memoize(:project_timeout) do + BuildTimeout.new(project&.build_timeout, :project_timeout_source) + end + end + + def job_timeout + return unless build.options + + strong_memoize(:job_timeout) do + if timeout_from_options = build.options[:job_timeout] + BuildTimeout.new(timeout_from_options, :job_timeout_source) + end + end + end + + def runner_timeout + return unless runner_timeout_set? + + strong_memoize(:runner_timeout) do + BuildTimeout.new(build.runner.maximum_timeout, :runner_timeout_source) + end + end + + def runner_timeout_set? + build.runner&.maximum_timeout.to_i > 0 + end end end diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 8b011bca72c..57118bf7a6b 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -33,16 +33,9 @@ module Routable # # Returns a single object, or nil. def find_by_full_path(path, follow_redirects: false) - routable_calls_counter.increment(method: 'find_by_full_path') - - if Feature.enabled?(:routable_two_step_lookup) - # Case sensitive match first (it's cheaper and the usual case) - # If we didn't have an exact match, we perform a case insensitive search - found = includes(:route).find_by(routes: { path: path }) || where_full_path_in([path]).take - else - order_sql = Arel.sql("(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)") - found = where_full_path_in([path]).reorder(order_sql).take - end + # Case sensitive match first (it's cheaper and the usual case) + # If we didn't have an exact match, we perform a case insensitive search + found = includes(:route).find_by(routes: { path: path }) || where_full_path_in([path]).take return found if found @@ -61,19 +54,12 @@ module Routable def where_full_path_in(paths) return none if paths.empty? - routable_calls_counter.increment(method: 'where_full_path_in') - wheres = paths.map do |path| "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))" end includes(:route).where(wheres.join(' OR ')).references(:routes) end - - # Temporary instrumentation of method calls - def routable_calls_counter - @routable_calls_counter ||= Gitlab::Metrics.counter(:gitlab_routable_calls_total, 'Number of calls to Routable by method') - end end def full_name diff --git a/app/models/project.rb b/app/models/project.rb index 96c715095f6..59c187fac31 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -58,6 +58,7 @@ class Project < ApplicationRecord ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT = 10 SORTING_PREFERENCE_FIELD = :projects_sort + MAX_BUILD_TIMEOUT = 1.month cache_markdown_field :description, pipeline: :description @@ -430,7 +431,7 @@ class Project < ApplicationRecord validates :build_timeout, allow_nil: true, numericality: { greater_than_or_equal_to: 10.minutes, - less_than: 1.month, + less_than: MAX_BUILD_TIMEOUT, only_integer: true, message: _('needs to be between 10 minutes and 1 month') } diff --git a/app/presenters/ci/build_metadata_presenter.rb b/app/presenters/ci/build_metadata_presenter.rb index 015b1f67db7..4871bb3a919 100644 --- a/app/presenters/ci/build_metadata_presenter.rb +++ b/app/presenters/ci/build_metadata_presenter.rb @@ -5,7 +5,8 @@ module Ci TIMEOUT_SOURCES = { unknown_timeout_source: nil, project_timeout_source: 'project', - runner_timeout_source: 'runner' + runner_timeout_source: 'runner', + job_timeout_source: 'job' }.freeze presents :metadata diff --git a/changelogs/unreleased/ms-implement-build-specific-timeout.yml b/changelogs/unreleased/ms-implement-build-specific-timeout.yml new file mode 100644 index 00000000000..bc66b0b2fba --- /dev/null +++ b/changelogs/unreleased/ms-implement-build-specific-timeout.yml @@ -0,0 +1,5 @@ +--- +title: Allow specifying timeout per-job in .gitlab-ci.yml +merge_request: 16777 +author: Michał Siwek +type: added diff --git a/changelogs/unreleased/update-rouge.yml b/changelogs/unreleased/update-rouge.yml deleted file mode 100644 index 6f44de02d76..00000000000 --- a/changelogs/unreleased/update-rouge.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Update rouge to v3.10.0 -merge_request: 32745 -author: -type: other diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 53b354d2f92..27794878cfe 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -635,7 +635,7 @@ Confirm the following are all true: UI, it immediatley fails with a red `401 Unauthorized` banner. - Creating a new project and [initializing it with a README](../../gitlab-basics/create-project.md#blank-projects) successfully creates the project but doesn't create the README. -- When [tailing the logs](https://docs.gitlab.com/omnibus/settings/logs.md#tail-logs-in-a-console-on-the-server) on an app node and reproducing the error, you get `401` errors +- When [tailing the logs](https://docs.gitlab.com/omnibus/settings/logs.html#tail-logs-in-a-console-on-the-server) on an app node and reproducing the error, you get `401` errors when reaching the `/api/v4/internal/allowed` endpoint: ```sh diff --git a/doc/ci/environments.md b/doc/ci/environments.md index b41fd7971d6..32f9ab2205a 100644 --- a/doc/ci/environments.md +++ b/doc/ci/environments.md @@ -679,7 +679,7 @@ fetch = +refs/environments/*:refs/remotes/origin/environments/* ### Scoping environments with specs > - [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2112) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.4. -> - [Moved](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30779) to Core in Gitlab 12.2. +> - [Scoping for environment variables was moved to Core](https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/30779) in GitLab 12.2. You can limit the environment scope of a variable by defining which environments it can be available for. diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 22244d5688e..83ac38f3aa3 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -110,6 +110,7 @@ The following table lists available parameters for jobs: | [`dependencies`](#dependencies) | Other jobs that a job depends on so that you can pass artifacts between them. | | [`coverage`](#coverage) | Code coverage settings for a given job. | | [`retry`](#retry) | When and how many times a job can be auto-retried in case of a failure. | +| [`timeout`](#timeout) | Define a custom timeout that would take precedence over the project-wide one. | | [`parallel`](#parallel) | How many instances of a job should be run in parallel. | | [`trigger`](#trigger-premium) | Defines a downstream pipeline trigger. | | [`include`](#include) | Allows this job to include external YAML files. Also available: `include:local`, `include:file`, `include:template`, and `include:remote`. | @@ -1995,6 +1996,20 @@ Possible values for `when` are: - `missing_dependency_failure`: Retry if a dependency was missing. - `runner_unsupported`: Retry if the runner was unsupported. +### timeout + +`timeout` allows you to configure a timeout for a specific job: + +```yaml +build: + script: build.sh + timeout: 3 hours 30 minutes + +test: + script: rspec + timeout: 3h 30m +``` + ### `parallel` > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22631) in GitLab 11.5. diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 147bd21e6c7..1a970549cbd 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -142,7 +142,7 @@ Component statuses are linked to configuration documentation for each component. | [GitLab self-monitoring: Prometheus](#prometheus) | Time-series database, metrics collection, and query service | [✅][prometheus-omnibus] | [✅][prometheus-charts] | [⚙][prometheus-charts] | [✅](../user/gitlab_com/index.md#prometheus) | ❌ | ❌ | CE & EE | | [GitLab self-monitoring: Alertmanager](#alertmanager) | Deduplicates, groups, and routes alerts from Prometheus | [⚙][alertmanager-omnibus] | [✅][alertmanager-charts] | [⚙][alertmanager-charts] | [✅](https://about.gitlab.com/handbook/engineering/monitoring/) | ❌ | ❌ | CE & EE | | [GitLab self-monitoring: Grafana](#grafana) | Metrics dashboard | [✅][grafana-omnibus] | [⤓][grafana-charts] | [⤓][grafana-charts] | [✅](https://dashboards.gitlab.com/d/RZmbBr7mk/gitlab-triage?refresh=30s) | ❌ | ❌ | CE & EE | -| [GitLab self-monitoring: Sentry](#sentry) | Track errors generated by the GitLab instance | [⤓][sentry-omnibus] | [❌][sentry-charts] | [❌][sentry-charts] | [✅](https://about.gitlab.com/handbook/support/workflows/services/gitlab_com/500_errors.html#searching-sentry) | [⤓][gitlab-yml] | [⤓][gitlab-yml] | CE & EE | +| [GitLab self-monitoring: Sentry](#sentry) | Track errors generated by the GitLab instance | [⤓][sentry-omnibus] | [❌][sentry-charts] | [❌][sentry-charts] | [✅](https://about.gitlab.com/handbook/support/workflows/500_errors.html#searching-sentry) | [⤓][gitlab-yml] | [⤓][gitlab-yml] | CE & EE | | [GitLab self-monitoring: Jaeger](#jaeger) | View traces generated by the GitLab instance | [❌][jaeger-omnibus] | [❌][jaeger-charts] | [❌][jaeger-charts] | [❌](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/4104) | [⤓][jaeger-source] | [⚙][jaeger-gdk] | CE & EE | | [Redis Exporter](#redis-exporter) | Prometheus endpoint with Redis metrics | [✅][redis-exporter-omnibus] | [✅][redis-exporter-charts] | [✅][redis-exporter-charts] | [✅](https://about.gitlab.com/handbook/engineering/monitoring/) | ❌ | ❌ | CE & EE | | [Postgres Exporter](#postgres-exporter) | Prometheus endpoint with PostgreSQL metrics | [✅][postgres-exporter-omnibus] | [✅][postgres-exporter-charts] | [✅][postgres-exporter-charts] | [✅](https://about.gitlab.com/handbook/engineering/monitoring/) | ❌ | ❌ | CE & EE | diff --git a/doc/university/support/README.md b/doc/university/support/README.md index fdeba89f9c8..0634e01764f 100644 --- a/doc/university/support/README.md +++ b/doc/university/support/README.md @@ -145,7 +145,7 @@ Zendesk is our Support Centre and our main communication line with our Customers Some tickets need specific knowledge or a deep understanding of a particular component and will need to be escalated to a Senior Service Engineer or Developer -- Read about [Escalation](https://about.gitlab.com/handbook/support/workflows/shared/support_workflows/issue_escalations.html) +- Read about [Escalation](https://about.gitlab.com/handbook/support/workflows/issue_escalations.html) - Find the macros in Zendesk for ticket escalations - Take a look at the [GitLab.com Team page](https://about.gitlab.com/team/) to find the resident experts in their fields diff --git a/doc/user/project/wiki/img/wiki_create_new_page_modal.png b/doc/user/project/wiki/img/wiki_create_new_page_modal.png Binary files differdeleted file mode 100644 index b800508901b..00000000000 --- a/doc/user/project/wiki/img/wiki_create_new_page_modal.png +++ /dev/null diff --git a/doc/user/project/wiki/index.md b/doc/user/project/wiki/index.md index d1bab47de25..c0159230fec 100644 --- a/doc/user/project/wiki/index.md +++ b/doc/user/project/wiki/index.md @@ -30,10 +30,12 @@ Requires Developer [permissions](../../permissions.md). Create a new page by clicking the **New page** button that can be found in all wiki pages. -You will be asked to fill in a title for your new wiki page. Wiki titles -also determine the path to the wiki page. You can specify a full path -(using "`/`" for subdirectories) for the new title and any missing -directories will be created automatically. +You will be asked to fill in a title for your new wiki page. + +You can specify a full path for the wiki page by using '/' in the +title to indicate subdirectories. Any missing directories will be created +automatically. For example, a title of `docs/my-page` will create a wiki +page with a path `/wikis/docs/my-page`. Once you enter the page name, it's time to fill in its content. GitLab wikis support Markdown, RDoc and AsciiDoc. For Markdown based pages, all the diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index f750886a8c5..5874b01ef2a 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,8 +14,8 @@ module Gitlab ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze ALLOWED_KEYS = %i[tags script only except rules type image services allow_failure type stage when start_in artifacts cache - dependencies needs before_script after_script variables - environment coverage retry parallel extends interruptible].freeze + dependencies before_script needs after_script variables + environment coverage retry parallel extends interruptible timeout].freeze REQUIRED_BY_NEEDS = %i[stage].freeze @@ -46,6 +46,8 @@ module Gitlab message: "should be one of: #{ALLOWED_WHEN.join(', ')}" } + validates :timeout, duration: { limit: ChronicDuration.output(Project::MAX_BUILD_TIMEOUT) } + validates :dependencies, array_of_strings: true validates :needs, array_of_strings: true validates :extends, array_of_strings_or_string: true @@ -127,7 +129,7 @@ module Gitlab attributes :script, :tags, :allow_failure, :when, :dependencies, :needs, :retry, :parallel, :extends, :start_in, :rules, - :interruptible + :interruptible, :timeout def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -218,6 +220,7 @@ module Gitlab retry: retry_defined? ? retry_value : nil, parallel: parallel_defined? ? parallel_value.to_i : nil, interruptible: interruptible_defined? ? interruptible_value : nil, + timeout: has_timeout? ? ChronicDuration.parse(timeout.to_s) : nil, artifacts: artifacts_value, after_script: after_script_value, ignore: ignored?, diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 56ce33203ad..7f9a7df2f31 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -2,57 +2,26 @@ container_scanning: stage: test - image: docker:stable + image: + name: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable + entrypoint: [] variables: - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - # Defining two new variables based on GitLab's CI/CD predefined variables - # https://docs.gitlab.com/ee/ci/variables/#predefined-environment-variables - CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG - CI_APPLICATION_TAG: $CI_COMMIT_SHA - # Prior to this, you need to have the Container Registry running for your project and setup a build job - # with at least the following steps: - # - # docker build -t $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG . - # docker push $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA - # - # Container Scanning deals with Docker images only so no need to import the project's Git repository: + # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here + # with a specific version to provide consistency for integration testing purposes + CLAIR_DB_IMAGE_TAG: latest + # Override this variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yaml` file. + # See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # for details GIT_STRATEGY: none - # Services and containers running in the same Kubernetes pod are all sharing the same localhost address - # https://docs.gitlab.com/runner/executors/kubernetes.html - DOCKER_SERVICE: docker - DOCKER_HOST: tcp://${DOCKER_SERVICE}:2375/ - # https://hub.docker.com/r/arminc/clair-local-scan/tags - CLAIR_LOCAL_SCAN_VERSION: v2.0.8_0ed98e9ead65a51ba53f7cc53fa5e80c92169207 - CLAIR_EXECUTABLE_VERSION: v12 - CLAIR_EXECUTABLE_SHA: 44f2a3fdd7b0d102c98510e7586f6956edc89ab72c6943980f92f4979f7f4081 - ## Disable the proxy for clair-local-scan, otherwise Container Scanning will - ## fail when a proxy is used. - NO_PROXY: ${DOCKER_SERVICE},localhost allow_failure: true services: - - docker:stable-dind + - name: arminc/clair-db:$CLAIR_DB_IMAGE_TAG + alias: clair-vulnerabilities-db script: - - if [[ -n "$KUBERNETES_PORT" ]]; then { export DOCKER_SERVICE="localhost" ; export DOCKER_HOST="tcp://${DOCKER_SERVICE}:2375" ; } fi - - | - if [[ -n "$CI_REGISTRY_USER" ]]; then - echo "Logging to GitLab Container Registry with CI credentials..." - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" - echo "" - fi - - docker run -d --name db arminc/clair-db:latest - - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION} - - apk add -U wget ca-certificates - - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} - - wget https://github.com/arminc/clair-scanner/releases/download/${CLAIR_EXECUTABLE_VERSION}/clair-scanner_linux_amd64 - - echo "${CLAIR_EXECUTABLE_SHA} clair-scanner_linux_amd64" | sha256sum -c - - mv clair-scanner_linux_amd64 clair-scanner - - chmod +x clair-scanner - - touch clair-whitelist.yml - - retries=0 - - echo "Waiting for clair daemon to start" - - while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done - - ./clair-scanner -c http://${DOCKER_SERVICE}:6060 --ip $(hostname -i) -r gl-container-scanning-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true + # the kubernetes executor currently ignores the Docker image entrypoint value, so the start.sh script must + # be explicitly executed here in order for this to work with both the kubernetes and docker executors + # see this issue for more details https://gitlab.com/gitlab-org/gitlab-runner/issues/4125 + - /container-scanner/start.sh artifacts: reports: container_scanning: gl-container-scanning-report.json diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 986605efdc3..f6a3abefcfb 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -49,6 +49,7 @@ module Gitlab artifacts: job[:artifacts], cache: job[:cache], dependencies: job[:dependencies], + job_timeout: job[:timeout], before_script: job[:before_script], script: job[:script], after_script: job[:after_script], diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index a8e993e087e..0678799b64b 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -34,6 +34,7 @@ module Gitlab end SAFE_HOOK_RELATIONS = %i[ + assignees labels total_time_spent ].freeze diff --git a/package.json b/package.json index 20747b910a1..864d13d4980 100644 --- a/package.json +++ b/package.json @@ -37,8 +37,8 @@ "@babel/plugin-syntax-dynamic-import": "^7.2.0", "@babel/plugin-syntax-import-meta": "^7.2.0", "@babel/preset-env": "^7.5.5", - "@gitlab/svgs": "^1.72.0", - "@gitlab/ui": "5.21.1", + "@gitlab/svgs": "^1.73.0", + "@gitlab/ui": "5.25.2", "@gitlab/visual-review-tools": "1.0.1", "apollo-cache-inmemory": "^1.5.1", "apollo-client": "^2.5.1", diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index 8b64dd3292c..a4fb4749720 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -197,17 +197,17 @@ function download_chart() { function deploy() { local name="$CI_ENVIRONMENT_SLUG" + local edition="${GITLAB_EDITION-ce}" echoinfo "Deploying ${name}..." true IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror" - IMAGE_VERSION="${CI_PROJECT_NAME#gitlab-}" - gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-${IMAGE_VERSION}" - gitlab_sidekiq_image_repository="${IMAGE_REPOSITORY}/gitlab-sidekiq-${IMAGE_VERSION}" - gitlab_unicorn_image_repository="${IMAGE_REPOSITORY}/gitlab-unicorn-${IMAGE_VERSION}" - gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-${IMAGE_VERSION}" + gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-${edition}" + gitlab_sidekiq_image_repository="${IMAGE_REPOSITORY}/gitlab-sidekiq-${edition}" + gitlab_unicorn_image_repository="${IMAGE_REPOSITORY}/gitlab-unicorn-${edition}" + gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-${edition}" gitlab_gitaly_image_repository="${IMAGE_REPOSITORY}/gitaly" gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell" - gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${IMAGE_VERSION}" + gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${edition}" create_application_secret diff --git a/spec/lib/gitlab/ci/build/step_spec.rb b/spec/lib/gitlab/ci/build/step_spec.rb index 84e6e0e177f..9c1a8cf5e91 100644 --- a/spec/lib/gitlab/ci/build/step_spec.rb +++ b/spec/lib/gitlab/ci/build/step_spec.rb @@ -4,39 +4,49 @@ require 'spec_helper' describe Gitlab::Ci::Build::Step do describe '#from_commands' do - shared_examples 'has correct script' do - subject { described_class.from_commands(job) } + subject { described_class.from_commands(job) } - before do - job.run! - end + before do + job.run! + end + shared_examples 'has correct script' do it 'fabricates an object' do expect(subject.name).to eq(:script) expect(subject.script).to eq(script) - expect(subject.timeout).to eq(job.metadata_timeout) expect(subject.when).to eq('on_success') expect(subject.allow_failure).to be_falsey end end context 'when script option is specified' do - it_behaves_like 'has correct script' do - let(:job) { create(:ci_build, :no_options, options: { script: ["ls -la\necho aaa", "date"] }) } - let(:script) { ["ls -la\necho aaa", 'date'] } - end + let(:job) { create(:ci_build, :no_options, options: { script: ["ls -la\necho aaa", "date"] }) } + let(:script) { ["ls -la\necho aaa", 'date'] } + + it_behaves_like 'has correct script' end context 'when before and script option is specified' do - it_behaves_like 'has correct script' do - let(:job) do - create(:ci_build, options: { - before_script: ["ls -la\necho aaa"], - script: ["date"] - }) - end - - let(:script) { ["ls -la\necho aaa", 'date'] } + let(:job) do + create(:ci_build, options: { + before_script: ["ls -la\necho aaa"], + script: ["date"] + }) + end + + let(:script) { ["ls -la\necho aaa", 'date'] } + + it_behaves_like 'has correct script' + end + + context 'when timeout option is specified in seconds' do + let(:job) { create(:ci_build, options: { job_timeout: 3, script: ["ls -la\necho aaa", 'date'] }) } + let(:script) { ["ls -la\necho aaa", 'date'] } + + it_behaves_like 'has correct script' + + it 'has job level timeout' do + expect(subject.timeout).to eq(3) end end end @@ -57,12 +67,12 @@ describe Gitlab::Ci::Build::Step do end context 'when after_script is not empty' do - let(:job) { create(:ci_build, options: { script: ['bash'], after_script: ['ls -la', 'date'] }) } + let(:job) { create(:ci_build, options: { job_timeout: 60, script: ['bash'], after_script: ['ls -la', 'date'] }) } it 'fabricates an object' do expect(subject.name).to eq(:after_script) expect(subject.script).to eq(['ls -la', 'date']) - expect(subject.timeout).to eq(job.metadata_timeout) + expect(subject.timeout).to eq(60) expect(subject.when).to eq('always') expect(subject.allow_failure).to be_truthy end diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 1853efde350..1c4887e87c4 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -417,6 +417,37 @@ describe Gitlab::Ci::Config::Entry::Job do end end end + + context 'when timeout value is not correct' do + context 'when it is higher than instance wide timeout' do + let(:config) { { timeout: '3 months' } } + + it 'returns error about value too high' do + expect(entry).not_to be_valid + expect(entry.errors) + .to include "job timeout should not exceed the limit" + end + end + + context 'when it is not a duration' do + let(:config) { { timeout: 100 } } + + it 'returns error about wrong value' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'job timeout should be a duration' + end + end + end + + context 'when timeout value is correct' do + let(:config) { { script: 'echo', timeout: '1m 1s' } } + + it 'returns correct timeout' do + expect(entry).to be_valid + expect(entry.errors).to be_empty + expect(entry.timeout).to eq('1m 1s') + end + end end end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 9d9a9ecda33..8f2f23f6110 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -1134,6 +1134,48 @@ module Gitlab end end + describe "Timeout" do + let(:config) do + { + deploy_to_production: { + stage: 'deploy', + script: 'test' + } + } + end + + let(:processor) { Gitlab::Ci::YamlProcessor.new(YAML.dump(config)) } + let(:builds) { processor.stage_builds_attributes('deploy') } + + context 'when no timeout was provided' do + it 'does not include job_timeout' do + expect(builds.size).to eq(1) + expect(builds.first[:options]).not_to include(:job_timeout) + end + end + + context 'when an invalid timeout was provided' do + before do + config[:deploy_to_production][:timeout] = 'not-a-number' + end + + it 'raises an error for invalid number' do + expect { builds }.to raise_error('jobs:deploy_to_production timeout should be a duration') + end + end + + context 'when some valid timeout was provided' do + before do + config[:deploy_to_production][:timeout] = '1m 3s' + end + + it 'returns provided timeout value' do + expect(builds.size).to eq(1) + expect(builds.first[:options]).to include(job_timeout: 63) + end + end + end + describe "Dependencies" do let(:config) do { diff --git a/spec/lib/gitlab/hook_data/issuable_builder_spec.rb b/spec/lib/gitlab/hook_data/issuable_builder_spec.rb index 569d5dcc757..97a89b319ea 100644 --- a/spec/lib/gitlab/hook_data/issuable_builder_spec.rb +++ b/spec/lib/gitlab/hook_data/issuable_builder_spec.rb @@ -42,7 +42,15 @@ describe Gitlab::HookData::IssuableBuilder do [{ id: 1, title: 'foo' }], [{ id: 1, title: 'foo' }, { id: 2, title: 'bar' }] ], - total_time_spent: [1, 2] + total_time_spent: [1, 2], + assignees: [ + [], + [{ + name: "Foo Bar", + username: "foobar", + avatar_url: "http://www.example.com/my-avatar.jpg" + }] + ] } end let(:data) { builder.build(user: user, changes: changes) } @@ -58,6 +66,14 @@ describe Gitlab::HookData::IssuableBuilder do total_time_spent: { previous: 1, current: 2 + }, + assignees: { + previous: [], + current: [{ + name: "Foo Bar", + username: "foobar", + avatar_url: "http://www.example.com/my-avatar.jpg" + }] } })) end diff --git a/spec/models/ci/build_metadata_spec.rb b/spec/models/ci/build_metadata_spec.rb index 917a65ddf21..67cd939b4c6 100644 --- a/spec/models/ci/build_metadata_spec.rb +++ b/spec/models/ci/build_metadata_spec.rb @@ -22,42 +22,72 @@ describe Ci::BuildMetadata do describe '#update_timeout_state' do subject { metadata } - context 'when runner is not assigned to the job' do - it "doesn't change timeout value" do - expect { subject.update_timeout_state }.not_to change { subject.reload.timeout } + shared_examples 'sets timeout' do |source, timeout| + it 'sets project_timeout_source' do + expect { subject.update_timeout_state }.to change { subject.reload.timeout_source }.to(source) end - it "doesn't change timeout_source value" do - expect { subject.update_timeout_state }.not_to change { subject.reload.timeout_source } + it 'sets project timeout' do + expect { subject.update_timeout_state }.to change { subject.reload.timeout }.to(timeout) end end - context 'when runner is assigned to the job' do - before do - build.update(runner: runner) + context 'when project timeout is set' do + context 'when runner is assigned to the job' do + before do + build.update!(runner: runner) + end + + context 'when runner timeout is not set' do + let(:runner) { create(:ci_runner, maximum_timeout: nil) } + + it_behaves_like 'sets timeout', 'project_timeout_source', 2000 + end + + context 'when runner timeout is lower than project timeout' do + let(:runner) { create(:ci_runner, maximum_timeout: 1900) } + + it_behaves_like 'sets timeout', 'runner_timeout_source', 1900 + end + + context 'when runner timeout is higher than project timeout' do + let(:runner) { create(:ci_runner, maximum_timeout: 2100) } + + it_behaves_like 'sets timeout', 'project_timeout_source', 2000 + end end - context 'when runner timeout is lower than project timeout' do - let(:runner) { create(:ci_runner, maximum_timeout: 1900) } + context 'when job timeout is set' do + context 'when job timeout is higher than project timeout' do + let(:build) { create(:ci_build, pipeline: pipeline, options: { job_timeout: 3000 }) } - it 'sets runner timeout' do - expect { subject.update_timeout_state }.to change { subject.reload.timeout }.to(1900) + it_behaves_like 'sets timeout', 'job_timeout_source', 3000 end - it 'sets runner_timeout_source' do - expect { subject.update_timeout_state }.to change { subject.reload.timeout_source }.to('runner_timeout_source') + context 'when job timeout is lower than project timeout' do + let(:build) { create(:ci_build, pipeline: pipeline, options: { job_timeout: 1000 }) } + + it_behaves_like 'sets timeout', 'job_timeout_source', 1000 end end - context 'when runner timeout is higher than project timeout' do - let(:runner) { create(:ci_runner, maximum_timeout: 2100) } + context 'when both runner and job timeouts are set' do + before do + build.update(runner: runner) + end + + context 'when job timeout is higher than runner timeout' do + let(:build) { create(:ci_build, pipeline: pipeline, options: { job_timeout: 3000 }) } + let(:runner) { create(:ci_runner, maximum_timeout: 2100) } - it 'sets project timeout' do - expect { subject.update_timeout_state }.to change { subject.reload.timeout }.to(2000) + it_behaves_like 'sets timeout', 'runner_timeout_source', 2100 end - it 'sets project_timeout_source' do - expect { subject.update_timeout_state }.to change { subject.reload.timeout_source }.to('project_timeout_source') + context 'when job timeout is lower than runner timeout' do + let(:build) { create(:ci_build, pipeline: pipeline, options: { job_timeout: 1900 }) } + let(:runner) { create(:ci_runner, maximum_timeout: 2100) } + + it_behaves_like 'sets timeout', 'job_timeout_source', 1900 end end end diff --git a/spec/models/concerns/routable_spec.rb b/spec/models/concerns/routable_spec.rb index cad705ee594..f78a089bc2e 100644 --- a/spec/models/concerns/routable_spec.rb +++ b/spec/models/concerns/routable_spec.rb @@ -58,7 +58,7 @@ describe Group, 'Routable' do end end - shared_examples_for '.find_by_full_path' do + context '.find_by_full_path' do let!(:nested_group) { create(:group, parent: group) } context 'without any redirect routes' do @@ -117,24 +117,6 @@ describe Group, 'Routable' do end end - describe '.find_by_full_path' do - context 'with routable_two_step_lookup feature' do - before do - stub_feature_flags(routable_two_step_lookup: true) - end - - it_behaves_like '.find_by_full_path' - end - - context 'without routable_two_step_lookup feature' do - before do - stub_feature_flags(routable_two_step_lookup: false) - end - - it_behaves_like '.find_by_full_path' - end - end - describe '.where_full_path_in' do context 'without any paths' do it 'returns an empty relation' do diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index 19cc2ddf7db..6eafe122697 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -795,6 +795,22 @@ describe Ci::CreatePipelineService do end end + context 'with timeout' do + context 'when builds with custom timeouts are configured' do + before do + config = YAML.dump(rspec: { script: 'rspec', timeout: '2m 3s' }) + stub_ci_pipeline_yaml_file(config) + end + + it 'correctly creates builds with custom timeout value configured' do + pipeline = execute_service + + expect(pipeline).to be_persisted + expect(pipeline.builds.find_by(name: 'rspec').options[:job_timeout]).to eq 123 + end + end + end + shared_examples 'when ref is protected' do let(:user) { create(:user) } diff --git a/yarn.lock b/yarn.lock index 18307d7abae..aa3dea56560 100644 --- a/yarn.lock +++ b/yarn.lock @@ -991,19 +991,18 @@ dependencies: vue-eslint-parser "^6.0.4" -"@gitlab/svgs@^1.72.0": - version "1.72.0" - resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.72.0.tgz#78a29fd383a5a2b31ef91670068a6fea05ba234e" - integrity sha512-EVKRqrXsCY6tUiVAh+lpFMJAyNXZwfEqv5NeKH5ginhALMlOunRkY5rsDllyNvgZ0DWHJS1KEKJj2oVU1ouwAg== +"@gitlab/svgs@^1.73.0": + version "1.73.0" + resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.73.0.tgz#e44b347e4be77b94474c80cf5c2ee26ca0325c2f" + integrity sha512-4on+l5CS8Ae8OOcrnxwkO5s2zq1kHl/YjnOrHaX7megr6jsTYsVzKGaEMe0ViMSIPXA2+QnGD6vElKMkeD2+YQ== -"@gitlab/ui@5.21.1": - version "5.21.1" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-5.21.1.tgz#8215ab3eae4296845596d5b3a987d5460b030569" - integrity sha512-TjPVhex9sQGUVwebaJK5XuopDHWx4+Sh9N7yH5u8eXSFWa8vk11voR4qYVt7DZB7powAO/+iiXxYMLLNtXmC/g== +"@gitlab/ui@5.25.2": + version "5.25.2" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-5.25.2.tgz#599954fefcc228d31a398dbe3c1e2287a0fcdb3e" + integrity sha512-mwwvEhVTomnZQjG0dADD+9Kg1UHZXAIb4s5QwQxwpgTkemILYIb1r96oKWfmPe8Pl/xrzAoMUtGEPT3XbxDqYQ== dependencies: "@babel/standalone" "^7.0.0" "@gitlab/vue-toasted" "^1.2.1" - bootstrap "4.3.1" bootstrap-vue "2.0.0-rc.27" copy-to-clipboard "^3.0.8" echarts "^4.2.1" |
