diff options
| -rw-r--r-- | GITLAB_SHELL_VERSION | 2 | ||||
| -rw-r--r-- | GITLAB_WORKHORSE_VERSION | 2 | ||||
| -rw-r--r-- | app/assets/stylesheets/framework/job_log.scss | 4 | ||||
| -rw-r--r-- | app/assets/stylesheets/framework/variables.scss | 7 | ||||
| -rw-r--r-- | app/models/ci/build.rb | 6 | ||||
| -rw-r--r-- | changelogs/unreleased/rd-fix-show-trial-errors-helper.yml | 5 | ||||
| -rw-r--r-- | changelogs/unreleased/workhorse-8-14.yml | 5 | ||||
| -rw-r--r-- | doc/administration/gitaly/praefect.md | 52 | ||||
| -rw-r--r-- | doc/ci/caching/index.md | 5 | ||||
| -rw-r--r-- | doc/development/contributing/merge_request_workflow.md | 2 | ||||
| -rw-r--r-- | doc/development/internal_api.md | 1 | ||||
| -rw-r--r-- | doc/user/group/index.md | 2 | ||||
| -rw-r--r-- | lib/api/internal/base.rb | 43 | ||||
| -rw-r--r-- | lib/gitlab/sidekiq_daemon/memory_killer.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/sidekiq_logging/structured_logger.rb | 23 | ||||
| -rw-r--r-- | spec/factories/ci/builds.rb | 32 | ||||
| -rw-r--r-- | spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb | 2 | ||||
| -rw-r--r-- | spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb | 49 |
18 files changed, 147 insertions, 97 deletions
diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index a13e7b9c87e..4149c39eec6 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -10.0.0 +10.1.0 diff --git a/GITLAB_WORKHORSE_VERSION b/GITLAB_WORKHORSE_VERSION index e51b3430127..2a5dd0d6389 100644 --- a/GITLAB_WORKHORSE_VERSION +++ b/GITLAB_WORKHORSE_VERSION @@ -1 +1 @@ -8.12.0 +8.14.0 diff --git a/app/assets/stylesheets/framework/job_log.scss b/app/assets/stylesheets/framework/job_log.scss index b93b224ae13..ba163aa46ae 100644 --- a/app/assets/stylesheets/framework/job_log.scss +++ b/app/assets/stylesheets/framework/job_log.scss @@ -12,7 +12,7 @@ } .log-line { - padding: 1px $gl-padding 1px $job-log-line-padding; + padding: 1px $gl-padding-8 1px $job-log-line-padding; min-height: $gl-line-height-20; } @@ -20,7 +20,7 @@ color: $gl-text-color-inverted; padding: 0 $gl-padding-8; min-width: $job-line-number-width; - margin-left: -$job-line-number-width; + margin-left: -$job-line-number-margin; padding-right: 1em; &:hover, diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss index e77527ac130..f352ee33535 100644 --- a/app/assets/stylesheets/framework/variables.scss +++ b/app/assets/stylesheets/framework/variables.scss @@ -606,9 +606,10 @@ $blame-blue: #254e77; */ $builds-trace-bg: #111; $job-log-highlight-height: 18px; -$job-log-line-padding: 62px; -$job-line-number-width: 40px; -$job-arrow-margin: 50px; +$job-log-line-padding: 55px; +$job-line-number-width: 50px; +$job-line-number-margin: 43px; +$job-arrow-margin: 55px; /* * Commit Page diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 5cca18024c1..fba14f0100c 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -128,6 +128,12 @@ module Ci scope :with_stale_live_trace, -> { with_live_trace.finished_before(12.hours.ago) } scope :finished_before, -> (date) { finished.where('finished_at < ?', date) } + scope :with_secure_reports_from_options, -> (job_type) { where('options like :job_type', job_type: "%:artifacts:%:reports:%:#{job_type}:%") } + + scope :with_secure_reports_from_config_options, -> (job_types) do + joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) + end + scope :matches_tag_ids, -> (tag_ids) do matcher = ::ActsAsTaggableOn::Tagging .where(taggable_type: CommitStatus.name) diff --git a/changelogs/unreleased/rd-fix-show-trial-errors-helper.yml b/changelogs/unreleased/rd-fix-show-trial-errors-helper.yml new file mode 100644 index 00000000000..ddcc5f34676 --- /dev/null +++ b/changelogs/unreleased/rd-fix-show-trial-errors-helper.yml @@ -0,0 +1,5 @@ +--- +title: Fix error where helper was incorrectly returning `true` +merge_request: 18231 +author: +type: fixed diff --git a/changelogs/unreleased/workhorse-8-14.yml b/changelogs/unreleased/workhorse-8-14.yml new file mode 100644 index 00000000000..5df25876201 --- /dev/null +++ b/changelogs/unreleased/workhorse-8-14.yml @@ -0,0 +1,5 @@ +--- +title: Update Workhorse to v8.14.0 +merge_request: 18391 +author: +type: other diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 24a5a6df768..dbe78d212e2 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -25,40 +25,43 @@ three Gitaly nodes, `Praefect-Git-1`, `Praefect-Git-2`, and `Praefect-Git-3`. There should be no knowledge other than with Praefect about the existence of the `Praefect-Git-X` nodes. -### Enable the daemon +### Setup -Praefect is expected to run on their own host, this means that no other service -other than the support services run on this machine. +In this setup guide, the Gitaly node will be added first, then Praefect, and +lastly we update the GitLab configuration. -Praefect is disabled by default, to enable praefect uncomment the following line -and set it to `true`: `# praefect['enable'] = false'` +#### Gitaly -```ruby -praefect['enable'] = true -``` - -By default praefect will listen on port `:2305`. It's recommended to enable -prometheus to expose metrics. Uncomment the line so it looks like: - -```ruby -praefect['prometheus_listen_addr'] = "localhost:9652" -``` +In their own machine, configure the Gitaly server as described in the +[gitaly documentation](index.md#3-gitaly-server-configuration). -Preafect needs at least one storage to store the Git data on. This node should -run Gitaly and should not be listed as storage for GitLab itself, that is, the -only way it receives traffic is through Praefect and it's not listed in the -`git_data_dirs` on any `gitlab.rb` in your GitLab cluster. +#### Praefect -To set the nodes as depicted in the diagram above, the configuration should look -like: +Next, Praefect has to be enabled on its own node. Disable all other services, +and add each Gitaly node that will be connected to Praefect. In the example below, +the Gitaly nodes are named `praefect-git-X`. Note that one node is designated as +primary, by setting the primary to `true`: ```ruby +# /etc/gitlab/gitlab.rb + +# Avoid running unnecessary services on the Gitaly server +postgresql['enable'] = false +redis['enable'] = false +nginx['enable'] = false +prometheus['enable'] = false +unicorn['enable'] = false +sidekiq['enable'] = false +gitlab_workhorse['enable'] = false +gitaly['enable'] = false + +praefect['enable'] = true praefect['storage_nodes'] = [ { 'storage' => 'praefect-git-1', 'address' => 'tcp://praefect-git-1.internal', 'primary' => true -} + }, { 'storage' => 'praefect-git-2', 'address' => 'tcp://praefect-git-2.internal' @@ -70,10 +73,9 @@ praefect['storage_nodes'] = [ ] ``` -Save the file, and run `gitlab-ctl reconfigure`. To test if Praefect is running, -you could run `gitlab-ctl status` which should list praefect as being up. +Save the file and [reconfigure Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure). -### Enable Preafect as storage backend in GitLab +#### GitLab When Praefect is running, it should be exposed as a storage to GitLab. This is done through setting the `git_data_dirs`. Assuming the default storage diff --git a/doc/ci/caching/index.md b/doc/ci/caching/index.md index 76ee4829a3d..6a7b60c2ba5 100644 --- a/doc/ci/caching/index.md +++ b/doc/ci/caching/index.md @@ -12,8 +12,9 @@ content of a previous job. It can be particularly useful when you are developing software that depends on other libraries which are fetched via the internet during build time. -If caching is enabled, it's shared between pipelines and jobs by default, -starting from GitLab 9.0. +If caching is enabled, it's shared between pipelines and jobs at the project +level by default, starting from GitLab 9.0. Caches are not shared across +projects. Make sure you read the [`cache` reference](../yaml/README.md#cache) to learn how it is defined in `.gitlab-ci.yml`. diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index 97d77cdad6a..86f17f4ecdb 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -220,6 +220,8 @@ requirements. 1. Working and clean code that is commented where needed. 1. [Unit, integration, and system tests](../testing_guide/index.md) that all pass on the CI server. +1. Regressions and bugs are covered with tests that reduce the risk of the issue happening + again. 1. Performance/scalability implications have been considered, addressed, and tested. 1. [Documented](../documentation/index.md) in the `/doc` directory. 1. [Changelog entry added](../changelog.md), if necessary. diff --git a/doc/development/internal_api.md b/doc/development/internal_api.md index 2b8a20939ef..b08112aacb2 100644 --- a/doc/development/internal_api.md +++ b/doc/development/internal_api.md @@ -47,6 +47,7 @@ POST /internal/allowed | `protocol` | string | yes | SSH when called from GitLab-shell, HTTP or SSH when called from Gitaly | | `action` | string | yes | Git command being run (`git-upload-pack`, `git-receive-pack`, `git-upload-archive`) | | `changes` | string | yes | `<oldrev> <newrev> <refname>` when called from Gitaly, The magic string `_any` when called from GitLab Shell | +| `check_ip` | string | no | Ip adress from which call to GitLab Shell was made | Example request: diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 0dbf5bdd156..258f1264b48 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -351,7 +351,7 @@ Add one or more whitelisted IP subnets using CIDR notation in comma separated fo coming from a different IP address won't be able to access the restricted content. -Restriction currently applies to UI and API access, Git actions via SSH are not restricted. +Restriction currently applies to UI, API access and Git actions via SSH. To avoid accidental lock-out, admins and group owners are are able to access the group regardless of the IP restriction. diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 00e9b56b932..d9a22484c1f 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -26,20 +26,11 @@ module API def ee_post_receive_response_hook(response) # Hook for EE to add messages end - end - namespace 'internal' do - # Check if git command is allowed for project - # - # Params: - # key_id - ssh key id for Git over SSH - # user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode - # username - user name for Git over SSH in keyless SSH cert mode - # protocol - Git access protocol being used, e.g. HTTP or SSH - # project - project full_path (not path on disk) - # action - git action (git-upload-pack or git-receive-pack) - # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList - post "/allowed" do + def check_allowed(params) + # This is a separate method so that EE can alter its behaviour more + # easily. + # Stores some Git-specific env thread-safely env = parse_env Gitlab::Git::HookEnv.set(gl_repository, env) if project @@ -53,11 +44,11 @@ module API @project ||= access_checker.project result rescue Gitlab::GitAccess::UnauthorizedError => e - break response_with_status(code: 401, success: false, message: e.message) + return response_with_status(code: 401, success: false, message: e.message) rescue Gitlab::GitAccess::TimeoutError => e - break response_with_status(code: 503, success: false, message: e.message) + return response_with_status(code: 503, success: false, message: e.message) rescue Gitlab::GitAccess::NotFoundError => e - break response_with_status(code: 404, success: false, message: e.message) + return response_with_status(code: 404, success: false, message: e.message) end log_user_activity(actor.user) @@ -91,6 +82,26 @@ module API response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR) end end + end + + namespace 'internal' do + # Check if git command is allowed for project + # + # Params: + # key_id - ssh key id for Git over SSH + # user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode + # username - user name for Git over SSH in keyless SSH cert mode + # protocol - Git access protocol being used, e.g. HTTP or SSH + # project - project full_path (not path on disk) + # action - git action (git-upload-pack or git-receive-pack) + # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList + # check_ip - optional, only in EE version, may limit access to + # group resources based on its IP restrictions + post "/allowed" do + # It was moved to a separate method so that EE can alter its behaviour more + # easily. + check_allowed(params) + end # rubocop: disable CodeReuse/ActiveRecord post "/lfs_authenticate" do diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index 25b91318dfe..eb58435e3f1 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -39,8 +39,8 @@ module Gitlab while enabled? begin - restart_sidekiq unless rss_within_range? sleep(CHECK_INTERVAL_SECONDS) + restart_sidekiq unless rss_within_range? rescue => e log_exception(e, __method__) rescue Exception => e # rubocop:disable Lint/RescueException diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 0a21e529ceb..853fb2777c3 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -70,10 +70,11 @@ module Gitlab end def add_time_keys!(time, payload) - payload['duration'] = time[:duration].round(3) - payload['system_s'] = time[:stime].round(3) - payload['user_s'] = time[:utime].round(3) - payload['child_s'] = time[:ctime].round(3) if time[:ctime] > 0 + payload['duration'] = time[:duration].round(6) + + # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0) + # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime + payload['cpu_s'] = time[:cputime].round(6) if time[:cputime] > 0 payload['completed_at'] = Time.now.utc end @@ -98,30 +99,24 @@ module Gitlab end def elapsed_by_absolute_time(start) - (Time.now.utc - start).to_f.round(3) + (Time.now.utc - start).to_f.round(6) end def elapsed(t0) t1 = get_time { duration: t1[:now] - t0[:now], - stime: t1[:times][:stime] - t0[:times][:stime], - utime: t1[:times][:utime] - t0[:times][:utime], - ctime: ctime(t1[:times]) - ctime(t0[:times]) + cputime: t1[:thread_cputime] - t0[:thread_cputime] } end def get_time { now: current_time, - times: Process.times + thread_cputime: defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 } end - def ctime(times) - times[:cstime] + times[:cutime] - end - def current_time Gitlab::Metrics::System.monotonic_time end @@ -129,7 +124,7 @@ module Gitlab def format_time(timestamp) return timestamp if timestamp.is_a?(String) - Time.at(timestamp).utc.iso8601(3) + Time.at(timestamp).utc.iso8601(6) end def limited_job_args(args) diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb index 6725cde08f2..c0f7948f963 100644 --- a/spec/factories/ci/builds.rb +++ b/spec/factories/ci/builds.rb @@ -330,6 +330,38 @@ FactoryBot.define do options { {} } end + trait :dast do + options do + { + artifacts: { reports: { dast: 'gl-dast-report.json' } } + } + end + end + + trait :sast do + options do + { + artifacts: { reports: { sast: 'gl-sast-report.json' } } + } + end + end + + trait :dependency_scanning do + options do + { + artifacts: { reports: { dependency_scanning: 'gl-dependency-scanning-report.json' } } + } + end + end + + trait :container_scanning do + options do + { + artifacts: { reports: { container_scanning: 'gl-container-scanning-report.json' } } + } + end + end + trait :non_playable do status { 'created' } self.when { 'manual' } diff --git a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb index 756c7947df0..263cc821c1a 100644 --- a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb +++ b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb @@ -40,6 +40,7 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do message: "Exception from start_working: My Exception") expect(memory_killer).to receive(:rss_within_range?).twice.and_raise(StandardError, 'My Exception') + expect(memory_killer).to receive(:sleep).twice.with(Gitlab::SidekiqDaemon::MemoryKiller::CHECK_INTERVAL_SECONDS) expect { subject }.not_to raise_exception end @@ -53,6 +54,7 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do expect(memory_killer).to receive(:rss_within_range?).once.and_raise(Exception, 'My Exception') + expect(memory_killer).to receive(:sleep).with(Gitlab::SidekiqDaemon::MemoryKiller::CHECK_INTERVAL_SECONDS) expect(Sidekiq.logger).to receive(:warn).once .with( class: described_class.to_s, diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb index 6e6a8e14fc9..46fbc069efb 100644 --- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb @@ -23,13 +23,15 @@ describe Gitlab::SidekiqLogging::StructuredLogger do end let(:logger) { double } + let(:clock_thread_cputime_start) { 0.222222299 } + let(:clock_thread_cputime_end) { 1.333333799 } let(:start_payload) do job.merge( 'message' => 'TestWorker JID-da883554ee4fe414012f5f42: start', 'job_status' => 'start', 'pid' => Process.pid, - 'created_at' => created_at.iso8601(3), - 'enqueued_at' => created_at.iso8601(3), + 'created_at' => created_at.iso8601(6), + 'enqueued_at' => created_at.iso8601(6), 'scheduling_latency_s' => scheduling_latency_s ) end @@ -38,9 +40,8 @@ describe Gitlab::SidekiqLogging::StructuredLogger do 'message' => 'TestWorker JID-da883554ee4fe414012f5f42: done: 0.0 sec', 'job_status' => 'done', 'duration' => 0.0, - "completed_at" => timestamp.iso8601(3), - "system_s" => 0.0, - "user_s" => 0.0 + "completed_at" => timestamp.iso8601(6), + "cpu_s" => 1.111112 ) end let(:exception_payload) do @@ -57,12 +58,7 @@ describe Gitlab::SidekiqLogging::StructuredLogger do allow(subject).to receive(:current_time).and_return(timestamp.to_f) - allow(Process).to receive(:times).and_return( - stime: 0.0, - utime: 0.0, - cutime: 0.0, - cstime: 0.0 - ) + allow(Process).to receive(:clock_gettime).with(Process::CLOCK_THREAD_CPUTIME_ID).and_return(clock_thread_cputime_start, clock_thread_cputime_end) end subject { described_class.new } @@ -187,31 +183,22 @@ describe Gitlab::SidekiqLogging::StructuredLogger do end end end + end - def ctime(times) - times[:cstime] + times[:cutime] - end + describe '#add_time_keys!' do + let(:time) { { duration: 0.1231234, cputime: 1.2342345 } } + let(:payload) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status' } } + let(:current_utc_time) { '2019-09-23 10:00:58 UTC' } + let(:payload_with_time_keys) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status', 'duration' => 0.123123, 'cpu_s' => 1.234235, 'completed_at' => current_utc_time } } - context 'with ctime value greater than 0' do - let(:times_start) { { stime: 0.04999, utime: 0.0483, cstime: 0.0188, cutime: 0.0188 } } - let(:times_end) { { stime: 0.0699, utime: 0.0699, cstime: 0.0399, cutime: 0.0399 } } + subject { described_class.new } - before do - end_payload['system_s'] = 0.02 - end_payload['user_s'] = 0.022 - end_payload['child_s'] = 0.042 + it 'update payload correctly' do + expect(Time).to receive_message_chain(:now, :utc).and_return(current_utc_time) - allow(Process).to receive(:times).and_return(times_start, times_end) - end + subject.send(:add_time_keys!, time, payload) - it 'logs with ctime data and other cpu data' do - Timecop.freeze(timestamp) do - expect(logger).to receive(:info).with(start_payload.except('args')).ordered - expect(logger).to receive(:info).with(end_payload.except('args')).ordered - - subject.call(job, 'test_queue') { } - end - end + expect(payload).to eq(payload_with_time_keys) end end end |