summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--GITLAB_SHELL_VERSION2
-rw-r--r--GITLAB_WORKHORSE_VERSION2
-rw-r--r--app/assets/stylesheets/framework/job_log.scss4
-rw-r--r--app/assets/stylesheets/framework/variables.scss7
-rw-r--r--app/models/ci/build.rb6
-rw-r--r--changelogs/unreleased/rd-fix-show-trial-errors-helper.yml5
-rw-r--r--changelogs/unreleased/workhorse-8-14.yml5
-rw-r--r--doc/administration/gitaly/praefect.md52
-rw-r--r--doc/ci/caching/index.md5
-rw-r--r--doc/development/contributing/merge_request_workflow.md2
-rw-r--r--doc/development/internal_api.md1
-rw-r--r--doc/user/group/index.md2
-rw-r--r--lib/api/internal/base.rb43
-rw-r--r--lib/gitlab/sidekiq_daemon/memory_killer.rb2
-rw-r--r--lib/gitlab/sidekiq_logging/structured_logger.rb23
-rw-r--r--spec/factories/ci/builds.rb32
-rw-r--r--spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb2
-rw-r--r--spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb49
18 files changed, 147 insertions, 97 deletions
diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION
index a13e7b9c87e..4149c39eec6 100644
--- a/GITLAB_SHELL_VERSION
+++ b/GITLAB_SHELL_VERSION
@@ -1 +1 @@
-10.0.0
+10.1.0
diff --git a/GITLAB_WORKHORSE_VERSION b/GITLAB_WORKHORSE_VERSION
index e51b3430127..2a5dd0d6389 100644
--- a/GITLAB_WORKHORSE_VERSION
+++ b/GITLAB_WORKHORSE_VERSION
@@ -1 +1 @@
-8.12.0
+8.14.0
diff --git a/app/assets/stylesheets/framework/job_log.scss b/app/assets/stylesheets/framework/job_log.scss
index b93b224ae13..ba163aa46ae 100644
--- a/app/assets/stylesheets/framework/job_log.scss
+++ b/app/assets/stylesheets/framework/job_log.scss
@@ -12,7 +12,7 @@
}
.log-line {
- padding: 1px $gl-padding 1px $job-log-line-padding;
+ padding: 1px $gl-padding-8 1px $job-log-line-padding;
min-height: $gl-line-height-20;
}
@@ -20,7 +20,7 @@
color: $gl-text-color-inverted;
padding: 0 $gl-padding-8;
min-width: $job-line-number-width;
- margin-left: -$job-line-number-width;
+ margin-left: -$job-line-number-margin;
padding-right: 1em;
&:hover,
diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss
index e77527ac130..f352ee33535 100644
--- a/app/assets/stylesheets/framework/variables.scss
+++ b/app/assets/stylesheets/framework/variables.scss
@@ -606,9 +606,10 @@ $blame-blue: #254e77;
*/
$builds-trace-bg: #111;
$job-log-highlight-height: 18px;
-$job-log-line-padding: 62px;
-$job-line-number-width: 40px;
-$job-arrow-margin: 50px;
+$job-log-line-padding: 55px;
+$job-line-number-width: 50px;
+$job-line-number-margin: 43px;
+$job-arrow-margin: 55px;
/*
* Commit Page
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 5cca18024c1..fba14f0100c 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -128,6 +128,12 @@ module Ci
scope :with_stale_live_trace, -> { with_live_trace.finished_before(12.hours.ago) }
scope :finished_before, -> (date) { finished.where('finished_at < ?', date) }
+ scope :with_secure_reports_from_options, -> (job_type) { where('options like :job_type', job_type: "%:artifacts:%:reports:%:#{job_type}:%") }
+
+ scope :with_secure_reports_from_config_options, -> (job_types) do
+ joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types)
+ end
+
scope :matches_tag_ids, -> (tag_ids) do
matcher = ::ActsAsTaggableOn::Tagging
.where(taggable_type: CommitStatus.name)
diff --git a/changelogs/unreleased/rd-fix-show-trial-errors-helper.yml b/changelogs/unreleased/rd-fix-show-trial-errors-helper.yml
new file mode 100644
index 00000000000..ddcc5f34676
--- /dev/null
+++ b/changelogs/unreleased/rd-fix-show-trial-errors-helper.yml
@@ -0,0 +1,5 @@
+---
+title: Fix error where helper was incorrectly returning `true`
+merge_request: 18231
+author:
+type: fixed
diff --git a/changelogs/unreleased/workhorse-8-14.yml b/changelogs/unreleased/workhorse-8-14.yml
new file mode 100644
index 00000000000..5df25876201
--- /dev/null
+++ b/changelogs/unreleased/workhorse-8-14.yml
@@ -0,0 +1,5 @@
+---
+title: Update Workhorse to v8.14.0
+merge_request: 18391
+author:
+type: other
diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md
index 24a5a6df768..dbe78d212e2 100644
--- a/doc/administration/gitaly/praefect.md
+++ b/doc/administration/gitaly/praefect.md
@@ -25,40 +25,43 @@ three Gitaly nodes, `Praefect-Git-1`, `Praefect-Git-2`, and `Praefect-Git-3`.
There should be no knowledge other than with Praefect about the existence of
the `Praefect-Git-X` nodes.
-### Enable the daemon
+### Setup
-Praefect is expected to run on their own host, this means that no other service
-other than the support services run on this machine.
+In this setup guide, the Gitaly node will be added first, then Praefect, and
+lastly we update the GitLab configuration.
-Praefect is disabled by default, to enable praefect uncomment the following line
-and set it to `true`: `# praefect['enable'] = false'`
+#### Gitaly
-```ruby
-praefect['enable'] = true
-```
-
-By default praefect will listen on port `:2305`. It's recommended to enable
-prometheus to expose metrics. Uncomment the line so it looks like:
-
-```ruby
-praefect['prometheus_listen_addr'] = "localhost:9652"
-```
+In their own machine, configure the Gitaly server as described in the
+[gitaly documentation](index.md#3-gitaly-server-configuration).
-Preafect needs at least one storage to store the Git data on. This node should
-run Gitaly and should not be listed as storage for GitLab itself, that is, the
-only way it receives traffic is through Praefect and it's not listed in the
-`git_data_dirs` on any `gitlab.rb` in your GitLab cluster.
+#### Praefect
-To set the nodes as depicted in the diagram above, the configuration should look
-like:
+Next, Praefect has to be enabled on its own node. Disable all other services,
+and add each Gitaly node that will be connected to Praefect. In the example below,
+the Gitaly nodes are named `praefect-git-X`. Note that one node is designated as
+primary, by setting the primary to `true`:
```ruby
+# /etc/gitlab/gitlab.rb
+
+# Avoid running unnecessary services on the Gitaly server
+postgresql['enable'] = false
+redis['enable'] = false
+nginx['enable'] = false
+prometheus['enable'] = false
+unicorn['enable'] = false
+sidekiq['enable'] = false
+gitlab_workhorse['enable'] = false
+gitaly['enable'] = false
+
+praefect['enable'] = true
praefect['storage_nodes'] = [
{
'storage' => 'praefect-git-1',
'address' => 'tcp://praefect-git-1.internal',
'primary' => true
-}
+ },
{
'storage' => 'praefect-git-2',
'address' => 'tcp://praefect-git-2.internal'
@@ -70,10 +73,9 @@ praefect['storage_nodes'] = [
]
```
-Save the file, and run `gitlab-ctl reconfigure`. To test if Praefect is running,
-you could run `gitlab-ctl status` which should list praefect as being up.
+Save the file and [reconfigure Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure).
-### Enable Preafect as storage backend in GitLab
+#### GitLab
When Praefect is running, it should be exposed as a storage to GitLab. This
is done through setting the `git_data_dirs`. Assuming the default storage
diff --git a/doc/ci/caching/index.md b/doc/ci/caching/index.md
index 76ee4829a3d..6a7b60c2ba5 100644
--- a/doc/ci/caching/index.md
+++ b/doc/ci/caching/index.md
@@ -12,8 +12,9 @@ content of a previous job. It can be particularly useful when you are
developing software that depends on other libraries which are fetched via the
internet during build time.
-If caching is enabled, it's shared between pipelines and jobs by default,
-starting from GitLab 9.0.
+If caching is enabled, it's shared between pipelines and jobs at the project
+level by default, starting from GitLab 9.0. Caches are not shared across
+projects.
Make sure you read the [`cache` reference](../yaml/README.md#cache) to learn
how it is defined in `.gitlab-ci.yml`.
diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md
index 97d77cdad6a..86f17f4ecdb 100644
--- a/doc/development/contributing/merge_request_workflow.md
+++ b/doc/development/contributing/merge_request_workflow.md
@@ -220,6 +220,8 @@ requirements.
1. Working and clean code that is commented where needed.
1. [Unit, integration, and system tests](../testing_guide/index.md) that all pass
on the CI server.
+1. Regressions and bugs are covered with tests that reduce the risk of the issue happening
+ again.
1. Performance/scalability implications have been considered, addressed, and tested.
1. [Documented](../documentation/index.md) in the `/doc` directory.
1. [Changelog entry added](../changelog.md), if necessary.
diff --git a/doc/development/internal_api.md b/doc/development/internal_api.md
index 2b8a20939ef..b08112aacb2 100644
--- a/doc/development/internal_api.md
+++ b/doc/development/internal_api.md
@@ -47,6 +47,7 @@ POST /internal/allowed
| `protocol` | string | yes | SSH when called from GitLab-shell, HTTP or SSH when called from Gitaly |
| `action` | string | yes | Git command being run (`git-upload-pack`, `git-receive-pack`, `git-upload-archive`) |
| `changes` | string | yes | `<oldrev> <newrev> <refname>` when called from Gitaly, The magic string `_any` when called from GitLab Shell |
+| `check_ip` | string | no | Ip adress from which call to GitLab Shell was made |
Example request:
diff --git a/doc/user/group/index.md b/doc/user/group/index.md
index 0dbf5bdd156..258f1264b48 100644
--- a/doc/user/group/index.md
+++ b/doc/user/group/index.md
@@ -351,7 +351,7 @@ Add one or more whitelisted IP subnets using CIDR notation in comma separated fo
coming from a different IP address won't be able to access the restricted
content.
-Restriction currently applies to UI and API access, Git actions via SSH are not restricted.
+Restriction currently applies to UI, API access and Git actions via SSH.
To avoid accidental lock-out, admins and group owners are are able to access
the group regardless of the IP restriction.
diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb
index 00e9b56b932..d9a22484c1f 100644
--- a/lib/api/internal/base.rb
+++ b/lib/api/internal/base.rb
@@ -26,20 +26,11 @@ module API
def ee_post_receive_response_hook(response)
# Hook for EE to add messages
end
- end
- namespace 'internal' do
- # Check if git command is allowed for project
- #
- # Params:
- # key_id - ssh key id for Git over SSH
- # user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode
- # username - user name for Git over SSH in keyless SSH cert mode
- # protocol - Git access protocol being used, e.g. HTTP or SSH
- # project - project full_path (not path on disk)
- # action - git action (git-upload-pack or git-receive-pack)
- # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
- post "/allowed" do
+ def check_allowed(params)
+ # This is a separate method so that EE can alter its behaviour more
+ # easily.
+
# Stores some Git-specific env thread-safely
env = parse_env
Gitlab::Git::HookEnv.set(gl_repository, env) if project
@@ -53,11 +44,11 @@ module API
@project ||= access_checker.project
result
rescue Gitlab::GitAccess::UnauthorizedError => e
- break response_with_status(code: 401, success: false, message: e.message)
+ return response_with_status(code: 401, success: false, message: e.message)
rescue Gitlab::GitAccess::TimeoutError => e
- break response_with_status(code: 503, success: false, message: e.message)
+ return response_with_status(code: 503, success: false, message: e.message)
rescue Gitlab::GitAccess::NotFoundError => e
- break response_with_status(code: 404, success: false, message: e.message)
+ return response_with_status(code: 404, success: false, message: e.message)
end
log_user_activity(actor.user)
@@ -91,6 +82,26 @@ module API
response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR)
end
end
+ end
+
+ namespace 'internal' do
+ # Check if git command is allowed for project
+ #
+ # Params:
+ # key_id - ssh key id for Git over SSH
+ # user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode
+ # username - user name for Git over SSH in keyless SSH cert mode
+ # protocol - Git access protocol being used, e.g. HTTP or SSH
+ # project - project full_path (not path on disk)
+ # action - git action (git-upload-pack or git-receive-pack)
+ # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
+ # check_ip - optional, only in EE version, may limit access to
+ # group resources based on its IP restrictions
+ post "/allowed" do
+ # It was moved to a separate method so that EE can alter its behaviour more
+ # easily.
+ check_allowed(params)
+ end
# rubocop: disable CodeReuse/ActiveRecord
post "/lfs_authenticate" do
diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb
index 25b91318dfe..eb58435e3f1 100644
--- a/lib/gitlab/sidekiq_daemon/memory_killer.rb
+++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb
@@ -39,8 +39,8 @@ module Gitlab
while enabled?
begin
- restart_sidekiq unless rss_within_range?
sleep(CHECK_INTERVAL_SECONDS)
+ restart_sidekiq unless rss_within_range?
rescue => e
log_exception(e, __method__)
rescue Exception => e # rubocop:disable Lint/RescueException
diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index 0a21e529ceb..853fb2777c3 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -70,10 +70,11 @@ module Gitlab
end
def add_time_keys!(time, payload)
- payload['duration'] = time[:duration].round(3)
- payload['system_s'] = time[:stime].round(3)
- payload['user_s'] = time[:utime].round(3)
- payload['child_s'] = time[:ctime].round(3) if time[:ctime] > 0
+ payload['duration'] = time[:duration].round(6)
+
+ # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0)
+ # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime
+ payload['cpu_s'] = time[:cputime].round(6) if time[:cputime] > 0
payload['completed_at'] = Time.now.utc
end
@@ -98,30 +99,24 @@ module Gitlab
end
def elapsed_by_absolute_time(start)
- (Time.now.utc - start).to_f.round(3)
+ (Time.now.utc - start).to_f.round(6)
end
def elapsed(t0)
t1 = get_time
{
duration: t1[:now] - t0[:now],
- stime: t1[:times][:stime] - t0[:times][:stime],
- utime: t1[:times][:utime] - t0[:times][:utime],
- ctime: ctime(t1[:times]) - ctime(t0[:times])
+ cputime: t1[:thread_cputime] - t0[:thread_cputime]
}
end
def get_time
{
now: current_time,
- times: Process.times
+ thread_cputime: defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0
}
end
- def ctime(times)
- times[:cstime] + times[:cutime]
- end
-
def current_time
Gitlab::Metrics::System.monotonic_time
end
@@ -129,7 +124,7 @@ module Gitlab
def format_time(timestamp)
return timestamp if timestamp.is_a?(String)
- Time.at(timestamp).utc.iso8601(3)
+ Time.at(timestamp).utc.iso8601(6)
end
def limited_job_args(args)
diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb
index 6725cde08f2..c0f7948f963 100644
--- a/spec/factories/ci/builds.rb
+++ b/spec/factories/ci/builds.rb
@@ -330,6 +330,38 @@ FactoryBot.define do
options { {} }
end
+ trait :dast do
+ options do
+ {
+ artifacts: { reports: { dast: 'gl-dast-report.json' } }
+ }
+ end
+ end
+
+ trait :sast do
+ options do
+ {
+ artifacts: { reports: { sast: 'gl-sast-report.json' } }
+ }
+ end
+ end
+
+ trait :dependency_scanning do
+ options do
+ {
+ artifacts: { reports: { dependency_scanning: 'gl-dependency-scanning-report.json' } }
+ }
+ end
+ end
+
+ trait :container_scanning do
+ options do
+ {
+ artifacts: { reports: { container_scanning: 'gl-container-scanning-report.json' } }
+ }
+ end
+ end
+
trait :non_playable do
status { 'created' }
self.when { 'manual' }
diff --git a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb
index 756c7947df0..263cc821c1a 100644
--- a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb
+++ b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb
@@ -40,6 +40,7 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do
message: "Exception from start_working: My Exception")
expect(memory_killer).to receive(:rss_within_range?).twice.and_raise(StandardError, 'My Exception')
+ expect(memory_killer).to receive(:sleep).twice.with(Gitlab::SidekiqDaemon::MemoryKiller::CHECK_INTERVAL_SECONDS)
expect { subject }.not_to raise_exception
end
@@ -53,6 +54,7 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do
expect(memory_killer).to receive(:rss_within_range?).once.and_raise(Exception, 'My Exception')
+ expect(memory_killer).to receive(:sleep).with(Gitlab::SidekiqDaemon::MemoryKiller::CHECK_INTERVAL_SECONDS)
expect(Sidekiq.logger).to receive(:warn).once
.with(
class: described_class.to_s,
diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb
index 6e6a8e14fc9..46fbc069efb 100644
--- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb
+++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb
@@ -23,13 +23,15 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
end
let(:logger) { double }
+ let(:clock_thread_cputime_start) { 0.222222299 }
+ let(:clock_thread_cputime_end) { 1.333333799 }
let(:start_payload) do
job.merge(
'message' => 'TestWorker JID-da883554ee4fe414012f5f42: start',
'job_status' => 'start',
'pid' => Process.pid,
- 'created_at' => created_at.iso8601(3),
- 'enqueued_at' => created_at.iso8601(3),
+ 'created_at' => created_at.iso8601(6),
+ 'enqueued_at' => created_at.iso8601(6),
'scheduling_latency_s' => scheduling_latency_s
)
end
@@ -38,9 +40,8 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
'message' => 'TestWorker JID-da883554ee4fe414012f5f42: done: 0.0 sec',
'job_status' => 'done',
'duration' => 0.0,
- "completed_at" => timestamp.iso8601(3),
- "system_s" => 0.0,
- "user_s" => 0.0
+ "completed_at" => timestamp.iso8601(6),
+ "cpu_s" => 1.111112
)
end
let(:exception_payload) do
@@ -57,12 +58,7 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
allow(subject).to receive(:current_time).and_return(timestamp.to_f)
- allow(Process).to receive(:times).and_return(
- stime: 0.0,
- utime: 0.0,
- cutime: 0.0,
- cstime: 0.0
- )
+ allow(Process).to receive(:clock_gettime).with(Process::CLOCK_THREAD_CPUTIME_ID).and_return(clock_thread_cputime_start, clock_thread_cputime_end)
end
subject { described_class.new }
@@ -187,31 +183,22 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
end
end
end
+ end
- def ctime(times)
- times[:cstime] + times[:cutime]
- end
+ describe '#add_time_keys!' do
+ let(:time) { { duration: 0.1231234, cputime: 1.2342345 } }
+ let(:payload) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status' } }
+ let(:current_utc_time) { '2019-09-23 10:00:58 UTC' }
+ let(:payload_with_time_keys) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status', 'duration' => 0.123123, 'cpu_s' => 1.234235, 'completed_at' => current_utc_time } }
- context 'with ctime value greater than 0' do
- let(:times_start) { { stime: 0.04999, utime: 0.0483, cstime: 0.0188, cutime: 0.0188 } }
- let(:times_end) { { stime: 0.0699, utime: 0.0699, cstime: 0.0399, cutime: 0.0399 } }
+ subject { described_class.new }
- before do
- end_payload['system_s'] = 0.02
- end_payload['user_s'] = 0.022
- end_payload['child_s'] = 0.042
+ it 'update payload correctly' do
+ expect(Time).to receive_message_chain(:now, :utc).and_return(current_utc_time)
- allow(Process).to receive(:times).and_return(times_start, times_end)
- end
+ subject.send(:add_time_keys!, time, payload)
- it 'logs with ctime data and other cpu data' do
- Timecop.freeze(timestamp) do
- expect(logger).to receive(:info).with(start_payload.except('args')).ordered
- expect(logger).to receive(:info).with(end_payload.except('args')).ordered
-
- subject.call(job, 'test_queue') { }
- end
- end
+ expect(payload).to eq(payload_with_time_keys)
end
end
end