blob: 8434b0c8e5b15fb6cc295010a4b303e73cd3256f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
require 'spec_helper'
describe StuckCiJobsWorker do
let!(:runner) { create :ci_runner }
let!(:job) { create :ci_build, runner: runner }
let(:worker) { described_class.new }
let(:exclusive_lease_uuid) { SecureRandom.uuid }
subject do
job.reload
job.status
end
before do
job.update!(status: status, updated_at: updated_at)
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(exclusive_lease_uuid)
end
shared_examples 'job is dropped' do
it 'changes status' do
worker.perform
is_expected.to eq('failed')
end
end
shared_examples 'job is unchanged' do
it "doesn't change status" do
worker.perform
is_expected.to eq(status)
end
end
context 'when job is pending' do
let(:status) { 'pending' }
context 'when job is not stuck' do
before { allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(false) }
context 'when job was not updated for more than 1 day ago' do
let(:updated_at) { 2.days.ago }
it_behaves_like 'job is dropped'
end
context 'when job was updated in less than 1 day ago' do
let(:updated_at) { 6.hours.ago }
it_behaves_like 'job is unchanged'
end
context 'when job was not updated for more than 1 hour ago' do
let(:updated_at) { 2.hours.ago }
it_behaves_like 'job is unchanged'
end
end
context 'when job is stuck' do
before { allow_any_instance_of(Ci::Build).to receive(:stuck?).and_return(true) }
context 'when job was not updated for more than 1 hour ago' do
let(:updated_at) { 2.hours.ago }
it_behaves_like 'job is dropped'
end
context 'when job was updated in less than 1 hour ago' do
let(:updated_at) { 30.minutes.ago }
it_behaves_like 'job is unchanged'
end
end
end
context 'when job is running' do
let(:status) { 'running' }
context 'when job was not updated for more than 1 hour ago' do
let(:updated_at) { 2.hours.ago }
it_behaves_like 'job is dropped'
end
context 'when job was updated in less than 1 hour ago' do
let(:updated_at) { 30.minutes.ago }
it_behaves_like 'job is unchanged'
end
end
%w(success skipped failed canceled).each do |status|
context "when job is #{status}" do
let(:status) { status }
let(:updated_at) { 2.days.ago }
it_behaves_like 'job is unchanged'
end
end
context 'for deleted project' do
let(:status) { 'running' }
let(:updated_at) { 2.days.ago }
before { job.project.update(pending_delete: true) }
it 'does not drop job' do
expect_any_instance_of(Ci::Build).not_to receive(:drop)
worker.perform
end
end
describe 'exclusive lease' do
let(:status) { 'running' }
let(:updated_at) { 2.days.ago }
let(:worker2) { described_class.new }
it 'is guard by exclusive lease when executed concurrently' do
expect(worker).to receive(:drop).at_least(:once)
expect(worker2).not_to receive(:drop)
worker.perform
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(false)
worker2.perform
end
it 'can be executed in sequence' do
expect(worker).to receive(:drop).at_least(:once)
expect(worker2).to receive(:drop).at_least(:once)
worker.perform
worker2.perform
end
it 'cancels exclusive lease after worker perform' do
expect(Gitlab::ExclusiveLease).to receive(:cancel).with(described_class::EXCLUSIVE_LEASE_KEY, exclusive_lease_uuid)
worker.perform
end
end
end
|
