blob: bb88003ce204e9fe50fe68d429d90d71ae25b358 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
|
# frozen_string_literal: true
require 'fast_spec_helper'
require 'rspec-parameterized'
RSpec.describe Gitlab::RobotsTxt::Parser do
describe '#disallowed?' do
subject { described_class.new(content).disallowed?(path) }
context 'a simple robots.txt file' do
using RSpec::Parameterized::TableSyntax
let(:content) do
<<~TXT
User-Agent: *
Disallow: /autocomplete/users
Disallow: /search
Disallow: /api
TXT
end
where(:path, :result) do
'/autocomplete/users' | true
'/autocomplete/users/a.html' | true
'/search' | true
'/search.html' | true
'/api' | true
'/api/grapql' | true
'/api/index.html' | true
'/projects' | false
end
with_them do
it { is_expected.to eq(result), "#{path} expected to be #{result}" }
end
end
context 'robots.txt file with wildcard' do
using RSpec::Parameterized::TableSyntax
let(:content) do
<<~TXT
User-Agent: *
Disallow: /search
User-Agent: *
Disallow: /*/*.git
Disallow: /*/archive/
Disallow: /*/repository/archive*
TXT
end
where(:path, :result) do
'/search' | true
'/namespace/project.git' | true
'/project/archive/' | true
'/project/archive/file.gz' | true
'/project/repository/archive' | true
'/project/repository/archive.gz' | true
'/project/repository/archive/file.gz' | true
'/projects' | false
'/git' | false
'/projects/git' | false
end
with_them do
it { is_expected.to eq(result), "#{path} expected to be #{result}" }
end
end
end
end
|
