Add latest changes from gitlab-org/gitlab@master

4 files changed, 93 insertions, 18 deletions

diff --git a/spec/features/invites_spec.rb b/spec/features/invites_spec.rb
index bbe0e64b422..26b9b398692 100644
--- a/spec/features/invites_spec.rb
+++ b/spec/features/invites_spec.rb
@@ -26,7 +26,6 @@ RSpec.describe 'Invites', :aggregate_failures do
     fill_in 'new_user_name', with: new_user.name
     fill_in 'new_user_username', with: new_user.username
     fill_in 'new_user_email', with: new_user.email
-    fill_in 'new_user_email_confirmation', with: new_user.email
     fill_in 'new_user_password', with: new_user.password
     click_button 'Register'
   end
diff --git a/spec/features/projects/show/user_sees_readme_spec.rb b/spec/features/projects/show/user_sees_readme_spec.rb
index 250f707948e..6a5b9472be8 100644
--- a/spec/features/projects/show/user_sees_readme_spec.rb
+++ b/spec/features/projects/show/user_sees_readme_spec.rb
@@ -14,4 +14,25 @@ RSpec.describe 'Projects > Show > User sees README' do
       expect(page).to have_content 'testme'
     end
   end
+
+  context 'obeying robots.txt' do
+    before do
+      Gitlab::Testing::RobotsBlockerMiddleware.block_requests!
+    end
+
+    after do
+      Gitlab::Testing::RobotsBlockerMiddleware.allow_requests!
+    end
+
+    # For example, see this regression we had in
+    # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39520
+    it 'does not block the requests necessary to load the project README', :js do
+      visit project_path(project)
+      wait_for_requests
+
+      page.within('.readme-holder') do
+        expect(page).to have_content 'testme'
+      end
+    end
+  end
 end
diff --git a/spec/features/users/signup_spec.rb b/spec/features/users/signup_spec.rb
index b9752f6676e..7b54a590805 100644
--- a/spec/features/users/signup_spec.rb
+++ b/spec/features/users/signup_spec.rb
@@ -152,7 +152,6 @@ RSpec.shared_examples 'Signup' do
             fill_in 'new_user_last_name', with: new_user.last_name
           else
             fill_in 'new_user_name', with: new_user.name
-            fill_in 'new_user_email_confirmation', with: new_user.email
           end
 
           fill_in 'new_user_password', with: new_user.password
@@ -180,7 +179,6 @@ RSpec.shared_examples 'Signup' do
             fill_in 'new_user_last_name', with: new_user.last_name
           else
             fill_in 'new_user_name', with: new_user.name
-            fill_in 'new_user_email_confirmation', with: new_user.email
           end
 
           fill_in 'new_user_password', with: new_user.password
@@ -204,7 +202,6 @@ RSpec.shared_examples 'Signup' do
           fill_in 'new_user_last_name', with: new_user.last_name
         else
           fill_in 'new_user_name', with: new_user.name
-          fill_in 'new_user_email_confirmation', with: new_user.email.capitalize
         end
 
         fill_in 'new_user_password', with: new_user.password
@@ -230,7 +227,6 @@ RSpec.shared_examples 'Signup' do
           fill_in 'new_user_last_name', with: new_user.last_name
         else
           fill_in 'new_user_name', with: new_user.name
-          fill_in 'new_user_email_confirmation', with: new_user.email
         end
 
         fill_in 'new_user_password', with: new_user.password
@@ -260,14 +256,7 @@ RSpec.shared_examples 'Signup' do
       click_button "Register"
 
       expect(current_path).to eq user_registration_path
-
-      if Gitlab::Experimentation.enabled?(:signup_flow)
-        expect(page).to have_content("error prohibited this user from being saved")
-      else
-        expect(page).to have_content("errors prohibited this user from being saved")
-        expect(page).to have_content("Email confirmation doesn't match")
-      end
-
+      expect(page).to have_content("error prohibited this user from being saved")
       expect(page).to have_content("Email has already been taken")
     end
 
@@ -309,7 +298,6 @@ RSpec.shared_examples 'Signup' do
         fill_in 'new_user_last_name', with: new_user.last_name
       else
         fill_in 'new_user_name', with: new_user.name
-        fill_in 'new_user_email_confirmation', with: new_user.email
       end
 
       fill_in 'new_user_password', with: new_user.password
@@ -331,7 +319,6 @@ RSpec.shared_examples 'Signup' do
         fill_in 'new_user_last_name', with: new_user.last_name
       else
         fill_in 'new_user_name', with: new_user.name
-        fill_in 'new_user_email_confirmation', with: new_user.email
       end
 
       fill_in 'new_user_password', with: new_user.password
@@ -374,7 +361,6 @@ RSpec.shared_examples 'Signup' do
           fill_in 'new_user_last_name', with: new_user.last_name
         else
           fill_in 'new_user_name', with: new_user.name
-          fill_in 'new_user_email_confirmation', with: new_user.email
         end
 
         fill_in 'new_user_password', with: new_user.password
@@ -396,7 +382,6 @@ RSpec.shared_examples 'Signup' do
           fill_in 'new_user_last_name', with: new_user.last_name
         else
           fill_in 'new_user_name', with: new_user.name
-          fill_in 'new_user_email_confirmation', with: new_user.email
         end
 
         fill_in 'new_user_password', with: new_user.password
@@ -419,7 +404,6 @@ RSpec.shared_examples 'Signup' do
       fill_in 'new_user_last_name', with: new_user.last_name
     else
       fill_in 'new_user_name', with: new_user.name
-      fill_in 'new_user_email_confirmation', with: new_user.email
     end
 
     fill_in 'new_user_password', with: new_user.password
diff --git a/spec/lib/gitlab/robots_txt/parser_spec.rb b/spec/lib/gitlab/robots_txt/parser_spec.rb
new file mode 100644
index 00000000000..bb88003ce20
--- /dev/null
+++ b/spec/lib/gitlab/robots_txt/parser_spec.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+require 'rspec-parameterized'
+
+RSpec.describe Gitlab::RobotsTxt::Parser do
+  describe '#disallowed?' do
+    subject { described_class.new(content).disallowed?(path) }
+
+    context 'a simple robots.txt file' do
+      using RSpec::Parameterized::TableSyntax
+
+      let(:content) do
+        <<~TXT
+          User-Agent: *
+          Disallow: /autocomplete/users
+          Disallow: /search
+          Disallow: /api
+        TXT
+      end
+
+      where(:path, :result) do
+        '/autocomplete/users' | true
+        '/autocomplete/users/a.html' | true
+        '/search' | true
+        '/search.html' | true
+        '/api' | true
+        '/api/grapql' | true
+        '/api/index.html' | true
+        '/projects' | false
+      end
+
+      with_them do
+        it { is_expected.to eq(result), "#{path} expected to be #{result}" }
+      end
+    end
+
+    context 'robots.txt file with wildcard' do
+      using RSpec::Parameterized::TableSyntax
+
+      let(:content) do
+        <<~TXT
+          User-Agent: *
+          Disallow: /search
+
+          User-Agent: *
+          Disallow: /*/*.git
+          Disallow: /*/archive/
+          Disallow: /*/repository/archive*
+        TXT
+      end
+
+      where(:path, :result) do
+        '/search' | true
+        '/namespace/project.git' | true
+        '/project/archive/' | true
+        '/project/archive/file.gz' | true
+        '/project/repository/archive' | true
+        '/project/repository/archive.gz' | true
+        '/project/repository/archive/file.gz' | true
+        '/projects' | false
+        '/git' | false
+        '/projects/git' | false
+      end
+
+      with_them do
+        it { is_expected.to eq(result), "#{path} expected to be #{result}" }
+      end
+    end
+  end
+end