summaryrefslogtreecommitdiff
path: root/public/robots.txt
diff options
context:
space:
mode:
authorBen Bodenmiller <bbodenmiller@hotmail.com>2015-08-17 23:11:16 -0700
committerBen Bodenmiller <bbodenmiller@hotmail.com>2015-08-17 23:11:16 -0700
commit595a93ee2c89be666725aa793a013179537d98b0 (patch)
treef8b6795172c020dd2f581c286c7ac44484c3dc9f /public/robots.txt
parent5fad0b585eb50e86aa09b18e99c5efdc4357ba2c (diff)
downloadgitlab-ce-595a93ee2c89be666725aa793a013179537d98b0.tar.gz
disallow irrelevant pages by default in robots
Update default robots.txt rules to disallow irrelevant pages that search engines should not care about. This will still allow important pages like the files, commit details, merge requests, issues, comments, etc. to be crawled.
Diffstat (limited to 'public/robots.txt')
-rw-r--r--public/robots.txt63
1 files changed, 62 insertions, 1 deletions
diff --git a/public/robots.txt b/public/robots.txt
index 085187fa58b..528f421083e 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,5 +1,66 @@
-# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
+# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /
+
+User-Agent: *
+
+# Add a 1 second delay between successive requests to the same server, limits resources used by crawler
+# Only some crawlers respect this setting, e.g. Googlebot does not
+# Crawl-delay: 1
+
+# Based on details in https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/routes.rb, https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/routing, and using application
+Disallow: /autocomplete/users
+Disallow: /search
+Disallow: /api
+Disallow: /admin
+Disallow: /profile
+Disallow: /dashboard
+Disallow: /projects/new
+Disallow: /groups/new
+Disallow: /groups/*/edit
+Disallow: /users
+
+# Global snippets
+Disallow: /s
+Disallow: /snippets/new
+Disallow: /snippets/*/edit
+Disallow: /snippets/*/raw
+
+# Project details
+Disallow: /*/*.git
+Disallow: /*/*/fork/new
+Disallow: /*/*/repository/archive*
+Disallow: /*/*/activity
+Disallow: /*/*/new
+Disallow: /*/*/edit
+Disallow: /*/*/raw
+Disallow: /*/*/blame
+Disallow: /*/*/commits/*/*
+Disallow: /*/*/commit
+Disallow: /*/*/compare
+Disallow: /*/*/branches/new
+Disallow: /*/*/tags/new
+Disallow: /*/*/network
+Disallow: /*/*/graphs
+Disallow: /*/*/milestones/new
+Disallow: /*/*/milestones/*/edit
+Disallow: /*/*/issues/new
+Disallow: /*/*/issues/*/edit
+Disallow: /*/*/merge_requests/new
+Disallow: /*/*/merge_requests/*.patch
+Disallow: /*/*/merge_requests/*.diff
+Disallow: /*/*/merge_requests/*/edit
+Disallow: /*/*/merge_requests/*/diffs
+Disallow: /*/*/project_members/import
+Disallow: /*/*/labels/new
+Disallow: /*/*/labels/*/edit
+Disallow: /*/*/wikis/*/edit
+Disallow: /*/*/snippets/new
+Disallow: /*/*/snippets/*/edit
+Disallow: /*/*/snippets/*/raw
+Disallow: /*/*/deploy_keys
+Disallow: /*/*/hooks
+Disallow: /*/*/services
+Disallow: /*/*/protected_branches