From abf18abb52dbdb409d481854bde24ed880c76f56 Mon Sep 17 00:00:00 2001 From: Ben Bodenmiller Date: Sun, 4 Oct 2015 23:39:24 -0700 Subject: allow crawling of commit page but not patch/diffs Commit page has valuable information that search engines should be allowed to crawl however the .patch and .diff pages have no new information that is not on commit page --- public/robots.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'public/robots.txt') diff --git a/public/robots.txt b/public/robots.txt index 528f421083e..4f616c7f4c1 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -38,7 +38,8 @@ Disallow: /*/*/edit Disallow: /*/*/raw Disallow: /*/*/blame Disallow: /*/*/commits/*/* -Disallow: /*/*/commit +Disallow: /*/*/commit/*.patch +Disallow: /*/*/commit/*.diff Disallow: /*/*/compare Disallow: /*/*/branches/new Disallow: /*/*/tags/new -- cgit v1.2.1