From 1be6b52ebfd46b98538dcc491a35e6431ea55465 Mon Sep 17 00:00:00 2001
From: Ivan Fraixedes <ivan@fraixed.es>
Date: Tue, 8 Dec 2015 21:13:09 +0000
Subject: [PATCH] Add embeded template for robots.txt

---
 content/commands/hugo.md          |  1 +
 content/extras/robots-txt.md      | 34 +++++++++++++++++++++++++++++++
 content/extras/urls.md            |  2 +-
 content/overview/configuration.md |  2 ++
 4 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 content/extras/robots-txt.md

diff --git a/content/commands/hugo.md b/content/commands/hugo.md
index d36cb0bab..190f67a14 100644
--- a/content/commands/hugo.md
+++ b/content/commands/hugo.md
@@ -34,6 +34,7 @@ hugo
   -d, --destination="": filesystem path to write files to
       --disableRSS[=false]: Do not build RSS files
       --disableSitemap[=false]: Do not build Sitemap file
+      --disableRobotsTXT[=false]: Do not build robots TXT file
       --editor="": edit new content with this editor, if provided
       --ignoreCache[=false]: Ignores the cache directory for reading but still writes to it
       --log[=false]: Enable Logging
diff --git a/content/extras/robots-txt.md b/content/extras/robots-txt.md
new file mode 100644
index 000000000..336999d1c
--- /dev/null
+++ b/content/extras/robots-txt.md
@@ -0,0 +1,34 @@
+---
+date: 2013-07-09
+menu:
+  main:
+    parent: extras
+next: /community/mailing-list
+prev: /extras/urls
+title: Table of Contents
+weight: 120
+---
+
+Hugo can generated customized [robots.txt](http://www.robotstxt.org/) in the
+[same way than any other template]({{< ref "templates/go-templates.md" >}}).
+
+By default it generates a robots.txt which allows everything, it looks exactly
+
+  User-agent: *
+
+To disable it just set `disableRobotsTXT` option to false in the [command line]({{< ref "commands/hugo.md" >}}) or [configuration file]({{< ref "overview/configuration.md" >}}).
+
+Hugo will use the template `robots.txt` following the list starting with the one with more priority
+
+* /layouts/robots.txt
+* /themes/`THEME`/layout/robots.txt
+
+An example of a robots.txt layout is:
+
+    User-agent: *
+
+    {{range .Data.Pages}}
+    Disallow: {{.RelPermalink}}{{end}}
+
+
+This template disallows and all the pages of the site creating one `Disallow` entry for each one.
diff --git a/content/extras/urls.md b/content/extras/urls.md
index 497caa0fe..fb53b8f13 100644
--- a/content/extras/urls.md
+++ b/content/extras/urls.md
@@ -5,7 +5,7 @@ date: 2014-01-03
 menu:
   main:
     parent: extras
-next: /community/mailing-list
+next: /extras/robots-txt
 notoc: true
 prev: /extras/toc
 title: URLs
diff --git a/content/overview/configuration.md b/content/overview/configuration.md
index 89a2642c5..d5db7520e 100644
--- a/content/overview/configuration.md
+++ b/content/overview/configuration.md
@@ -96,6 +96,8 @@ Following is a list of Hugo-defined variables that you can configure and their c
     disableRSS:                 false
     # Do not build Sitemap file
     disableSitemap:             false
+    # Do not build robots.txt file
+    disableRobotsTXT:           false
     # edit new content with this editor, if provided
     editor:                     ""
     footnoteAnchorPrefix:       ""