Add robots.txt and staging config file (#2184)

config.staging.toml

@@ -0,0 +1,30 @@
+baseURL = "https://test2.docs.influxdata.com/"
+languageCode = "en-us"
+title = "InfluxDB Documentation"
+
+# Git history information for lastMod-type functionality
+enableGitInfo = true
+
+# Syntax Highlighting
+pygmentsCodefences = true
+pygmentsUseClasses = true
+
+# Preserve case in article tags
+preserveTaxonomyNames = true
+
+# Generate a robots.txt
+enableRobotsTXT = true
+
+# Markdown rendering options
+[blackfriday]
+hrefTargetBlank = true
+smartDashes = false
+
+[taxonomies]
+  "influxdb/v2.0/tag" = "influxdb/v2.0/tags"
+  "influxdb/cloud/tag" = "influxdb/cloud/tags"
+
+[markup]
+  [markup.goldmark]
+    [markup.goldmark.renderer]
+      unsafe = true

config.toml

@@ -12,6 +12,9 @@ pygmentsUseClasses = true
 # Preserve case in article tags
 preserveTaxonomyNames = true
 
+# Generate a robots.txt
+enableRobotsTXT = true
+
 # Markdown rendering options
 [blackfriday]
 hrefTargetBlank = true

layouts/robots.txt

@@ -0,0 +1 @@
+User-agent: *{{ if ne .Site.BaseURL "https://docs.influxdata.com/" }} Disallow: /{{ end }}