Add _headers file checking logic (#9879)

* Add _headers file checking logic

* Remove netlify_noindex_headers.txt file

Makefile

@@ -8,14 +8,22 @@ DOCKER_RUN   = $(DOCKER) run --rm --interactive --tty --volume $(PWD):/src
 help: ## Show this help.
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
 
-all: build ## Build site with production settings and put deliverables in _site.
+all: build ## Build site with production settings and put deliverables in ./public
 
-build: ## Build site with production settings and put deliverables in _site.
+build: ## Build site with production settings and put deliverables in ./public
 	hugo
 
-build-preview: ## Build site with drafts and future posts enabled.
+build-preview: ## Build site with drafts and future posts enabled
 	hugo -D -F
 
+check-headers-file:
+	scripts/check-headers-file.sh
+
+production-build: build check-headers-file ## Build the production site and ensure that noindex headers aren't added
+
+non-production-build: ## Build the non-production site, which adds noindex headers to prevent indexing
+	hugo --enableGitInfo
+
 serve: ## Boot the development server.
 	hugo server --ignoreCache --disableFastRender
 

layouts/index.headers

@@ -1,3 +1,4 @@
+{{- if eq (getenv "HUGO_ENV") "production" }}
 {{- $cssFilesFromConfig := .Site.Params.pushAssets.css -}}
 {{- $jsFilesFromConfig := .Site.Params.pushAssets.js -}}
 {{- $pages := .Site.RegularPages -}}
@@ -33,4 +34,8 @@
   {{- end -}}
   {{- end -}}
 {{- end }}
-{{- end -}}
+{{- end -}}
+{{- else }}
+/*
+  X-Robots-Tag: noindex
+{{- end }}

netlify.toml

@@ -3,7 +3,7 @@
 # It is turned off for only for the production site by using [context.master] below
 # DO NOT REMOVE THIS (contact @chenopis or @sig-docs-maintainers)
 publish = "public"
-command = "hugo --enableGitInfo && cp netlify_noindex_headers.txt public/_headers"
+command = "make non-production-build"
 
 [build.environment]
 HUGO_VERSION = "0.47.1"
@@ -23,4 +23,4 @@ command = "hugo --enableGitInfo -b $DEPLOY_PRIME_URL"
 # This context is triggered by the `master` branch and allows search indexing
 # DO NOT REMOVE THIS (contact @chenopis or @sig-docs-maintainers)
 publish = "public"
-command = "hugo"
+command = "make production-build"

netlify_noindex_headers.txt

@@ -1,3 +0,0 @@
-# Prevent bots from indexing site
-/*
-  X-Robots-Tag: noindex

scripts/check-headers-file.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+if [ "$HUGO_ENV" == "production" ]; then
+  echo "INFO: Production environment. Checking the _headers file for noindex headers."
+
+  if grep -q "noindex" public/_headers; then
+    echo "PANIC: noindex headers were found in the _headers file. This build has failed."
+    exit 1
+  else
+    echo "INFO: noindex headers were not found in the _headers file. All clear."
+    exit 0
+  fi
+else
+  echo "Non-production environment. Skipping the _headers file check."
+  exit 0
+fi