From 6fe67a65799b376bdf0563bdb1b1e065f7ccab35 Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Thu, 27 Apr 2017 12:36:29 -0700 Subject: [PATCH] Refactor robots.txt (#3534) * Test robots.txt * refactor robots.txt to use /* * clarify note * verified and cleaned up robots.txt entries * remove unnecessary wildcards --- robots.txt | 261 +++++++++++++++++++++-------------------------------- 1 file changed, 105 insertions(+), 156 deletions(-) diff --git a/robots.txt b/robots.txt index 83b4eee580..2c8f39131b 100644 --- a/robots.txt +++ b/robots.txt @@ -6,162 +6,111 @@ Disallow: /v1.1/ Disallow: /404/ Disallow: 404.html -Disallow: /docs/user-guide/configuring-containers.html -Disallow: /docs/user-guide/containers.html -Disallow: /docs/user-guide/deploying-applications.html -Disallow: /docs/user-guide/getting-into-containers.html -Disallow: /docs/user-guide/liveness/index.html -Disallow: /docs/user-guide/pod-states.html -Disallow: /docs/user-guide/production-pods.html -Disallow: /docs/user-guide/quick-start.html +# NOTE: site renders /topic.md into /topic/index.html, effectively /topic/ -Disallow: /docs/user-guide/downward-api/index.html -Disallow: /docs/user-guide/downward-api/volume/index.html - -Disallow: /docs/user-guide/persistent-volumes/walkthrough.html - -Disallow: /docs/user-guide/pods/init-container.html -Disallow: /docs/user-guide/pods/single-container.html - -Disallow: /docs/user-guide/secrets/walkthrough.html - -Disallow: /docs/user-guide/accessing-the-cluster.html -Disallow: /docs/user-guide/working-with-resources.html -Disallow: /docs/user-guide/garbage-collection.html -Disallow: /docs/user-guide/services/operations.html -Disallow: /docs/user-guide/load-balancer.html -Disallow: /docs/user-guide/services-firewalls.html -Disallow: /docs/user-guide/federation/federated-services.html -Disallow: /docs/user-guide/pods/multi-container.html -Disallow: /docs/user-guide/environment-guide/index.html -Disallow: /docs/user-guide/compute-resources.html - -Disallow: /docs/whatisk8s.html - -Disallow: /docs/user-guide/petset/bootstrapping/index.html -Disallow: /docs/user-guide/monitoring.html -Disallow: /docs/user-guide/logging/overview.html -Disallow: /docs/user-guide/logging/stackdriver.html -Disallow: /docs/user-guide/logging/elasticsearch.html -Disallow: /docs/user-guide/connecting-to-applications-proxy.html -Disallow: /docs/user-guide/connecting-to-applications-port-forward.html - -Disallow: /docs/admin/audit.html -Disallow: /docs/admin/disruptions.html -Disallow: /docs/admin/resourcequota/index.html -Disallow: /docs/admin/resourcequota/walkthrough.html -Disallow: /docs/admin/resourcequota/limitstorageconsumption.html -Disallow: /docs/admin/rescheduler.html -Disallow: /docs/admin/sysctls.html -Disallow: /docs/admin/cluster-components.html - -Disallow: /docs/admin/multi-cluster.html -Disallow: /docs/admin/networking.html -Disallow: /docs/admin/dns.html -Disallow: /docs/admin/network-plugins.html -Disallow: /docs/admin/static-pods.html -Disallow: /docs/admin/out-of-resource.html -Disallow: /docs/admin/node-problem.html -Disallow: /docs/admin/apparmor/index.html - -Disallow: /docs/user-guide/federation/configmap.html -Disallow: /docs/user-guide/federation/daemonsets.html -Disallow: /docs/user-guide/federation/deployment.html -Disallow: /docs/user-guide/federation/events.html -Disallow: /docs/user-guide/federation/federated-ingress.html -Disallow: /docs/user-guide/federation/namespaces.html -Disallow: /docs/user-guide/federation/replicasets.html -Disallow: /docs/user-guide/federation/secrets.html - -Disallow: /docs/getting-started-guides/kubectl.html -Disallow: /docs/user-guide/sharing-clusters.html -Disallow: /docs/user-guide/kubeconfig-file.html -Disallow: /docs/user-guide/prereqs.html - -Disallow: /docs/user-guide/managing-deployments.html -Disallow: /docs/user-guide/replication-controller/operations.html -Disallow: /docs/user-guide/resizing-a-replication-controller.html -Disallow: /docs/user-guide/rolling-updates.html -Disallow: /docs/user-guide/update-demo/index.html - -Disallow: /docs/user-guide/jobs.html -Disallow: /docs/user-guide/jobs/work-queue-1/index.html -Disallow: /docs/user-guide/jobs/work-queue-2/index.html - -Disallow: /docs/user-guide/container-environment.html -Disallow: /docs/user-guide/images.html -Disallow: /docs/user-guide/volumes.html -Disallow: /docs/user-guide/node-selection/index.html - -Disallow: /docs/user-guide/config-best-practices.html -Disallow: /docs/user-guide/connecting-applications.html -Disallow: /docs/user-guide/pod-templates.html -Disallow: /docs/user-guide/persistent-volumes/index.html - -Disallow: /docs/admin/index.html -Disallow: /docs/admin/cluster-management.html -Disallow: /docs/admin/limitrange/index.html -Disallow: /docs/admin/multiple-schedulers.html -Disallow: /docs/admin/master-node-communication.html - -Disallow: /docs/admin/federation/kubefed.html -Disallow: /docs/user-guide/federation/index.html - -Disallow: /docs/user-guide/debugging-pods-and-replication-controllers.html -Disallow: /docs/user-guide/introspection-and-debugging.html -Disallow: /docs/user-guide/application-troubleshooting.html -Disallow: /docs/admin/cluster-troubleshooting.html -Disallow: /docs/user-guide/debugging-services.html -Disallow: /docs/contribute/create-pull-request.html -Disallow: /docs/contribute/write-new-topic.html -Disallow: /docs/contribute/stage-documentation-changes.html -Disallow: /docs/contribute/page-templates.html -Disallow: /docs/contribute/review-issues.html -Disallow: /docs/contribute/style-guide.html -Disallow: /docs/deprecation-policy.html - -Disallow: /docs/admin/daemons.html -Disallow: /docs/user-guide/deployments.html -Disallow: /docs/user-guide/replicasets.html -Disallow: /docs/user-guide/replication-controller/index.html -Disallow: /docs/user-guide/ingress.html -Disallow: /docs/user-guide/namespaces.html -Disallow: /docs/user-guide/identifiers.html -Disallow: /docs/user-guide/networkpolicies.html -Disallow: /docs/admin/node.html -Disallow: /docs/user-guide/petset.html -Disallow: /docs/user-guide/pods/intex.html - -Disallow: /docs/user-guide/replication-controller/index.html -Disallow: /docs/user-guide/secrets/index.html -Disallow: /docs/user-guide/security-context.html -Disallow: /docs/user-guide/services/index.html -Disallow: /docs/user-guide/service-accounts.html -Disallow: /docs/user-guide/thirdpartyresources.html -Disallow: /docs/user-guide/pod-preset/index.html -Disallow: /docs/user-guide/pod-security-policy/index.html -Disallow: /docs/user-guide/horizontal-pod-autoscaling/index.html -Disallow: /docs/user-guide/annotations.html - -Disallow: /docs/user-guide/configmap/index.html -Disallow: /docs/user-guide/horizontal-pod-autoscaling/walkthrough.html -Disallow: /docs/getting-started-guides/network-policy/walkthrough.html -Disallow: /docs/getting-started-guides/network-policy/calico.html -Disallow: /docs/getting-started-guides/network-policy/romana.html -Disallow: /docs/getting-started-guides/network-policy/weave.html -Disallow: /docs/user-guide/cron-jobs.html -Disallow: /docs/user-guide/gpus.html -Disallow: /docs/user-guide/projected-volume/index.html -Disallow: /docs/admin/upgrade-1-6.html -Disallow: /docs/admin/addons.html -Disallow: /docs/admin/ha-master-gce.html -Disallow: /docs/admin/namespaces/index.html -Disallow: /docs/admin/namespaces/walkthrough.html -Disallow: /docs/admin/etcd_upgrade.html -Disallow: /docs/admin/etcd.html -Disallow: /docs/admin/garbage-collection.html -Disallow: /docs/admin/node-allocatable.html -Disallow: /docs/user-guide/jobs/expansions/index.html -Disallow: /docs/user-guide/ui/index.html +Disallow: /docs/admin/addons/ +Disallow: /docs/admin/apparmor/ +Disallow: /docs/admin/audit/ +Disallow: /docs/admin/cluster-components/ +Disallow: /docs/admin/cluster-management/ +Disallow: /docs/admin/cluster-troubleshooting/ +Disallow: /docs/admin/daemons/ +Disallow: /docs/admin/disruptions/ +Disallow: /docs/admin/dns/ +Disallow: /docs/admin/etcd/ +Disallow: /docs/admin/etcd_upgrade/ +Disallow: /docs/admin/federation/kubefed/ +Disallow: /docs/admin/garbage-collection/ +Disallow: /docs/admin/ha-master-gce/ +Disallow: /docs/admin/limitrange/ +Disallow: /docs/admin/master-node-communication/ +Disallow: /docs/admin/multi-cluster/ +Disallow: /docs/admin/multiple-schedulers/ +Disallow: /docs/admin/namespaces/ +Disallow: /docs/admin/namespaces/walkthrough/ +Disallow: /docs/admin/network-plugins/ +Disallow: /docs/admin/networking/ +Disallow: /docs/admin/node-allocatable/ +Disallow: /docs/admin/node-problem/ +Disallow: /docs/admin/node/ +Disallow: /docs/admin/out-of-resource/ +Disallow: /docs/admin/rescheduler/ +Disallow: /docs/admin/resourcequota/ +Disallow: /docs/admin/resourcequota/limitstorageconsumption/ +Disallow: /docs/admin/resourcequota/walkthrough/ +Disallow: /docs/admin/static-pods/ +Disallow: /docs/admin/sysctls/ +Disallow: /docs/admin/upgrade-1-6/ +Disallow: /docs/api/ +Disallow: /docs/contribute/ +Disallow: /docs/deprecation-policy/ +Disallow: /docs/getting-started-guides/kubectl/ +Disallow: /docs/getting-started-guides/network-policy/ +Disallow: /docs/getting-started-guides/running-cloud-controller/ +Disallow: /docs/user-guide/accessing-the-cluster/ +Disallow: /docs/user-guide/annotations/ +Disallow: /docs/user-guide/application-troubleshooting/ +Disallow: /docs/user-guide/compute-resources/ +Disallow: /docs/user-guide/config-best-practices/ +Disallow: /docs/user-guide/configmap/ +Disallow: /docs/user-guide/configuring-containers/ +Disallow: /docs/user-guide/connecting-applications/ +Disallow: /docs/user-guide/connecting-to-applications-port-forward/ +Disallow: /docs/user-guide/connecting-to-applications-proxy/ +Disallow: /docs/user-guide/container-environment/ +Disallow: /docs/user-guide/containers/ +Disallow: /docs/user-guide/cron-jobs/ +Disallow: /docs/user-guide/debugging-pods-and-replication-controllers/ +Disallow: /docs/user-guide/debugging-services/ +Disallow: /docs/user-guide/deploying-applications/ +Disallow: /docs/user-guide/deployments/ +Disallow: /docs/user-guide/downward-api/ +Disallow: /docs/user-guide/environment-guide/ +Disallow: /docs/user-guide/federation/ +Disallow: /docs/user-guide/garbage-collection/ +Disallow: /docs/user-guide/getting-into-containers/ +Disallow: /docs/user-guide/gpus/ +Disallow: /docs/user-guide/horizontal-pod-autoscaling/ +Disallow: /docs/user-guide/identifiers/ +Disallow: /docs/user-guide/images/ +Disallow: /docs/user-guide/ingress/ +Disallow: /docs/user-guide/introspection-and-debugging/ +Disallow: /docs/user-guide/jobs/ +Disallow: /docs/user-guide/kubeconfig-file/ +Disallow: /docs/user-guide/liveness/ +Disallow: /docs/user-guide/load-balancer/ +Disallow: /docs/user-guide/logging/ +Disallow: /docs/user-guide/managing-deployments/ +Disallow: /docs/user-guide/monitoring/ +Disallow: /docs/user-guide/namespaces/ +Disallow: /docs/user-guide/networkpolicies/ +Disallow: /docs/user-guide/node-selection/ +Disallow: /docs/user-guide/persistent-volumes/ +Disallow: /docs/user-guide/petset/ +Disallow: /docs/user-guide/pod-preset/ +Disallow: /docs/user-guide/pod-security-policy/ +Disallow: /docs/user-guide/pod-states/ +Disallow: /docs/user-guide/pod-templates/ +Disallow: /docs/user-guide/pods/ +Disallow: /docs/user-guide/prereqs/ +Disallow: /docs/user-guide/production-pods/ +Disallow: /docs/user-guide/projected-volume/ +Disallow: /docs/user-guide/quick-start/ +Disallow: /docs/user-guide/replicasets/ +Disallow: /docs/user-guide/replication-controller/ +Disallow: /docs/user-guide/resizing-a-replication-controller/ +Disallow: /docs/user-guide/rolling-updates/ +Disallow: /docs/user-guide/secrets/ +Disallow: /docs/user-guide/security-context/ +Disallow: /docs/user-guide/service-accounts/ +Disallow: /docs/user-guide/services-firewalls/ +Disallow: /docs/user-guide/services/ +Disallow: /docs/user-guide/sharing-clusters/ +Disallow: /docs/user-guide/thirdpartyresources/ +Disallow: /docs/user-guide/ui/ +Disallow: /docs/user-guide/update-demo/ +Disallow: /docs/user-guide/volumes/ +Disallow: /docs/user-guide/working-with-resources/ +Disallow: /docs/whatisk8s/ SITEMAP: http://kubernetes.io/sitemap.xml