From 5ca070478ebd8e684f547e7ca372e03b22175c96 Mon Sep 17 00:00:00 2001 From: Andy Zheng Date: Thu, 5 Nov 2015 17:14:47 -0800 Subject: [PATCH] Refactor cluster/gce/trusty/node.yaml The node.yaml has some logic that will be also used by the kubernetes master on trusty work (issue #16702). This change moves the code shared by the master and node configuration to a separate script, and the master and node configuration can source it to use the code. Moreover, this change stages the script for GKE use. --- build/common.sh | 2 +- cluster/gce/trusty/configure.sh | 221 ++++++++++++++++++++++++++++++++ cluster/gce/trusty/helper.sh | 5 +- cluster/gce/trusty/node.yaml | 192 ++++----------------------- cluster/gce/util.sh | 3 +- 5 files changed, 252 insertions(+), 171 deletions(-) create mode 100644 cluster/gce/trusty/configure.sh diff --git a/build/common.sh b/build/common.sh index 0718cdb263..17a97e1933 100755 --- a/build/common.sh +++ b/build/common.sh @@ -1000,7 +1000,7 @@ function kube::release::gcs::copy_release_artifacts() { # deploy hosted with the release is useful for GKE. kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/configure-vm.sh" extra/gce || return 1 kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/trusty/node.yaml" extra/gce || return 1 - + kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/trusty/configure.sh" extra/gce || return 1 # Upload the "naked" binaries to GCS. This is useful for install scripts that # download the binaries directly and don't need tars. diff --git a/cluster/gce/trusty/configure.sh b/cluster/gce/trusty/configure.sh new file mode 100644 index 0000000000..222c1c421d --- /dev/null +++ b/cluster/gce/trusty/configure.sh @@ -0,0 +1,221 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script contains functions for configuring instances to run kubernetes +# nodes. It is uploaded to GCE metadata server when a VM instance is created, +# and then downloaded by the instance. The upstart jobs in +# cluster/gce/trusty/node.yaml source this script to make use of needed +# functions. The script itself is not supposed to be executed in other manners. + +config_hostname() { + # Set the hostname to the short version. + short_hostname=$(hostname -s) + hostname $short_hostname +} + +config_ip_firewall() { + # We have seen that GCE image may have strict host firewall rules which drop + # most inbound/forwarded packets. In such a case, add rules to accept all + # TCP/UDP packets. + if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then + echo "Add rules to accpet all inbound TCP/UDP packets" + iptables -A INPUT -w -p TCP -j ACCEPT + iptables -A INPUT -w -p UDP -j ACCEPT + fi + if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then + echo "Add rules to accpet all forwarded TCP/UDP packets" + iptables -A FORWARD -w -p TCP -j ACCEPT + iptables -A FORWARD -w -p UDP -j ACCEPT + fi +} + +create_dirs() { + # Create required directories. + mkdir -p /var/lib/kubelet + mkdir -p /var/lib/kube-proxy + mkdir -p /etc/kubernetes/manifests +} + +download_kube_env() { + # Fetch kube-env from GCE metadata server. + curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /tmp/kube-env-yaml \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env + # Convert the yaml format file into a shell-style file. + eval $(python -c ''' +import pipes,sys,yaml +for k,v in yaml.load(sys.stdin).iteritems(): + print "readonly {var}={value}".format(var = k, value = pipes.quote(str(v))) +''' < /tmp/kube-env-yaml > /etc/kube-env) +} + +create_kubelet_kubeconfig() { + # Create the kubelet kubeconfig file. + . /etc/kube-env + if [ -z "${KUBELET_CA_CERT:-}" ]; then + KUBELET_CA_CERT="${CA_CERT}" + fi + cat > /var/lib/kubelet/kubeconfig << EOF +apiVersion: v1 +kind: Config +users: +- name: kubelet + user: + client-certificate-data: ${KUBELET_CERT} + client-key-data: ${KUBELET_KEY} +clusters: +- name: local + cluster: + certificate-authority-data: ${KUBELET_CA_CERT} +contexts: +- context: + cluster: local + user: kubelet + name: service-account-context +current-context: service-account-context +EOF +} + +create_kubeproxy_kubeconfig() { + # Create the kube-proxy config file. + cat > /var/lib/kube-proxy/kubeconfig << EOF +apiVersion: v1 +kind: Config +users: +- name: kube-proxy + user: + token: ${KUBE_PROXY_TOKEN} +clusters: +- name: local + cluster: + certificate-authority-data: ${CA_CERT} +contexts: +- context: + cluster: local + user: kube-proxy + name: service-account-context +current-context: service-account-context +EOF +} + +# Installs the critical packages that are required by spinning up a cluster. +install_critical_packages() { + apt-get update + # Install docker and brctl if they are not in the image. + if ! which docker > /dev/null; then + echo "Do not find docker. Install it." + # We should install the latest qualified docker, which is version 1.8.3 at present. + curl -sSL https://get.docker.com/ | DOCKER_VERSION=1.8.3 sh + fi + if ! which brctl > /dev/null; then + echo "Do not find brctl. Install it." + apt-get install --yes bridge-utils + fi +} + +# Install the packages that are useful but not required by spinning up a cluster. +install_additional_packages() { + # Socat and nsenter are not required for spinning up a cluster. We move the + # installation here to be in parallel with the cluster creation. + if ! which socat > /dev/null; then + echo "Do not find socat. Install it." + apt-get install --yes socat + fi + if ! which nsenter > /dev/null; then + echo "Do not find nsenter. Install it." + # Note: this is an easy way to install nsenter, but may not be the fastest + # way. In addition, this may not be a trusted source. So, replace it if + # we have a better solution. + docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter + fi +} + +# Downloads kubernetes binaries and salt tarball, unpacks them, and places them +# to suitable directories. +install_kube_binary_config() { + . /etc/kube-env + # For a testing cluster, we pull kubelet, kube-proxy, and kubectl binaries, + # and place them in /usr/local/bin. For a non-test cluster, we use the binaries + # pre-installed in the image, or pull and place them in /usr/bin if they are + # not pre-installed. + BINARY_PATH="/usr/bin/" + if [ "${TEST_CLUSTER:-}" = "true" ]; then + BINARY_PATH="/usr/local/bin/" + fi + if ! which kubelet > /dev/null || ! which kube-proxy > /dev/null || [ "${TEST_CLUSTER:-}" = "true" ]; then + cd /tmp + k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1" + echo "Downloading k8s tar sha1 file ${k8s_sha1}" + curl -Lo "${k8s_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}.sha1" + k8s_tar="${SERVER_BINARY_TAR_URL##*/}" + echo "Downloading k8s tar file ${k8s_tar}" + curl -Lo "${k8s_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}" + # Validate hash. + actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true + if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then + echo "== ${k8s_tar} corrupted, sha1 ${actual} doesn't match expected ${SERVER_BINARY_TAR_HASH} ==" + else + echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}" + fi + tar xzf "/tmp/${k8s_tar}" -C /tmp/ --overwrite + cp /tmp/kubernetes/server/bin/kubelet ${BINARY_PATH} + cp /tmp/kubernetes/server/bin/kube-proxy ${BINARY_PATH} + cp /tmp/kubernetes/server/bin/kubectl ${BINARY_PATH} + rm -rf "/tmp/kubernetes" + rm "/tmp/${k8s_tar}" + rm "/tmp/${k8s_sha1}" + fi + + # Put saltbase configuration files in /etc/saltbase. We will use the add-on yaml files. + mkdir -p /etc/saltbase + cd /etc/saltbase + salt_sha1="${SALT_TAR_URL##*/}.sha1" + echo "Downloading Salt tar sha1 file ${salt_sha1}" + curl -Lo "${salt_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}.sha1" + salt_tar="${SALT_TAR_URL##*/}" + echo "Downloading Salt tar file ${salt_tar}" + curl -Lo "${salt_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}" + # Validate hash. + actual=$(sha1sum ${salt_tar} | awk '{ print $1 }') || true + if [ "${actual}" != "${SALT_TAR_HASH}" ]; then + echo "== ${salt_tar} corrupted, sha1 ${actual} doesn't match expected ${SALT_TAR_HASH} ==" + else + echo "Validated ${SALT_TAR_URL} SHA1 = ${SALT_TAR_HASH}" + fi + tar xzf "/etc/saltbase/${salt_tar}" -C /etc/saltbase/ --overwrite + rm "/etc/saltbase/${salt_sha1}" + rm "/etc/saltbase/${salt_tar}" +} + +restart_docker_daemon() { + . /etc/kube-env + # Assemble docker deamon options + DOCKER_OPTS="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false" + if [ "${TEST_CLUSTER:-}" = "true" ]; then + DOCKER_OPTS="${DOCKER_OPTS} --log-level=debug" + fi + echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS}\"" > /etc/default/docker + # Make sure the network interface cbr0 is created before restarting docker daemon + while ! [ -L /sys/class/net/cbr0 ]; do + echo "Sleep 1 second to wait for cbr0" + sleep 1 + done + initctl restart docker + # Remove docker0 + ifconfig docker0 down + brctl delbr docker0 +} diff --git a/cluster/gce/trusty/helper.sh b/cluster/gce/trusty/helper.sh index 72062dc829..58a6fe0732 100755 --- a/cluster/gce/trusty/helper.sh +++ b/cluster/gce/trusty/helper.sh @@ -31,6 +31,7 @@ source "${KUBE_ROOT}/cluster/gce/debian/helper.sh" function create-node-instance-template { local template_name="$1" create-node-template "$template_name" "${scope_flags[*]}" \ - "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \ - "user-data=${KUBE_ROOT}/cluster/gce/trusty/node.yaml" + "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \ + "user-data=${KUBE_ROOT}/cluster/gce/trusty/node.yaml" \ + "configure-sh=${KUBE_ROOT}/cluster/gce/trusty/configure.sh" } diff --git a/cluster/gce/trusty/node.yaml b/cluster/gce/trusty/node.yaml index 1f37c98c85..fde84c1e5b 100644 --- a/cluster/gce/trusty/node.yaml +++ b/cluster/gce/trusty/node.yaml @@ -18,84 +18,25 @@ script set -o errexit set -o nounset - # Set the hostname to the short version. - short_hostname=$(hostname -s) - hostname $short_hostname - - # We have seen that GCE image may have strict host firewall rules which drop - # most inbound/forwarded packets. In such a case, add rules to accept all - # TCP/UDP packets. - if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then - echo "Add rules to accpet all inbound TCP/UDP packets" - iptables -A INPUT -w -p TCP -j ACCEPT - iptables -A INPUT -w -p UDP -j ACCEPT - fi - if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then - echo "Add rules to accpet all forwarded TCP/UDP packets" - iptables -A FORWARD -w -p TCP -j ACCEPT - iptables -A FORWARD -w -p UDP -j ACCEPT - fi - # Create required directories. - mkdir -p /var/lib/kubelet - mkdir -p /var/lib/kube-proxy - mkdir -p /etc/kubernetes/manifests - - # Fetch kube-env from GCE metadata server. + # Fetch the script for configuring the instance. curl --fail --silent --show-error \ -H "X-Google-Metadata-Request: True" \ - -o /tmp/kube-env-yaml \ - http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env - # Convert the yaml format file into a shell-style file. - eval $(python -c ''' -import pipes,sys,yaml -for k,v in yaml.load(sys.stdin).iteritems(): - print "readonly {var}={value}".format(var = k, value = pipes.quote(str(v))) -''' < /tmp/kube-env-yaml > /etc/kube-env) + -o /etc/kube-configure.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + . /etc/kube-configure.sh - #Create the kubelet kubeconfig file. - . /etc/kube-env - if [ -z "${KUBELET_CA_CERT:-}" ]; then - KUBELET_CA_CERT="${CA_CERT}" - fi - cat > /var/lib/kubelet/kubeconfig << EOF -apiVersion: v1 -kind: Config -users: -- name: kubelet - user: - client-certificate-data: ${KUBELET_CERT} - client-key-data: ${KUBELET_KEY} -clusters: -- name: local - cluster: - certificate-authority-data: ${KUBELET_CA_CERT} -contexts: -- context: - cluster: local - user: kubelet - name: service-account-context -current-context: service-account-context -EOF - - # Create the kube-proxy config file. - cat > /var/lib/kube-proxy/kubeconfig << EOF -apiVersion: v1 -kind: Config -users: -- name: kube-proxy - user: - token: ${KUBE_PROXY_TOKEN} -clusters: -- name: local - cluster: - certificate-authority-data: ${CA_CERT} -contexts: -- context: - cluster: local - user: kube-proxy - name: service-account-context -current-context: service-account-context -EOF + echo "Configuring hostname" + config_hostname + echo "Configuring IP firewall rules" + config_ip_firewall + echo "Creating required directories" + create_dirs + echo "Downloading kube-env file" + download_kube_env + echo "Creating kubelet kubeconfig file" + create_kubelet_kubeconfig + echo "Creating kube-proxy kubeconfig file" + create_kubeproxy_kubeconfig end script --==================================== @@ -108,23 +49,14 @@ Content-Disposition: attachment; filename="kube-install-packages.conf" description "Install packages needed to run kubernetes" -start on cloud-config +start on stopped kube-env script set -o errexit set -o nounset - apt-get update - # Install docker and brctl if they are not in the image. - if ! which docker > /dev/null; then - echo "Do not find docker. Install it." - # We should install the latest qualified docker, which is version 1.8.3 at present. - curl -sSL https://get.docker.com/ | DOCKER_VERSION=1.8.3 sh - fi - if ! which brctl > /dev/null; then - echo "Do not find brctl. Install it." - apt-get install --yes bridge-utils - fi + . /etc/kube-configure.sh + install_critical_packages end script --==================================== @@ -143,19 +75,8 @@ script set -o errexit set -o nounset - # Socat and nsenter are not required for spinning up a cluster. We move the - # installation here to be in parallel with the cluster creation. - if ! which socat > /dev/null; then - echo "Do not find socat. Install it." - apt-get install --yes socat - fi - if ! which nsenter > /dev/null; then - echo "Do not find nsenter. Install it." - # Note: this is an easy way to install nsenter, but may not be the fastest way. - # In addition, this may not be a trusted source. So, replace it if we have a - # better solution. - docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter - fi + . /etc/kube-configure.sh + install_additional_packages end script --==================================== @@ -174,56 +95,8 @@ script set -o errexit set -o nounset - . /etc/kube-env - # For a testing cluster, we pull kubelet and kube-proxy binaries, and place them - # in /usr/local/bin. For a non-test cluster, we use the binaries pre-installed - # in the image, or pull and place them in /usr/bin if they are not pre-installed. - BINARY_PATH="/usr/bin/" - if [ "${TEST_CLUSTER:-}" = "true" ]; then - BINARY_PATH="/usr/local/bin/" - fi - if ! which kubelet > /dev/null || ! which kube-proxy > /dev/null || [ "${TEST_CLUSTER:-}" = "true" ]; then - cd /tmp - k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1" - echo "Downloading k8s tar sha1 file ${k8s_sha1}" - curl -Lo "${k8s_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}.sha1" - k8s_tar="${SERVER_BINARY_TAR_URL##*/}" - echo "Downloading k8s tar file ${k8s_tar}" - curl -Lo "${k8s_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}" - # Validate hash. - actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true - if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then - echo "== ${k8s_tar} corrupted, sha1 ${actual} doesn't match expected ${SERVER_BINARY_TAR_HASH} ==" - else - echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}" - fi - tar xzf "/tmp/${k8s_tar}" -C /tmp/ --overwrite - cp /tmp/kubernetes/server/bin/kubelet ${BINARY_PATH} - cp /tmp/kubernetes/server/bin/kube-proxy ${BINARY_PATH} - rm -rf "/tmp/kubernetes" - rm "/tmp/${k8s_tar}" - rm "/tmp/${k8s_sha1}" - fi - - # Put saltbase configuration files in /etc/saltbase. We will use the add-on yaml files. - mkdir -p /etc/saltbase - cd /etc/saltbase - salt_sha1="${SALT_TAR_URL##*/}.sha1" - echo "Downloading Salt tar sha1 file ${salt_sha1}" - curl -Lo "${salt_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}.sha1" - salt_tar="${SALT_TAR_URL##*/}" - echo "Downloading Salt tar file ${salt_tar}" - curl -Lo "${salt_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}" - # Validate hash. - actual=$(sha1sum ${salt_tar} | awk '{ print $1 }') || true - if [ "${actual}" != "${SALT_TAR_HASH}" ]; then - echo "== ${salt_tar} corrupted, sha1 ${actual} doesn't match expected ${SALT_TAR_HASH} ==" - else - echo "Validated ${SALT_TAR_URL} SHA1 = ${SALT_TAR_HASH}" - fi - tar xzf "/etc/saltbase/${salt_tar}" -C /etc/saltbase/ --overwrite - rm "/etc/saltbase/${salt_sha1}" - rm "/etc/saltbase/${salt_tar}" + . /etc/kube-configure.sh + install_kube_binary_config end script --==================================== @@ -325,22 +198,8 @@ script set -o errexit set -o nounset - . /etc/kube-env - # Assemble docker deamon options - DOCKER_OPTS="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false" - if [ "${TEST_CLUSTER:-}" = "true" ]; then - DOCKER_OPTS="${DOCKER_OPTS} --log-level=debug" - fi - echo "DOCKER_OPTS=\"${DOCKER_OPTS} ${EXTRA_DOCKER_OPTS}\"" > /etc/default/docker - # Make sure the network interface cbr0 is created before restarting docker daemon - while ! [ -L /sys/class/net/cbr0 ]; do - echo "Sleep 1 second to wait for cbr0" - sleep 1 - done - initctl restart docker - # Remove docker0 - ifconfig docker0 down - brctl delbr docker0 + . /etc/kube-configure.sh + restart_docker_daemon end script --==================================== @@ -416,4 +275,3 @@ script end script --====================================-- - diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index e481eb40ab..d1e80d4783 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -351,6 +351,7 @@ function get-template-name-from-version { # $2: The scopes flag. # $3: The minion start script metadata from file. # $4: The kube-env metadata. +# $5(optional): Additional metadata for Ubuntu trusty nodes. function create-node-template { detect-project local template_name="$1" @@ -386,7 +387,7 @@ function create-node-template { ${preemptible_minions} \ $2 \ --can-ip-forward \ - --metadata-from-file "$3","$4" >&2; then + --metadata-from-file $(echo ${@:3} | tr ' ' ',') >&2; then if (( attempt > 5 )); then echo -e "${color_red}Failed to create instance template $template_name ${color_norm}" >&2 exit 2