diff --git a/_site/css/styles.css b/_site/css/styles.css index 222904f218..12fb3ed2e0 100755 --- a/_site/css/styles.css +++ b/_site/css/styles.css @@ -1 +1,1258 @@ -*,.button{box-sizing:border-box;background:none;margin:0;border:0}*,.button,body{font-family:Roboto,sans-serif}h1,h2,h5,p{font-weight:300}h3,h4{font-weight:400}body.overlay-on{overflow:hidden}.pi-modal-close-button{position:absolute;top:50px;right:50px;width:36px;height:36px;border:0}.pi-modal-close-button:after,.pi-modal-close-button:before{content:"";position:absolute;top:0;width:50.91169px;height:2px;background-color:#fff}.pi-modal-close-button:before{left:0;-webkit-transform-origin:top left;transform-origin:top left;-webkit-transform:rotate(45deg);transform:rotate(45deg)}.pi-modal-close-button:after{right:0;-webkit-transform-origin:top right;transform-origin:top right;-webkit-transform:rotate(-45deg);transform:rotate(-45deg)}.pi-modal-overlay{position:fixed;top:0;left:0;width:100%;height:100%;color:#fff;background-color:rgba(0,0,0,.8);display:none;opacity:0;transition:.3s}.pi-modal-overlay.on{opacity:1}.pi-multi-frame-display{position:relative;width:80vw;height:80vh;margin:10vh auto}.pi-multi-frame-display .item-wrapper{position:relative;width:100%;height:100%}.pi-multi-frame-display .item{position:absolute;top:50%;left:50%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%);display:none}.pi-multi-frame-display .item.on{display:block}.pi-multi-frame-display .pager{position:absolute;bottom:10%;left:0;width:100%;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-justify-content:center;-ms-flex-pack:center;justify-content:center}.pi-multi-frame-display .pager .pager-button{width:20px;height:20px;border-radius:50%;background-color:#000;border:0;margin:0 10px;transition:.3s}.pi-multi-frame-display .pager .pager-button.yah{background-color:#0f9}.pi-multi-frame-display .next-button,.pi-multi-frame-display .prev-button{position:absolute;top:50%;width:30px;height:30px;border:0;border-color:#0f9;z-index:2;-webkit-transform:translateY(-50%) rotate(45deg);transform:translateY(-50%) rotate(45deg);transition:.3s}.pi-multi-frame-display .next-button:hover,.pi-multi-frame-display .prev-button:hover{border-color:red}.pi-multi-frame-display .prev-button{left:30px;border-bottom:1px solid;border-left:1px solid}.pi-multi-frame-display .next-button{right:30px;border-top:1px solid;border-right:1px solid}.pi-accordion{margin-left:-20px}.pi-accordion .container{padding-left:20px}.pi-accordion .item{overflow:hidden}.pi-accordion .title{color:#303030;position:relative;padding:5px 10px 10px 20px;cursor:pointer;transition:.3s}.pi-accordion .title:hover{color:#3371e3}.pi-accordion a.item>.title{color:#000;padding-left:0}.pi-accordion a.item>.title:hover{color:#3371e3}.pi-accordion div.item>.title{font-weight:700}.pi-accordion div.item>.title:before{content:"";position:absolute;top:9px;left:2px;border-style:solid;border-width:5px 0 5px 8px;border-color:transparent transparent transparent #3371e3;-webkit-transform:rotate(0deg);transform:rotate(0deg);transition:.3s}.pi-accordion .wrapper{position:relative;width:100%;transition:.3s}.pi-accordion .content{padding-left:20px;opacity:0;transition:.3s;padding-bottom:20px}.pi-accordion .item.on>.title:before{-webkit-transform:rotate(90deg);transform:rotate(90deg)}.pi-accordion .item.on>.wrapper>.content{opacity:1}.pi-dialog .dialog-box{position:absolute;top:50%;left:50%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%)}.pi-dialog .title{font-size:36px;text-align:center;padding:30px 0}.pi-dialog .content-box{color:#222;max-width:600px;overflow:hidden}.pi-dialog .content-box .content{padding:30px;border-radius:4px;background-color:#fff}.pi-dialog .content-box .buttons{border-top:1px solid #000;text-align:right;margin-top:30px}.pi-dialog .content-box .buttons button{color:#0f9;border-color:#0f9;border-radius:4px;transition:.3s}.pi-dialog .content-box .buttons button:hover{background-color:#0f9;color:#000}.pi-pushmenu{display:none;position:fixed;top:0;width:100%;height:100%;opacity:0;transition:opacity .3s}.pi-pushmenu.on{opacity:1}.pi-pushmenu .overlay{position:fixed;top:0;left:0;width:100%;height:100%;background-color:rgba(0,0,0,.4)}.pi-pushmenu .sled{position:absolute;top:0;width:0;height:100%;background-color:#fff;overflow:auto;transition:width .3s}.pi-pushmenu.on .sled{width:320px}.pi-pushmenu .top-bar{height:0;line-height:60px;background-color:#444}.pi-pushmenu .pi-modal-close-button{position:absolute;top:10px;right:10px;z-index:1}.pi-pushmenu .pi-modal-close-button:after,.pi-pushmenu .pi-modal-close-button:before{background-color:#222}.pi-pushmenu ul{padding:0 60px 0 20px}.pi-pushmenu li{position:relative;display:block;width:100%;min-height:44px}.pi-pushmenu a{display:inline-block;width:100%;height:44px;line-height:44px;font-weight:700;font-size:20px;color:#3371e3}.pi-pushmenu .button{background:none;padding:0}.pi-pushmenu ul ul{padding:0 20px}.pi-pushmenu ul ul li{min-height:40px}.pi-pushmenu ul ul a{height:40px;line-height:40px;font-size:18px;color:#555}body,html{margin:0;padding:0}button,input{outline:none}button{cursor:pointer}li,ul{list-style:none}ul{margin:0;padding:0}a{text-decoration:none}.clear{display:block;clear:both}.light-text{color:#fff}.right{float:right}.left{float:left}.center{text-align:center}h1{font-size:40px;line-height:50px}h2{font-size:42px;line-height:60px}h3{line-height:32px}h3,h4{font-size:26px}h4{line-height:40px}h5{font-size:18px;line-height:36px}p{font-size:20px;line-height:28px}#vendorStrip,footer,header,section{padding-left:20px;padding-right:20px}#vendorStrip main,footer main,header main,section main{width:100%;max-width:100%}.logo,header{height:50px}.logo{width:120px}.nav-buttons{height:50px;line-height:50px}.nav-buttons .button+*{margin-left:30px}#hamburger{width:50px;height:50px}#mainNav{padding:140px 0 30px}#mainNav h5{margin-bottom:1em}#mainNav h3{margin-bottom:.6em}#mainNav .nav-box{width:20%}#mainNav .nav-box+.nav-box{margin-left:6.66667%}#mainNav main+main{margin-top:60px}#mainNav .left .button{height:50px;line-height:50px;font-size:18px}.open-nav #tryKubernetes,.y-enough #tryKubernetes{margin-left:30px}#hero{padding-top:80px}#docs #hero h1,#docs #hero h5{padding-left:20px;padding-right:20px}#headlineWrapper,#quickstartButton{margin-bottom:40px}#quickstartButton{padding:0 50px}#vendorStrip{height:60px;line-height:60px;font-size:18px}#vendorStrip img{max-height:30px;max-width:100px;margin-left:50px}#oceanNodes{padding-top:60px;padding-bottom:60px}#video{height:200px}#features h3{margin-bottom:20px}#features .feature-box{margin-bottom:0}#features .feature-box>div{width:100%;margin-bottom:40px}body,section{background-color:#fff}section{position:relative}footer main,header main,section main{position:relative;margin:auto}h3{text-align:center;color:#3371e3}.button{display:inline-block;border-radius:6px;padding:0 20px;line-height:40px;color:#fff;background-color:#3371e3;text-decoration:none}#cellophane{height:100%;display:none}#cellophane,header{position:fixed;top:0;left:0;width:100%}header{z-index:4;background-color:transparent;box-shadow:0 0 0 transparent;overflow:hidden;transition:.3s}.logo{float:left;background-image:url(../images/nav_logo.svg);background-size:contain;background-position:center center;background-repeat:no-repeat}.nav-buttons{float:right}#tryKubernetes,#viewDocs{display:none}#viewDocs{border:1px solid #fff;background-color:transparent;transition:.3s}#viewDocs:hover{background-color:#fff;color:#303030}#tryKubernetes{width:0;margin-left:0;text-align:center;overflow:hidden;white-space:nowrap;transition:.3s}#hamburger,#tryKubernetes{padding:0;vertical-align:middle}#hamburger{display:inline-block;position:relative;border:0;background:none}#hamburger:after,#hamburger:before,#hamburger div{position:absolute;left:15%;width:70%;height:2px;background-color:#3371e3;transition:.3s;content:""}#hamburger div{top:calc(50% - 1px)}#hamburger:before{top:24%}#hamburger:after{bottom:24%}#hamburger:hover:after,#hamburger:hover:before,#hamburger:hover div{background-color:#fff}#mainNav h5{color:#3371e3;font-weight:400}#mainNav main{white-space:nowrap;overflow:hidden;clear:both}#mainNav .nav-box{float:left;white-space:normal}#mainNav h3 a{color:#3371e3;text-decoration:none}.flip-nav header{background-color:#fff}.open-nav body{overflow:hidden}.open-nav #cellophane{display:block;z-index:5}.open-nav header{background-color:#f7f7f7;z-index:6}.open-nav #hamburger div{opacity:0}.open-nav #hamburger:after,.open-nav #hamburger:before{left:12px;-webkit-transform-origin:0 1px;transform-origin:0 1px}.open-nav #hamburger:before{-webkit-transform:rotate(45deg);transform:rotate(45deg)}.open-nav #hamburger:after{-webkit-transform:rotate(-45deg);transform:rotate(-45deg)}.open-nav #tryKubernetes,.y-enough #tryKubernetes{width:160px}.flip-nav header,.open-nav header{box-shadow:0 1px 2px #4c4c4c}.flip-nav header .logo,.open-nav header .logo{background-image:url(../images/nav_logo2.svg)}.flip-nav #viewDocs,.open-nav #viewDocs{border-color:#303030;color:#303030}.flip-nav #viewDocs:hover,.open-nav #viewDocs:hover{border-color:#3371e3;background-color:#3371e3;color:#fff}#hero,.flip-nav #hamburger:hover:after,.flip-nav #hamburger:hover:before,.flip-nav #hamburger:hover div,.open-nav #hamburger:hover:after,.open-nav #hamburger:hover:before,.open-nav #hamburger:hover div{background-color:#303030}#hero{background-image:url(../images/texture.png);text-align:center;padding-left:0;padding-right:0}#headlineWrapper{text-align:center;padding:0 20px}#quickstartButton{display:none;font-size:24px;line-height:50px;padding:0 50px}#vendorStrip{display:none;background-color:hsla(0,0%,100%,.1);font-weight:100;white-space:nowrap;text-align:left}#vendorStrip div{height:100%;display:inline-block}#vendorStrip div,#vendorStrip img{vertical-align:middle}#oceanNodes main{margin-bottom:60px}#oceanNodes .image-wrapper{max-width:75%;margin:0 auto 20px}#oceanNodes .image-wrapper img{width:100%}#oceanNodes main:first-child .image-wrapper{max-width:100%}#oceanNodes h3{margin-bottom:30px}#video{width:100%;position:relative;background-image:url(../images/kub_video_thm.jpg);background-position:center center;background-size:cover}#video>.light-text{display:none;position:absolute;top:50%;left:75%;width:50%;max-width:600px;min-width:500px;padding-right:10%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%);color:#fff}#video h2{font-size:32px;line-height:44px}#video h2,#video p{margin-bottom:20px}#video #desktopShowVideoButton{position:relative;font-size:24px;background-color:#fff;border-radius:8px;color:#3371e3;padding:15px 30px 15px 80px}#video #desktopShowVideoButton:before{content:"";position:absolute;top:50%;left:40px;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%);width:0;height:0;border-style:solid;border-width:10px 0 10px 20px;border-color:transparent transparent transparent #3371e3}#video #mobileShowVideoButton{left:50%;width:80px;height:80px;border-radius:50%;background-color:transparent;border:5px solid hsla(0,0%,100%,.2);overflow:visible}#video #mobileShowVideoButton,#video #mobileShowVideoButton:after{position:absolute;top:50%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%)}#video #mobileShowVideoButton:after{left:50%;left:40px;content:"";width:0;height:0;border-style:solid;border-width:20px 0 20px 30px;border-color:transparent transparent transparent #fff}#features{padding-top:140px;background-color:#f7f7f7;background-image:url(../images/wheel.png);background-position:center 60px;background-repeat:no-repeat;background-size:auto}.feature-box{width:100%;overflow:hidden;clear:both}.feature-box h4{line-height:normal;margin-bottom:15px}.feature-box>div:first-child{float:left}.feature-box>div:last-child{float:right}#talkToUs main{padding:50px 0}#talkToUs h3,#talkToUs h4{text-align:center}#talkToUs h3{margin-bottom:15px}#talkToUs h4{line-height:normal;margin-bottom:50px}#talkToUs #bigSocial{overflow:hidden}#talkToUs #bigSocial div{width:calc(25% - 15px);float:left;padding:30px;padding-top:110px;background-position:center top;background-size:auto;background-repeat:no-repeat}#talkToUs #bigSocial div:nth-child(1){background-image:url(../images/twitter_icon.png)}#talkToUs #bigSocial div:nth-child(2){background-image:url(../images/github_icon.png)}#talkToUs #bigSocial div:nth-child(3){background-image:url(../images/slack_icon.png)}#talkToUs #bigSocial div:nth-child(4){background-image:url(../images/stackoverflow_icon.png)}#talkToUs #bigSocial div+div{margin-left:20px}#talkToUs #bigSocial a{display:inline-block;color:#3371e3;font-size:24px;font-weight:400;text-decoration:none;margin-bottom:15px}footer{width:100%;background-image:url(../images/texture.png);background-color:#303030}footer main{padding:40px 0}footer nav{text-align:center;white-space:nowrap}footer nav a{display:inline-block;font-size:32px;font-weight:300;color:#fff;text-decoration:none}footer nav a+a{margin-left:10%}footer .social{position:relative;margin:80px 0 60px}footer .social label{float:right;display:inline-block;height:50px;line-height:50px;font-weight:100;white-space:nowrap}footer .social label input{margin-left:8px;width:300px}#search,#wishField{background-color:transparent;padding:10px;font-size:16px;font-weight:100;color:#fff;border:1px solid #fff;transition:.3s}#search:focus,#wishField:focus{background-color:#f7f7f7;color:#303030}.social a{display:inline-block;background-image:url(../images/social_sprite.png);background-repeat:no-repeat;background-size:auto;width:50px;height:50px;border-radius:5px;margin-right:30px}.social a:hover{background-color:#fff}.social a span{position:absolute;display:block;height:0;overflow:hidden}a.twitter{background-position:0 0}a.twitter:hover{background-position:0 100%}a.stack-overflow{background-position:-50px 0}a.stack-overflow:hover{background-position:-50px 100%}a.slack{background-position:-100px 0}a.slack:hover{background-position:-100px 100%}a.github{background-position:-150px 0}a.github:hover{background-position:-150px 100%}a.mailing-list{background-position:-200px 0}a.mailing-list:hover{background-position:-200px 100%}iframe{width:80vw;height:45vw;max-width:142.22222vh;max-height:80vh;position:absolute;top:50%;left:50%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%)}#docs #viewDocs{display:none}#docs section{background-color:#fff;min-width:1024px}#docs #hero{background-color:#303030}#docs #hero h5{margin-bottom:40px}#docs #vendorStrip{position:relative;white-space:nowrap}#docs #vendorStrip ul{float:left}#docs #vendorStrip li{display:inline-block;height:100%}#docs #vendorStrip a{display:block;height:100%;color:#fff}#docs #vendorStrip #search,#docs #vendorStrip li+li{margin-left:60px}#docs #vendorStrip #search{position:relative;float:right;width:250px;padding:0 10px;margin:10px 30px 0 0;height:40px;line-height:40px;border:1px solid #f7f7f7;font-size:16px;vertical-align:middle}#docs #vendorStrip .dropdown{position:relative;float:right;height:60px;font-size:20px;color:#fff;line-height:60px;cursor:pointer;overflow:hidden;z-index:3}#docs #vendorStrip .dropdown.on{overflow:visible}#docs #vendorStrip .dropdown .readout{padding-left:10px;padding-right:40px;transition:.3s}#docs #vendorStrip .dropdown .readout:after{content:"";position:absolute;top:30px;right:10px;width:0;height:0;border-style:solid;border-width:6px 6px 0;border-color:#fff transparent transparent;transition:.3s}#docs #vendorStrip .dropdown .readout:hover{background-color:hsla(0,0%,100%,.1)}#docs #vendorStrip .dropdown.on .readout{background-color:#f7f7f7;color:#303030}#docs #vendorStrip .dropdown.on .readout:after{border-top-color:#303030}#docs #vendorStrip .dropdown .wrapper{position:absolute;top:100%;left:0;width:100%}#docs #vendorStrip .dropdown a{display:block;width:100%;height:30px;line-height:30px;padding:0 10px;font-size:16px;font-weight:300;background-color:#f7f7f7;color:#303030;opacity:0;transition:.3s}#docs #vendorStrip .dropdown a:hover{background-color:#ccc}#docs #vendorStrip .dropdown.on a{opacity:1}#docs #encyclopedia{position:relative;padding:50px;overflow:hidden}#docs #encyclopedia>div{height:100%}#docs #encyclopedia #docsToc{float:left;width:350px;padding:0 20px 0 0;border-right:1px solid #ccc}#docs #encyclopedia #docsContent{float:right;width:calc(100% - 400px)}.fixed footer{position:fixed;bottom:0}@media screen and (min-width:750px){h1{font-size:60px;line-height:90px}h2{font-size:42px;line-height:60px}h3{font-size:30px}h3,h4{line-height:40px}h4{font-size:26px}h5{font-size:22px;line-height:36px}p{font-size:20px;line-height:28px}#vendorStrip,footer,header,section{padding-left:20px;padding-right:20px}#vendorStrip main,footer main,header main,section main{width:100%;max-width:100%}.logo,header{height:60px}.logo{width:120px}.nav-buttons{height:60px;line-height:60px}.nav-buttons .button+*{margin-left:30px}#hamburger{width:50px;height:50px}#mainNav{padding:140px 0 30px}#mainNav h5{margin-bottom:1em}#mainNav h3{margin-bottom:.6em}#mainNav .nav-box{width:20%}#mainNav .nav-box+.nav-box{margin-left:6.66667%}#mainNav main+main{margin-top:60px}#mainNav .left .button{height:50px;line-height:50px;font-size:18px}.open-nav #tryKubernetes,.y-enough #tryKubernetes{margin-left:30px}#hero{padding-top:80px}#docs #hero h1,#docs #hero h5{padding-left:20px;padding-right:20px}#headlineWrapper,#quickstartButton{margin-bottom:40px}#quickstartButton{padding:0 50px}#vendorStrip{height:60px;line-height:60px;font-size:18px}#vendorStrip img{max-height:30px;max-width:100px;margin-left:50px}#oceanNodes{padding-top:60px;padding-bottom:60px}#oceanNodes main{margin-bottom:60px}#video{height:400px}#features h3{margin-bottom:40px}#features .feature-box{margin-bottom:60px}#features .feature-box>div{width:45%;margin-bottom:0}#docs #hero h1,#docs #hero h5{text-align:left}#tryKubernetes,#viewDocs{display:inline-block}#vendorStrip{display:block}#quickstartButton{display:inline-block}#oceanNodes h3{text-align:left}#oceanNodes main{position:relative;clear:both}#oceanNodes main .image-wrapper{position:absolute;top:50%;max-width:25%;max-height:100%;-webkit-transform:translateY(-50%);transform:translateY(-50%)}#oceanNodes main:nth-child(odd) .content{padding-right:30%}#oceanNodes main:nth-child(odd) .image-wrapper{right:0}#oceanNodes main:nth-child(even) .content{padding-left:30%}#oceanNodes main:nth-child(even) .image-wrapper{left:0}#oceanNodes main:nth-child(1) .content{padding-right:0}#oceanNodes main:nth-child(1) .image-wrapper{position:relative;max-width:100%;-webkit-transform:none;transform:none}#oceanNodes main img{width:100%}#video,#video>.light-text{display:block}#mobileShowVideoButton{display:none}#features{padding-bottom:60px}#features .feature-box:last-child{margin-bottom:0}}@media screen and (min-width:1025px){h1{font-size:60px;line-height:90px}h2{font-size:42px;line-height:60px}h3{font-size:30px}h3,h4{line-height:40px}h4{font-size:26px}h5{font-size:22px;line-height:36px}p{font-size:20px;line-height:28px}#vendorStrip,footer,header,section{padding-left:50px;padding-right:50px}#vendorStrip main,footer main,header main,section main{width:100%;max-width:1200px}.logo,header{height:80px}.logo{width:180px}.nav-buttons{height:80px;line-height:80px}.nav-buttons .button+*{margin-left:30px}#hamburger{width:50px;height:50px}#mainNav{padding:140px 0 30px}#mainNav h5{margin-bottom:1em}#mainNav h3{margin-bottom:.6em}#mainNav .nav-box{width:20%}#mainNav .nav-box+.nav-box{margin-left:6.66667%}#mainNav main+main{margin-top:60px}#mainNav .left .button{height:50px;line-height:50px;font-size:18px}.open-nav #tryKubernetes,.y-enough #tryKubernetes{margin-left:30px}#hero{padding-top:80px}#docs #hero h1,#docs #hero h5{padding-left:50px;padding-right:50px}#headlineWrapper,#quickstartButton{margin-bottom:40px}#quickstartButton{padding:0 50px}#vendorStrip{height:60px;line-height:60px;font-size:18px}#vendorStrip img{max-height:30px;max-width:100px;margin-left:50px}#oceanNodes{padding-top:60px;padding-bottom:60px}#oceanNodes main{margin-bottom:60px}#video{height:550px}#features h3{margin-bottom:40px}#features .feature-box{margin-bottom:60px}#features .feature-box>div{width:45%;margin-bottom:0}#oceanNodes main:nth-child(1){padding-right:50%}#oceanNodes main:nth-child(1) .image-wrapper{position:absolute;max-width:50%;-webkit-transform:translateY(-50%);transform:translateY(-50%)}#video{position:relative;background-image:url(../images/kub_video_thm.jpg);background-position:center center;background-size:cover;height:550px}} +html, body { + margin: 0; + padding: 0; } + +input, button { + outline: none; } + +button { + cursor: pointer; } + +ul, li { + list-style: none; } + +ul { + margin: 0; + padding: 0; } + +a { + text-decoration: none; } + +.clear { + display: block; + clear: both; } + +.light-text { + color: white; } + +.right { + float: right; } + +.left { + float: left; } + +.center { + text-align: center; } + +*, .button { + box-sizing: border-box; + font-family: "Roboto", sans-serif; + background: none; + margin: 0; + border: 0; } + +body { + font-family: "Roboto", sans-serif; } + +h1, h2, h5, p { + font-weight: 300; } + +h3, h4 { + font-weight: 400; } + +body.overlay-on { + overflow: hidden; } + +.pi-modal-close-button { + position: absolute; + top: 50px; + right: 50px; + width: 36px; + height: 36px; + border: 0; } + .pi-modal-close-button:before, .pi-modal-close-button:after { + content: ""; + position: absolute; + top: 0; + width: 50.91169px; + height: 2px; + background-color: white; } + .pi-modal-close-button:before { + left: 0; + -webkit-transform-origin: top left; + -ms-transform-origin: top left; + transform-origin: top left; + -webkit-transform: rotateZ(45deg); + -ms-transform: rotate(45deg); + transform: rotateZ(45deg); } + .pi-modal-close-button:after { + right: 0; + -webkit-transform-origin: top right; + -ms-transform-origin: top right; + transform-origin: top right; + -webkit-transform: rotateZ(-45deg); + -ms-transform: rotate(-45deg); + transform: rotateZ(-45deg); } + +.pi-modal-overlay { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + color: white; + background-color: rgba(0, 0, 0, 0.8); + display: none; + opacity: 0; + transition: 0.3s; } + .pi-modal-overlay.on { + opacity: 1; } + +.pi-multi-frame-display { + position: relative; + width: 80vw; + height: 80vh; + margin: 10vh auto; } + .pi-multi-frame-display .item-wrapper { + position: relative; + width: 100%; + height: 100%; } + .pi-multi-frame-display .item { + position: absolute; + top: 50%; + left: 50%; + -webkit-transform: translate(-50%, -50%); + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); + display: none; } + .pi-multi-frame-display .item.on { + display: block; } + .pi-multi-frame-display .pager { + position: absolute; + bottom: 10%; + left: 0; + width: 100%; + display: -webkit-flex; + display: -ms-flexbox; + display: flex; + -webkit-justify-content: center; + -ms-flex-pack: center; + justify-content: center; } + .pi-multi-frame-display .pager .pager-button { + width: 20px; + height: 20px; + border-radius: 50%; + background-color: black; + border: 0; + margin: 0 10px; + transition: 0.3s; } + .pi-multi-frame-display .pager .pager-button.yah { + background-color: #00ff99; } + .pi-multi-frame-display .prev-button, .pi-multi-frame-display .next-button { + position: absolute; + top: 50%; + width: 30px; + height: 30px; + border: 0; + border-color: #00ff99; + z-index: 3; + -webkit-transform: translateY(-50%) rotateZ(45deg); + -ms-transform: translateY(-50%) rotate(45deg); + transform: translateY(-50%) rotateZ(45deg); + transition: 0.3s; } + .pi-multi-frame-display .prev-button:hover, .pi-multi-frame-display .next-button:hover { + border-color: red; } + .pi-multi-frame-display .prev-button { + left: 30px; + border-bottom: 1px solid; + border-left: 1px solid; } + .pi-multi-frame-display .next-button { + right: 30px; + border-top: 1px solid; + border-right: 1px solid; } + +.pi-accordion { + margin-left: -20px; } + .pi-accordion .container { + padding-left: 20px; } + .pi-accordion .item { + overflow: hidden; } + .pi-accordion .title { + color: #303030; + position: relative; + padding: 5px 10px 10px 20px; + cursor: pointer; + transition: 0.3s; } + .pi-accordion .title:hover { + color: #3371e3; } + .pi-accordion a.item > .title { + color: black; + padding-left: 0; } + .pi-accordion a.item > .title:hover { + color: #3371e3; } + .pi-accordion div.item > .title { + font-weight: 700; } + .pi-accordion div.item > .title:before { + content: ""; + position: absolute; + top: 9px; + left: 2px; + border-style: solid; + border-width: 5px 0 5px 8px; + border-color: transparent transparent transparent #3371e3; + -webkit-transform: rotate(0deg); + -ms-transform: rotate(0deg); + transform: rotate(0deg); + transition: 0.3s; } + .pi-accordion .wrapper { + position: relative; + width: 100%; + transition: 0.3s; } + .pi-accordion .content { + padding-left: 20px; + opacity: 0; + transition: 0.3s; + padding-bottom: 20px; } + .pi-accordion .item.on > .title:before { + -webkit-transform: rotate(90deg); + -ms-transform: rotate(90deg); + transform: rotate(90deg); } + .pi-accordion .item.on > .wrapper > .content { + opacity: 1; } + +.pi-dialog .dialog-box { + position: absolute; + top: 50%; + left: 50%; + -webkit-transform: translate(-50%, -50%); + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); } + +.pi-dialog .title { + font-size: 36px; + text-align: center; + padding: 30px 0; } + +.pi-dialog .content-box { + color: #222; + max-width: 600px; + overflow: hidden; } + .pi-dialog .content-box .content { + padding: 30px; + border-radius: 4px; + background-color: white; } + .pi-dialog .content-box .buttons { + border-top: 1px solid black; + text-align: right; + margin-top: 30px; } + .pi-dialog .content-box .buttons button { + color: #00ff99; + border-color: #00ff99; + border-radius: 4px; + transition: 0.3s; } + .pi-dialog .content-box .buttons button:hover { + background-color: #00ff99; + color: black; } + +.pi-pushmenu { + display: none; + position: fixed; + top: 0; + width: 100%; + height: 100%; + opacity: 0; + transition: opacity 0.3s; } + .pi-pushmenu.on { + opacity: 1; } + .pi-pushmenu .overlay { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: rgba(0, 0, 0, 0.4); } + .pi-pushmenu .sled { + position: absolute; + top: 0; + width: 0; + height: 100%; + background-color: white; + overflow: auto; + transition: width 0.3s; } + .pi-pushmenu.on .sled { + width: 320px; } + .pi-pushmenu .top-bar { + height: 0; + line-height: 60px; + background-color: #444; } + .pi-pushmenu .pi-modal-close-button { + position: absolute; + top: 10px; + right: 10px; + z-index: 1; } + .pi-pushmenu .pi-modal-close-button:before, .pi-pushmenu .pi-modal-close-button:after { + background-color: #222222; } + .pi-pushmenu ul { + padding: 0 60px 0 20px; } + .pi-pushmenu li { + position: relative; + display: block; + width: 100%; + min-height: 44px; } + .pi-pushmenu a { + display: inline-block; + width: 100%; + height: 44px; + line-height: 44px; + font-weight: bold; + font-size: 20px; + color: #3371e3; } + .pi-pushmenu .button { + background: none; + padding: 0; } + .pi-pushmenu ul ul { + padding: 0 20px; } + .pi-pushmenu ul ul li { + min-height: 40px; } + .pi-pushmenu ul ul a { + height: 40px; + line-height: 40px; + font-size: 18px; + color: #555555; } + +html, body { + margin: 0; + padding: 0; } + +input, button { + outline: none; } + +button { + cursor: pointer; } + +ul, li { + list-style: none; } + +ul { + margin: 0; + padding: 0; } + +a { + text-decoration: none; } + +.clear { + display: block; + clear: both; } + +.light-text { + color: white; } + +.right { + float: right; } + +.left { + float: left; } + +.center { + text-align: center; } + +h1 { + font-size: 40px; + line-height: 50px; } + +h2 { + font-size: 42px; + line-height: 60px; } + +h3 { + font-size: 26px; + line-height: 32px; } + +h4 { + font-size: 26px; + line-height: 40px; } + +h5 { + font-size: 18px; + line-height: 36px; } + +p { + font-size: 20px; + line-height: 28px; } + +section, header, footer, #vendorStrip { + padding-left: 20px; + padding-right: 20px; } + section main, header main, footer main, #vendorStrip main { + width: 100%; + max-width: 100%; } + +header { + height: 50px; } + +.logo { + width: 120px; + height: 50px; } + +.nav-buttons { + height: 50px; + line-height: 50px; } + .nav-buttons .button + * { + margin-left: 30px; } + +#hamburger { + width: 50px; + height: 50px; } + +#mainNav { + padding: 140px 0 30px; } + #mainNav h5 { + margin-bottom: 1em; } + #mainNav h3 { + margin-bottom: 0.6em; } + #mainNav .nav-box { + width: 20%; } + #mainNav .nav-box + .nav-box { + margin-left: calc(20% / 3); } + #mainNav main + main { + margin-top: 60px; } + #mainNav .left .button { + height: 50px; + line-height: 50px; + font-size: 18px; } + +.open-nav #tryKubernetes, .y-enough #tryKubernetes { + margin-left: 30px; } + +#hero { + padding-top: 80px; } + +#docs #hero h1, #docs #hero h5 { + padding-left: 20px; + padding-right: 20px; } + +#headlineWrapper, #quickstartButton { + margin-bottom: 40px; } + +#quickstartButton { + padding: 0 50px; } + +#vendorStrip { + height: 60px; + line-height: 60px; + font-size: 18px; } + #vendorStrip img { + max-height: 30px; + max-width: 100px; + margin-left: 50px; } + +#oceanNodes { + padding-top: 60px; + padding-bottom: 60px; } + #oceanNodes main { + margin-bottom: 60px; } + +#video { + height: 200px; } + +#features h3 { + margin-bottom: 20px; } + +#features .feature-box { + margin-bottom: 0; } + #features .feature-box > div { + width: 100%; + margin-bottom: 40px; } + +body { + background-color: white; } + +section { + position: relative; + background-color: white; } + +section main, header main, footer main { + position: relative; + margin: auto; } + +h3 { + text-align: center; + color: #3371e3; } + +.button { + display: inline-block; + border-radius: 6px; + padding: 0 20px; + line-height: 40px; + color: white; + background-color: #3371e3; + text-decoration: none; } + +#cellophane { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: none; } + +header { + position: fixed; + top: 0; + left: 0; + width: 100%; + z-index: 8888; + background-color: transparent; + box-shadow: 0 0 0 transparent; + overflow: hidden; + transition: 0.3s; } + +.logo { + float: left; + background-image: url(../images/nav_logo.svg); + background-size: contain; + background-position: center center; + background-repeat: no-repeat; } + +.nav-buttons { + float: right; } + +#viewDocs, #tryKubernetes { + display: none; } + +#viewDocs { + border: 1px solid white; + background-color: transparent; + transition: 0.3s; } + #viewDocs:hover { + background-color: white; + color: #303030; } + +#tryKubernetes { + width: 0; + padding: 0; + margin-left: 0; + vertical-align: middle; + text-align: center; + overflow: hidden; + white-space: nowrap; + transition: 0.3s; } + +#hamburger { + display: inline-block; + position: relative; + vertical-align: middle; + padding: 0; + border: 0; + background: none; } + #hamburger div, #hamburger:before, #hamburger:after { + position: absolute; + left: 15%; + width: 70%; + height: 2px; + background-color: #3371e3; + transition: 0.3s; + content: ""; } + #hamburger div { + top: calc(50% - 1px); } + #hamburger:before { + top: 24%; } + #hamburger:after { + bottom: 24%; } + #hamburger:hover div, #hamburger:hover:before, #hamburger:hover:after { + background-color: white; } + +#mainNav h5 { + color: #3371e3; + font-weight: normal; } + +#mainNav main { + white-space: nowrap; + overflow: hidden; + clear: both; } + +#mainNav .nav-box { + float: left; + white-space: normal; } + +#mainNav h3 a { + color: #3371e3; + text-decoration: none; } + +.flip-nav header { + background-color: white; } + +.open-nav body { + overflow: hidden; } + +.open-nav #cellophane { + display: block; + z-index: 9998; } + +.open-nav header { + background-color: #f7f7f7; + z-index: 9999; } + +.open-nav #hamburger div { + opacity: 0; } + +.open-nav #hamburger:before, .open-nav #hamburger:after { + left: 12px; + -webkit-transform-origin: 0 1px; + -ms-transform-origin: 0 1px; + transform-origin: 0 1px; } + +.open-nav #hamburger:before { + -webkit-transform: rotate(45deg); + -ms-transform: rotate(45deg); + transform: rotate(45deg); } + +.open-nav #hamburger:after { + -webkit-transform: rotate(-45deg); + -ms-transform: rotate(-45deg); + transform: rotate(-45deg); } + +.open-nav #tryKubernetes, .y-enough #tryKubernetes { + width: 160px; } + +.flip-nav header, .open-nav header { + box-shadow: 0 1px 2px #4c4c4c; } + .flip-nav header .logo, .open-nav header .logo { + background-image: url(../images/nav_logo2.svg); } + +.flip-nav #viewDocs, .open-nav #viewDocs { + border-color: #303030; + color: #303030; } + .flip-nav #viewDocs:hover, .open-nav #viewDocs:hover { + border-color: #3371e3; + background-color: #3371e3; + color: white; } + +.flip-nav #hamburger:hover div, .flip-nav #hamburger:hover:before, .flip-nav #hamburger:hover:after, .open-nav #hamburger:hover div, .open-nav #hamburger:hover:before, .open-nav #hamburger:hover:after { + background-color: #303030; } + +#hero { + background-image: url(../images/texture.png); + background-color: #303030; + text-align: center; + padding-left: 0; + padding-right: 0; } + +#headlineWrapper { + text-align: center; + padding: 0 20px; } + +#quickstartButton { + display: none; + font-size: 24px; + line-height: 50px; + padding: 0 50px; } + +#vendorStrip { + display: none; + background-color: rgba(255, 255, 255, 0.1); + font-weight: 100; + white-space: nowrap; + text-align: left; } + #vendorStrip div { + vertical-align: middle; + height: 100%; + display: inline-block; } + #vendorStrip img { + vertical-align: middle; } + +#oceanNodes main { + margin-bottom: 60px; } + +#oceanNodes .image-wrapper { + max-width: 75%; + margin: 0 auto 20px; } + #oceanNodes .image-wrapper img { + width: 100%; } + +#oceanNodes main:first-child .image-wrapper { + max-width: 100%; } + +#oceanNodes h3 { + margin-bottom: 30px; } + +#video { + width: 100%; + position: relative; + background-image: url(../images/kub_video_thm.jpg); + background-position: center center; + background-size: cover; } + #video > .light-text { + display: none; + position: absolute; + top: 50%; + left: 75%; + width: 50%; + max-width: 600px; + min-width: 500px; + padding-right: 10%; + -webkit-transform: translate(-50%, -50%); + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); + color: white; } + #video h2 { + font-size: 32px; + line-height: 44px; + margin-bottom: 20px; } + #video p { + margin-bottom: 20px; } + #video #desktopShowVideoButton { + position: relative; + font-size: 24px; + background-color: white; + border-radius: 8px; + color: #3371e3; + padding: 15px 30px 15px 80px; } + #video #desktopShowVideoButton:before { + content: ""; + position: absolute; + position: absolute; + top: 50%; + left: 40px; + -webkit-transform: translate(-50%, -50%); + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); + width: 0; + height: 0; + border-style: solid; + border-width: 10px 0 10px 20px; + border-color: transparent transparent transparent #3371e3; } + #video #mobileShowVideoButton { + position: absolute; + top: 50%; + left: 50%; + -webkit-transform: translate(-50%, -50%); + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); + width: 80px; + height: 80px; + border-radius: 50%; + background-color: transparent; + border: 5px solid rgba(255, 255, 255, 0.2); + overflow: visible; } + #video #mobileShowVideoButton:after { + position: absolute; + top: 50%; + left: 50%; + -webkit-transform: translate(-50%, -50%); + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); + left: 40px; + content: ""; + width: 0; + height: 0; + border-style: solid; + border-width: 20px 0 20px 30px; + border-color: transparent transparent transparent #ffffff; } + +#features { + padding-top: 140px; + background-color: #f7f7f7; + background-image: url(../images/wheel.png); + background-position: center 60px; + background-repeat: no-repeat; + background-size: auto; } + +.feature-box { + width: 100%; + overflow: hidden; + clear: both; } + .feature-box h4 { + line-height: normal; + margin-bottom: 15px; } + .feature-box > div:first-child { + float: left; } + .feature-box > div:last-child { + float: right; } + +#talkToUs main { + padding: 50px 0; } + +#talkToUs h3, #talkToUs h4 { + text-align: center; } + +#talkToUs h3 { + margin-bottom: 15px; } + +#talkToUs h4 { + line-height: normal; + margin-bottom: 50px; } + +#talkToUs #bigSocial { + overflow: hidden; } + #talkToUs #bigSocial div { + width: calc(25% - 15px); + float: left; + padding: 30px; + padding-top: 110px; + background-position: center top; + background-size: auto; + background-repeat: no-repeat; } + #talkToUs #bigSocial div:nth-child(1) { + background-image: url(../images/twitter_icon.png); } + #talkToUs #bigSocial div:nth-child(2) { + background-image: url(../images/github_icon.png); } + #talkToUs #bigSocial div:nth-child(3) { + background-image: url(../images/slack_icon.png); } + #talkToUs #bigSocial div:nth-child(4) { + background-image: url(../images/stackoverflow_icon.png); } + #talkToUs #bigSocial div + div { + margin-left: 20px; } + #talkToUs #bigSocial a { + display: inline-block; + color: #3371e3; + font-size: 24px; + font-weight: 400; + text-decoration: none; + margin-bottom: 15px; } + +footer { + width: 100%; + background-image: url(../images/texture.png); + background-color: #303030; } + footer main { + padding: 40px 0; } + footer nav { + text-align: center; + white-space: nowrap; } + footer nav a { + display: inline-block; + font-size: 32px; + font-weight: 300; + color: white; + text-decoration: none; } + footer nav a + a { + margin-left: 10%; } + footer .social { + position: relative; + margin: 80px 0 60px; } + footer .social label { + float: right; + display: inline-block; + height: 50px; + line-height: 50px; + font-weight: 100; + white-space: nowrap; } + footer .social label input { + margin-left: 8px; + width: 300px; } + +#search, #wishField { + background-color: transparent; + padding: 10px; + font-size: 16px; + font-weight: 100; + color: white; + border: 1px solid white; + transition: 0.3s; } + #search:focus, #wishField:focus { + background-color: #f7f7f7; + color: #303030; } + +.social a { + display: inline-block; + background-image: url(../images/social_sprite.png); + background-repeat: no-repeat; + background-size: auto; + width: 50px; + height: 50px; + border-radius: 5px; + margin-right: 30px; } + .social a:hover { + background-color: #fff; } + .social a span { + position: absolute; + display: block; + height: 0; + overflow: hidden; } + +a.twitter { + background-position: 0 0; } + a.twitter:hover { + background-position: 0 100%; } + +a.stack-overflow { + background-position: -50px 0; } + a.stack-overflow:hover { + background-position: -50px 100%; } + +a.slack { + background-position: -100px 0; } + a.slack:hover { + background-position: -100px 100%; } + +a.github { + background-position: -150px 0; } + a.github:hover { + background-position: -150px 100%; } + +a.mailing-list { + background-position: -200px 0; } + a.mailing-list:hover { + background-position: -200px 100%; } + +iframe { + width: 80vw; + height: 45vw; + max-width: 142.22222vh; + max-height: 80vh; + position: absolute; + top: 50%; + left: 50%; + -webkit-transform: translate(-50%, -50%); + -ms-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); } + +#docs #viewDocs { + display: none; } + +#docs section { + background-color: white; + min-width: 1024px; } + +#docs #hero { + background-color: #303030; } + #docs #hero h5 { + margin-bottom: 40px; } + +#docs #vendorStrip { + position: relative; + white-space: nowrap; } + #docs #vendorStrip ul { + float: left; } + #docs #vendorStrip li { + display: inline-block; + height: 100%; } + #docs #vendorStrip a { + display: block; + height: 100%; + color: white; } + #docs #vendorStrip li + li, #docs #vendorStrip #search { + margin-left: 60px; } + #docs #vendorStrip #search { + position: relative; + float: right; + width: 250px; + padding: 0 10px; + margin: 10px 30px 0 0; + height: 40px; + line-height: 40px; + border: 1px solid #f7f7f7; + font-size: 16px; + vertical-align: middle; } + #docs #vendorStrip .dropdown { + position: relative; + float: right; + height: 60px; + font-size: 20px; + color: white; + line-height: 60px; + cursor: pointer; + overflow: hidden; + z-index: 5000; } + #docs #vendorStrip .dropdown.on { + overflow: visible; } + #docs #vendorStrip .dropdown .readout { + padding-left: 10px; + padding-right: 40px; + transition: 0.3s; } + #docs #vendorStrip .dropdown .readout:after { + content: ""; + position: absolute; + top: 30px; + right: 10px; + width: 0; + height: 0; + border-style: solid; + border-width: 6px 6px 0 6px; + border-color: white transparent transparent transparent; + transition: 0.3s; } + #docs #vendorStrip .dropdown .readout:hover { + background-color: rgba(255, 255, 255, 0.1); } + #docs #vendorStrip .dropdown.on .readout { + background-color: #f7f7f7; + color: #303030; } + #docs #vendorStrip .dropdown.on .readout:after { + border-top-color: #303030; } + #docs #vendorStrip .dropdown .wrapper { + position: absolute; + top: 100%; + left: 0; + width: 100%; } + #docs #vendorStrip .dropdown a { + display: block; + width: 100%; + height: 30px; + line-height: 30px; + padding: 0 10px; + font-size: 16px; + font-weight: 300; + background-color: #f7f7f7; + color: #303030; + opacity: 0; + transition: 0.3s; } + #docs #vendorStrip .dropdown a:hover { + background-color: #cccccc; } + #docs #vendorStrip .dropdown.on a { + opacity: 1; } + +#docs #encyclopedia { + position: relative; + padding: 50px; + overflow: hidden; } + #docs #encyclopedia > div { + height: 100%; } + #docs #encyclopedia #docsToc { + float: left; + width: 350px; + padding: 0 20px 0 0; + border-right: 1px solid #cccccc; } + #docs #encyclopedia #docsToc .yah > .title { + background-color: rgba(51, 113, 227, 0.2); + padding-left: 10px; + padding-right: 10px; } + #docs #encyclopedia #docsContent { + float: right; + width: calc(100% - 400px); } + +.fixed footer { + position: fixed; + bottom: 0; } + +@media screen and (min-width: 750px) { + h1 { + font-size: 60px; + line-height: 90px; } + h2 { + font-size: 42px; + line-height: 60px; } + h3 { + font-size: 30px; + line-height: 40px; } + h4 { + font-size: 26px; + line-height: 40px; } + h5 { + font-size: 22px; + line-height: 36px; } + p { + font-size: 20px; + line-height: 28px; } + section, header, footer, #vendorStrip { + padding-left: 20px; + padding-right: 20px; } + section main, header main, footer main, #vendorStrip main { + width: 100%; + max-width: 100%; } + header { + height: 60px; } + .logo { + width: 120px; + height: 60px; } + .nav-buttons { + height: 60px; + line-height: 60px; } + .nav-buttons .button + * { + margin-left: 30px; } + #hamburger { + width: 50px; + height: 50px; } + #mainNav { + padding: 140px 0 30px; } + #mainNav h5 { + margin-bottom: 1em; } + #mainNav h3 { + margin-bottom: 0.6em; } + #mainNav .nav-box { + width: 20%; } + #mainNav .nav-box + .nav-box { + margin-left: calc(20% / 3); } + #mainNav main + main { + margin-top: 60px; } + #mainNav .left .button { + height: 50px; + line-height: 50px; + font-size: 18px; } + .open-nav #tryKubernetes, .y-enough #tryKubernetes { + margin-left: 30px; } + #hero { + padding-top: 80px; } + #docs #hero h1, #docs #hero h5 { + padding-left: 20px; + padding-right: 20px; } + #headlineWrapper, #quickstartButton { + margin-bottom: 40px; } + #quickstartButton { + padding: 0 50px; } + #vendorStrip { + height: 60px; + line-height: 60px; + font-size: 18px; } + #vendorStrip img { + max-height: 30px; + max-width: 100px; + margin-left: 50px; } + #oceanNodes { + padding-top: 60px; + padding-bottom: 60px; } + #oceanNodes main { + margin-bottom: 60px; } + #video { + height: 400px; } + #features h3 { + margin-bottom: 40px; } + #features .feature-box { + margin-bottom: 60px; } + #features .feature-box > div { + width: 45%; + margin-bottom: 0; } + #docs #hero h1, #docs #hero h5 { + text-align: left; } + #viewDocs, #tryKubernetes { + display: inline-block; } + #vendorStrip { + display: block; } + #quickstartButton { + display: inline-block; } + #oceanNodes h3 { + text-align: left; } + #oceanNodes main { + position: relative; + clear: both; } + #oceanNodes main .image-wrapper { + position: absolute; + top: 50%; + max-width: 25%; + max-height: 100%; + -webkit-transform: translateY(-50%); + -ms-transform: translateY(-50%); + transform: translateY(-50%); } + #oceanNodes main:nth-child(odd) .content { + padding-right: 30%; } + #oceanNodes main:nth-child(odd) .image-wrapper { + right: 0; } + #oceanNodes main:nth-child(even) .content { + padding-left: 30%; } + #oceanNodes main:nth-child(even) .image-wrapper { + left: 0; } + #oceanNodes main:nth-child(1) .content { + padding-right: 0; } + #oceanNodes main:nth-child(1) .image-wrapper { + position: relative; + max-width: 100%; + -webkit-transform: none; + -ms-transform: none; + transform: none; } + #oceanNodes main img { + width: 100%; } + #video { + display: block; } + #video > .light-text { + display: block; } + #mobileShowVideoButton { + display: none; } + #features { + padding-bottom: 60px; } + #features .feature-box:last-child { + margin-bottom: 0; } } + +@media screen and (min-width: 1025px) { + h1 { + font-size: 60px; + line-height: 90px; } + h2 { + font-size: 42px; + line-height: 60px; } + h3 { + font-size: 30px; + line-height: 40px; } + h4 { + font-size: 26px; + line-height: 40px; } + h5 { + font-size: 22px; + line-height: 36px; } + p { + font-size: 20px; + line-height: 28px; } + section, header, footer, #vendorStrip { + padding-left: 50px; + padding-right: 50px; } + section main, header main, footer main, #vendorStrip main { + width: 100%; + max-width: 1200px; } + header { + height: 80px; } + .logo { + width: 180px; + height: 80px; } + .nav-buttons { + height: 80px; + line-height: 80px; } + .nav-buttons .button + * { + margin-left: 30px; } + #hamburger { + width: 50px; + height: 50px; } + #mainNav { + padding: 140px 0 30px; } + #mainNav h5 { + margin-bottom: 1em; } + #mainNav h3 { + margin-bottom: 0.6em; } + #mainNav .nav-box { + width: 20%; } + #mainNav .nav-box + .nav-box { + margin-left: calc(20% / 3); } + #mainNav main + main { + margin-top: 60px; } + #mainNav .left .button { + height: 50px; + line-height: 50px; + font-size: 18px; } + .open-nav #tryKubernetes, .y-enough #tryKubernetes { + margin-left: 30px; } + #hero { + padding-top: 80px; } + #docs #hero h1, #docs #hero h5 { + padding-left: 50px; + padding-right: 50px; } + #headlineWrapper, #quickstartButton { + margin-bottom: 40px; } + #quickstartButton { + padding: 0 50px; } + #vendorStrip { + height: 60px; + line-height: 60px; + font-size: 18px; } + #vendorStrip img { + max-height: 30px; + max-width: 100px; + margin-left: 50px; } + #oceanNodes { + padding-top: 60px; + padding-bottom: 60px; } + #oceanNodes main { + margin-bottom: 60px; } + #video { + height: 550px; } + #features h3 { + margin-bottom: 40px; } + #features .feature-box { + margin-bottom: 60px; } + #features .feature-box > div { + width: 45%; + margin-bottom: 0; } + #oceanNodes main { + position: relative; } + #oceanNodes main:nth-child(1) .image-wrapper { + position: absolute; + max-width: 48%; + -webkit-transform: translateY(-50%); + -ms-transform: translateY(-50%); + transform: translateY(-50%); } + #oceanNodes main:nth-child(1) .content { + width: 50%; } + #video { + position: relative; + background-image: url(../images/kub_video_thm.jpg); + background-position: center center; + background-size: cover; + height: 550px; } } + +/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInBhcnRpYWxzL19yZXNldC5zYXNzIiwicGFydGlhbHMvX3NraW4uc2FzcyIsIi4uL2Jhc2VDb21wb25lbnRzL1/PgC1iYXNlQ29tcG9uZW50cy5zYXNzIiwiLi4vbW9kdWxlcy/PgC1hY2NvcmRpb24vX8+ALWFjY29yZGlvbi5zYXNzIiwiLi4vbW9kdWxlcy/PgC1kaWFsb2cvX8+ALWRpYWxvZy5zYXNzIiwiLi4vbW9kdWxlcy/PgC1wdXNobWVudS9fz4AtcHVzaG1lbnUuc2FzcyIsInBhcnRpYWxzL19zaXplLnNhc3MiLCJtZWRpYVF1ZXJpZXMvX2Jhc2Uuc2FzcyIsIm1lZGlhUXVlcmllcy9fdGFibGV0LnNhc3MiLCJtZWRpYVF1ZXJpZXMvX2Rlc2t0b3Auc2FzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTtFQUNDLFVBQVU7RUFDVixXQUFXLEVBQUc7O0FBR2Y7RUFDQyxjQUFjLEVBQUc7O0FBRWxCO0VBQ0MsZ0JBQWdCLEVBQUc7O0FBRXBCO0VBQ0MsaUJBQWlCLEVBQUc7O0FBRXJCO0VBQ0MsVUFBVTtFQUNWLFdBQVcsRUFBRzs7QUFFZjtFQUNDLHNCQUFzQixFQUFHOztBQUcxQjtFQUNDLGVBQWU7RUFDZixZQUFZLEVBQUc7O0FBRWhCO0VBQ0MsYUFBYSxFQUFHOztBQUVqQjtFQUNDLGFBQWEsRUFBRzs7QUFFakI7RUFDQyxZQUFZLEVBQUc7O0FBRWhCO0VBQ0MsbUJBQW1CLEVBQUc7O0FDNUJ2QjtFQUNDLHVCQUF1QjtFQUN2QixrQ0FMOEI7RUFNOUIsaUJBQWlCO0VBQ2pCLFVBQVU7RUFDVixVQUFVLEVBQUc7O0FBRWQ7RUFDQyxrQ0FYOEIsRUFXTDs7QUFFMUI7RUFDQyxpQkFBaUIsRUFBRzs7QUFFckI7RUFDQyxpQkFBaUIsRUFBRzs7QUN0QnJCO0VBQ0MsaUJBQWlCLEVBQUc7O0FBTXJCO0VBSUMsbUJBQW1CO0VBQ25CLFVBQVU7RUFDVixZQUFZO0VBQ1osWUFOeUI7RUFPekIsYUFQeUI7RUFRekIsVUFBVSxFQWtCc0I7RUEzQmpDO0lBWUUsWUFBWTtJQUNaLG1CQUFtQjtJQUNuQixPQUFPO0lBQ1Asa0JBYnNDO0lBY3RDLFlBQVk7SUFDWix3QkFBd0IsRUFBRztFQWpCN0I7SUFvQkUsUUFBUTtJQUNSLG1DQUEyQjtRQUEzQiwrQkFBMkI7WUFBM0IsMkJBQTJCO0lBQzNCLGtDQUFrQjtRQUFsQiw2QkFBa0I7WUFBbEIsMEJBQWtCLEVBQVU7RUF0QjlCO0lBeUJFLFNBQVM7SUFDVCxvQ0FBNEI7UUFBNUIsZ0NBQTRCO1lBQTVCLDRCQUE0QjtJQUM1QixtQ0FBa0I7UUFBbEIsOEJBQWtCO1lBQWxCLDJCQUFrQixFQUFXOztBQUcvQjtFQUNDLGdCQUFnQjtFQUNoQixPQUFPO0VBQ1AsUUFBUTtFQUNSLFlBQVk7RUFDWixhQUFhO0VBQ2IsYUFBYTtFQUNiLHFDQUFzQjtFQUN0QixjQUFjO0VBQ2QsV0FBVztFQUNYLGlCQUFpQixFQUdBO0VBYmxCO0lBYUUsV0FBVyxFQUFHOztBQUdoQjtFQUNDLG1CQUFtQjtFQUNuQixZQUFZO0VBQ1osYUFBYTtFQUNiLGtCQUFrQixFQTREeUI7RUFoRTVDO0lBT0UsbUJBQW1CO0lBQ25CLFlBQVk7SUFDWixhQUFhLEVBQUc7RUFUbEI7SUFZRSxtQkFBbUI7SUFDbkIsU0FBUztJQUNULFVBQVU7SUFDVix5Q0FBb0I7UUFBcEIscUNBQW9CO1lBQXBCLGlDQUFvQjtJQUNwQixjQUFjLEVBR087SUFuQnZCO01BbUJHLGVBQWUsRUFBRztFQW5CckI7SUFzQkUsbUJBQW1CO0lBQ25CLFlBQVk7SUFDWixRQUFRO0lBQ1IsWUFBWTtJQUNaLHNCQUFjO0lBQWQscUJBQWM7SUFBZCxjQUFjO0lBQ2QsZ0NBQXdCO1FBQXhCLHNCQUF3QjtZQUF4Qix3QkFBd0IsRUFZaUI7SUF2QzNDO01BOEJHLFlBQVk7TUFDWixhQUFhO01BQ2IsbUJBQW1CO01BQ25CLHdCQUF3QjtNQUN4QixVQUFVO01BQ1YsZUFBZTtNQUNmLGlCQUFpQixFQUdxQjtNQXZDekM7UUF1Q0ksMEJBeEZtQixFQXdGZ0I7RUF2Q3ZDO0lBMkNFLG1CQUFtQjtJQUNuQixTQUFTO0lBQ1QsWUFBWTtJQUNaLGFBQWE7SUFDYixVQUFVO0lBQ1Ysc0JBakdxQjtJQWtHckIsV0FBVztJQUNYLG1EQUFtQztRQUFuQyw4Q0FBbUM7WUFBbkMsMkNBQW1DO0lBQ25DLGlCQUFpQixFQUdPO0lBdEQxQjtNQXNERyxrQkFBa0IsRUFBRztFQXREeEI7SUF5REUsV0FBVztJQUNYLHlCQWpCaUM7SUFrQmpDLHVCQWxCaUMsRUFrQk07RUEzRHpDO0lBOERFLFlBQVk7SUFDWixzQkF0QmlDO0lBdUJqQyx3QkF2QmlDLEVBdUJPOztBQ3JIMUM7RUFDQyxtQkFBbUIsRUF1REM7RUF4RHJCO0lBSUUsbUJBQW1CLEVBQUc7RUFKeEI7SUFPRSxpQkFBaUIsRUFBRztFQVB0QjtJQVVFLGVGUmlCO0lFU2pCLG1CQUFtQjtJQUNuQiw0QkFBNEI7SUFDNUIsZ0JBQWdCO0lBQ2hCLGlCQUFpQixFQUdDO0lBakJwQjtNQWlCRyxlRmpCVyxFRWlCSTtFQWpCbEI7SUFvQkUsYUFBYTtJQUNiLGdCQUFnQixFQUdFO0lBeEJwQjtNQXdCRyxlRnhCVyxFRXdCSTtFQXhCbEI7SUEyQkUsaUJBQWlCLEVBV007SUF0Q3pCO01BOEJHLFlBQVk7TUFDWixtQkFBbUI7TUFDbkIsU0FBUztNQUNULFVBQVU7TUFDVixvQkFBb0I7TUFDcEIsNEJBQTRCO01BQzVCLDBERnBDVztNRXFDWCxnQ0FBaUI7VUFBakIsNEJBQWlCO2NBQWpCLHdCQUFpQjtNQUNqQixpQkFBaUIsRUFBRztFQXRDdkI7SUF5Q0UsbUJBQW1CO0lBQ25CLFlBQVk7SUFDWixpQkFBaUIsRUFBRztFQTNDdEI7SUE4Q0UsbUJBQW1CO0lBQ25CLFdBQVc7SUFDWCxpQkFBaUI7SUFDakIscUJBQXFCLEVBQUc7RUFqRDFCO0lBcURHLGlDQUFpQjtRQUFqQiw2QkFBaUI7WUFBakIseUJBQWlCLEVBQVU7RUFyRDlCO0lBd0RHLFdBQVcsRUFBRzs7QUN0RGpCO0VBR0UsbUJBQW1CO0VBQ25CLFNBQVM7RUFDVCxVQUFVO0VBQ1YseUNBQW9CO01BQXBCLHFDQUFvQjtVQUFwQixpQ0FBb0IsRUFBZTs7QUFOckM7RUFTRSxnQkFBZ0I7RUFDaEIsbUJBQW1CO0VBQ25CLGdCQUFnQixFQUFHOztBQVhyQjtFQWNFLFlBQVk7RUFDWixpQkFBaUI7RUFDakIsaUJBQWlCLEVBb0JRO0VBcEMzQjtJQW1CRyxjQUFjO0lBQ2QsbUJBQW1CO0lBQ25CLHdCQUF3QixFQUFHO0VBckI5QjtJQXdCRyw0QkFBNEI7SUFDNUIsa0JBQWtCO0lBQ2xCLGlCQUFpQixFQVVLO0lBcEN6QjtNQTZCSSxlQS9CbUI7TUFnQ25CLHNCQWhDbUI7TUFpQ25CLG1CQUFtQjtNQUNuQixpQkFBaUIsRUFJRTtNQXBDdkI7UUFtQ0ssMEJBckNrQjtRQXNDbEIsYUFBYSxFQUFHOztBQ3RDckI7RUFDQyxjQUFjO0VBQ2QsZ0JBQWdCO0VBQ2hCLE9BQU87RUFDUCxZQUFZO0VBQ1osYUFBYTtFQUNiLFdBQVc7RUFDWCx5QkFBeUIsRUF5RVk7RUFoRnRDO0lBVUUsV0FBVyxFQUFHO0VBVmhCO0lBYUUsZ0JBQWdCO0lBQ2hCLE9BQU87SUFDUCxRQUFRO0lBQ1IsWUFBWTtJQUNaLGFBQWE7SUFDYixxQ0FBc0IsRUFBaUI7RUFsQnpDO0lBcUJFLG1CQUFtQjtJQUNuQixPQUFPO0lBQ1AsU0FBUztJQUNULGFBQWE7SUFDYix3QkFBd0I7SUFDeEIsZUFBZTtJQUNmLHVCQUF1QixFQUFHO0VBM0I1QjtJQThCRSxhQUFhLEVBQUc7RUE5QmxCO0lBaUNFLFVBQVU7SUFDVixrQkFBa0I7SUFDbEIsdUJBQXVCLEVBQUc7RUFuQzVCO0lBc0NFLG1CQUFtQjtJQUNuQixVQUFVO0lBQ1YsWUFBWTtJQUNaLFdBQVcsRUFHcUI7SUE1Q2xDO01BNENHLDBCQUEwQixFQUFHO0VBNUNoQztJQWdERSx1QkFBdUIsRUFBRztFQWhENUI7SUFrREUsbUJBQW1CO0lBQ25CLGVBQWU7SUFDZixZQUFZO0lBQ1osaUJBQWlCLEVBQUc7RUFyRHRCO0lBd0RFLHNCQUFzQjtJQUN0QixZQUFZO0lBQ1osYUFBYTtJQUNiLGtCQUFrQjtJQUNsQixrQkFBa0I7SUFDbEIsZ0JBQWdCO0lBQ2hCLGVKOURZLEVJOERHO0VBOURqQjtJQWlFRSxpQkFBaUI7SUFDakIsV0FBVyxFQUFHO0VBbEVoQjtJQXVFRSxnQkFBZ0IsRUFTa0I7SUFoRnBDO01BMEVHLGlCQUFpQixFQUFHO0lBMUV2QjtNQTZFRyxhQUFhO01BQ2Isa0JBQWtCO01BQ2xCLGdCQUFnQjtNQUNoQixlQUFjLEVBQWlCOztBTGhGbEM7RUFDQyxVQUFVO0VBQ1YsV0FBVyxFQUFHOztBQUdmO0VBQ0MsY0FBYyxFQUFHOztBQUVsQjtFQUNDLGdCQUFnQixFQUFHOztBQUVwQjtFQUNDLGlCQUFpQixFQUFHOztBQUVyQjtFQUNDLFVBQVU7RUFDVixXQUFXLEVBQUc7O0FBRWY7RUFDQyxzQkFBc0IsRUFBRzs7QUFHMUI7RUFDQyxlQUFlO0VBQ2YsWUFBWSxFQUFHOztBQUVoQjtFQUNDLGFBQWEsRUFBRzs7QUFFakI7RUFDQyxhQUFhLEVBQUc7O0FBRWpCO0VBQ0MsWUFBWSxFQUFHOztBQUVoQjtFQUNDLG1CQUFtQixFQUFHOztBTXBDdkI7RUFDQyxnQkNFa0I7RUREbEIsa0JDRW9CLEVERlc7O0FBRWhDO0VBQ0MsZ0JDQ2tCO0VEQWxCLGtCQ0NvQixFRERXOztBQUVoQztFQUNDLGdCQ0FrQjtFRENsQixrQkNBb0IsRURBVzs7QUFFaEM7RUFDQyxnQkNEa0I7RURFbEIsa0JDRG9CLEVEQ1c7O0FBRWhDO0VBQ0MsZ0JDRmtCO0VER2xCLGtCQ0ZvQixFREVXOztBQUVoQztFQUNDLGdCQ0hpQjtFRElqQixrQkNIbUIsRURHVzs7QUFFL0I7RUFDQyxtQkNGeUI7RURHekIsb0JDSHlCLEVET087RUFOakM7SUFLRSxZQ0plO0lES2YsZ0JDSm1CLEVESVU7O0FBRS9CO0VBQ0MsYUNMbUIsRURLTTs7QUFFMUI7RUFDQyxhQ05pQjtFRE9qQixhQ1RtQixFRFNNOztBQUUxQjtFQUNDLGFDWm1CO0VEYW5CLGtCQ2JtQixFRGdCd0I7RUFMNUM7SUFLRSxrQkNaNEIsRURZWTs7QUFFMUM7RUFDQyxZQ2JvQjtFRGNwQixhQ2RvQixFRGNNOztBQUUzQjtFQUNDLHNCQ2I4QixFRGlDa0I7RUFyQmpEO0lBSUUsbUJDZjZCLEVEZWU7RUFKOUM7SUFPRSxxQkNqQitCLEVEaUJhO0VBUDlDO0lBVUUsV0NuQmlCLEVEbUJPO0VBVjFCO0lBYUUsMkJDckJnQyxFRHFCWTtFQWI5QztJQWdCRSxpQkN2QnFDLEVEdUJXO0VBaEJsRDtJQW1CRSxhQ3pCOEI7SUQwQjlCLGtCQzFCOEI7SUQyQjlCLGdCQzFCbUMsRUQwQlU7O0FBRS9DO0VBRUUsa0JDM0M0QixFRDJDWTs7QUFFMUM7RUFDQyxrQkM3QnNCLEVENkJXOztBQUVsQztFQUdHLG1CQzVEdUI7RUQ2RHZCLG9CQzdEdUIsRUQ2RGU7O0FBRXpDO0VBQ0Msb0JDckNvQyxFRHFDYTs7QUFFbEQ7RUFDQyxnQkN2Q2lDLEVEdUNLOztBQUV2QztFQUNDLGFDekN5QjtFRDBDekIsa0JDMUN5QjtFRDJDekIsZ0JDMUM0QixFRCtDb0I7RUFSakQ7SUFNRSxpQkM1Q2dDO0lENkNoQyxpQkM1Q2dDO0lENkNoQyxrQkM1Q2lDLEVENENZOztBQUUvQztFQUNDLGtCQzNDMkI7RUQ0QzNCLHFCQzVDMkIsRUQrQ2dCO0VBTDVDO0lBS0Usb0JDL0MwQixFRCtDYzs7QUFHMUM7RUFDQyxjQzdDMkIsRUQ2Q0s7O0FBR2pDO0VBRUUsb0JDL0M4QixFRCtDYzs7QUFGOUM7RUFLRSxpQkNoRDJCLEVEb0R3QjtFQVRyRDtJQVFHLFlDcER5QjtJRHFEekIsb0JDbkRpQyxFRG1EZTs7QUN0Q25EO0VBQ0Msd0JBQXdCLEVBQUc7O0FBRTVCO0VBQ0MsbUJBQW1CO0VBQ25CLHdCQUF3QixFQUFHOztBQUU1QjtFQUVFLG1CQUFtQjtFQUNuQixhQUFhLEVBQUc7O0FBRWxCO0VBQ0MsbUJBQW1CO0VBQ25CLGVObkdhLEVNbUdFOztBQUVoQjtFQUNDLHNCQUFzQjtFQUN0QixtQkFBbUI7RUFDbkIsZ0JBQWdCO0VBQ2hCLGtCQUFrQjtFQUNsQixhQUFhO0VBQ2IsMEJOM0dhO0VNNEdiLHNCQUFzQixFQUFHOztBQUUxQjtFQUNDLGdCQUFnQjtFQUNoQixPQUFPO0VBQ1AsUUFBUTtFQUNSLFlBQVk7RUFDWixhQUFhO0VBQ2IsY0FBYyxFQUFHOztBQUlsQjtFQUNDLGdCQUFnQjtFQUNoQixPQUFPO0VBQ1AsUUFBUTtFQUNSLFlBQVk7RUFDWixjQUFjO0VBQ2QsOEJBQThCO0VBQzlCLDhCQUE4QjtFQUM5QixpQkFBaUI7RUFDakIsaUJBQWlCLEVBQUc7O0FBRXJCO0VBQ0MsWUFBWTtFQUNaLGlEQUFnRDtFQUNoRCx5QkFBeUI7RUFDekIsbUNBQW1DO0VBQ25DLDZCQUE2QixFQUFHOztBQUVqQztFQUNDLGFBQWEsRUFBRzs7QUFFakI7RUFDQyxjQUFjLEVBQUc7O0FBRWxCO0VBQ0Msd0JBQXdCO0VBQ3hCLDhCQUE4QjtFQUM5QixpQkFBaUIsRUFJTTtFQVB4QjtJQU1FLHdCQUF3QjtJQUN4QixlTnJKaUIsRU1xSkc7O0FBRXRCO0VBQ0MsU0FBUztFQUNULFdBQVc7RUFDWCxlQUFlO0VBQ2YsdUJBQXVCO0VBQ3ZCLG1CQUFtQjtFQUNuQixpQkFBaUI7RUFDakIsb0JBQW9CO0VBQ3BCLGlCQUFpQixFQUFHOztBQUdyQjtFQUNDLHNCQUFzQjtFQUN0QixtQkFBbUI7RUFDbkIsdUJBQXVCO0VBQ3ZCLFdBQVc7RUFDWCxVQUFVO0VBQ1YsaUJBQWlCLEVBc0JnQjtFQTVCbEM7SUFTRSxtQkFBbUI7SUFDbkIsVUFBVTtJQUNWLFdBQVc7SUFDWCxZQUFZO0lBQ1osMEJOakxZO0lNa0xaLGlCQUFpQjtJQUNqQixZQUFZLEVBQUc7RUFmakI7SUFrQkUscUJBQVMsRUFBYztFQWxCekI7SUFxQkUsU0FBUyxFQUFHO0VBckJkO0lBd0JFLFlBQVksRUFBRztFQXhCakI7SUE0Qkcsd0JBQXdCLEVBQUc7O0FBRTlCO0VBRUUsZU5wTVk7RU1xTVosb0JBQW9CLEVBQUc7O0FBSHpCO0VBTUUsb0JBQW9CO0VBQ3BCLGlCQUFpQjtFQUNqQixZQUFZLEVBQUc7O0FBUmpCO0VBV0UsWUFBWTtFQUNaLG9CQUFvQixFQUFHOztBQVp6QjtFQWdCRyxlTmxOVztFTW1OWCxzQkFBc0IsRUFBRzs7QUFHNUI7RUFFRSx3QkFBd0IsRUFBRzs7QUFJN0I7RUFFRSxpQkFBaUIsRUFBRzs7QUFGdEI7RUFLRSxlQUFlO0VBQ2YsY0FBYyxFQUFHOztBQU5uQjtFQVNFLDBCTnBPa0I7RU1xT2xCLGNBQWMsRUFBRzs7QUFWbkI7RUFjRyxXQUFXLEVBQUc7O0FBZGpCO0VBaUJHLFdBQVc7RUFDWCxnQ0FBd0I7TUFBeEIsNEJBQXdCO1VBQXhCLHdCQUF3QixFQUFHOztBQWxCOUI7RUFxQkcsaUNBQWlCO01BQWpCLDZCQUFpQjtVQUFqQix5QkFBaUIsRUFBVTs7QUFyQjlCO0VBd0JHLGtDQUFpQjtNQUFqQiw4QkFBaUI7VUFBakIsMEJBQWlCLEVBQVc7O0FBRS9CO0VBRUUsYUFBYSxFQUFHOztBQUdsQjtFQUVFLDhCTjFQbUIsRU02UG9DO0VBTHpEO0lBS0csa0RBQWlELEVBQUc7O0FBTHZEO0VBUUUsc0JOalFpQjtFTWtRakIsZU5sUWlCLEVNdVFFO0VBZHJCO0lBWUcsc0JOdlFXO0lNd1FYLDBCTnhRVztJTXlRWCxhQUFhLEVBQUc7O0FBZG5CO0VBa0JHLDBCTjNRZ0IsRU0yUWU7O0FBS2xDO0VBQ0MsZ0RBQStDO0VBQy9DLDBCTmxSa0I7RU1tUmxCLG1CQUFtQjtFQUNuQixnQkFBZ0I7RUFDaEIsaUJBQWlCLEVBQUc7O0FBRXJCO0VBQ0MsbUJBQW1CO0VBQ25CLGdCQUFnQixFQUFHOztBQUVwQjtFQUNDLGNBQWM7RUFDZCxnQkFBZ0I7RUFDaEIsa0JBQWtCO0VBQ2xCLGdCQUFnQixFQUFHOztBQUlwQjtFQUNDLGNBQWM7RUFDZCwyQ0FBZ0M7RUFDaEMsaUJBQWlCO0VBQ2pCLG9CQUFvQjtFQUNwQixpQkFBaUIsRUFTWTtFQWQ5QjtJQVFFLHVCQUF1QjtJQUN2QixhQUFhO0lBQ2Isc0JBQXNCLEVBQUc7RUFWM0I7SUFjRSx1QkFBdUIsRUFBRzs7QUFJNUI7RUFFRSxvQkE1UG1DLEVBNFBjOztBQUZuRDtFQUtFLGVBQWU7RUFDZixvQkFBb0IsRUFHRjtFQVRwQjtJQVNHLFlBQVksRUFBRzs7QUFUbEI7RUFhRyxnQkFBZ0IsRUFBRzs7QUFidEI7RUFnQkUsb0JBelFpQyxFQXlRYzs7QUFHakQ7RUFDQyxZQUFZO0VBQ1osbUJBQW1CO0VBQ25CLHNEQUFxRDtFQUNyRCxtQ0FBbUM7RUFDbkMsdUJBQXVCLEVBMEQ0QztFQS9EcEU7SUFTRSxjQUFjO0lBQ2QsbUJBQW1CO0lBQ25CLFNBQVM7SUFDVCxVQUFVO0lBQ1YsV0FBVztJQUNYLGlCQUFpQjtJQUNqQixpQkFBaUI7SUFDakIsbUJBQW1CO0lBQ25CLHlDQUFvQjtRQUFwQixxQ0FBb0I7WUFBcEIsaUNBQW9CO0lBQ3BCLGFBQWEsRUFBRztFQWxCbEI7SUFxQkUsZ0JBQWdCO0lBQ2hCLGtCQUFrQjtJQUNsQixvQkFBb0IsRUFBRztFQXZCekI7SUEwQkUsb0JBQW9CLEVBQUc7RUExQnpCO0lBNkJFLG1CQUFtQjtJQUNuQixnQkFBZ0I7SUFDaEIsd0JBQXdCO0lBQ3hCLG1CQUFtQjtJQUNuQixlTjNXWTtJTTRXWiw2QkFBNkIsRUFVZ0M7SUE1Qy9EO01BcUNHLFlBQVk7TUFDWixtQkFBbUI7TVB2VXJCLG1CQUFtQjtNQUNuQixTQUZzQztNQUd0QyxXT3NVMEI7TVByVTFCLHlDQUFvQjtVQUFwQixxQ0FBb0I7Y0FBcEIsaUNBQW9CO01Pc1VsQixTQUFTO01BQ1QsVUFBVTtNQUNWLG9CQUFvQjtNQUNwQiwrQkFBK0I7TUFDL0IsMEROdFhXLEVNc1grQztFQTVDN0Q7SVBqU0MsbUJBQW1CO0lBQ25CLFNBRnNDO0lBR3RDLFVBSDJCO0lBSTNCLHlDQUFvQjtRQUFwQixxQ0FBb0I7WUFBcEIsaUNBQW9CO0lPOFVuQixZQUFZO0lBQ1osYUFBYTtJQUNiLG1CQUFtQjtJQUNuQiw4QkFBOEI7SUFDOUIsMkNBQXNCO0lBQ3RCLGtCQUFrQixFQVU4QztJQS9EbEU7TVBqU0MsbUJBQW1CO01BQ25CLFNBRnNDO01BR3RDLFVBSDJCO01BSTNCLHlDQUFvQjtVQUFwQixxQ0FBb0I7Y0FBcEIsaUNBQW9CO01PdVZsQixXQUFXO01BQ1gsWUFBWTtNQUNaLFNBQVM7TUFDVCxVQUFVO01BQ1Ysb0JBQW9CO01BQ3BCLCtCQUErQjtNQUMvQiwwREFBMEQsRUFBRzs7QUFJaEU7RUFDQyxtQkFBbUI7RUFDbkIsMEJOOVltQjtFTStZbkIsOENBQTZDO0VBQzdDLGlDQUFpQztFQUNqQyw2QkFBNkI7RUFDN0Isc0JBQXNCLEVBQUc7O0FBRTFCO0VBRUMsWUFBWTtFQUNaLGlCQUFpQjtFQUNqQixZQUFZLEVBVU87RUFkcEI7SUFPRSxvQkFBb0I7SUFDcEIsb0JBQW9CLEVBQUc7RUFSekI7SUFXRSxZQUFZLEVBQUc7RUFYakI7SUFjRSxhQUFhLEVBQUc7O0FBSWxCO0VBRUUsZ0JBQWdCLEVBQUc7O0FBRnJCO0VBS0UsbUJBQW1CLEVBQUc7O0FBTHhCO0VBUUUsb0JBQW9CLEVBQUc7O0FBUnpCO0VBV0Usb0JBQW9CO0VBQ3BCLG9CQUFvQixFQUFHOztBQVp6QjtFQWVFLGlCQUFpQixFQWdDUztFQS9DNUI7SUFrQkcsd0JBQVc7SUFDWCxZQUFZO0lBQ1osY0FBYztJQUNkLG1CQUFtQjtJQUNuQixnQ0FBZ0M7SUFDaEMsc0JBQXNCO0lBQ3RCLDZCQUE2QixFQUFHO0VBeEJuQztJQTJCRyxxREFBb0QsRUFBRztFQTNCMUQ7SUE4Qkcsb0RBQW1ELEVBQUc7RUE5QnpEO0lBaUNHLG1EQUFrRCxFQUFHO0VBakN4RDtJQW9DRywyREFBMEQsRUFBRztFQXBDaEU7SUF1Q0csa0JBQWtCLEVBQUc7RUF2Q3hCO0lBMENHLHNCQUFzQjtJQUN0QixlTmxkVztJTW1kWCxnQkFBZ0I7SUFDaEIsaUJBQWlCO0lBQ2pCLHNCQUFzQjtJQUN0QixvQkFBb0IsRUFBRzs7QUFJMUI7RUFDQyxZQUFZO0VBQ1osZ0RBQStDO0VBQy9DLDBCTjNka0IsRU00Zk87RUFwQzFCO0lBTUUsZ0JBQWdCLEVBQUc7RUFOckI7SUFTRSxtQkFBbUI7SUFDbkIsb0JBQW9CLEVBVUc7SUFwQnpCO01BYUcsc0JBQXNCO01BQ3RCLGdCQUFnQjtNQUNoQixpQkFBaUI7TUFDakIsYUFBYTtNQUNiLHNCQUFzQixFQUFHO0lBakI1QjtNQW9CRyxpQkFBaUIsRUFBRztFQXBCdkI7SUF1QkUsbUJBQW1CO0lBQ25CLG9CQUFvQixFQVlFO0lBcEN4QjtNQTJCRyxhQUFhO01BQ2Isc0JBQXNCO01BQ3RCLGFBQWE7TUFDYixrQkFBa0I7TUFDbEIsaUJBQWlCO01BQ2pCLG9CQUFvQixFQUlEO01BcEN0QjtRQW1DSSxpQkFBaUI7UUFDakIsYUFBYSxFQUFHOztBQUdwQjtFQUNDLDhCQUE4QjtFQUM5QixjQUFjO0VBQ2QsZ0JBQWdCO0VBQ2hCLGlCQUFpQjtFQUNqQixhQUFhO0VBQ2Isd0JBQXdCO0VBQ3hCLGlCQUFpQixFQUlNO0VBWHhCO0lBVUUsMEJOMWdCa0I7SU0yZ0JsQixlTjFnQmlCLEVNMGdCRzs7QUFFdEI7RUFDQyxzQkFBc0I7RUFDdEIsc0RBQXFEO0VBQ3JELDZCQUE2QjtFQUM3QixzQkFBc0I7RUFDdEIsWUFBWTtFQUNaLGFBQWE7RUFDYixtQkFBbUI7RUFDbkIsbUJBQW1CLEVBU0k7RUFqQnhCO0lBV0UsdUJBQXVCLEVBQUc7RUFYNUI7SUFjRSxtQkFBbUI7SUFDbkIsZUFBZTtJQUNmLFVBQVU7SUFDVixpQkFBaUIsRUFBRzs7QUFFdEI7RUFDQyx5QkFBeUIsRUFHUztFQUpuQztJQUlFLDRCQUE0QixFQUFHOztBQUVqQztFQUNDLDZCQUE2QixFQUdTO0VBSnZDO0lBSUUsZ0NBQWdDLEVBQUc7O0FBRXJDO0VBQ0MsOEJBQThCLEVBR1M7RUFKeEM7SUFJRSxpQ0FBaUMsRUFBRzs7QUFFdEM7RUFDQyw4QkFBOEIsRUFHUztFQUp4QztJQUlFLGlDQUFpQyxFQUFHOztBQUV0QztFQUNDLDhCQUE4QixFQUdTO0VBSnhDO0lBSUUsaUNBQWlDLEVBQUc7O0FBR3RDO0VQaGhCQyxZQURzRjtFQUV0RixhQUFxQjtFQUNyQix1QkFBeUI7RUFDekIsaUJBSjRHO0VBTjVHLG1CQUFtQjtFQUNuQixTQUZzQztFQUd0QyxVQUgyQjtFQUkzQix5Q0FBb0I7TUFBcEIscUNBQW9CO1VBQXBCLGlDQUFvQixFT3NoQkU7O0FBU3ZCO0VBRUUsY0FBYyxFQUFHOztBQUZuQjtFQU9FLHdCQUF3QjtFQUN4QixrQkFBa0IsRUFBRzs7QUFSdkI7RUFXRSwwQk5wbEJpQixFTXVsQlM7RUFkNUI7SUFjRyxvQkFBb0IsRUFBRzs7QUFkMUI7RUFpQkUsbUJBQW1CO0VBQ25CLG9CQUFvQixFQTZGQTtFQS9HdEI7SUFxQkcsWUFBWSxFQUFHO0VBckJsQjtJQXdCRyxzQkFBc0I7SUFDdEIsYUFBYSxFQUFHO0VBekJuQjtJQTRCRyxlQUFlO0lBQ2YsYUFBYTtJQUNiLGFBQWEsRUFBRztFQTlCbkI7SUFpQ0csa0JBQWtCLEVBQUc7RUFqQ3hCO0lBb0NHLG1CQUFtQjtJQUNuQixhQUFhO0lBQ2IsYUFBYTtJQUNiLGdCQUFnQjtJQUNoQixzQkFBc0I7SUFDdEIsYUFBYTtJQUNiLGtCQUFrQjtJQUNsQiwwQk5ybkJpQjtJTXNuQmpCLGdCQUFnQjtJQUNoQix1QkFBdUIsRUFBRztFQTdDN0I7SUFnREcsbUJBQW1CO0lBQ25CLGFBQWE7SUFDYixhQUFhO0lBQ2IsZ0JBQWdCO0lBQ2hCLGFBQWE7SUFDYixrQkFBa0I7SUFDbEIsZ0JBQWdCO0lBQ2hCLGlCQUFpQjtJQUNqQixjQUFjLEVBdURHO0lBL0dwQjtNQTJESSxrQkFBa0IsRUFBRztJQTNEekI7TUE4REksbUJBQW1CO01BQ25CLG9CQUFvQjtNQUNwQixpQkFBaUIsRUFZTTtNQTVFM0I7UUFtRUssWUFBWTtRQUNaLG1CQUFtQjtRQUNuQixVQUFVO1FBQ1YsWUFBWTtRQUNaLFNBQVM7UUFDVCxVQUFVO1FBQ1Ysb0JBQW9CO1FBQ3BCLDRCQUE0QjtRQUM1Qix3REFBd0Q7UUFDeEQsaUJBQWlCLEVBQUc7SUE1RXpCO01BK0VJLDJDQUFnQyxFQUFlO0lBL0VuRDtNQWtGSSwwQk41cEJnQjtNTTZwQmhCLGVONXBCZSxFTTRwQks7SUFuRnhCO01Bc0ZJLDBCTi9wQmUsRU0rcEJnQjtJQXRGbkM7TUF5RkksbUJBQW1CO01BQ25CLFVBQVU7TUFDVixRQUFRO01BQ1IsWUFBWSxFQUFHO0lBNUZuQjtNQStGSSxlQUFlO01BQ2YsWUFBWTtNQUNaLGFBQWE7TUFDYixrQkFBa0I7TUFDbEIsZ0JBQWdCO01BQ2hCLGdCQUFnQjtNQUNoQixpQkFBaUI7TUFDakIsMEJOaHJCZ0I7TU1pckJoQixlTmhyQmU7TU1pckJmLFdBQVc7TUFDWCxpQkFBaUIsRUFHZTtNQTVHcEM7UUE0R0ssMEJBQTBCLEVBQUc7SUE1R2xDO01BK0dJLFdBQVcsRUFBRzs7QUEvR2xCO0VBa0hFLG1CQUFtQjtFQUNuQixjQUFjO0VBQ2QsaUJBQWlCLEVBbUJjO0VBdklqQztJQXVIRyxhQUFhLEVBQUc7RUF2SG5CO0lBMEhHLFlBQVk7SUFDWixhQUFhO0lBQ2Isb0JBQW9CO0lBQ3BCLGdDQUFnQyxFQU1IO0lBbkloQztNQWlJSywwQ0FBZ0M7TUFDaEMsbUJBQW1CO01BQ25CLG9CQUFvQixFQUFHO0VBbkk1QjtJQXNJRyxhQUFhO0lBQ2IsMEJBQVcsRUFBaUI7O0FBRy9CO0VBQ0MsZ0JBQWdCO0VBQ2hCLFVBQVUsRUFBRzs7QUM5b0JkO0VGekVBO0lBQ0MsZ0JFQWtCO0lGQ2xCLGtCRUFvQixFRkFXO0VBRWhDO0lBQ0MsZ0JFRGtCO0lGRWxCLGtCRURvQixFRkNXO0VBRWhDO0lBQ0MsZ0JFRmtCO0lGR2xCLGtCRUZvQixFRkVXO0VBRWhDO0lBQ0MsZ0JFSGtCO0lGSWxCLGtCRUhvQixFRkdXO0VBRWhDO0lBQ0MsZ0JFSmtCO0lGS2xCLGtCRUpvQixFRklXO0VBRWhDO0lBQ0MsZ0JFTGlCO0lGTWpCLGtCRUxtQixFRktXO0VBRS9CO0lBQ0MsbUJFSnlCO0lGS3pCLG9CRUx5QixFRlNPO0lBTmpDO01BS0UsWUVOZTtNRk9mLGdCRU5tQixFRk1VO0VBRS9CO0lBQ0MsYUVQbUIsRUZPTTtFQUUxQjtJQUNDLGFFUmlCO0lGU2pCLGFFWG1CLEVGV007RUFFMUI7SUFDQyxhRWRtQjtJRmVuQixrQkVmbUIsRUZrQndCO0lBTDVDO01BS0Usa0JFZDRCLEVGY1k7RUFFMUM7SUFDQyxZRWZvQjtJRmdCcEIsYUVoQm9CLEVGZ0JNO0VBRTNCO0lBQ0Msc0JFZjhCLEVGbUNrQjtJQXJCakQ7TUFJRSxtQkVqQjZCLEVGaUJlO0lBSjlDO01BT0UscUJFbkIrQixFRm1CYTtJQVA5QztNQVVFLFdFckJpQixFRnFCTztJQVYxQjtNQWFFLDJCRXZCZ0MsRUZ1Qlk7SUFiOUM7TUFnQkUsaUJFekJxQyxFRnlCVztJQWhCbEQ7TUFtQkUsYUUzQjhCO01GNEI5QixrQkU1QjhCO01GNkI5QixnQkU1Qm1DLEVGNEJVO0VBRS9DO0lBRUUsa0JFN0M0QixFRjZDWTtFQUUxQztJQUNDLGtCRS9Cc0IsRUYrQlc7RUFFbEM7SUFHRyxtQkU5RHVCO0lGK0R2QixvQkUvRHVCLEVGK0RlO0VBRXpDO0lBQ0Msb0JFdkNvQyxFRnVDYTtFQUVsRDtJQUNDLGdCRXpDaUMsRUZ5Q0s7RUFFdkM7SUFDQyxhRTNDeUI7SUY0Q3pCLGtCRTVDeUI7SUY2Q3pCLGdCRTVDNEIsRUZpRG9CO0lBUmpEO01BTUUsaUJFOUNnQztNRitDaEMsaUJFOUNnQztNRitDaEMsa0JFOUNpQyxFRjhDWTtFQUUvQztJQUNDLGtCRTlDMkI7SUYrQzNCLHFCRS9DMkIsRUZrRGdCO0lBTDVDO01BS0Usb0JFbEQwQixFRmtEYztFQUcxQztJQUNDLGNFaEQyQixFRmdESztFQUdqQztJQUVFLG9CRWxEOEIsRUZrRGM7RUFGOUM7SUFLRSxvQkVwRDhCLEVGd0RxQjtJQVRyRDtNQVFHLFdFckR3QjtNRnNEeEIsaUJFdkQ4QixFRnVEa0I7RUUzQ2xEO0lBRUUsaUJBQWlCLEVBQUc7RUFFdEI7SUFDQyxzQkFBc0IsRUFBRztFQUUxQjtJQUNDLGVBQWUsRUFBRztFQUVuQjtJQUNDLHNCQUFzQixFQUFHO0VBRzFCO0lBRUUsaUJBQWlCLEVBQUc7RUFGdEI7SUFLRSxtQkFBbUI7SUFDbkIsWUFBWSxFQWlDTTtJQXZDcEI7TUFTRyxtQkFBbUI7TUFDbkIsU0FBUztNQUNULGVBQWU7TUFDZixpQkFBaUI7TUFDakIsb0NBQXFCO1VBQXJCLGdDQUFxQjtjQUFyQiw0QkFBcUIsRUFBUztJQWJqQztNQWlCSSxtQkFBbUIsRUFBRztJQWpCMUI7TUFvQkksU0FBUyxFQUFHO0lBcEJoQjtNQXdCSSxrQkFBa0IsRUFBRztJQXhCekI7TUEyQkksUUFBUSxFQUFHO0lBM0JmO01BK0JJLGlCQUFpQixFQUFHO0lBL0J4QjtNQWtDSSxtQkFBbUI7TUFDbkIsZ0JBQWdCO01BQ2hCLHdCQUFnQjtVQUFoQixvQkFBZ0I7Y0FBaEIsZ0JBQWdCLEVBQUc7SUFwQ3ZCO01BdUNHLFlBQVksRUFBRztFQUdsQjtJQUNDLGVBQWUsRUFHTTtJQUp0QjtNQUlFLGVBQWUsRUFBRztFQUVwQjtJQUNDLGNBQWMsRUFBRztFQUVsQjtJQUNDLHFCQUFxQixFQUlLO0lBTDNCO01BS0csaUJBQWlCLEVBQUcsRUFBQTs7QUN0RnhCO0VIaEVBO0lBQ0MsZ0JHQWtCO0lIQ2xCLGtCR0FvQixFSEFXO0VBRWhDO0lBQ0MsZ0JHRGtCO0lIRWxCLGtCR0RvQixFSENXO0VBRWhDO0lBQ0MsZ0JHRmtCO0lIR2xCLGtCR0ZvQixFSEVXO0VBRWhDO0lBQ0MsZ0JHSGtCO0lISWxCLGtCR0hvQixFSEdXO0VBRWhDO0lBQ0MsZ0JHSmtCO0lIS2xCLGtCR0pvQixFSElXO0VBRWhDO0lBQ0MsZ0JHTGlCO0lITWpCLGtCR0xtQixFSEtXO0VBRS9CO0lBQ0MsbUJHSnlCO0lIS3pCLG9CR0x5QixFSFNPO0lBTmpDO01BS0UsWUdOZTtNSE9mLGtCR05xQixFSE1RO0VBRS9CO0lBQ0MsYUdQbUIsRUhPTTtFQUUxQjtJQUNDLGFHUmlCO0lIU2pCLGFHWG1CLEVIV007RUFFMUI7SUFDQyxhR2RtQjtJSGVuQixrQkdmbUIsRUhrQndCO0lBTDVDO01BS0Usa0JHZDRCLEVIY1k7RUFFMUM7SUFDQyxZR2ZvQjtJSGdCcEIsYUdoQm9CLEVIZ0JNO0VBRTNCO0lBQ0Msc0JHZjhCLEVIbUNrQjtJQXJCakQ7TUFJRSxtQkdqQjZCLEVIaUJlO0lBSjlDO01BT0UscUJHbkIrQixFSG1CYTtJQVA5QztNQVVFLFdHckJpQixFSHFCTztJQVYxQjtNQWFFLDJCR3ZCZ0MsRUh1Qlk7SUFiOUM7TUFnQkUsaUJHekJxQyxFSHlCVztJQWhCbEQ7TUFtQkUsYUczQjhCO01INEI5QixrQkc1QjhCO01INkI5QixnQkc1Qm1DLEVINEJVO0VBRS9DO0lBRUUsa0JHN0M0QixFSDZDWTtFQUUxQztJQUNDLGtCRy9Cc0IsRUgrQlc7RUFFbEM7SUFHRyxtQkc5RHVCO0lIK0R2QixvQkcvRHVCLEVIK0RlO0VBRXpDO0lBQ0Msb0JHdkNvQyxFSHVDYTtFQUVsRDtJQUNDLGdCR3pDaUMsRUh5Q0s7RUFFdkM7SUFDQyxhRzNDeUI7SUg0Q3pCLGtCRzVDeUI7SUg2Q3pCLGdCRzVDNEIsRUhpRG9CO0lBUmpEO01BTUUsaUJHOUNnQztNSCtDaEMsaUJHOUNnQztNSCtDaEMsa0JHOUNpQyxFSDhDWTtFQUUvQztJQUNDLGtCRzlDMkI7SUgrQzNCLHFCRy9DMkIsRUhrRGdCO0lBTDVDO01BS0Usb0JHbEQwQixFSGtEYztFQUcxQztJQUNDLGNHakQyQixFSGlESztFQUdqQztJQUVFLG9CRWxEOEIsRUZrRGM7RUFGOUM7SUFLRSxvQkVwRDhCLEVGd0RxQjtJQVRyRDtNQVFHLFdFckR3QjtNRnNEeEIsaUJFdkQ4QixFRnVEa0I7RUdwRGxEO0lBRUUsbUJBQW1CLEVBVUM7SUFadEI7TUFPSSxtQkFBbUI7TUFDbkIsZUFBZTtNQUNmLG9DQUFxQjtVQUFyQixnQ0FBcUI7Y0FBckIsNEJBQXFCLEVBQVM7SUFUbEM7TUFZSSxXQUFXLEVBQUc7RUFJbEI7SUFDQyxtQkFBbUI7SUFDbkIsc0RBQXFEO0lBQ3JELG1DQUFtQztJQUNuQyx1QkFBdUI7SUFDdkIsY0FBYyxFQUFHLEVBQUEiLCJmaWxlIjoic3R5bGVzLmNzcyIsInNvdXJjZXNDb250ZW50IjpbImh0bWwsIGJvZHkge1xuXHRtYXJnaW46IDA7XG5cdHBhZGRpbmc6IDA7IH1cblxuXG5pbnB1dCwgYnV0dG9uIHtcblx0b3V0bGluZTogbm9uZTsgfVxuXG5idXR0b24ge1xuXHRjdXJzb3I6IHBvaW50ZXI7IH1cblxudWwsIGxpIHtcblx0bGlzdC1zdHlsZTogbm9uZTsgfVxuXG51bCB7XG5cdG1hcmdpbjogMDtcblx0cGFkZGluZzogMDsgfVxuXG5hIHtcblx0dGV4dC1kZWNvcmF0aW9uOiBub25lOyB9XG5cbi8vIENPTlZFTklFTkNFXG4uY2xlYXIge1xuXHRkaXNwbGF5OiBibG9jaztcblx0Y2xlYXI6IGJvdGg7IH1cblxuLmxpZ2h0LXRleHQge1xuXHRjb2xvcjogd2hpdGU7IH1cblxuLnJpZ2h0IHtcblx0ZmxvYXQ6IHJpZ2h0OyB9XG5cbi5sZWZ0IHtcblx0ZmxvYXQ6IGxlZnQ7IH1cblxuLmNlbnRlciB7XG5cdHRleHQtYWxpZ246IGNlbnRlcjsgfVxuXG5cbi8vbWl4aW5zXG5AbWl4aW4gcHVyZUNlbnRlcigkbGVmdDogNTAlLCAkdG9wOiA1MCUpIHtcblx0cG9zaXRpb246IGFic29sdXRlO1xuXHR0b3A6ICR0b3A7XG5cdGxlZnQ6ICRsZWZ0O1xuXHR0cmFuc2Zvcm06IHRyYW5zbGF0ZSgtNTAlLCAtNTAlKTsgfVxuXG5cbkBtaXhpbiBtYWludGFpbi1hc3BlY3QtcmF0aW8gKCR3aWR0aC1mYWN0b3I6IDE2LCAkaGVpZ2h0LWZhY3RvcjogOSwgJHRhcmdldC13aWR0aDogODB2dywgJHRhcmdldC1oZWlnaHQ6IDgwdmgpIHtcblx0d2lkdGg6ICR0YXJnZXQtd2lkdGg7XG5cdGhlaWdodDogJHRhcmdldC13aWR0aCAqICgkaGVpZ2h0LWZhY3RvciAvICR3aWR0aC1mYWN0b3IpO1xuXHRtYXgtd2lkdGg6ICR0YXJnZXQtaGVpZ2h0ICogKCR3aWR0aC1mYWN0b3IgLyAkaGVpZ2h0LWZhY3Rvcik7XG5cdG1heC1oZWlnaHQ6ICR0YXJnZXQtaGVpZ2h0OyB9XG5cbiIsIiRibHVlOiAjMzM3MWUzO1xuJGxpZ2h0LWdyZXk6ICNmN2Y3Zjc7XG4kZGFyay1ncmV5OiAjMzAzMDMwO1xuJG1lZGl1bS1ncmV5OiAjNGM0YzRjO1xuXG4kYmFzZUZvbnQ6ICdSb2JvdG8nLCBzYW5zLXNlcmlmO1xuXG5cbiosIC5idXR0b24ge1xuXHRib3gtc2l6aW5nOiBib3JkZXItYm94O1xuXHRmb250LWZhbWlseTogJGJhc2VGb250O1xuXHRiYWNrZ3JvdW5kOiBub25lO1xuXHRtYXJnaW46IDA7XG5cdGJvcmRlcjogMDsgfVxuXG5ib2R5IHtcblx0Zm9udC1mYW1pbHk6ICRiYXNlRm9udDsgfVxuXG5oMSwgaDIsIGg1LCBwIHtcblx0Zm9udC13ZWlnaHQ6IDMwMDsgfVxuXG5oMywgaDQge1xuXHRmb250LXdlaWdodDogNDAwOyB9XG4iLCJib2R5Lm92ZXJsYXktb24ge1xuXHRvdmVyZmxvdzogaGlkZGVuOyB9XG5cblxuJGF3ZXNvbWUtZ3JlZW46ICMwMGZmOTk7XG5cbi8vIG1vZGFsIGNsb3NlIGJ1dHRvblxuLnBpLW1vZGFsLWNsb3NlLWJ1dHRvbiB7XG5cdCRidXR0b24tc2lkZS1sZW5ndGg6IDM2cHg7XG5cdCRoeXBvdGVudXNlLWxlbmd0aDogNTAuOTExNjg4MjQ1NDMxNDJweDtcblxuXHRwb3NpdGlvbjogYWJzb2x1dGU7XG5cdHRvcDogNTBweDtcblx0cmlnaHQ6IDUwcHg7XG5cdHdpZHRoOiAkYnV0dG9uLXNpZGUtbGVuZ3RoO1xuXHRoZWlnaHQ6ICRidXR0b24tc2lkZS1sZW5ndGg7XG5cdGJvcmRlcjogMDtcblxuXHQmOmJlZm9yZSwgJjphZnRlciB7XG5cdFx0Y29udGVudDogXCJcIjtcblx0XHRwb3NpdGlvbjogYWJzb2x1dGU7XG5cdFx0dG9wOiAwO1xuXHRcdHdpZHRoOiAkaHlwb3RlbnVzZS1sZW5ndGg7XG5cdFx0aGVpZ2h0OiAycHg7XG5cdFx0YmFja2dyb3VuZC1jb2xvcjogd2hpdGU7IH1cblxuXHQmOmJlZm9yZSB7XG5cdFx0bGVmdDogMDtcblx0XHR0cmFuc2Zvcm0tb3JpZ2luOiB0b3AgbGVmdDtcblx0XHR0cmFuc2Zvcm06IHJvdGF0ZVooNDVkZWcpOyB9XG5cblx0JjphZnRlciB7XG5cdFx0cmlnaHQ6IDA7XG5cdFx0dHJhbnNmb3JtLW9yaWdpbjogdG9wIHJpZ2h0O1xuXHRcdHRyYW5zZm9ybTogcm90YXRlWigtNDVkZWcpOyB9IH1cblxuLy8gbW9kYWwgb3ZlcmxheVxuLnBpLW1vZGFsLW92ZXJsYXkge1xuXHRwb3NpdGlvbjogZml4ZWQ7XG5cdHRvcDogMDtcblx0bGVmdDogMDtcblx0d2lkdGg6IDEwMCU7XG5cdGhlaWdodDogMTAwJTtcblx0Y29sb3I6IHdoaXRlO1xuXHRiYWNrZ3JvdW5kLWNvbG9yOiByZ2JhKDAsIDAsIDAsIDAuOCk7XG5cdGRpc3BsYXk6IG5vbmU7XG5cdG9wYWNpdHk6IDA7XG5cdHRyYW5zaXRpb246IDAuM3M7XG5cblx0Ji5vbiB7XG5cdFx0b3BhY2l0eTogMTsgfSB9XG5cbi8vIG11bHRpRnJhbWVEaXNwbGF5XG4ucGktbXVsdGktZnJhbWUtZGlzcGxheSB7XG5cdHBvc2l0aW9uOiByZWxhdGl2ZTtcblx0d2lkdGg6IDgwdnc7XG5cdGhlaWdodDogODB2aDtcblx0bWFyZ2luOiAxMHZoIGF1dG87XG5cblx0Lml0ZW0td3JhcHBlciB7XG5cdFx0cG9zaXRpb246IHJlbGF0aXZlO1xuXHRcdHdpZHRoOiAxMDAlO1xuXHRcdGhlaWdodDogMTAwJTsgfVxuXG5cdC5pdGVtIHtcblx0XHRwb3NpdGlvbjogYWJzb2x1dGU7XG5cdFx0dG9wOiA1MCU7XG5cdFx0bGVmdDogNTAlO1xuXHRcdHRyYW5zZm9ybTogdHJhbnNsYXRlKC01MCUsIC01MCUpO1xuXHRcdGRpc3BsYXk6IG5vbmU7XG5cblx0XHQmLm9uIHtcblx0XHRcdGRpc3BsYXk6IGJsb2NrOyB9IH1cblxuXHQucGFnZXIge1xuXHRcdHBvc2l0aW9uOiBhYnNvbHV0ZTtcblx0XHRib3R0b206IDEwJTtcblx0XHRsZWZ0OiAwO1xuXHRcdHdpZHRoOiAxMDAlO1xuXHRcdGRpc3BsYXk6IGZsZXg7XG5cdFx0anVzdGlmeS1jb250ZW50OiBjZW50ZXI7XG5cblx0XHQucGFnZXItYnV0dG9uIHtcblx0XHRcdHdpZHRoOiAyMHB4O1xuXHRcdFx0aGVpZ2h0OiAyMHB4O1xuXHRcdFx0Ym9yZGVyLXJhZGl1czogNTAlO1xuXHRcdFx0YmFja2dyb3VuZC1jb2xvcjogYmxhY2s7XG5cdFx0XHRib3JkZXI6IDA7XG5cdFx0XHRtYXJnaW46IDAgMTBweDtcblx0XHRcdHRyYW5zaXRpb246IDAuM3M7XG5cblx0XHRcdCYueWFoIHtcblx0XHRcdFx0YmFja2dyb3VuZC1jb2xvcjogJGF3ZXNvbWUtZ3JlZW47IH0gfSB9XG5cblx0JHByZXYtbmV4dC1ib3JkZXItc3R5bGU6IDFweCBzb2xpZDtcblx0LnByZXYtYnV0dG9uLCAubmV4dC1idXR0b24ge1xuXHRcdHBvc2l0aW9uOiBhYnNvbHV0ZTtcblx0XHR0b3A6IDUwJTtcblx0XHR3aWR0aDogMzBweDtcblx0XHRoZWlnaHQ6IDMwcHg7XG5cdFx0Ym9yZGVyOiAwO1xuXHRcdGJvcmRlci1jb2xvcjogJGF3ZXNvbWUtZ3JlZW47XG5cdFx0ei1pbmRleDogMztcblx0XHR0cmFuc2Zvcm06IHRyYW5zbGF0ZVkoLTUwJSkgcm90YXRlWig0NWRlZyk7XG5cdFx0dHJhbnNpdGlvbjogMC4zcztcblxuXHRcdCY6aG92ZXIge1xuXHRcdFx0Ym9yZGVyLWNvbG9yOiByZWQ7IH0gfVxuXG5cdC5wcmV2LWJ1dHRvbiB7XG5cdFx0bGVmdDogMzBweDtcblx0XHRib3JkZXItYm90dG9tOiAkcHJldi1uZXh0LWJvcmRlci1zdHlsZTtcblx0XHRib3JkZXItbGVmdDogJHByZXYtbmV4dC1ib3JkZXItc3R5bGU7IH1cblxuXHQubmV4dC1idXR0b24ge1xuXHRcdHJpZ2h0OiAzMHB4O1xuXHRcdGJvcmRlci10b3A6ICRwcmV2LW5leHQtYm9yZGVyLXN0eWxlO1xuXHRcdGJvcmRlci1yaWdodDogJHByZXYtbmV4dC1ib3JkZXItc3R5bGU7IH0gfVxuXG4iLCIucGktYWNjb3JkaW9uIHtcblx0bWFyZ2luLWxlZnQ6IC0yMHB4O1xuXG5cdC5jb250YWluZXIge1xuXHRcdHBhZGRpbmctbGVmdDogMjBweDsgfVxuXG5cdC5pdGVtIHtcblx0XHRvdmVyZmxvdzogaGlkZGVuOyB9XG5cblx0LnRpdGxlIHtcblx0XHRjb2xvcjogJGRhcmstZ3JleTtcblx0XHRwb3NpdGlvbjogcmVsYXRpdmU7XG5cdFx0cGFkZGluZzogNXB4IDEwcHggMTBweCAyMHB4O1xuXHRcdGN1cnNvcjogcG9pbnRlcjtcblx0XHR0cmFuc2l0aW9uOiAwLjNzO1xuXG5cdFx0Jjpob3ZlciB7XG5cdFx0XHRjb2xvcjogJGJsdWU7IH0gfVxuXG5cdGEuaXRlbSA+IC50aXRsZSB7XG5cdFx0Y29sb3I6IGJsYWNrO1xuXHRcdHBhZGRpbmctbGVmdDogMDtcblxuXHRcdCY6aG92ZXIge1xuXHRcdFx0Y29sb3I6ICRibHVlOyB9IH1cblxuXHRkaXYuaXRlbSA+IC50aXRsZSB7XG5cdFx0Zm9udC13ZWlnaHQ6IDcwMDtcblxuXHRcdCY6YmVmb3JlIHtcblx0XHRcdGNvbnRlbnQ6IFwiXCI7XG5cdFx0XHRwb3NpdGlvbjogYWJzb2x1dGU7XG5cdFx0XHR0b3A6IDlweDtcblx0XHRcdGxlZnQ6IDJweDtcblx0XHRcdGJvcmRlci1zdHlsZTogc29saWQ7XG5cdFx0XHRib3JkZXItd2lkdGg6IDVweCAwIDVweCA4cHg7XG5cdFx0XHRib3JkZXItY29sb3I6IHRyYW5zcGFyZW50IHRyYW5zcGFyZW50IHRyYW5zcGFyZW50ICRibHVlO1xuXHRcdFx0dHJhbnNmb3JtOiByb3RhdGUoMGRlZyk7XG5cdFx0XHR0cmFuc2l0aW9uOiAwLjNzOyB9IH1cblxuXHQud3JhcHBlciB7XG5cdFx0cG9zaXRpb246IHJlbGF0aXZlO1xuXHRcdHdpZHRoOiAxMDAlO1xuXHRcdHRyYW5zaXRpb246IDAuM3M7IH1cblxuXHQuY29udGVudCB7XG5cdFx0cGFkZGluZy1sZWZ0OiAyMHB4O1xuXHRcdG9wYWNpdHk6IDA7XG5cdFx0dHJhbnNpdGlvbjogMC4zcztcblx0XHRwYWRkaW5nLWJvdHRvbTogMjBweDsgfVxuXG5cdC5pdGVtLm9uIHtcblx0XHQmID4gLnRpdGxlOmJlZm9yZSB7XG5cdFx0XHR0cmFuc2Zvcm06IHJvdGF0ZSg5MGRlZyk7IH1cblxuXHRcdCYgPiAud3JhcHBlciA+IC5jb250ZW50IHtcblx0XHRcdG9wYWNpdHk6IDE7IH0gfSB9XG5cbiIsIiRhd2Vzb21lLWdyZWVuOiAjMDBmZjk5O1xuXG4ucGktZGlhbG9nIHtcblxuXHQuZGlhbG9nLWJveCB7XG5cdFx0cG9zaXRpb246IGFic29sdXRlO1xuXHRcdHRvcDogNTAlO1xuXHRcdGxlZnQ6IDUwJTtcblx0XHR0cmFuc2Zvcm06IHRyYW5zbGF0ZSgtNTAlLCAtNTAlKTsgfVxuXG5cdC50aXRsZSB7XG5cdFx0Zm9udC1zaXplOiAzNnB4O1xuXHRcdHRleHQtYWxpZ246IGNlbnRlcjtcblx0XHRwYWRkaW5nOiAzMHB4IDA7IH1cblxuXHQuY29udGVudC1ib3gge1xuXHRcdGNvbG9yOiAjMjIyO1xuXHRcdG1heC13aWR0aDogNjAwcHg7XG5cdFx0b3ZlcmZsb3c6IGhpZGRlbjtcblxuXHRcdC5jb250ZW50IHtcblx0XHRcdHBhZGRpbmc6IDMwcHg7XG5cdFx0XHRib3JkZXItcmFkaXVzOiA0cHg7XG5cdFx0XHRiYWNrZ3JvdW5kLWNvbG9yOiB3aGl0ZTsgfVxuXG5cdFx0LmJ1dHRvbnMge1xuXHRcdFx0Ym9yZGVyLXRvcDogMXB4IHNvbGlkIGJsYWNrO1xuXHRcdFx0dGV4dC1hbGlnbjogcmlnaHQ7XG5cdFx0XHRtYXJnaW4tdG9wOiAzMHB4O1xuXG5cdFx0XHRidXR0b24ge1xuXHRcdFx0XHRjb2xvcjogJGF3ZXNvbWUtZ3JlZW47XG5cdFx0XHRcdGJvcmRlci1jb2xvcjogJGF3ZXNvbWUtZ3JlZW47XG5cdFx0XHRcdGJvcmRlci1yYWRpdXM6IDRweDtcblx0XHRcdFx0dHJhbnNpdGlvbjogMC4zcztcblxuXHRcdFx0XHQmOmhvdmVyIHtcblx0XHRcdFx0XHRiYWNrZ3JvdW5kLWNvbG9yOiAkYXdlc29tZS1ncmVlbjtcblx0XHRcdFx0XHRjb2xvcjogYmxhY2s7IH0gfSB9IH0gfVxuIiwiLnBpLXB1c2htZW51IHtcblx0ZGlzcGxheTogbm9uZTtcblx0cG9zaXRpb246IGZpeGVkO1xuXHR0b3A6IDA7XG5cdHdpZHRoOiAxMDAlO1xuXHRoZWlnaHQ6IDEwMCU7XG5cdG9wYWNpdHk6IDA7XG5cdHRyYW5zaXRpb246IG9wYWNpdHkgMC4zcztcblxuXHQmLm9uIHtcblx0XHRvcGFjaXR5OiAxOyB9XG5cblx0Lm92ZXJsYXkge1xuXHRcdHBvc2l0aW9uOiBmaXhlZDtcblx0XHR0b3A6IDA7XG5cdFx0bGVmdDogMDtcblx0XHR3aWR0aDogMTAwJTtcblx0XHRoZWlnaHQ6IDEwMCU7XG5cdFx0YmFja2dyb3VuZC1jb2xvcjogcmdiYSgwLCAwLCAwLCAwLjQpOyB9XG5cblx0LnNsZWQge1xuXHRcdHBvc2l0aW9uOiBhYnNvbHV0ZTtcblx0XHR0b3A6IDA7XG5cdFx0d2lkdGg6IDA7XG5cdFx0aGVpZ2h0OiAxMDAlO1xuXHRcdGJhY2tncm91bmQtY29sb3I6IHdoaXRlO1xuXHRcdG92ZXJmbG93OiBhdXRvO1xuXHRcdHRyYW5zaXRpb246IHdpZHRoIDAuM3M7IH1cblxuXHQmLm9uIC5zbGVkIHtcblx0XHR3aWR0aDogMzIwcHg7IH1cblxuXHQudG9wLWJhciB7XG5cdFx0aGVpZ2h0OiAwO1xuXHRcdGxpbmUtaGVpZ2h0OiA2MHB4O1xuXHRcdGJhY2tncm91bmQtY29sb3I6ICM0NDQ7IH1cblxuXHQucGktbW9kYWwtY2xvc2UtYnV0dG9uIHtcblx0XHRwb3NpdGlvbjogYWJzb2x1dGU7XG5cdFx0dG9wOiAxMHB4O1xuXHRcdHJpZ2h0OiAxMHB4O1xuXHRcdHotaW5kZXg6IDE7XG5cblx0XHQmOmJlZm9yZSwgJjphZnRlciB7XG5cdFx0XHRiYWNrZ3JvdW5kLWNvbG9yOiAjMjIyMjIyOyB9IH1cblxuXG5cdHVsIHtcblx0XHRwYWRkaW5nOiAwIDYwcHggMCAyMHB4OyB9XG5cdGxpIHtcblx0XHRwb3NpdGlvbjogcmVsYXRpdmU7XG5cdFx0ZGlzcGxheTogYmxvY2s7XG5cdFx0d2lkdGg6IDEwMCU7XG5cdFx0bWluLWhlaWdodDogNDRweDsgfVxuXG5cdGEge1xuXHRcdGRpc3BsYXk6IGlubGluZS1ibG9jaztcblx0XHR3aWR0aDogMTAwJTtcblx0XHRoZWlnaHQ6IDQ0cHg7XG5cdFx0bGluZS1oZWlnaHQ6IDQ0cHg7XG5cdFx0Zm9udC13ZWlnaHQ6IGJvbGQ7XG5cdFx0Zm9udC1zaXplOiAyMHB4O1xuXHRcdGNvbG9yOiAkYmx1ZTsgfVxuXG5cdC5idXR0b24ge1xuXHRcdGJhY2tncm91bmQ6IG5vbmU7XG5cdFx0cGFkZGluZzogMDsgfVxuXG5cblxuXHR1bCB1bCB7XG5cdFx0cGFkZGluZzogMCAyMHB4O1xuXG5cdFx0bGkge1xuXHRcdFx0bWluLWhlaWdodDogNDBweDsgfVxuXG5cdFx0YSB7XG5cdFx0XHRoZWlnaHQ6IDQwcHg7XG5cdFx0XHRsaW5lLWhlaWdodDogNDBweDtcblx0XHRcdGZvbnQtc2l6ZTogMThweDtcblx0XHRcdGNvbG9yOiBsaWdodGVuKCMyMjIyMjIsIDIwJSk7IH0gfSB9XG4iLCJoMSB7XG5cdGZvbnQtc2l6ZTogJGgxLWZvbnQtc2l6ZTtcblx0bGluZS1oZWlnaHQ6ICRoMS1saW5lLWhlaWdodDsgfVxuXG5oMiB7XG5cdGZvbnQtc2l6ZTogJGgyLWZvbnQtc2l6ZTtcblx0bGluZS1oZWlnaHQ6ICRoMi1saW5lLWhlaWdodDsgfVxuXG5oMyB7XG5cdGZvbnQtc2l6ZTogJGgzLWZvbnQtc2l6ZTtcblx0bGluZS1oZWlnaHQ6ICRoMy1saW5lLWhlaWdodDsgfVxuXG5oNCB7XG5cdGZvbnQtc2l6ZTogJGg0LWZvbnQtc2l6ZTtcblx0bGluZS1oZWlnaHQ6ICRoNC1saW5lLWhlaWdodDsgfVxuXG5oNSB7XG5cdGZvbnQtc2l6ZTogJGg1LWZvbnQtc2l6ZTtcblx0bGluZS1oZWlnaHQ6ICRoNS1saW5lLWhlaWdodDsgfVxuXG5wIHtcblx0Zm9udC1zaXplOiAkcC1mb250LXNpemU7XG5cdGxpbmUtaGVpZ2h0OiAkcC1saW5lLWhlaWdodDsgfVxuXG5zZWN0aW9uLCBoZWFkZXIsIGZvb3RlciwgI3ZlbmRvclN0cmlwIHtcblx0cGFkZGluZy1sZWZ0OiAkZnVsbC13aWR0aC1wYWRkaW5nWDtcblx0cGFkZGluZy1yaWdodDogJGZ1bGwtd2lkdGgtcGFkZGluZ1g7XG5cblx0bWFpbiB7XG5cdFx0d2lkdGg6ICRtYWluLXdpZHRoO1xuXHRcdG1heC13aWR0aDogJG1haW4tbWF4LXdpZHRoOyB9IH1cblxuaGVhZGVyIHtcblx0aGVpZ2h0OiAkaGVhZGVyLWhlaWdodDsgfVxuXG4ubG9nbyB7XG5cdHdpZHRoOiAkbG9nby13aWR0aDtcblx0aGVpZ2h0OiAkaGVhZGVyLWhlaWdodDsgfVxuXG4ubmF2LWJ1dHRvbnMge1xuXHRoZWlnaHQ6ICRoZWFkZXItaGVpZ2h0O1xuXHRsaW5lLWhlaWdodDogJGhlYWRlci1oZWlnaHQ7XG5cblx0LmJ1dHRvbiArICoge1xuXHRcdG1hcmdpbi1sZWZ0OiAkbmF2LWJ1dHRvbnMtbWFyZ2luLWxlZnQ7IH0gfVxuXG4jaGFtYnVyZ2VyIHtcblx0d2lkdGg6ICRoYW1idXJnZXItc2l6ZTtcblx0aGVpZ2h0OiAkaGFtYnVyZ2VyLXNpemU7IH1cblxuI21haW5OYXYge1xuXHRwYWRkaW5nOiAkbWFpbi1uYXYtcGFkZGluZztcblxuXHRoNSB7XG5cdFx0bWFyZ2luLWJvdHRvbTogJG1haW4tbmF2LWg1LW1hcmdpbi1ib3R0b207IH1cblxuXHRoMyB7XG5cdFx0bWFyZ2luLWJvdHRvbTogJG1haW4tbmF2LWgzLW1hcmdpbi1ib3R0b207IH1cblxuXHQubmF2LWJveCB7XG5cdFx0d2lkdGg6ICRuYXYtYm94LXdpZHRoOyB9XG5cblx0Lm5hdi1ib3ggKyAubmF2LWJveCB7XG5cdFx0bWFyZ2luLWxlZnQ6ICRuYXYtYm94LXNpYmxpbmctbWFyZ2luLWxlZnQ7IH1cblxuXHRtYWluICsgbWFpbiB7XG5cdFx0bWFyZ2luLXRvcDogJG1haW4tbmF2LW1haW4tc2libGluZy1tYXJnaW4tdG9wOyB9XG5cblx0LmxlZnQgLmJ1dHRvbiB7XG5cdFx0aGVpZ2h0OiAkbWFpbi1uYXYtbGVmdC1idXR0b24tc2l6ZTtcblx0XHRsaW5lLWhlaWdodDogJG1haW4tbmF2LWxlZnQtYnV0dG9uLXNpemU7XG5cdFx0Zm9udC1zaXplOiAkbWFpbi1uYXYtbGVmdC1idXR0b24tZm9udC1zaXplOyB9IH1cblxuLm9wZW4tbmF2LCAueS1lbm91Z2gge1xuXHQjdHJ5S3ViZXJuZXRlcyB7XG5cdFx0bWFyZ2luLWxlZnQ6ICRuYXYtYnV0dG9ucy1tYXJnaW4tbGVmdDsgfSB9XG5cbiNoZXJvIHtcblx0cGFkZGluZy10b3A6ICRoZXJvLXBhZGRpbmctdG9wOyB9XG5cbiNkb2NzIHtcblx0I2hlcm8ge1xuXHRcdGgxLCBoNSB7XG5cdFx0XHRwYWRkaW5nLWxlZnQ6ICRmdWxsLXdpZHRoLXBhZGRpbmdYO1xuXHRcdFx0cGFkZGluZy1yaWdodDogJGZ1bGwtd2lkdGgtcGFkZGluZ1g7IH0gfSB9XG5cbiNoZWFkbGluZVdyYXBwZXIsICNxdWlja3N0YXJ0QnV0dG9uIHtcblx0bWFyZ2luLWJvdHRvbTogJGhlYWRsaW5lLXdyYXBwZXItbWFyZ2luLWJvdHRvbTsgfVxuXG4jcXVpY2tzdGFydEJ1dHRvbiB7XG5cdHBhZGRpbmc6ICRxdWlja3N0YXJ0LWJ1dHRvbi1wYWRkaW5nOyB9XG5cbiN2ZW5kb3JTdHJpcCB7XG5cdGhlaWdodDogJHZlbmRvci1zdHJpcC1oZWlnaHQ7XG5cdGxpbmUtaGVpZ2h0OiAkdmVuZG9yLXN0cmlwLWhlaWdodDtcblx0Zm9udC1zaXplOiAkdmVuZG9yLXN0cmlwLWZvbnQtc2l6ZTtcblxuXHRpbWcge1xuXHRcdG1heC1oZWlnaHQ6ICR2ZW5kb3Itc3RyaXAtaW1nLW1heC1oZWlnaHQ7XG5cdFx0bWF4LXdpZHRoOiAkdmVuZG9yLXN0cmlwLWltZy1tYXgtd2lkdGg7XG5cdFx0bWFyZ2luLWxlZnQ6ICR2ZW5kb3Itc3RyaXAtaW1nLW1hcmdpbi1sZWZ0OyB9IH1cblxuI29jZWFuTm9kZXMge1xuXHRwYWRkaW5nLXRvcDogJG9jZWFuLW5vZGVzLXBhZGRpbmctWTtcblx0cGFkZGluZy1ib3R0b206ICRvY2Vhbi1ub2Rlcy1wYWRkaW5nLVk7XG5cblx0bWFpbiB7XG5cdFx0bWFyZ2luLWJvdHRvbTogJG9jZWFuLW5vZGVzLXBhZGRpbmctWTsgfSB9XG5cblxuI3ZpZGVvIHtcblx0aGVpZ2h0OiAkdmlkZW8tc2VjdGlvbi1oZWlnaHQ7IH1cblxuXG4jZmVhdHVyZXMge1xuXHRoMyB7XG5cdFx0bWFyZ2luLWJvdHRvbTogJGZlYXR1cmVzLWgzLW1hcmdpbi1ib3R0b207IH1cblxuXHQuZmVhdHVyZS1ib3gge1xuXHRcdG1hcmdpbi1ib3R0b206ICRmZWF0dXJlLWJveC1tYXJnaW4tYm90dG9tO1xuXG5cdFx0JiA+IGRpdiB7XG5cdFx0XHR3aWR0aDogJGZlYXR1cmUtYm94LWRpdi13aWR0aDtcblx0XHRcdG1hcmdpbi1ib3R0b206ICRmZWF0dXJlLWJveC1kaXYtbWFyZ2luLWJvdHRvbTsgfSB9IH1cblxuIiwiQGltcG9ydCBcIi4uL3BhcnRpYWxzL3Jlc2V0XCI7XG5cbi8vIFRFWFRcbiRoMS1mb250LXNpemU6IDQwcHg7XG4kaDEtbGluZS1oZWlnaHQ6IDUwcHg7XG5cbiRoMi1mb250LXNpemU6IDQycHg7XG4kaDItbGluZS1oZWlnaHQ6IDYwcHg7XG5cbiRoMy1mb250LXNpemU6IDI2cHg7XG4kaDMtbGluZS1oZWlnaHQ6IDMycHg7XG5cbiRoNC1mb250LXNpemU6IDI2cHg7XG4kaDQtbGluZS1oZWlnaHQ6IDQwcHg7XG5cbiRoNS1mb250LXNpemU6IDE4cHg7XG4kaDUtbGluZS1oZWlnaHQ6IDM2cHg7XG5cbiRwLWZvbnQtc2l6ZTogMjBweDtcbiRwLWxpbmUtaGVpZ2h0OiAyOHB4O1xuXG5cbi8vIGhlYWRlclxuJGZ1bGwtd2lkdGgtcGFkZGluZ1g6IDIwcHg7XG5cbiRtYWluLXdpZHRoOiAxMDAlO1xuJG1haW4tbWF4LXdpZHRoOiAxMDAlO1xuXG4kaGVhZGVyLWhlaWdodDogNTBweDtcblxuJGxvZ28td2lkdGg6IDEyMHB4O1xuXG4kbmF2LWJ1dHRvbnMtbWFyZ2luLWxlZnQ6IDMwcHg7XG5cbiRoYW1idXJnZXItc2l6ZTogNTBweDtcblxuXG4vLyBtYWluIG5hdlxuJG1haW4tbmF2LXBhZGRpbmc6IDE0MHB4IDAgMzBweDtcbiRtYWluLW5hdi1oNS1tYXJnaW4tYm90dG9tOiAxZW07XG4kbWFpbi1uYXYtaDMtbWFyZ2luLWJvdHRvbTogMC42ZW07XG4kbmF2LWJveC13aWR0aDogMjAlO1xuJG5hdi1ib3gtc2libGluZy1tYXJnaW4tbGVmdDogY2FsYygyMCUgLyAzKTtcbiRtYWluLW5hdi1tYWluLXNpYmxpbmctbWFyZ2luLXRvcDogNjBweDtcbiRtYWluLW5hdi1sZWZ0LWJ1dHRvbi1zaXplOiA1MHB4O1xuJG1haW4tbmF2LWxlZnQtYnV0dG9uLWZvbnQtc2l6ZTogMThweDtcblxuXG4vLyBoZXJvXG4kaGVyby1wYWRkaW5nLXRvcDogODBweDtcbiRoZWFkbGluZS13cmFwcGVyLW1hcmdpbi1ib3R0b206IDQwcHg7XG4kcXVpY2tzdGFydC1idXR0b24tcGFkZGluZzogMCA1MHB4O1xuJHZlbmRvci1zdHJpcC1oZWlnaHQ6IDYwcHg7XG4kdmVuZG9yLXN0cmlwLWZvbnQtc2l6ZTogMThweDtcbiR2ZW5kb3Itc3RyaXAtaW1nLW1heC1oZWlnaHQ6IDMwcHg7XG4kdmVuZG9yLXN0cmlwLWltZy1tYXgtd2lkdGg6IDEwMHB4O1xuJHZlbmRvci1zdHJpcC1pbWctbWFyZ2luLWxlZnQ6IDUwcHg7XG5cblxuLy8gb2NlYW4gbm9kZXNcbiRvY2Vhbi1ub2Rlcy1wYWRkaW5nLVk6IDYwcHg7XG4kb2NlYW4tbm9kZXMtbWFpbi1tYXJnaW4tYm90dG9tOiA2MHB4O1xuJG9jZWFuLW5vZGVzLWgzLW1hcmdpbi1ib3R0b206IDMwcHg7XG5cblxuLy8gdmlkZW9cbiR2aWRlby1zZWN0aW9uLWhlaWdodDogMjAwcHg7XG5cbi8vIGZlYXR1cmVzXG4kZmVhdHVyZXMtaDMtbWFyZ2luLWJvdHRvbTogMjBweDtcbiRmZWF0dXJlLWJveC1kaXYtd2lkdGg6IDEwMCU7XG4kZmVhdHVyZS1ib3gtbWFyZ2luLWJvdHRvbTogMDtcbiRmZWF0dXJlLWJveC1kaXYtbWFyZ2luLWJvdHRvbTogNDBweDtcblxuXG4vLyB0YWxrIHRvIHVzXG5cblxuXG5AaW1wb3J0IFwiLi4vcGFydGlhbHMvc2l6ZVwiO1xuXG4vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vXG4vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vXG4vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vXG5cbmJvZHkge1xuXHRiYWNrZ3JvdW5kLWNvbG9yOiB3aGl0ZTsgfVxuXG5zZWN0aW9uIHtcblx0cG9zaXRpb246IHJlbGF0aXZlO1xuXHRiYWNrZ3JvdW5kLWNvbG9yOiB3aGl0ZTsgfVxuXG5zZWN0aW9uLCBoZWFkZXIsIGZvb3RlciB7XG5cdG1haW4ge1xuXHRcdHBvc2l0aW9uOiByZWxhdGl2ZTtcblx0XHRtYXJnaW46IGF1dG87IH0gfVxuXG5oMyB7XG5cdHRleHQtYWxpZ246IGNlbnRlcjtcblx0Y29sb3I6ICRibHVlOyB9XG5cbi5idXR0b24ge1xuXHRkaXNwbGF5OiBpbmxpbmUtYmxvY2s7XG5cdGJvcmRlci1yYWRpdXM6IDZweDtcblx0cGFkZGluZzogMCAyMHB4O1xuXHRsaW5lLWhlaWdodDogNDBweDtcblx0Y29sb3I6IHdoaXRlO1xuXHRiYWNrZ3JvdW5kLWNvbG9yOiAkYmx1ZTtcblx0dGV4dC1kZWNvcmF0aW9uOiBub25lOyB9XG5cbiNjZWxsb3BoYW5lIHtcblx0cG9zaXRpb246IGZpeGVkO1xuXHR0b3A6IDA7XG5cdGxlZnQ6IDA7XG5cdHdpZHRoOiAxMDAlO1xuXHRoZWlnaHQ6IDEwMCU7XG5cdGRpc3BsYXk6IG5vbmU7IH1cblxuXG4vLyBIRUFERVJcbmhlYWRlciB7XG5cdHBvc2l0aW9uOiBmaXhlZDtcblx0dG9wOiAwO1xuXHRsZWZ0OiAwO1xuXHR3aWR0aDogMTAwJTtcblx0ei1pbmRleDogODg4ODtcblx0YmFja2dyb3VuZC1jb2xvcjogdHJhbnNwYXJlbnQ7XG5cdGJveC1zaGFkb3c6IDAgMCAwIHRyYW5zcGFyZW50O1xuXHRvdmVyZmxvdzogaGlkZGVuO1xuXHR0cmFuc2l0aW9uOiAwLjNzOyB9XG5cbi5sb2dvIHtcblx0ZmxvYXQ6IGxlZnQ7XG5cdGJhY2tncm91bmQtaW1hZ2U6IHVybCguLi8uLi9pbWFnZXMvbmF2X2xvZ28uc3ZnKTtcblx0YmFja2dyb3VuZC1zaXplOiBjb250YWluO1xuXHRiYWNrZ3JvdW5kLXBvc2l0aW9uOiBjZW50ZXIgY2VudGVyO1xuXHRiYWNrZ3JvdW5kLXJlcGVhdDogbm8tcmVwZWF0OyB9XG5cbi5uYXYtYnV0dG9ucyB7XG5cdGZsb2F0OiByaWdodDsgfVxuXG4jdmlld0RvY3MsICN0cnlLdWJlcm5ldGVzIHtcblx0ZGlzcGxheTogbm9uZTsgfVxuXG4jdmlld0RvY3Mge1xuXHRib3JkZXI6IDFweCBzb2xpZCB3aGl0ZTtcblx0YmFja2dyb3VuZC1jb2xvcjogdHJhbnNwYXJlbnQ7XG5cdHRyYW5zaXRpb246IDAuM3M7XG5cblx0Jjpob3ZlciB7XG5cdFx0YmFja2dyb3VuZC1jb2xvcjogd2hpdGU7XG5cdFx0Y29sb3I6ICRkYXJrLWdyZXk7IH0gfVxuXG4jdHJ5S3ViZXJuZXRlcyB7XG5cdHdpZHRoOiAwO1xuXHRwYWRkaW5nOiAwO1xuXHRtYXJnaW4tbGVmdDogMDtcblx0dmVydGljYWwtYWxpZ246IG1pZGRsZTtcblx0dGV4dC1hbGlnbjogY2VudGVyO1xuXHRvdmVyZmxvdzogaGlkZGVuO1xuXHR3aGl0ZS1zcGFjZTogbm93cmFwO1xuXHR0cmFuc2l0aW9uOiAwLjNzOyB9XG5cblxuI2hhbWJ1cmdlciB7XG5cdGRpc3BsYXk6IGlubGluZS1ibG9jaztcblx0cG9zaXRpb246IHJlbGF0aXZlO1xuXHR2ZXJ0aWNhbC1hbGlnbjogbWlkZGxlO1xuXHRwYWRkaW5nOiAwO1xuXHRib3JkZXI6IDA7XG5cdGJhY2tncm91bmQ6IG5vbmU7XG5cblx0ZGl2LCAmOmJlZm9yZSwgJjphZnRlciB7XG5cdFx0cG9zaXRpb246IGFic29sdXRlO1xuXHRcdGxlZnQ6IDE1JTtcblx0XHR3aWR0aDogNzAlO1xuXHRcdGhlaWdodDogMnB4O1xuXHRcdGJhY2tncm91bmQtY29sb3I6ICRibHVlO1xuXHRcdHRyYW5zaXRpb246IDAuM3M7XG5cdFx0Y29udGVudDogXCJcIjsgfVxuXG5cdGRpdiB7XG5cdFx0dG9wOiBjYWxjKDUwJSAtIDFweCk7IH1cblxuXHQmOmJlZm9yZSB7XG5cdFx0dG9wOiAyNCU7IH1cblxuXHQmOmFmdGVyIHtcblx0XHRib3R0b206IDI0JTsgfVxuXG5cdCY6aG92ZXIge1xuXHRcdGRpdiwgJjpiZWZvcmUsICY6YWZ0ZXIge1xuXHRcdFx0YmFja2dyb3VuZC1jb2xvcjogd2hpdGU7IH0gfSB9XG5cbiNtYWluTmF2IHtcblx0aDUge1xuXHRcdGNvbG9yOiAkYmx1ZTtcblx0XHRmb250LXdlaWdodDogbm9ybWFsOyB9XG5cblx0bWFpbiB7XG5cdFx0d2hpdGUtc3BhY2U6IG5vd3JhcDtcblx0XHRvdmVyZmxvdzogaGlkZGVuO1xuXHRcdGNsZWFyOiBib3RoOyB9XG5cblx0Lm5hdi1ib3gge1xuXHRcdGZsb2F0OiBsZWZ0O1xuXHRcdHdoaXRlLXNwYWNlOiBub3JtYWw7IH1cblxuXHRoMyB7XG5cdFx0YSB7XG5cdFx0XHRjb2xvcjogJGJsdWU7XG5cdFx0XHR0ZXh0LWRlY29yYXRpb246IG5vbmU7IH0gfSB9XG5cbi8vIEZMSVAgTkFWXG4uZmxpcC1uYXYge1xuXHRoZWFkZXIge1xuXHRcdGJhY2tncm91bmQtY29sb3I6IHdoaXRlOyB9IH1cblxuXG4vLyBPUEVOIE5BVlxuLm9wZW4tbmF2IHtcblx0Ym9keSB7XG5cdFx0b3ZlcmZsb3c6IGhpZGRlbjsgfVxuXG5cdCNjZWxsb3BoYW5lIHtcblx0XHRkaXNwbGF5OiBibG9jaztcblx0XHR6LWluZGV4OiA5OTk4OyB9XG5cblx0aGVhZGVyIHtcblx0XHRiYWNrZ3JvdW5kLWNvbG9yOiAkbGlnaHQtZ3JleTtcblx0XHR6LWluZGV4OiA5OTk5OyB9XG5cblx0I2hhbWJ1cmdlciB7XG5cdFx0ZGl2IHtcblx0XHRcdG9wYWNpdHk6IDA7IH1cblxuXHRcdCY6YmVmb3JlLCAmOmFmdGVyIHtcblx0XHRcdGxlZnQ6IDEycHg7XG5cdFx0XHR0cmFuc2Zvcm0tb3JpZ2luOiAwIDFweDsgfVxuXG5cdFx0JjpiZWZvcmUge1xuXHRcdFx0dHJhbnNmb3JtOiByb3RhdGUoNDVkZWcpOyB9XG5cblx0XHQmOmFmdGVyIHtcblx0XHRcdHRyYW5zZm9ybTogcm90YXRlKC00NWRlZyk7IH0gfSB9XG5cbi5vcGVuLW5hdiwgLnktZW5vdWdoIHtcblx0I3RyeUt1YmVybmV0ZXMge1xuXHRcdHdpZHRoOiAxNjBweDsgfSB9XG5cblxuLmZsaXAtbmF2LCAub3Blbi1uYXYge1xuXHRoZWFkZXIge1xuXHRcdGJveC1zaGFkb3c6IDAgMXB4IDJweCAkbWVkaXVtLWdyZXk7XG5cblx0XHQubG9nbyB7XG5cdFx0XHRiYWNrZ3JvdW5kLWltYWdlOiB1cmwoLi4vLi4vaW1hZ2VzL25hdl9sb2dvMi5zdmcpOyB9IH1cblxuXHQjdmlld0RvY3Mge1xuXHRcdGJvcmRlci1jb2xvcjogJGRhcmstZ3JleTtcblx0XHRjb2xvcjogJGRhcmstZ3JleTtcblxuXHRcdCY6aG92ZXIge1xuXHRcdFx0Ym9yZGVyLWNvbG9yOiAkYmx1ZTtcblx0XHRcdGJhY2tncm91bmQtY29sb3I6ICRibHVlO1xuXHRcdFx0Y29sb3I6IHdoaXRlOyB9IH1cblxuXHQjaGFtYnVyZ2VyOmhvdmVyIHtcblx0XHRkaXYsICY6YmVmb3JlLCAmOmFmdGVyIHtcblx0XHRcdGJhY2tncm91bmQtY29sb3I6ICRkYXJrLWdyZXk7IH0gfSB9XG5cblxuXG4vLyBIRVJPXG4jaGVybyB7XG5cdGJhY2tncm91bmQtaW1hZ2U6IHVybCguLi8uLi9pbWFnZXMvdGV4dHVyZS5wbmcpO1xuXHRiYWNrZ3JvdW5kLWNvbG9yOiAkZGFyay1ncmV5O1xuXHR0ZXh0LWFsaWduOiBjZW50ZXI7XG5cdHBhZGRpbmctbGVmdDogMDtcblx0cGFkZGluZy1yaWdodDogMDsgfVxuXG4jaGVhZGxpbmVXcmFwcGVyIHtcblx0dGV4dC1hbGlnbjogY2VudGVyO1xuXHRwYWRkaW5nOiAwIDIwcHg7IH1cblxuI3F1aWNrc3RhcnRCdXR0b24ge1xuXHRkaXNwbGF5OiBub25lO1xuXHRmb250LXNpemU6IDI0cHg7XG5cdGxpbmUtaGVpZ2h0OiA1MHB4O1xuXHRwYWRkaW5nOiAwIDUwcHg7IH1cblxuXG4vLyBWRU5ET1IgU1RSSVBcbiN2ZW5kb3JTdHJpcCB7XG5cdGRpc3BsYXk6IG5vbmU7XG5cdGJhY2tncm91bmQtY29sb3I6IHRyYW5zcGFyZW50aXplKHdoaXRlLCAwLjkpO1xuXHRmb250LXdlaWdodDogMTAwO1xuXHR3aGl0ZS1zcGFjZTogbm93cmFwO1xuXHR0ZXh0LWFsaWduOiBsZWZ0O1xuXG5cdGRpdiB7XG5cdFx0dmVydGljYWwtYWxpZ246IG1pZGRsZTtcblx0XHRoZWlnaHQ6IDEwMCU7XG5cdFx0ZGlzcGxheTogaW5saW5lLWJsb2NrOyB9XG5cblxuXHRpbWcge1xuXHRcdHZlcnRpY2FsLWFsaWduOiBtaWRkbGU7IH0gfVxuXG5cbi8vIE9DRUFOIE5PREVTXG4jb2NlYW5Ob2RlcyB7XG5cdG1haW4ge1xuXHRcdG1hcmdpbi1ib3R0b206ICRvY2Vhbi1ub2Rlcy1tYWluLW1hcmdpbi1ib3R0b207IH1cblxuXHQuaW1hZ2Utd3JhcHBlciB7XG5cdFx0bWF4LXdpZHRoOiA3NSU7XG5cdFx0bWFyZ2luOiAwIGF1dG8gMjBweDtcblxuXHRcdGltZyB7XG5cdFx0XHR3aWR0aDogMTAwJTsgfSB9XG5cblx0bWFpbjpmaXJzdC1jaGlsZCB7XG5cdFx0LmltYWdlLXdyYXBwZXIge1xuXHRcdFx0bWF4LXdpZHRoOiAxMDAlOyB9IH1cblxuXHRoMyB7XG5cdFx0bWFyZ2luLWJvdHRvbTogJG9jZWFuLW5vZGVzLWgzLW1hcmdpbi1ib3R0b207IH0gfVxuXG4vLyBWaWRlbyB0aGluZ3lcbiN2aWRlbyB7XG5cdHdpZHRoOiAxMDAlO1xuXHRwb3NpdGlvbjogcmVsYXRpdmU7XG5cdGJhY2tncm91bmQtaW1hZ2U6IHVybCguLi8uLi9pbWFnZXMva3ViX3ZpZGVvX3RobS5qcGcpO1xuXHRiYWNrZ3JvdW5kLXBvc2l0aW9uOiBjZW50ZXIgY2VudGVyO1xuXHRiYWNrZ3JvdW5kLXNpemU6IGNvdmVyO1xuXG5cblx0JiA+IC5saWdodC10ZXh0IHtcblx0XHRkaXNwbGF5OiBub25lO1xuXHRcdHBvc2l0aW9uOiBhYnNvbHV0ZTtcblx0XHR0b3A6IDUwJTtcblx0XHRsZWZ0OiA3NSU7XG5cdFx0d2lkdGg6IDUwJTtcblx0XHRtYXgtd2lkdGg6IDYwMHB4O1xuXHRcdG1pbi13aWR0aDogNTAwcHg7XG5cdFx0cGFkZGluZy1yaWdodDogMTAlO1xuXHRcdHRyYW5zZm9ybTogdHJhbnNsYXRlKC01MCUsIC01MCUpO1xuXHRcdGNvbG9yOiB3aGl0ZTsgfVxuXG5cdGgyIHtcblx0XHRmb250LXNpemU6IDMycHg7XG5cdFx0bGluZS1oZWlnaHQ6IDQ0cHg7XG5cdFx0bWFyZ2luLWJvdHRvbTogMjBweDsgfVxuXG5cdHAge1xuXHRcdG1hcmdpbi1ib3R0b206IDIwcHg7IH1cblxuXHQjZGVza3RvcFNob3dWaWRlb0J1dHRvbiB7XG5cdFx0cG9zaXRpb246IHJlbGF0aXZlO1xuXHRcdGZvbnQtc2l6ZTogMjRweDtcblx0XHRiYWNrZ3JvdW5kLWNvbG9yOiB3aGl0ZTtcblx0XHRib3JkZXItcmFkaXVzOiA4cHg7XG5cdFx0Y29sb3I6ICRibHVlO1xuXHRcdHBhZGRpbmc6IDE1cHggMzBweCAxNXB4IDgwcHg7XG5cblx0XHQmOmJlZm9yZSB7XG5cdFx0XHRjb250ZW50OiBcIlwiO1xuXHRcdFx0cG9zaXRpb246IGFic29sdXRlO1xuXHRcdFx0QGluY2x1ZGUgcHVyZUNlbnRlcig0MHB4KTtcblx0XHRcdHdpZHRoOiAwO1xuXHRcdFx0aGVpZ2h0OiAwO1xuXHRcdFx0Ym9yZGVyLXN0eWxlOiBzb2xpZDtcblx0XHRcdGJvcmRlci13aWR0aDogMTBweCAwIDEwcHggMjBweDtcblx0XHRcdGJvcmRlci1jb2xvcjogdHJhbnNwYXJlbnQgdHJhbnNwYXJlbnQgdHJhbnNwYXJlbnQgJGJsdWU7IH0gfVxuXG5cdCNtb2JpbGVTaG93VmlkZW9CdXR0b24ge1xuXHRcdEBpbmNsdWRlIHB1cmVDZW50ZXI7XG5cdFx0d2lkdGg6IDgwcHg7XG5cdFx0aGVpZ2h0OiA4MHB4O1xuXHRcdGJvcmRlci1yYWRpdXM6IDUwJTtcblx0XHRiYWNrZ3JvdW5kLWNvbG9yOiB0cmFuc3BhcmVudDtcblx0XHRib3JkZXI6IDVweCBzb2xpZCByZ2JhKDI1NSwgMjU1LCAyNTUsIDAuMik7XG5cdFx0b3ZlcmZsb3c6IHZpc2libGU7XG5cblx0XHQmOmFmdGVyIHtcblx0XHRcdEBpbmNsdWRlIHB1cmVDZW50ZXI7XG5cdFx0XHRsZWZ0OiA0MHB4O1xuXHRcdFx0Y29udGVudDogXCJcIjtcblx0XHRcdHdpZHRoOiAwO1xuXHRcdFx0aGVpZ2h0OiAwO1xuXHRcdFx0Ym9yZGVyLXN0eWxlOiBzb2xpZDtcblx0XHRcdGJvcmRlci13aWR0aDogMjBweCAwIDIwcHggMzBweDtcblx0XHRcdGJvcmRlci1jb2xvcjogdHJhbnNwYXJlbnQgdHJhbnNwYXJlbnQgdHJhbnNwYXJlbnQgI2ZmZmZmZjsgfSB9IH1cblxuXG4vLyBGZWF0dXJlc1xuI2ZlYXR1cmVzIHtcblx0cGFkZGluZy10b3A6IDE0MHB4O1xuXHRiYWNrZ3JvdW5kLWNvbG9yOiAkbGlnaHQtZ3JleTtcblx0YmFja2dyb3VuZC1pbWFnZTogdXJsKC4uLy4uL2ltYWdlcy93aGVlbC5wbmcpO1xuXHRiYWNrZ3JvdW5kLXBvc2l0aW9uOiBjZW50ZXIgNjBweDtcblx0YmFja2dyb3VuZC1yZXBlYXQ6IG5vLXJlcGVhdDtcblx0YmFja2dyb3VuZC1zaXplOiBhdXRvOyB9XG5cbi5mZWF0dXJlLWJveCB7XG5cdC8vcGFkZGluZzogNTBweCAwXG5cdHdpZHRoOiAxMDAlO1xuXHRvdmVyZmxvdzogaGlkZGVuO1xuXHRjbGVhcjogYm90aDtcblxuXHRoNCB7XG5cdFx0bGluZS1oZWlnaHQ6IG5vcm1hbDtcblx0XHRtYXJnaW4tYm90dG9tOiAxNXB4OyB9XG5cblx0JiA+IGRpdjpmaXJzdC1jaGlsZCB7XG5cdFx0ZmxvYXQ6IGxlZnQ7IH1cblxuXHQmID4gZGl2Omxhc3QtY2hpbGQge1xuXHRcdGZsb2F0OiByaWdodDsgfSB9XG5cblxuLy8gVGFsayB0byB1c1xuI3RhbGtUb1VzIHtcblx0bWFpbiB7XG5cdFx0cGFkZGluZzogNTBweCAwOyB9XG5cblx0aDMsIGg0IHtcblx0XHR0ZXh0LWFsaWduOiBjZW50ZXI7IH1cblxuXHRoMyB7XG5cdFx0bWFyZ2luLWJvdHRvbTogMTVweDsgfVxuXG5cdGg0IHtcblx0XHRsaW5lLWhlaWdodDogbm9ybWFsO1xuXHRcdG1hcmdpbi1ib3R0b206IDUwcHg7IH1cblxuXHQjYmlnU29jaWFsIHtcblx0XHRvdmVyZmxvdzogaGlkZGVuO1xuXG5cdFx0ZGl2IHtcblx0XHRcdHdpZHRoOiBjYWxjKDI1JSAtIDE1cHgpO1xuXHRcdFx0ZmxvYXQ6IGxlZnQ7XG5cdFx0XHRwYWRkaW5nOiAzMHB4O1xuXHRcdFx0cGFkZGluZy10b3A6IDExMHB4O1xuXHRcdFx0YmFja2dyb3VuZC1wb3NpdGlvbjogY2VudGVyIHRvcDtcblx0XHRcdGJhY2tncm91bmQtc2l6ZTogYXV0bztcblx0XHRcdGJhY2tncm91bmQtcmVwZWF0OiBuby1yZXBlYXQ7IH1cblxuXHRcdGRpdjpudGgtY2hpbGQoMSkge1xuXHRcdFx0YmFja2dyb3VuZC1pbWFnZTogdXJsKC4uLy4uL2ltYWdlcy90d2l0dGVyX2ljb24ucG5nKTsgfVxuXG5cdFx0ZGl2Om50aC1jaGlsZCgyKSB7XG5cdFx0XHRiYWNrZ3JvdW5kLWltYWdlOiB1cmwoLi4vLi4vaW1hZ2VzL2dpdGh1Yl9pY29uLnBuZyk7IH1cblxuXHRcdGRpdjpudGgtY2hpbGQoMykge1xuXHRcdFx0YmFja2dyb3VuZC1pbWFnZTogdXJsKC4uLy4uL2ltYWdlcy9zbGFja19pY29uLnBuZyk7IH1cblxuXHRcdGRpdjpudGgtY2hpbGQoNCkge1xuXHRcdFx0YmFja2dyb3VuZC1pbWFnZTogdXJsKC4uLy4uL2ltYWdlcy9zdGFja292ZXJmbG93X2ljb24ucG5nKTsgfVxuXG5cdFx0ZGl2ICsgZGl2IHtcblx0XHRcdG1hcmdpbi1sZWZ0OiAyMHB4OyB9XG5cblx0XHRhIHtcblx0XHRcdGRpc3BsYXk6IGlubGluZS1ibG9jaztcblx0XHRcdGNvbG9yOiAkYmx1ZTtcblx0XHRcdGZvbnQtc2l6ZTogMjRweDtcblx0XHRcdGZvbnQtd2VpZ2h0OiA0MDA7XG5cdFx0XHR0ZXh0LWRlY29yYXRpb246IG5vbmU7XG5cdFx0XHRtYXJnaW4tYm90dG9tOiAxNXB4OyB9IH0gfVxuXG5cbi8vIEZPT1RFUlxuZm9vdGVyIHtcblx0d2lkdGg6IDEwMCU7XG5cdGJhY2tncm91bmQtaW1hZ2U6IHVybCguLi8uLi9pbWFnZXMvdGV4dHVyZS5wbmcpO1xuXHRiYWNrZ3JvdW5kLWNvbG9yOiAkZGFyay1ncmV5O1xuXG5cdG1haW4ge1xuXHRcdHBhZGRpbmc6IDQwcHggMDsgfVxuXG5cdG5hdiB7XG5cdFx0dGV4dC1hbGlnbjogY2VudGVyO1xuXHRcdHdoaXRlLXNwYWNlOiBub3dyYXA7XG5cblx0XHRhIHtcblx0XHRcdGRpc3BsYXk6IGlubGluZS1ibG9jaztcblx0XHRcdGZvbnQtc2l6ZTogMzJweDtcblx0XHRcdGZvbnQtd2VpZ2h0OiAzMDA7XG5cdFx0XHRjb2xvcjogd2hpdGU7XG5cdFx0XHR0ZXh0LWRlY29yYXRpb246IG5vbmU7IH1cblxuXHRcdGEgKyBhIHtcblx0XHRcdG1hcmdpbi1sZWZ0OiAxMCU7IH0gfVxuXG5cdC5zb2NpYWwge1xuXHRcdHBvc2l0aW9uOiByZWxhdGl2ZTtcblx0XHRtYXJnaW46IDgwcHggMCA2MHB4O1xuXG5cdFx0bGFiZWwge1xuXHRcdFx0ZmxvYXQ6IHJpZ2h0O1xuXHRcdFx0ZGlzcGxheTogaW5saW5lLWJsb2NrO1xuXHRcdFx0aGVpZ2h0OiA1MHB4O1xuXHRcdFx0bGluZS1oZWlnaHQ6IDUwcHg7XG5cdFx0XHRmb250LXdlaWdodDogMTAwO1xuXHRcdFx0d2hpdGUtc3BhY2U6IG5vd3JhcDtcblxuXHRcdFx0aW5wdXQge1xuXHRcdFx0XHRtYXJnaW4tbGVmdDogOHB4O1xuXHRcdFx0XHR3aWR0aDogMzAwcHg7IH0gfSB9IH1cblxuXG4jc2VhcmNoLCAjd2lzaEZpZWxkIHtcblx0YmFja2dyb3VuZC1jb2xvcjogdHJhbnNwYXJlbnQ7XG5cdHBhZGRpbmc6IDEwcHg7XG5cdGZvbnQtc2l6ZTogMTZweDtcblx0Zm9udC13ZWlnaHQ6IDEwMDtcblx0Y29sb3I6IHdoaXRlO1xuXHRib3JkZXI6IDFweCBzb2xpZCB3aGl0ZTtcblx0dHJhbnNpdGlvbjogMC4zcztcblxuXHQmOmZvY3VzIHtcblx0XHRiYWNrZ3JvdW5kLWNvbG9yOiAkbGlnaHQtZ3JleTtcblx0XHRjb2xvcjogJGRhcmstZ3JleTsgfSB9XG5cbi5zb2NpYWwgYSB7XG5cdGRpc3BsYXk6IGlubGluZS1ibG9jaztcblx0YmFja2dyb3VuZC1pbWFnZTogdXJsKC4uLy4uL2ltYWdlcy9zb2NpYWxfc3ByaXRlLnBuZyk7XG5cdGJhY2tncm91bmQtcmVwZWF0OiBuby1yZXBlYXQ7XG5cdGJhY2tncm91bmQtc2l6ZTogYXV0bztcblx0d2lkdGg6IDUwcHg7XG5cdGhlaWdodDogNTBweDtcblx0Ym9yZGVyLXJhZGl1czogNXB4O1xuXHRtYXJnaW4tcmlnaHQ6IDMwcHg7XG5cblx0Jjpob3ZlciB7XG5cdFx0YmFja2dyb3VuZC1jb2xvcjogI2ZmZjsgfVxuXG5cdHNwYW4ge1xuXHRcdHBvc2l0aW9uOiBhYnNvbHV0ZTtcblx0XHRkaXNwbGF5OiBibG9jaztcblx0XHRoZWlnaHQ6IDA7XG5cdFx0b3ZlcmZsb3c6IGhpZGRlbjsgfSB9XG5cbmEudHdpdHRlciB7XG5cdGJhY2tncm91bmQtcG9zaXRpb246IDAgMDtcblxuXHQmOmhvdmVyIHtcblx0XHRiYWNrZ3JvdW5kLXBvc2l0aW9uOiAwIDEwMCU7IH0gfVxuXG5hLnN0YWNrLW92ZXJmbG93IHtcblx0YmFja2dyb3VuZC1wb3NpdGlvbjogLTUwcHggMDtcblxuXHQmOmhvdmVyIHtcblx0XHRiYWNrZ3JvdW5kLXBvc2l0aW9uOiAtNTBweCAxMDAlOyB9IH1cblxuYS5zbGFjayB7XG5cdGJhY2tncm91bmQtcG9zaXRpb246IC0xMDBweCAwO1xuXG5cdCY6aG92ZXIge1xuXHRcdGJhY2tncm91bmQtcG9zaXRpb246IC0xMDBweCAxMDAlOyB9IH1cblxuYS5naXRodWIge1xuXHRiYWNrZ3JvdW5kLXBvc2l0aW9uOiAtMTUwcHggMDtcblxuXHQmOmhvdmVyIHtcblx0XHRiYWNrZ3JvdW5kLXBvc2l0aW9uOiAtMTUwcHggMTAwJTsgfSB9XG5cbmEubWFpbGluZy1saXN0IHtcblx0YmFja2dyb3VuZC1wb3NpdGlvbjogLTIwMHB4IDA7XG5cblx0Jjpob3ZlciB7XG5cdFx0YmFja2dyb3VuZC1wb3NpdGlvbjogLTIwMHB4IDEwMCU7IH0gfVxuXG5cbmlmcmFtZSB7XG5cdEBpbmNsdWRlIG1haW50YWluLWFzcGVjdC1yYXRpbztcblx0QGluY2x1ZGUgcHVyZUNlbnRlcjsgfVxuXG5cblxuXG4vLyBET0NTXG5cblxuXG4jZG9jcyB7XG5cdCN2aWV3RG9jcyB7XG5cdFx0ZGlzcGxheTogbm9uZTsgfVxuXG5cblxuXHRzZWN0aW9uIHtcblx0XHRiYWNrZ3JvdW5kLWNvbG9yOiB3aGl0ZTtcblx0XHRtaW4td2lkdGg6IDEwMjRweDsgfVxuXG5cdCNoZXJvIHtcblx0XHRiYWNrZ3JvdW5kLWNvbG9yOiAkZGFyay1ncmV5O1xuXG5cdFx0aDUge1xuXHRcdFx0bWFyZ2luLWJvdHRvbTogNDBweDsgfSB9XG5cblx0I3ZlbmRvclN0cmlwIHtcblx0XHRwb3NpdGlvbjogcmVsYXRpdmU7XG5cdFx0d2hpdGUtc3BhY2U6IG5vd3JhcDtcblxuXHRcdHVsIHtcblx0XHRcdGZsb2F0OiBsZWZ0OyB9XG5cblx0XHRsaSB7XG5cdFx0XHRkaXNwbGF5OiBpbmxpbmUtYmxvY2s7XG5cdFx0XHRoZWlnaHQ6IDEwMCU7IH1cblxuXHRcdGEge1xuXHRcdFx0ZGlzcGxheTogYmxvY2s7XG5cdFx0XHRoZWlnaHQ6IDEwMCU7XG5cdFx0XHRjb2xvcjogd2hpdGU7IH1cblxuXHRcdGxpICsgbGksICNzZWFyY2gge1xuXHRcdFx0bWFyZ2luLWxlZnQ6IDYwcHg7IH1cblxuXHRcdCNzZWFyY2gge1xuXHRcdFx0cG9zaXRpb246IHJlbGF0aXZlO1xuXHRcdFx0ZmxvYXQ6IHJpZ2h0O1xuXHRcdFx0d2lkdGg6IDI1MHB4O1xuXHRcdFx0cGFkZGluZzogMCAxMHB4O1xuXHRcdFx0bWFyZ2luOiAxMHB4IDMwcHggMCAwO1xuXHRcdFx0aGVpZ2h0OiA0MHB4O1xuXHRcdFx0bGluZS1oZWlnaHQ6IDQwcHg7XG5cdFx0XHRib3JkZXI6IDFweCBzb2xpZCAkbGlnaHQtZ3JleTtcblx0XHRcdGZvbnQtc2l6ZTogMTZweDtcblx0XHRcdHZlcnRpY2FsLWFsaWduOiBtaWRkbGU7IH1cblxuXHRcdC5kcm9wZG93biB7XG5cdFx0XHRwb3NpdGlvbjogcmVsYXRpdmU7XG5cdFx0XHRmbG9hdDogcmlnaHQ7XG5cdFx0XHRoZWlnaHQ6IDYwcHg7XG5cdFx0XHRmb250LXNpemU6IDIwcHg7XG5cdFx0XHRjb2xvcjogd2hpdGU7XG5cdFx0XHRsaW5lLWhlaWdodDogNjBweDtcblx0XHRcdGN1cnNvcjogcG9pbnRlcjtcblx0XHRcdG92ZXJmbG93OiBoaWRkZW47XG5cdFx0XHR6LWluZGV4OiA1MDAwO1xuXG5cdFx0XHQmLm9uIHtcblx0XHRcdFx0b3ZlcmZsb3c6IHZpc2libGU7IH1cblxuXHRcdFx0LnJlYWRvdXQge1xuXHRcdFx0XHRwYWRkaW5nLWxlZnQ6IDEwcHg7XG5cdFx0XHRcdHBhZGRpbmctcmlnaHQ6IDQwcHg7XG5cdFx0XHRcdHRyYW5zaXRpb246IDAuM3M7XG5cblx0XHRcdFx0JjphZnRlciB7XG5cdFx0XHRcdFx0Y29udGVudDogXCJcIjtcblx0XHRcdFx0XHRwb3NpdGlvbjogYWJzb2x1dGU7XG5cdFx0XHRcdFx0dG9wOiAzMHB4O1xuXHRcdFx0XHRcdHJpZ2h0OiAxMHB4O1xuXHRcdFx0XHRcdHdpZHRoOiAwO1xuXHRcdFx0XHRcdGhlaWdodDogMDtcblx0XHRcdFx0XHRib3JkZXItc3R5bGU6IHNvbGlkO1xuXHRcdFx0XHRcdGJvcmRlci13aWR0aDogNnB4IDZweCAwIDZweDtcblx0XHRcdFx0XHRib3JkZXItY29sb3I6IHdoaXRlIHRyYW5zcGFyZW50IHRyYW5zcGFyZW50IHRyYW5zcGFyZW50O1xuXHRcdFx0XHRcdHRyYW5zaXRpb246IDAuM3M7IH0gfVxuXG5cdFx0XHQucmVhZG91dDpob3ZlciB7XG5cdFx0XHRcdGJhY2tncm91bmQtY29sb3I6IHRyYW5zcGFyZW50aXplKHdoaXRlLCAwLjkpOyB9XG5cblx0XHRcdCYub24gLnJlYWRvdXQge1xuXHRcdFx0XHRiYWNrZ3JvdW5kLWNvbG9yOiAkbGlnaHQtZ3JleTtcblx0XHRcdFx0Y29sb3I6ICRkYXJrLWdyZXk7IH1cblxuXHRcdFx0Ji5vbiAucmVhZG91dDphZnRlciB7XG5cdFx0XHRcdGJvcmRlci10b3AtY29sb3I6ICRkYXJrLWdyZXk7IH1cblxuXHRcdFx0LndyYXBwZXIge1xuXHRcdFx0XHRwb3NpdGlvbjogYWJzb2x1dGU7XG5cdFx0XHRcdHRvcDogMTAwJTtcblx0XHRcdFx0bGVmdDogMDtcblx0XHRcdFx0d2lkdGg6IDEwMCU7IH1cblxuXHRcdFx0YSB7XG5cdFx0XHRcdGRpc3BsYXk6IGJsb2NrO1xuXHRcdFx0XHR3aWR0aDogMTAwJTtcblx0XHRcdFx0aGVpZ2h0OiAzMHB4O1xuXHRcdFx0XHRsaW5lLWhlaWdodDogMzBweDtcblx0XHRcdFx0cGFkZGluZzogMCAxMHB4O1xuXHRcdFx0XHRmb250LXNpemU6IDE2cHg7XG5cdFx0XHRcdGZvbnQtd2VpZ2h0OiAzMDA7XG5cdFx0XHRcdGJhY2tncm91bmQtY29sb3I6ICRsaWdodC1ncmV5O1xuXHRcdFx0XHRjb2xvcjogJGRhcmstZ3JleTtcblx0XHRcdFx0b3BhY2l0eTogMDtcblx0XHRcdFx0dHJhbnNpdGlvbjogMC4zcztcblxuXHRcdFx0XHQmOmhvdmVyIHtcblx0XHRcdFx0XHRiYWNrZ3JvdW5kLWNvbG9yOiAjY2NjY2NjOyB9IH1cblxuXHRcdFx0Ji5vbiBhIHtcblx0XHRcdFx0b3BhY2l0eTogMTsgfSB9IH1cblxuXHQjZW5jeWNsb3BlZGlhIHtcblx0XHRwb3NpdGlvbjogcmVsYXRpdmU7XG5cdFx0cGFkZGluZzogNTBweDtcblx0XHRvdmVyZmxvdzogaGlkZGVuO1xuXG5cdFx0JiA+IGRpdiB7XG5cdFx0XHRoZWlnaHQ6IDEwMCU7IH1cblxuXHRcdCNkb2NzVG9jIHtcblx0XHRcdGZsb2F0OiBsZWZ0O1xuXHRcdFx0d2lkdGg6IDM1MHB4O1xuXHRcdFx0cGFkZGluZzogMCAyMHB4IDAgMDtcblx0XHRcdGJvcmRlci1yaWdodDogMXB4IHNvbGlkICNjY2NjY2M7XG5cblx0XHRcdC55YWgge1xuXHRcdFx0XHQmID4gLnRpdGxlIHtcblx0XHRcdFx0XHRiYWNrZ3JvdW5kLWNvbG9yOiB0cmFuc3BhcmVudGl6ZSgkYmx1ZSwgMC44KTtcblx0XHRcdFx0XHRwYWRkaW5nLWxlZnQ6IDEwcHg7XG5cdFx0XHRcdFx0cGFkZGluZy1yaWdodDogMTBweDsgfSB9IH1cblxuXHRcdCNkb2NzQ29udGVudCB7XG5cdFx0XHRmbG9hdDogcmlnaHQ7XG5cdFx0XHR3aWR0aDogY2FsYygxMDAlIC0gNDAwcHgpOyB9IH0gfVxuXG5cbi5maXhlZCBmb290ZXIge1xuXHRwb3NpdGlvbjogZml4ZWQ7XG5cdGJvdHRvbTogMDsgfVxuIiwiLy8gVEVYVFxuJGgxLWZvbnQtc2l6ZTogNjBweDtcbiRoMS1saW5lLWhlaWdodDogOTBweDtcblxuJGgyLWZvbnQtc2l6ZTogNDJweDtcbiRoMi1saW5lLWhlaWdodDogNjBweDtcblxuJGgzLWZvbnQtc2l6ZTogMzBweDtcbiRoMy1saW5lLWhlaWdodDogNDBweDtcblxuJGg0LWZvbnQtc2l6ZTogMjZweDtcbiRoNC1saW5lLWhlaWdodDogNDBweDtcblxuJGg1LWZvbnQtc2l6ZTogMjJweDtcbiRoNS1saW5lLWhlaWdodDogMzZweDtcblxuJHAtZm9udC1zaXplOiAyMHB4O1xuJHAtbGluZS1oZWlnaHQ6IDI4cHg7XG5cblxuLy8gaGVhZGVyXG4kZnVsbC13aWR0aC1wYWRkaW5nWDogMjBweDtcblxuJG1haW4td2lkdGg6IDEwMCU7XG4kbWFpbi1tYXgtd2lkdGg6IDEwMCU7XG5cbiRoZWFkZXItaGVpZ2h0OiA2MHB4O1xuXG4kbG9nby13aWR0aDogMTIwcHg7XG5cbiRuYXYtYnV0dG9ucy1tYXJnaW4tbGVmdDogMzBweDtcblxuJGhhbWJ1cmdlci1zaXplOiA1MHB4O1xuXG5cbi8vIG1haW4gbmF2XG4kbWFpbi1uYXYtcGFkZGluZzogMTQwcHggMCAzMHB4O1xuJG1haW4tbmF2LWg1LW1hcmdpbi1ib3R0b206IDFlbTtcbiRtYWluLW5hdi1oMy1tYXJnaW4tYm90dG9tOiAwLjZlbTtcbiRuYXYtYm94LXdpZHRoOiAyMCU7XG4kbmF2LWJveC1zaWJsaW5nLW1hcmdpbi1sZWZ0OiBjYWxjKDIwJSAvIDMpO1xuJG1haW4tbmF2LW1haW4tc2libGluZy1tYXJnaW4tdG9wOiA2MHB4O1xuJG1haW4tbmF2LWxlZnQtYnV0dG9uLXNpemU6IDUwcHg7XG4kbWFpbi1uYXYtbGVmdC1idXR0b24tZm9udC1zaXplOiAxOHB4O1xuXG5cbi8vIGhlcm9cbiRoZXJvLXBhZGRpbmctdG9wOiA4MHB4O1xuJGhlYWRsaW5lLXdyYXBwZXItbWFyZ2luLWJvdHRvbTogNDBweDtcbiRxdWlja3N0YXJ0LWJ1dHRvbi1wYWRkaW5nOiAwIDUwcHg7XG4kdmVuZG9yLXN0cmlwLWhlaWdodDogNjBweDtcbiR2ZW5kb3Itc3RyaXAtZm9udC1zaXplOiAxOHB4O1xuJHZlbmRvci1zdHJpcC1pbWctbWF4LWhlaWdodDogMzBweDtcbiR2ZW5kb3Itc3RyaXAtaW1nLW1heC13aWR0aDogMTAwcHg7XG4kdmVuZG9yLXN0cmlwLWltZy1tYXJnaW4tbGVmdDogNTBweDtcblxuLy8gb2NlYW4gbm9kZXNcbiRvY2Vhbi1ub2Rlcy1wYWRkaW5nLVk6IDYwcHg7XG4kb2NlYW4tbm9kZXMtbWFpbi1tYXJnaW4tYm90dG9tOiA2MHB4O1xuJG9jZWFuLW5vZGVzLWgzLW1hcmdpbi1ib3R0b206IDMwcHg7XG5cblxuLy8gdmlkZW9cbiR2aWRlby1zZWN0aW9uLWhlaWdodDogNDAwcHg7XG5cbi8vIGZlYXR1cmVzXG4kZmVhdHVyZXMtaDMtbWFyZ2luLWJvdHRvbTogNDBweDtcbiRmZWF0dXJlLWJveC1tYXJnaW4tYm90dG9tOiA2MHB4O1xuJGZlYXR1cmUtYm94LWRpdi1tYXJnaW4tYm90dG9tOiAwO1xuJGZlYXR1cmUtYm94LWRpdi13aWR0aDogNDUlO1xuXG5cblxuQG1lZGlhIHNjcmVlbiBhbmQgKG1pbi13aWR0aDogNzUwcHgpIHtcblx0QGltcG9ydCBcIi4uL3BhcnRpYWxzL3NpemVcIjtcblxuXHQvLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vXG5cdC8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy9cblx0Ly8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vL1xuXG5cdCNkb2NzICNoZXJvIHtcblx0XHRoMSwgaDUge1xuXHRcdFx0dGV4dC1hbGlnbjogbGVmdDsgfSB9XG5cblx0I3ZpZXdEb2NzLCAjdHJ5S3ViZXJuZXRlcyB7XG5cdFx0ZGlzcGxheTogaW5saW5lLWJsb2NrOyB9XG5cblx0I3ZlbmRvclN0cmlwIHtcblx0XHRkaXNwbGF5OiBibG9jazsgfVxuXG5cdCNxdWlja3N0YXJ0QnV0dG9uIHtcblx0XHRkaXNwbGF5OiBpbmxpbmUtYmxvY2s7IH1cblxuXG5cdCNvY2Vhbk5vZGVzIHtcblx0XHRoMyB7XG5cdFx0XHR0ZXh0LWFsaWduOiBsZWZ0OyB9XG5cblx0XHRtYWluIHtcblx0XHRcdHBvc2l0aW9uOiByZWxhdGl2ZTtcblx0XHRcdGNsZWFyOiBib3RoO1xuXG5cdFx0XHQuaW1hZ2Utd3JhcHBlciB7XG5cdFx0XHRcdHBvc2l0aW9uOiBhYnNvbHV0ZTtcblx0XHRcdFx0dG9wOiA1MCU7XG5cdFx0XHRcdG1heC13aWR0aDogMjUlO1xuXHRcdFx0XHRtYXgtaGVpZ2h0OiAxMDAlO1xuXHRcdFx0XHR0cmFuc2Zvcm06IHRyYW5zbGF0ZVkoLTUwJSk7IH1cblxuXHRcdFx0JjpudGgtY2hpbGQob2RkKSB7XG5cdFx0XHRcdC5jb250ZW50IHtcblx0XHRcdFx0XHRwYWRkaW5nLXJpZ2h0OiAzMCU7IH1cblxuXHRcdFx0XHQuaW1hZ2Utd3JhcHBlciB7XG5cdFx0XHRcdFx0cmlnaHQ6IDA7IH0gfVxuXG5cdFx0XHQmOm50aC1jaGlsZChldmVuKSB7XG5cdFx0XHRcdC5jb250ZW50IHtcblx0XHRcdFx0XHRwYWRkaW5nLWxlZnQ6IDMwJTsgfVxuXG5cdFx0XHRcdC5pbWFnZS13cmFwcGVyIHtcblx0XHRcdFx0XHRsZWZ0OiAwOyB9IH1cblxuXHRcdFx0JjpudGgtY2hpbGQoMSkge1xuXHRcdFx0XHQuY29udGVudCB7XG5cdFx0XHRcdFx0cGFkZGluZy1yaWdodDogMDsgfVxuXG5cdFx0XHRcdC5pbWFnZS13cmFwcGVyIHtcblx0XHRcdFx0XHRwb3NpdGlvbjogcmVsYXRpdmU7XG5cdFx0XHRcdFx0bWF4LXdpZHRoOiAxMDAlO1xuXHRcdFx0XHRcdHRyYW5zZm9ybTogbm9uZTsgfSB9XG5cblx0XHRcdGltZyB7XG5cdFx0XHRcdHdpZHRoOiAxMDAlOyB9IH0gfVxuXG5cblx0I3ZpZGVvIHtcblx0XHRkaXNwbGF5OiBibG9jaztcblxuXHRcdCYgPiAubGlnaHQtdGV4dCB7XG5cdFx0XHRkaXNwbGF5OiBibG9jazsgfSB9XG5cblx0I21vYmlsZVNob3dWaWRlb0J1dHRvbiB7XG5cdFx0ZGlzcGxheTogbm9uZTsgfVxuXG5cdCNmZWF0dXJlcyB7XG5cdFx0cGFkZGluZy1ib3R0b206IDYwcHg7XG5cblx0XHQuZmVhdHVyZS1ib3gge1xuXHRcdFx0JjpsYXN0LWNoaWxkIHtcblx0XHRcdFx0bWFyZ2luLWJvdHRvbTogMDsgfSB9IH0gfVxuXG5cblxuXG4iLCIvLyBURVhUXG4kaDEtZm9udC1zaXplOiA2MHB4O1xuJGgxLWxpbmUtaGVpZ2h0OiA5MHB4O1xuXG4kaDItZm9udC1zaXplOiA0MnB4O1xuJGgyLWxpbmUtaGVpZ2h0OiA2MHB4O1xuXG4kaDMtZm9udC1zaXplOiAzMHB4O1xuJGgzLWxpbmUtaGVpZ2h0OiA0MHB4O1xuXG4kaDQtZm9udC1zaXplOiAyNnB4O1xuJGg0LWxpbmUtaGVpZ2h0OiA0MHB4O1xuXG4kaDUtZm9udC1zaXplOiAyMnB4O1xuJGg1LWxpbmUtaGVpZ2h0OiAzNnB4O1xuXG4kcC1mb250LXNpemU6IDIwcHg7XG4kcC1saW5lLWhlaWdodDogMjhweDtcblxuXG4vLyBoZWFkZXJcbiRmdWxsLXdpZHRoLXBhZGRpbmdYOiA1MHB4O1xuXG4kbWFpbi13aWR0aDogMTAwJTtcbiRtYWluLW1heC13aWR0aDogMTIwMHB4O1xuXG4kaGVhZGVyLWhlaWdodDogODBweDtcblxuJGxvZ28td2lkdGg6IDE4MHB4O1xuXG4kbmF2LWJ1dHRvbnMtbWFyZ2luLWxlZnQ6IDMwcHg7XG5cbiRoYW1idXJnZXItc2l6ZTogNTBweDtcblxuXG4vLyBtYWluIG5hdlxuJG1haW4tbmF2LXBhZGRpbmc6IDE0MHB4IDAgMzBweDtcbiRtYWluLW5hdi1oNS1tYXJnaW4tYm90dG9tOiAxZW07XG4kbWFpbi1uYXYtaDMtbWFyZ2luLWJvdHRvbTogMC42ZW07XG4kbmF2LWJveC13aWR0aDogMjAlO1xuJG5hdi1ib3gtc2libGluZy1tYXJnaW4tbGVmdDogY2FsYygyMCUgLyAzKTtcbiRtYWluLW5hdi1tYWluLXNpYmxpbmctbWFyZ2luLXRvcDogNjBweDtcbiRtYWluLW5hdi1sZWZ0LWJ1dHRvbi1zaXplOiA1MHB4O1xuJG1haW4tbmF2LWxlZnQtYnV0dG9uLWZvbnQtc2l6ZTogMThweDtcblxuXG4vLyBoZXJvXG4kaGVyby1wYWRkaW5nLXRvcDogODBweDtcbiRoZWFkbGluZS13cmFwcGVyLW1hcmdpbi1ib3R0b206IDQwcHg7XG4kcXVpY2tzdGFydC1idXR0b24tcGFkZGluZzogMCA1MHB4O1xuJHZlbmRvci1zdHJpcC1oZWlnaHQ6IDYwcHg7XG4kdmVuZG9yLXN0cmlwLWZvbnQtc2l6ZTogMThweDtcbiR2ZW5kb3Itc3RyaXAtaW1nLW1heC1oZWlnaHQ6IDMwcHg7XG4kdmVuZG9yLXN0cmlwLWltZy1tYXgtd2lkdGg6IDEwMHB4O1xuJHZlbmRvci1zdHJpcC1pbWctbWFyZ2luLWxlZnQ6IDUwcHg7XG5cbi8vIG9jZWFuIG5vZGVzXG4kb2NlYW4tbm9kZXMtcGFkZGluZy1ZOiA2MHB4O1xuJG9jZWFuLW5vZGVzLW1haW4tbWFyZ2luLWJvdHRvbTogNjBweDtcbiRvY2Vhbi1ub2Rlcy1oMy1tYXJnaW4tYm90dG9tOiAzMHB4O1xuXG4vLyB2aWRlb1xuJHZpZGVvLXNlY3Rpb24taGVpZ2h0OiA1NTBweDtcblxuQG1lZGlhIHNjcmVlbiBhbmQgKG1pbi13aWR0aDogMTAyNXB4KSB7XG5cdEBpbXBvcnQgXCIuLi9wYXJ0aWFscy9zaXplXCI7XG5cblx0Ly8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vL1xuXHQvLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vXG5cdC8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy9cblxuXHQjb2NlYW5Ob2RlcyB7XG5cdFx0bWFpbiB7XG5cdFx0XHRwb3NpdGlvbjogcmVsYXRpdmU7XG5cblx0XHRcdCY6bnRoLWNoaWxkKDEpIHtcblxuXHRcdFx0XHQuaW1hZ2Utd3JhcHBlciB7XG5cdFx0XHRcdFx0cG9zaXRpb246IGFic29sdXRlO1xuXHRcdFx0XHRcdG1heC13aWR0aDogNDglO1xuXHRcdFx0XHRcdHRyYW5zZm9ybTogdHJhbnNsYXRlWSgtNTAlKTsgfVxuXG5cdFx0XHRcdC5jb250ZW50IHtcblx0XHRcdFx0XHR3aWR0aDogNTAlOyB9IH0gfSB9XG5cblxuXG5cdCN2aWRlbyB7XG5cdFx0cG9zaXRpb246IHJlbGF0aXZlO1xuXHRcdGJhY2tncm91bmQtaW1hZ2U6IHVybCguLi8uLi9pbWFnZXMva3ViX3ZpZGVvX3RobS5qcGcpO1xuXHRcdGJhY2tncm91bmQtcG9zaXRpb246IGNlbnRlciBjZW50ZXI7XG5cdFx0YmFja2dyb3VuZC1zaXplOiBjb3Zlcjtcblx0XHRoZWlnaHQ6IDU1MHB4OyB9IH1cblxuXG4iXSwic291cmNlUm9vdCI6Ii9zb3VyY2UvIn0= */ diff --git a/_site/js/non-mini.js b/_site/js/non-mini.js index 580dec0351..a51389eb59 100644 --- a/_site/js/non-mini.js +++ b/_site/js/non-mini.js @@ -43,4 +43,4 @@ $(document).ready(function(){ var result = $('
Use these reference documents to learn how to interact with Kubernetes through the REST API.
+ +You can also view details about the Extensions API. For more about extensions, see API versioning.
+ +Table of Contents:
+The detailed walkthrough covers all the in-depth details and tasks for administering your applications in Kubernetes.
+ +Table of Contents:
+Use this quick walkthrough of Kubernetes to learn about the basic application administration tasks.
+ +Table of Contents:
+Use the following examples to learn how to deploy your application into a Kubernetes cluster.
+ +Table of Contents:
+The User’s guide is for anyone who wants to run programs and +services on an existing Kubernetes cluster.
+The Cluster Admin’s guide is for anyone setting up +a Kubernetes cluster or administering it.
+The Developer guide is for anyone wanting to write +programs that access the Kubernetes API, write plugins or extensions, or +modify the core code of Kubernetes.
+The Kubectl Command Line Interface is a detailed reference on
+the kubectl
CLI.
The API object documentation +is a detailed description of all fields found in core API objects.
+An overview of the Design of Kubernetes
+There are example files and walkthroughs in the examples +folder.
+If something went wrong, see the troubleshooting document for how to debug. +You should also check the known issues for the release you’re using.
+To report a security issue, see Reporting a Security Issue.
+The cluster admin guide is for anyone creating or administering a Kubernetes cluster. +It assumes some familiarity with concepts in the User Guide.
+ +This document describes what ports the Kubernetes apiserver +may serve on and how to reach them. The audience is +cluster administrators who want to customize their cluster +or understand the details.
+ +Most questions about accessing the cluster are covered +in Accessing the cluster.
+ +The Kubernetes API is served by the Kubernetes apiserver process. Typically, +there is one of these running on a single kubernetes-master node.
+ +By default the Kubernetes APIserver serves HTTP on 2 ports:
+ 1. Localhost Port
+ - serves HTTP
+ - default is port 8080, change with --insecure-port
flag.
+ - defaults IP is localhost, change with --insecure-bind-address
flag.
+ - no authentication or authorization checks in HTTP
+ - protected by need to have host access
+ 2. Secure Port
+ - default is port 6443, change with --secure-port
flag.
+ - default IP is first non-localhost network interface, change with --bind-address
flag.
+ - serves HTTPS. Set cert with --tls-cert-file
and key with --tls-private-key-file
flag.
+ - uses token-file or client-certificate based authentication.
+ - uses policy-based authorization.
+ 3. Removed: ReadOnly Port
+ - For security reasons, this had to be removed. Use the service account feature instead.
Additionally, in some configurations there is a proxy (nginx) running +on the same machine as the apiserver process. The proxy serves HTTPS protected +by Basic Auth on port 443, and proxies to the apiserver on localhost:8080. In +these configurations the secure port is typically set to 6443.
+ +A firewall rule is typically configured to allow external HTTPS access to port 443.
+ +The above are defaults and reflect how Kubernetes is deployed to Google Compute Engine using +kube-up.sh. Other cloud providers may vary.
+ +There are three differently configured serving ports because there are a
+variety of uses cases:
+ 1. Clients outside of a Kubernetes cluster, such as human running kubectl
+ on desktop machine. Currently, accesses the Localhost Port via a proxy (nginx)
+ running on the kubernetes-master
machine. The proxy can use cert-based authentication
+ or token-based authentication.
+ 2. Processes running in Containers on Kubernetes that need to read from
+ the apiserver. Currently, these can use a service account.
+ 3. Scheduler and Controller-manager processes, which need to do read-write
+ API operations. Currently, these have to run on the same host as the
+ apiserver and use the Localhost Port. In the future, these will be
+ switched to using service accounts to avoid the need to be co-located.
+ 4. Kubelets, which need to do read-write API operations and are necessarily
+ on different machines than the apiserver. Kubelet uses the Secure Port
+ to get their pods, to find the services that a pod can see, and to
+ write events. Credentials are distributed to kubelets at cluster
+ setup time. Kubelet and kube-proxy can use cert-based authentication or token-based
+ authentication.
Table of Contents +
+ +An admission control plug-in is a piece of code that intercepts requests to the Kubernetes +API server prior to persistence of the object, but after the request is authenticated +and authorized. The plug-in code is in the API server process +and must be compiled into the binary in order to be used at this time.
+ +Each admission control plug-in is run in sequence before a request is accepted into the cluster. If +any of the plug-ins in the sequence reject the request, the entire request is rejected immediately +and an error is returned to the end-user.
+ +Admission control plug-ins may mutate the incoming object in some cases to apply system configured +defaults. In addition, admission control plug-ins may mutate related resources as part of request +processing to do things like increment quota usage.
+ +Many advanced features in Kubernetes require an admission control plug-in to be enabled in order +to properly support the feature. As a result, a Kubernetes API server that is not properly +configured with the right set of admission control plug-ins is an incomplete server and will not +support all the features you expect.
+ +The Kubernetes API server supports a flag, admission-control
that takes a comma-delimited,
+ordered list of admission control choices to invoke prior to modifying objects in the cluster.
Use this plugin by itself to pass-through all requests.
+ +Rejects all requests. Used for testing.
+ +This plug-in will intercept all requests to exec a command in a pod if that pod has a privileged container.
+ +If your cluster supports privileged containers, and you want to restrict the ability of end-users to exec +commands in those containers, we strongly encourage enabling this plug-in.
+ +This functionality has been merged into DenyEscalatingExec.
+ +This plug-in will deny exec and attach commands to pods that run with escalated privileges that +allow host access. This includes pods that run as privileged, have access to the host IPC namespace, and +have access to the host PID namespace.
+ +If your cluster supports containers that run with escalated privileges, and you want to +restrict the ability of end-users to exec commands in those containers, we strongly encourage +enabling this plug-in.
+ +This plug-in implements automation for serviceAccounts.
+We strongly recommend using this plug-in if you intend to make use of Kubernetes ServiceAccount
objects.
This plug-in will deny any pod with a SecurityContext that defines options that were not available on the Container
.
This plug-in will observe the incoming request and ensure that it does not violate any of the constraints
+enumerated in the ResourceQuota
object in a Namespace
. If you are using ResourceQuota
+objects in your Kubernetes deployment, you MUST use this plug-in to enforce quota constraints.
See the resourceQuota design doc and the example of Resource Quota for more details.
+ +It is strongly encouraged that this plug-in is configured last in the sequence of admission control plug-ins. This is +so that quota is not prematurely incremented only for the request to be rejected later in admission control.
+ +This plug-in will observe the incoming request and ensure that it does not violate any of the constraints
+enumerated in the LimitRange
object in a Namespace
. If you are using LimitRange
objects in
+your Kubernetes deployment, you MUST use this plug-in to enforce those constraints. LimitRanger can also
+be used to apply default resource requests to Pods that don’t specify any; currently, the default LimitRanger
+applies a 0.1 CPU requirement to all Pods in the default
namespace.
See the limitRange design doc and the example of Limit Range for more details.
+ +This plug-in observes pod creation requests. If a container omits compute resource requests and limits, +then the plug-in auto-populates a compute resource request based on historical usage of containers running the same image. +If there is not enough data to make a decision the Request is left unchanged. +When the plug-in sets a compute resource request, it annotates the pod with information on what compute resources it auto-populated.
+ +See the InitialResouces proposal for more details.
+ +This plug-in will observe all incoming requests that attempt to create a resource in a Kubernetes Namespace
+and reject the request if the Namespace
was not previously created. We strongly recommend running
+this plug-in to ensure integrity of your data.
The functionality of this admission controller has been merged into NamespaceLifecycle
This plug-in will observe all incoming requests that attempt to create a resource in a Kubernetes Namespace
+and create a new Namespace
if one did not already exist previously.
We strongly recommend NamespaceLifecycle
over NamespaceAutoProvision
.
This plug-in enforces that a Namespace
that is undergoing termination cannot have new objects created in it,
+and ensures that requests in a non-existant Namespace
are rejected.
A Namespace
deletion kicks off a sequence of operations that remove all objects (pods, services, etc.) in that
+namespace. In order to enforce integrity of that process, we strongly recommend running this plug-in.
Yes.
+ +For Kubernetes 1.0, we strongly recommend running the following set of admission control plug-ins (order matters):
+ +
+--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+
+
+
+
+
+
+
+
+
+
+
+ Kubernetes uses client certificates, tokens, or http basic auth to authenticate users for API calls.
+ +Client certificate authentication is enabled by passing the --client-ca-file=SOMEFILE
+option to apiserver. The referenced file must contain one or more certificates authorities
+to use to validate client certificates presented to the apiserver. If a client certificate
+is presented and verified, the common name of the subject is used as the user name for the
+request.
Token File is enabled by passing the --token-auth-file=SOMEFILE
option
+to apiserver. Currently, tokens last indefinitely, and the token list cannot
+be changed without restarting apiserver.
The token file format is implemented in plugin/pkg/auth/authenticator/token/tokenfile/...
+and is a csv file with 3 columns: token, user name, user uid.
When using token authentication from an http client the apiserver expects an Authorization
+header with a value of Bearer SOMETOKEN
.
OpenID Connect ID Token is enabled by passing the following options to the apiserver:
+- --oidc-issuer-url
(required) tells the apiserver where to connect to the OpenID provider. Only HTTPS scheme will be accepted.
+- --oidc-client-id
(required) is used by apiserver to verify the audience of the token.
+A valid ID token MUST have this
+client-id in its aud
claims.
+- --oidc-ca-file
(optional) is used by apiserver to establish and verify the secure connection
+to the OpenID provider.
+- --oidc-username-claim
(optional, experimental) specifies which OpenID claim to use as the user name. By default, sub
+will be used, which should be unique and immutable under the issuer’s domain. Cluster administrator can
+choose other claims such as email
to use as the user name, but the uniqueness and immutability is not guaranteed.
Please note that this flag is still experimental until we settle more on how to handle the mapping of the OpenID user to the Kubernetes user. Thus further changes are possible.
+ +Currently, the ID token will be obtained by some third-party app. This means the app and apiserver
+MUST share the --oidc-client-id
.
Like Token File, when using token authentication from an http client the apiserver expects
+an Authorization
header with a value of Bearer SOMETOKEN
.
Basic authentication is enabled by passing the --basic-auth-file=SOMEFILE
+option to apiserver. Currently, the basic auth credentials last indefinitely,
+and the password cannot be changed without restarting apiserver. Note that basic
+authentication is currently supported for convenience while we finish making the
+more secure modes described above easier to use.
The basic auth file format is implemented in plugin/pkg/auth/authenticator/password/passwordfile/...
+and is a csv file with 3 columns: password, user name, user id.
When using basic authentication from an http client, the apiserver expects an Authorization
header
+with a value of Basic BASE64ENCODED(USER:PASSWORD)
.
Keystone authentication is enabled by passing the --experimental-keystone-url=<AuthURL>
+option to the apiserver during startup. The plugin is implemented in
+plugin/pkg/auth/authenticator/request/keystone/keystone.go
.
+For details on how to use keystone to manage projects and users, refer to the
+Keystone documentation. Please note that
+this plugin is still experimental which means it is subject to changes.
+Please refer to the discussion
+and the blueprint for more details
We plan for the Kubernetes API server to issue tokens +after the user has been (re)authenticated by a bedrock authentication +provider external to Kubernetes. We plan to make it easy to develop modules +that interface between Kubernetes and a bedrock authentication provider (e.g. +github.com, google.com, enterprise directory, kerberos, etc.)
+ +When using client certificate authentication, you can generate certificates manually or +using an existing deployment script.
+ +Deployment script is implemented at
+cluster/saltbase/salt/generate-cert/make-ca-cert.sh
.
+Execute this script with two parameters. First is the IP address of apiserver, the second is
+a list of subject alternate names in the form IP:<ip-address> or DNS:<dns-name>
.
+The script will generate three files:ca.crt, server.crt and server.key.
+Finally, add these parameters
+--client-ca-file=/srv/kubernetes/ca.crt
+--tls-cert-file=/srv/kubernetes/server.cert
+--tls-private-key-file=/srv/kubernetes/server.key
+into apiserver start parameters.
easyrsa can be used to manually generate certificates for your cluster.
+ +Download, unpack, and initialize the patched version of easyrsa3.
+ + curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz
+ tar xzf easy-rsa.tar.gz
+ cd easy-rsa-master/easyrsa3
+ ./easyrsa init-pki
+
+ Generate a CA. (--batch
set automatic mode. --req-cn
default CN to use.)
./easyrsa --batch "--req-cn=${MASTER_IP}@`date +%s`" build-ca nopass
+
+ Generate server certificate and key. +(build-server-full [filename]: Generate a keypair and sign locally for a client or server)
+ + ./easyrsa --subject-alt-name="IP:${MASTER_IP}" build-server-full kubernetes-master nopass
+
+ pki/ca.crt
pki/issued/kubernetes-master.crt
+pki/private/kubernetes-master.key
to your directory.--client-ca-file=/yourdirectory/ca.crt
+--tls-cert-file=/yourdirectory/server.cert
+--tls-private-key-file=/yourdirectory/server.key
+and add these into apiserver start parameters.openssl can also be use to manually generate certificates for your cluster.
+ +openssl genrsa -out ca.key 2048
openssl req -x509 -new -nodes -key ca.key -subj "/CN=${MASTER_IP}" -days 10000 -out ca.crt
openssl genrsa -out server.key 2048
openssl req -new -key server.key -subj "/CN=${MASTER_IP}" -out server.csr
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt
+-days 10000
openssl x509 -noout -text -in ./server.crt
+Finally, do not forget fill the same parameters and add parameters into apiserver start parameters.In Kubernetes, authorization happens as a separate step from authentication. +See the authentication documentation for an +overview of authentication.
+ +Authorization applies to all HTTP accesses on the main (secure) apiserver port.
+ +The authorization check for any request compares attributes of the context of +the request, (such as user, resource, and namespace) with access +policies. An API call must be allowed by some policy in order to proceed.
+ +The following implementations are available, and are selected by flag:
+ - --authorization-mode=AlwaysDeny
+ - --authorization-mode=AlwaysAllow
+ - --authorization-mode=ABAC
AlwaysDeny
blocks all requests (used in tests).
+AlwaysAllow
allows all requests; use if you don’t need authorization.
+ABAC
allows for user-configured authorization policy. ABAC stands for Attribute-Based Access Control.
A request has 5 attributes that can be considered for authorization:
+ - user (the user-string which a user was authenticated as).
+ - group (the list of group names the authenticated user is a member of).
+ - whether the request is readonly (GETs are readonly).
+ - what resource is being accessed.
+ - applies only to the API endpoints, such as
+ /api/v1/namespaces/default/pods
. For miscellaneous endpoints, like /version
, the
+ resource is the empty string.
+ - the namespace of the object being access, or the empty string if the
+ endpoint does not support namespaced objects.
We anticipate adding more attributes to allow finer grained access control and +to assist in policy management.
+ +For mode ABAC
, also specify --authorization-policy-file=SOME_FILENAME
.
The file format is one JSON object per line. There should be no enclosing list or map, just +one map per line.
+ +Each line is a “policy object”. A policy object is a map with the following properties:
+ - user
, type string; the user-string from --token-auth-file
. If you specify user
, it must match the username of the authenticated user.
+ - group
, type string; if you specify group
, it must match one of the groups of the authenticated user.
+ - readonly
, type boolean, when true, means that the policy only applies to GET
+ operations.
+ - resource
, type string; a resource from an URL, such as pods
.
+ - namespace
, type string; a namespace string.
An unset property is the same as a property set to the zero value for its type (e.g. empty string, 0, false). +However, unset should be preferred for readability.
+ +In the future, policies may be expressed in a JSON format, and managed via a REST +interface.
+ +A request has attributes which correspond to the properties of a policy object.
+ +When a request is received, the attributes are determined. Unknown attributes +are set to the zero value of its type (e.g. empty string, 0, false).
+ +An unset property will match any value of the corresponding +attribute. An unset attribute will match any value of the corresponding property.
+ +The tuple of attributes is checked for a match against every policy in the policy file. +If at least one line matches the request attributes, then the request is authorized (but may fail later validation).
+ +To permit any user to do something, write a policy with the user property unset. +To permit an action Policy with an unset namespace applies regardless of namespace.
+ +{"user":"alice"}
{"user":"kubelet", "resource": "pods", "readonly": true}
{"user":"kubelet", "resource": "events"}
{"user":"bob", "resource": "pods", "readonly": true, "namespace": "projectCaribou"}
A service account automatically generates a user. The user’s name is generated according to the naming convention:
+ +
+system:serviceaccount:<namespace>:<serviceaccountname>
+
+
+
+Creating a new namespace also causes a new service account to be created, of this form:*
+ +
+system:serviceaccount:<namespace>:default
+
+
+
+For example, if you wanted to grant the default service account in the kube-system full privilege to the API, you would add this line to your policy file:
+ +{"user":"system:serviceaccount:kube-system:default"}
+
+The apiserver will need to be restarted to pickup the new policy lines.
+ +Other implementations can be developed fairly easily. +The APIserver calls the Authorizer interface:
+ +type Authorizer interface {
+ Authorize(a Attributes) error
+}
+
+to determine whether or not to allow each API action.
+ +An authorization plugin is a module that implements this interface.
+Authorization plugin code goes in pkg/auth/authorizer/$MODULENAME
.
An authorization module can be completely implemented in go, or can call out +to a remote authorization service. Authorization modules can implement +their own caching to reduce the cost of repeated authorization calls with the +same or similar arguments. Developers should then consider the interaction between +caching and revocation of permissions.
+ + + + + + + + + +This document outlines the various binary components that need to run to +deliver a functioning Kubernetes cluster.
+ +Master components are those that provide the cluster’s control plane. For +example, master components are responsible for making global decisions about the +cluster (e.g., scheduling), and detecting and responding to cluster events +(e.g., starting up a new pod when a replication controller’s ‘replicas’ field is +unsatisfied).
+ +Master components could in theory be run on any node in the cluster. However, +for simplicity, current set up scripts typically start all master components on +the same VM, and does not run user containers on this VM. See +high-availability.md for an example multi-master-VM setup.
+ +Even in the future, when Kubernetes is fully self-hosting, it will probably be +wise to only allow master components to schedule on a subset of nodes, to limit +co-running with user-run pods, reducing the possible scope of a +node-compromising security exploit.
+ +kube-apiserver exposes the Kubernetes API; it is the front-end for the +Kubernetes control plane. It is designed to scale horizontally (i.e., one scales +it by running more of them– high-availability.md).
+ +etcd is used as Kubernetes’ backing store. All cluster data is stored here. +Proper administration of a Kubernetes cluster includes a backup plan for etcd’s +data.
+ +kube-controller-manager is a binary that runs controllers, which are the +background threads that handle routine tasks in the cluster. Logically, each +controller is a separate process, but to reduce the number of moving pieces in +the system, they are all compiled into a single binary and run in a single +process.
+ +These controllers include:
+ +kube-scheduler watches newly created pods that have no node assigned, and +selects a node for them to run on.
+ +Addons are pods and services that implement cluster features. They don’t run on +the master VM, but currently the default setup scripts that make the API calls +to create these pods and services does run on the master VM. See: +kube-master-addons
+ +Addon objects are created in the “kube-system” namespace.
+ +Example addons are: +* DNS provides cluster local DNS. +* kube-ui provides a graphical UI for the + cluster. +* fluentd-elasticsearch provides + log storage. Also see the gcp version. +* cluster-monitoring provides + monitoring for the cluster.
+ +Node components run on every node, maintaining running pods and providing them +the Kubernetes runtime environment.
+ +kubelet is the primary node agent. It: +* Watches for pods that have been assigned to its node (either by apiserver + or via local configuration file) and: + * Mounts the pod’s required volumes + * Downloads the pod’s secrets + * Run the pod’s containers via docker (or, experimentally, rkt). + * Periodically executes any requested container liveness probes. + * Reports the status of the pod back to the rest of the system, by creating a + “mirror pod” if necessary. +* Reports the status of the node back to the rest of the system.
+ +kube-proxy enables the Kubernetes service abstraction by maintaining +network rules on the host and performing connection forwarding.
+ +docker
is of course used for actually running containers.
rkt
is supported experimentally as an alternative to docker.
supervisord
is a lightweight process babysitting system for keeping kubelet and docker
+running.
At v1.0, Kubernetes supports clusters up to 100 nodes with 30 pods per node and 1-2 containers per pod.
+ +A cluster is a set of nodes (physical or virtual machines) running Kubernetes agents, managed by a “master” (the cluster-level control plane).
+ +Normally the number of nodes in a cluster is controlled by the the value NUM_MINIONS
in the platform-specific config-default.sh
file (for example, see GCE’s config-default.sh
).
Simply changing that value to something very large, however, may cause the setup script to fail for many cloud providers. A GCE deployment, for example, will run in to quota issues and fail to bring the cluster up.
+ +When setting up a large Kubernetes cluster, the following issues must be considered.
+ +To avoid running into cloud provider quota issues, when creating a cluster with many nodes, consider: +* Increase the quota for things like CPU, IPs, etc. + * In GCE, for example, you’ll want to increase the quota for: + * CPUs + * VM instances + * Total persistent disk reserved + * In-use IP addresses + * Firewall Rules + * Forwarding rules + * Routes + * Target pools +* Gating the setup script so that it brings up new node VMs in smaller batches with waits in between, because some cloud providers rate limit the creation of VMs.
+ +To prevent memory leaks or other resource issues in cluster addons from consuming all the resources available on a node, Kubernetes sets resource limits on addon containers to limit the CPU and Memory resources they can consume (See PR #10653 and #10778).
+ +For example:
+ +containers:
+ - image: gcr.io/google_containers/heapster:v0.15.0
+ name: heapster
+ resources:
+ limits:
+ cpu: 100m
+ memory: 200Mi
+
+These limits, however, are based on data collected from addons running on 4-node clusters (see #10335). The addons consume a lot more resources when running on large deployment clusters (see #5880). So, if a large cluster is deployed without adjusting these values, the addons may continuously get killed because they keep hitting the limits.
+ +To avoid running into cluster addon resource issues, when creating a cluster with many nodes, consider the following: +* Scale memory and CPU limits for each of the following addons, if used, along with the size of cluster (there is one replica of each handling the entire cluster so memory and CPU usage tends to grow proportionally with size/load on cluster): + * Heapster (GCM/GCL backed, InfluxDB backed, InfluxDB/GCL backed, standalone) + * InfluxDB and Grafana + * skydns, kube2sky, and dns etcd + * Kibana +* Scale number of replicas for the following addons, if used, along with the size of cluster (there are multiple replicas of each so increasing replicas should help handle increased load, but, since load per replica also increases slightly, also consider increasing CPU/memory limits): + * elasticsearch +* Increase memory and CPU limits slightly for each of the following addons, if used, along with the size of cluster (there is one replica per node but CPU/memory usage increases slightly along with cluster load/size as well): + * FluentD with ElasticSearch Plugin + * FluentD with GCP Plugin
+ +For directions on how to detect if addon containers are hitting resource limits, see the Troubleshooting section of Compute Resources.
+ + + + + + + + + +This document describes several topics related to the lifecycle of a cluster: creating a new cluster, +upgrading your cluster’s +master and worker nodes, performing node maintenance (e.g. kernel upgrades), and upgrading the Kubernetes API version of a +running cluster.
+ +To install Kubernetes on a set of machines, consult one of the existing Getting Started guides depending on your environment.
+ +The current state of cluster upgrades is provider dependent.
+ +Both Google Container Engine (GKE) and +Compute Engine Open Source (GCE-OSS) support node upgrades via a Managed Instance Group. +Managed Instance Group upgrades sequentially delete and recreate each virtual machine, while maintaining the same +Persistent Disk (PD) to ensure that data is retained across the upgrade.
+ +In contrast, the kube-push.sh
process used on other platforms attempts to upgrade the binaries in
+places, without recreating the virtual machines.
Node upgrades for GKE and GCE-OSS again use a Managed Instance Group, each node is sequentially destroyed and then recreated with new software. Any Pods that are running +on that node need to be controlled by a Replication Controller, or manually re-created after the roll out.
+ +For other platforms, kube-push.sh
is again used, performing an in-place binary upgrade on existing machines.
Google Container Engine automatically updates master components (e.g. kube-apiserver
, kube-scheduler
) to the latest
+version. It also handles upgrading the operating system and other components that the master runs on.
The node upgrade process is user-initiated and is described in the GKE documentation.
+ +Upgrades on open source Google Compute Engine (GCE) clusters are controlled by the cluster/gce/upgrade.sh
script.
Get its usage by running cluster/gce/upgrade.sh -h
.
For example, to upgrade just your master to a specific version (v1.0.2):
+ +cluster/gce/upgrade.sh -M v1.0.2
+
+Alternatively, to upgrade your entire cluster to the latest stable release:
+ +cluster/gce/upgrade.sh release/stable
+
+The cluster/kube-push.sh
script will do a rudimentary update. This process is still quite experimental, we
+recommend testing the upgrade on an experimental cluster before performing the update on a production cluster.
If your cluster runs short on resources you can easily add more machines to it if your cluster is running in Node self-registration mode.
+If you’re using GCE or GKE it’s done by resizing Instance Group managing your Nodes. It can be accomplished by modifying number of instances on Compute > Compute Engine > Instance groups > your group > Edit group
Google Cloud Console page or using gcloud CLI:
+gcloud compute instance-groups managed --zone compute-zone resize my-cluster-minon-group --new-size 42
+
+
+
+Instance Group will take care of putting appropriate image on new machines and start them, while Kubelet will register its Node with API server to make it available for scheduling. If you scale the instance group down, system will randomly choose Nodes to kill.
+ +In other environments you may need to configure the machine yourself and tell the Kubelet on which machine API server is running.
+ +If you are using GCE, you can configure your cluster so that the number of nodes will be automatically scaled based on their CPU and memory utilization.
+Before setting up the cluster by kube-up.sh
, you can set KUBE_ENABLE_NODE_AUTOSCALER
environment variable to true
and export it.
+The script will create an autoscaler for the instance group managing your nodes.
The autoscaler will try to maintain the average CPU and memory utilization of nodes within the cluster close to the target value.
+The target value can be configured by KUBE_TARGET_NODE_UTILIZATION
environment variable (default: 0.7) for kube-up.sh
when creating the cluster.
+The node utilization is the total node’s CPU/memory usage (OS + k8s + user load) divided by the node’s capacity.
+If the desired numbers of nodes in the cluster resulting from CPU utilization and memory utilization are different,
+the autoscaler will choose the bigger number.
+The number of nodes in the cluster set by the autoscaler will be limited from KUBE_AUTOSCALER_MIN_NODES
(default: 1)
+to KUBE_AUTOSCALER_MAX_NODES
(default: the initial number of nodes in the cluster).
The autoscaler is implemented as a Compute Engine Autoscaler.
+The initial values of the autoscaler parameters set by kube-up.sh
and some more advanced options can be tweaked on
+Compute > Compute Engine > Instance groups > your group > Edit group
Google Cloud Console page
+or using gcloud CLI:
+gcloud preview autoscaler --zone compute-zone <command>
+
+
+
+Note that autoscaling will work properly only if node metrics are accessible in Google Cloud Monitoring.
+To make the metrics accessible, you need to create your cluster with KUBE_ENABLE_CLUSTER_MONITORING
+equal to google
or googleinfluxdb
(googleinfluxdb
is the default value).
If you need to reboot a node (such as for a kernel upgrade, libc upgrade, hardware repair, etc.), and the downtime is +brief, then when the Kubelet restarts, it will attempt to restart the pods scheduled to it. If the reboot takes longer, +then the node controller will terminate the pods that are bound to the unavailable node. If there is a corresponding +replication controller, then a new copy of the pod will be started on a different node. So, in the case where all +pods are replicated, upgrades can be done without special coordination, assuming that not all nodes will go down at the same time.
+ +If you want more control over the upgrading process, you may use the following workflow:
+ +Mark the node to be rebooted as unschedulable:
+ +kubectl replace nodes $NODENAME --patch='{"apiVersion": "v1", "spec": {"unschedulable": true}}'
+
+This keeps new pods from landing on the node while you are trying to get them off.
+ +Get the pods off the machine, via any of the following strategies: + * Wait for finite-duration pods to complete. + * Delete pods with:
+ +kubectl delete pods $PODNAME
+
+For pods with a replication controller, the pod will eventually be replaced by a new pod which will be scheduled to a new node. Additionally, if the pod is part of a service, then clients will automatically be redirected to the new pod.
+ +For pods with no replication controller, you need to bring up a new copy of the pod, and assuming it is not part of a service, redirect clients to it.
+ +Perform maintenance work on the node.
+ +Make the node schedulable again:
+ +kubectl replace nodes $NODENAME --patch='{"apiVersion": "v1", "spec": {"unschedulable": false}}'
+
+If you deleted the node’s VM instance and created a new one, then a new schedulable node resource will +be created automatically when you create a new VM instance (if you’re using a cloud provider that supports +node discovery; currently this is only Google Compute Engine, not including CoreOS on Google Compute Engine using kube-register). See Node for more details.
+ +When a new API version is released, you may need to upgrade a cluster to support the new API version (e.g. switching from ‘v1’ to ‘v2’ when ‘v2’ is launched)
+ +This is an infrequent event, but it requires careful management. There is a sequence of steps to upgrade to a new API version.
+ +cluster/update-storage-objects.sh
.Specific API versions can be turned on or off by passing –runtime-config=api/
The objects that are stored to disk for a cluster’s internal representation of the Kubernetes resources active in the cluster are written using a particular version of the API. +When the supported API changes, these objects may need to be rewritten in the newer API. Failure to do this will eventually result in resources that are no longer decodable or usable +by the kubernetes API server.
+ +KUBE_API_VERSIONS
environment variable for the kube-apiserver
binary which controls the API versions that are supported in the cluster. The first version in the list is used as the cluster’s storage version. Hence, to set a specific version as the storage version, bring it to the front of list of versions in the value of KUBE_API_VERSIONS
. You need to restart the kube-apiserver
binary
+for changes to this variable to take effect.
You can use the kube-version-change
utility to convert config files between different API versions.
$ hack/build-go.sh cmd/kube-version-change
+$ _output/local/go/bin/kube-version-change -i myPod.v1beta3.yaml -o myPod.v1.yaml
+
+This doc is about cluster troubleshooting; we assume you have already ruled out your application as the root cause of the +problem you are experiencing. See +the application troubleshooting guide for tips on application debugging. +You may also visit troubleshooting document for more information.
+ +The first thing to debug in your cluster is if your nodes are all registered correctly.
+ +Run
+ +kubectl get nodes
+
+And verify that all of the nodes you expect to see are present and that they are all in the Ready
state.
For now, digging deeper into the cluster requires logging into the relevant machines. Here are the locations
+of the relevant log files. (note that on systemd-based systems, you may need to use journalctl
instead)
This is an incomplete list of things that could go wrong, and how to adjust your cluster setup to mitigate the problems.
+ +Root causes: + - VM(s) shutdown + - Network partition within cluster, or between cluster and users + - Crashes in Kubernetes software + - Data loss or unavailability of persistent storage (e.g. GCE PD or AWS EBS volume) + - Operator error, e.g. misconfigured Kubernetes software or application software
+ +Specific scenarios: + - Apiserver VM shutdown or apiserver crashing + - Results + - unable to stop, update, or start new pods, services, replication controller + - existing pods and services should continue to work normally, unless they depend on the Kubernetes API + - Apiserver backing storage lost + - Results + - apiserver should fail to come up + - kubelets will not be able to reach it but will continue to run the same pods and provide the same service proxying + - manual recovery or recreation of apiserver state necessary before apiserver is restarted + - Supporting services (node controller, replication controller manager, scheduler, etc) VM shutdown or crashes + - currently those are colocated with the apiserver, and their unavailability has similar consequences as apiserver + - in future, these will be replicated as well and may not be co-located + - they do not have their own persistent state + - Individual node (VM or physical machine) shuts down + - Results + - pods on that Node stop running + - Network partition + - Results + - partition A thinks the nodes in partition B are down; partition B thinks the apiserver is down. (Assuming the master VM ends up in partition A.) + - Kubelet software fault + - Results + - crashing kubelet cannot start new pods on the node + - kubelet might delete the pods or not + - node marked unhealthy + - replication controllers start new pods elsewhere + - Cluster operator error + - Results + - loss of pods, services, etc + - lost of apiserver backing store + - users unable to read API + - etc.
+ +Mitigations: +- Action: Use IaaS provider’s automatic VM restarting feature for IaaS VMs + - Mitigates: Apiserver VM shutdown or apiserver crashing + - Mitigates: Supporting services VM shutdown or crashes
+ +Table of Contents +
+ +A Daemon Set ensures that all (or some) nodes run a copy of a pod. As nodes are added to the +cluster, pods are added to them. As nodes are removed from the cluster, those pods are garbage +collected. Deleting a Daemon Set will clean up the pods it created.
+ +Some typical uses of a Daemon Set are:
+ +glusterd
, ceph
, on each node.fluentd
or logstash
.collectd
, New Relic agent, or Ganglia gmond
.In a simple case, one Daemon Set, covering all nodes, would be used for each type of daemon. +A more complex setup might use multiple DaemonSets would be used for a single type of daemon, +but with different flags and/or different memory and cpu requests for different hardware types.
+ +As with all other Kubernetes config, a DaemonSet needs apiVersion
, kind
, and metadata
fields. For
+general information about working with config files, see here,
+here, and here.
A DaemonSet also needs a .spec
section.
The .spec.template
is the only required field of the .spec
.
The .spec.template
is a pod template.
+It has exactly the same schema as a pod, except
+it is nested and does not have an apiVersion
or kind
.
In addition to required fields for a pod, a pod template in a DaemonSet has to specify appropriate +labels (see pod selector).
+ +A pod template in a DaemonSet must have a RestartPolicy
+ equal to Always
, or be unspecified, which defaults to Always
.
The .spec.selector
field is a pod selector. It works the same as the .spec.selector
of
+a ReplicationController or
+Job.
If the .spec.selector
is specified, it must equal the .spec.template.metadata.labels
. If not
+specified, the are default to be equal. Config with these unequal will be rejected by the API.
Also you should not normally create any pods whose labels match this selector, either directly, via +another DaemonSet, or via other controller such as ReplicationController. Otherwise, the DaemonSet +controller will think that those pods were created by it. Kubernetes will not stop you from doing +this. Once case where you might want to do this is manually create a pod with a different value on +a node for testing.
+ +If you specify a .spec.template.spec.nodeSelector
, then the DaemonSet controller will
+create pods on nodes which match that node
+selector.
If you do not specify a .spec.template.spec.nodeSelector
, then the DaemonSet controller will
+create pods on all nodes.
Normally, the machine that a pod runs on is selected by the Kubernetes scheduler. However, pods
+created by the Daemon controller have the machine already selected (.spec.nodeName
is specified
+when the pod is created, so it is ignored by the scheduler). Therefore:
unschedulable
field of a node is not respected
+by the daemon set controller.Some possible patterns for communicating with pods in a DaemonSet are:
+ +hostPort
, so that the pods are reachable
+via the node IPs. Clients knows the the list of nodes ips somehow, and know the port by convention.endpoints
resource or retrieve multiple A records from
+DNS.If node labels are changed, the DaemonSet will promptly add pods to newly matching nodes and delete +pods from newly not-matching nodes.
+ +You can modify the pods that a DaemonSet creates. However, pods do not allow all +fields to be updated. Also, the DeamonSet controller will use the original template the next +time a node (even with the same name) is created.
+ +You can delete a DeamonSet. If you specify --cascade=false
with kubectl
, then the pods
+will be left on the nodes. You can then create a new DaemonSet with a different template.
+the new DaemonSet with the different template will recognize all the existing pods as having
+matching labels. It will not modify or delete them despite a mismatch in the pod template.
+You will need to force new pod creation by deleting the pod or deleting the node.
You cannot update a DaemonSet.
+ +Support for updating DaemonSets and controlled updating of nodes is planned.
+ +It is certainly possible to run daemon processes by directly starting them on a node (e.g using
+init
, upstartd
, or systemd
). This is perfectly fine. However, there are several advantages to
+running such processes via a DaemonSet:
kubectl
) for daemons and applications.It is possible to create pods directly which specify a particular node to run on. However, +a Daemon Set replaces pods that are deleted or terminated for any reason, such as in the case of +node failure or disruptive node maintenance, such as a kernel upgrade. For this reason, you should +use a Daemon Set rather than creating individual pods.
+ +It is possible to create pods by writing a file to a certain directory watched by Kubelet. These +are called static pods. +Unlike DaemonSet, static pods cannot be managed with kubectl +or other Kubernetes API clients. Static pods do not depend on the apiserver, making them useful +in cluster bootstrapping cases. Also, static pods may be deprecated in the future.
+ +Daemon Set are similar to Replication Controllers in that +they both create pods, and those pods have processes which are not expected to terminate (e.g. web servers, +storage servers).
+ +Use a replication controller for stateless services, like frontends, where scaling up and down the +number of replicas and rolling out updates are more important than controlling exactly which host +the pod runs on. Use a Daemon Controller when it is important that a copy of a pod always run on +all or certain hosts, and when it needs to start before other pods.
+ +DaemonSet objects are in the extensions
API Group.
+DaemonSet is not enabled by default. Enable it by setting
+--runtime-config=extensions/v1beta1/daemonsets=true
on the api server. This can be
+achieved by exporting ENABLE_DAEMONSETS=true before running kube-up.sh script
+on GCE.
DaemonSet objects effectively have API version v1alpha1
.
+ Alpha objects may change or even be discontinued in future software releases.
+However, due to to a known issue, they will appear as API version v1beta1
if enabled.
As of Kubernetes 0.8, DNS is offered as a cluster add-on. +If enabled, a DNS Pod and Service will be scheduled on the cluster, and the kubelets will be +configured to tell individual containers to use the DNS Service’s IP to resolve DNS names.
+ +Every Service defined in the cluster (including the DNS server itself) will be +assigned a DNS name. By default, a client Pod’s DNS search list will +include the Pod’s own namespace and the cluster’s default domain. This is best +illustrated by example:
+ +Assume a Service named foo
in the Kubernetes namespace bar
. A Pod running
+in namespace bar
can look up this service by simply doing a DNS query for
+foo
. A Pod running in namespace quux
can look up this service by doing a
+DNS query for foo.bar
.
The cluster DNS server (SkyDNS) +supports forward lookups (A records) and service lookups (SRV records).
+ +The running DNS pod holds 3 containers - skydns, etcd (a private instance which skydns uses), +and a Kubernetes-to-skydns bridge called kube2sky. The kube2sky process +watches the Kubernetes master for changes in Services, and then writes the +information to etcd, which skydns reads. This etcd instance is not linked to +any other etcd clusters that might exist, including the Kubernetes master.
+ +The skydns service is reachable directly from Kubernetes nodes (outside +of any container) and DNS resolution works if the skydns service is targeted +explicitly. However, nodes are not configured to use the cluster DNS service or +to search the cluster’s DNS domain by default. This may be resolved at a later +time.
+ +etcd is a highly-available key value +store which Kubernetes uses for persistent storage of all of its REST API +objects.
+ +Access Control: give only kube-apiserver read/write access to etcd. You do not +want apiserver’s etcd exposed to every node in your cluster (or worse, to the +internet at large), because access to etcd is equivalent to root in your +cluster.
+ +Data Reliability: for reasonable safety, either etcd needs to be run as a +cluster (multiple machines each running +etcd) or etcd’s data directory should be located on durable storage (e.g., GCE’s +persistent disk). In either case, if high availability is required–as it might +be in a production cluster–the data directory ought to be backed up +periodically, +to reduce downtime in case of corruption.
+ +The default setup scripts use kubelet’s file-based static pods feature to run etcd in a
+pod. This manifest should only
+be run on master VMs. The default location that kubelet scans for manifests is
+/etc/kubernetes/manifests/
.
By default, Kubernetes objects are stored under the /registry
key in etcd.
+This path can be prefixed by using the kube-apiserver flag
+--etcd-prefix="/foo"
.
etcd
is the only place that Kubernetes keeps state.
To test whether etcd
is running correctly, you can try writing a value to a
+test key. On your master VM (or somewhere with firewalls configured such that
+you can talk to your cluster’s etcd), try:
curl -fs -X PUT "http://${host}:${port}/v2/keys/_test"
+
+Garbage collection is managed by kubelet automatically, mainly including unreferenced +images and dead containers. kubelet applies container garbage collection every minute +and image garbage collection every 5 minutes. +Note that we don’t recommend external garbage collection tool generally, since it could +break the behavior of kubelet potentially if it attempts to remove all of the containers +which acts as the tombstone kubelet relies on. Yet those garbage collector aims to deal +with the docker leaking issues would be appreciated.
+ +kubernetes manages lifecycle of all images through imageManager, with the cooperation
+of cadvisor.
+The policy for garbage collecting images we apply takes two factors into consideration,
+HighThresholdPercent
and LowThresholdPercent
. Disk usage above the the high threshold
+will trigger garbage collection, which attempts to delete unused images until the low
+threshold is met. Least recently used images are deleted first.
The policy for garbage collecting containers we apply takes on three variables, which can
+be user-defined. MinAge
is the minimum age at which a container can be garbage collected,
+zero for no limit. MaxPerPodContainer
is the max number of dead containers any single
+pod (UID, container name) pair is allowed to have, less than zero for no limit.
+MaxContainers
is the max number of total dead containers, less than zero for no limit as well.
kubelet sorts out containers which are unidentified or stay out of bounds set by previous
+mentioned three flags. Gernerally the oldest containers are removed first. Since we take both
+MaxPerPodContainer
and MaxContainers
into consideration, it could happen when they
+have conflict – retaining the max number of containers per pod goes out of range set by max
+number of global dead containers. In this case, we would sacrifice the MaxPerPodContainer
+a little bit. For the worst case, we first downgrade it to 1 container per pod, and then
+evict the oldest containers for the greater good.
When kubelet removes the dead containers, all the files inside the container will be cleaned up as well. +Note that we will skip the containers that are not managed by kubelet.
+ +Users are free to set their own value to address image garbage collection.
+ +image-gc-high-threshold
, the percent of disk usage which triggers image garbage collection.
+Default is 90%.image-gc-low-threshold
, the percent of disk usage to which image garbage collection attempts
+to free. Default is 80%.We also allow users to customize garbage collection policy, basically via following three flags.
+ +minimum-container-ttl-duration
, minimum age for a finished container before it is
+garbage collected. Default is 1 minute.maximum-dead-containers-per-container
, maximum number of old instances to retain
+per container. Default is 2.maximum-dead-containers
, maximum number of old instances of containers to retain globally.
+Default is 100.Note that we highly recommend a large enough value for maximum-dead-containers-per-container
+to allow at least 2 dead containers retaining per expected container when you customize the flag
+configuration. A loose value for maximum-dead-containers
also assumes importance for a similar reason.
+See this issue for more details.
Table of Contents +
+ +This document describes how to build a high-availability (HA) Kubernetes cluster. This is a fairly advanced topic. +Users who merely want to experiment with Kubernetes are encouraged to use configurations that are simpler to set up such as +the simple Docker based single node cluster instructions, +or try Google Container Engine for hosted Kubernetes.
+ +Also, at this time high availability support for Kubernetes is not continuously tested in our end-to-end (e2e) testing. We will +be working to add this continuous testing, but for now the single-node master installations are more heavily tested.
+ +Setting up a truly reliable, highly available distributed system requires a number of steps, it is akin to +wearing underwear, pants, a belt, suspenders, another pair of underwear, and another pair of pants. We go into each +of these steps in detail, but a summary is given here to help guide and orient the user.
+ +The steps involved are as follows: + * Creating the reliable constituent nodes that collectively form our HA master implementation. + * Setting up a redundant, reliable storage layer with clustered etcd. + * Starting replicated, load balanced Kubernetes API servers + * Setting up master-elected Kubernetes scheduler and controller-manager daemons
+ +Here’s what the system should look like when it’s finished: +
+ +Ready? Let’s get started.
+ +The remainder of this guide assumes that you are setting up a 3-node clustered master, where each machine is running some flavor of Linux. +Examples in the guide are given for Debian distributions, but they should be easily adaptable to other distributions. +Likewise, this set up should work whether you are running in a public or private cloud provider, or if you are running +on bare metal.
+ +The easiest way to implement an HA Kubernetes cluster is to start with an existing single-master cluster. The +instructions at https://get.k8s.io +describe easy installation for single-master clusters on a variety of platforms.
+ +On each master node, we are going to run a number of processes that implement the Kubernetes API. The first step in making these reliable is
+to make sure that each automatically restarts when it fails. To achieve this, we need to install a process watcher. We choose to use
+the kubelet
that we run on each of the worker nodes. This is convenient, since we can use containers to distribute our binaries, we can
+establish resource limits, and introspect the resource usage of each daemon. Of course, we also need something to monitor the kubelet
+itself (insert who watches the watcher jokes here). For Debian systems, we choose monit, but there are a number of alternate
+choices. For example, on systemd-based systems (e.g. RHEL, CentOS), you can run ‘systemctl enable kubelet’.
If you are extending from a standard Kubernetes installation, the kubelet
binary should already be present on your system. You can run
+which kubelet
to determine if the binary is in fact installed. If it is not installed,
+you should install the kubelet binary, the
+kubelet init file and high-availability/default-kubelet
+scripts.
If you are using monit, you should also install the monit daemon (apt-get install monit
) and the high-availability/monit-kubelet and
+high-availability/monit-docker configs.
On systemd systems you systemctl enable kubelet
and systemctl enable docker
.
The central foundation of a highly available solution is a redundant, reliable storage layer. The number one rule of high-availability is +to protect the data. Whatever else happens, whatever catches on fire, if you have the data, you can rebuild. If you lose the data, you’re +done.
+ +Clustered etcd already replicates your storage to all master instances in your cluster. This means that to lose data, all three nodes would need +to have their physical (or virtual) disks fail at the same time. The probability that this occurs is relatively low, so for many people +running a replicated etcd cluster is likely reliable enough. You can add additional reliability by increasing the +size of the cluster from three to five nodes. If that is still insufficient, you can add +even more redundancy to your storage layer.
+ +The full details of clustering etcd are beyond the scope of this document, lots of details are given on the +etcd clustering page. This example walks through +a simple cluster set up, using etcd’s built in discovery to build our cluster.
+ +First, hit the etcd discovery service to create a new token:
+ +curl https://discovery.etcd.io/new?size=3
+
+On each node, copy the etcd.yaml file into /etc/kubernetes/manifests/etcd.yaml
The kubelet on each node actively monitors the contents of that directory, and it will create an instance of the etcd
+server from the definition of the pod specified in etcd.yaml
.
Note that in etcd.yaml
you should substitute the token URL you got above for ${DISCOVERY_TOKEN}
on all three machines,
+and you should substitute a different name (e.g. node-1
) for ${NODE_NAME} and the correct IP address
+for ${NODE_IP}
on each machine.
Once you copy this into all three nodes, you should have a clustered etcd set up. You can validate with
+ +etcdctl member list
+
+and
+ +etcdctl cluster-health
+
+You can also validate that this is working with etcdctl set foo bar
on one node, and etcd get foo
+on a different node.
Of course, if you are interested in increased data reliability, there are further options which makes the place where etcd +installs it’s data even more reliable than regular disks (belts and suspenders, ftw!).
+ +If you use a cloud provider, then they usually provide this +for you, for example Persistent Disk on the Google Cloud Platform. These +are block-device persistent storage that can be mounted onto your virtual machine. Other cloud providers provide similar solutions.
+ +If you are running on physical machines, you can also use network attached redundant storage using an iSCSI or NFS interface. +Alternatively, you can run a clustered file system like Gluster or Ceph. Finally, you can also run a RAID array on each physical machine.
+ +Regardless of how you choose to implement it, if you chose to use one of these options, you should make sure that your storage is mounted
+to each machine. If your storage is shared between the three masters in your cluster, you should create a different directory on the storage
+for each node. Throughout these instructions, we assume that this storage is mounted to your machine in /var/etcd/data
Once you have replicated etcd set up correctly, we will also install the apiserver using the kubelet.
+ +First you need to create the initial log file, so that Docker mounts a file instead of a directory:
+ +touch /var/log/kube-apiserver.log
+
+Next, you need to create a /srv/kubernetes/
directory on each node. This directory includes:
+ * basic_auth.csv - basic auth user and password
+ * ca.crt - Certificate Authority cert
+ * known_tokens.csv - tokens that entities (e.g. the kubelet) can use to talk to the apiserver
+ * kubecfg.crt - Client certificate, public key
+ * kubecfg.key - Client certificate, private key
+ * server.cert - Server certificate, public key
+ * server.key - Server certificate, private key
The easiest way to create this directory, may be to copy it from the master node of a working cluster, or you can manually generate these files yourself.
+ +Once these files exist, copy the kube-apiserver.yaml into /etc/kubernetes/manifests/
on each master node.
The kubelet monitors this directory, and will automatically create an instance of the kube-apiserver
container using the pod definition specified
+in the file.
At this point, you should have 3 apiservers all working correctly. If you set up a network load balancer, you should +be able to access your cluster via that load balancer, and see traffic balancing between the apiserver instances. Setting +up a load balancer will depend on the specifics of your platform, for example instructions for the Google Cloud +Platform can be found here
+ +Note, if you are using authentication, you may need to regenerate your certificate to include the IP address of the balancer, +in addition to the IP addresses of the individual nodes.
+ +For pods that you deploy into the cluster, the kubernetes
service/dns name should provide a load balanced endpoint for the master automatically.
For external users of the API (e.g. the kubectl
command line interface, continuous build pipelines, or other clients) you will want to configure
+them to talk to the external load balancer’s IP address.
So far we have set up state storage, and we have set up the API server, but we haven’t run anything that actually modifies
+cluster state, such as the controller manager and scheduler. To achieve this reliably, we only want to have one actor modifying state at a time, but we want replicated
+instances of these actors, in case a machine dies. To achieve this, we are going to use a lease-lock in etcd to perform
+master election. On each of the three apiserver nodes, we run a small utility application named podmaster
. It’s job is to implement a master
+election protocol using etcd “compare and swap”. If the apiserver node wins the election, it starts the master component it is managing (e.g. the scheduler), if it
+loses the election, it ensures that any master components running on the node (e.g. the scheduler) are stopped.
In the future, we expect to more tightly integrate this lease-locking into the scheduler and controller-manager binaries directly, as described in the high availability design proposal
+ +First, create empty log files on each node, so that Docker will mount the files not make new directories:
+ +touch /var/log/kube-scheduler.log
+touch /var/log/kube-controller-manager.log
+
+Next, set up the descriptions of the scheduler and controller manager pods on each node.
+by copying kube-scheduler.yaml and kube-controller-manager.yaml into the /srv/kubernetes/
+ directory.
Now that the configuration files are in place, copy the podmaster.yaml config file into /etc/kubernetes/manifests/
As before, the kubelet on the node monitors this directory, and will start an instance of the podmaster using the pod specification provided in podmaster.yaml
.
Now you will have one instance of the scheduler process running on a single master node, and likewise one +controller-manager process running on a single (possibly different) master node. If either of these processes fail, +the kubelet will restart them. If any of these nodes fail, the process will move to a different instance of a master +node.
+ +At this point, you are done (yeah!) with the master components, but you still need to add worker nodes (boo!).
+ +If you have an existing cluster, this is as simple as reconfiguring your kubelets to talk to the load-balanced endpoint, and +restarting the kubelets on each node.
+ +If you are turning up a fresh cluster, you will need to install the kubelet and kube-proxy on each worker node, and
+set the --apiserver
flag to your replicated endpoint.
We indeed have an initial proof of concept tester for this, which is available here.
+ +It implements the major concepts (with a few minor reductions for simplicity), of the podmaster HA implementation alongside a quick smoke test using k8petstore.
+ + + + + + + + + +The cluster admin guide is for anyone creating or administering a Kubernetes cluster. +It assumes some familiarity with concepts in the User Guide.
+ +The cluster admin guide is for anyone creating or administering a Kubernetes cluster. +It assumes some familiarity with concepts in the User Guide.
+ +There are many different examples of how to setup a kubernetes cluster. Many of them are listed in this +matrix. We call each of the combinations in this matrix a distro.
+ +Before choosing a particular guide, here are some things to consider:
+ +Pick one of the Getting Started Guides from the matrix and follow it. +If none of the Getting Started Guides fits, you may want to pull ideas from several of the guides.
+ +One option for custom networking is OpenVSwitch GRE/VxLAN networking (ovs-networking.md), which +uses OpenVSwitch to set up networking between pods across + Kubernetes nodes.
+ +If you are modifying an existing guide which uses Salt, this document explains how Salt is used in the Kubernetes +project.
+ +DNS Integration with SkyDNS (dns.md): +Resolving a DNS name directly to a Kubernetes service.
+Logging with Kibana
+Kubernetes Container Environment (docs/user-guide/container-environment.md): +Describes the environment for Kubelet managed containers on a Kubernetes +node.
+Securing access to the API Server accessing the api
+Authentication authentication
+Authorization authorization
+Admission Controllers admission_controllers
+The Kubernetes API server validates and configures data +for the api objects which include pods, services, replicationcontrollers, and +others. The API Server services REST operations and provides the frontend to the +cluster’s shared state through which all other components interact.
+ +
+kube-apiserver
+
+
+
+
+ --admission-control="AlwaysAdmit": Ordered list of plug-ins to do admission control of resources into cluster. Comma-delimited list of: AlwaysAdmit, AlwaysDeny, DenyEscalatingExec, DenyExecOnPrivileged, InitialResources, LimitRanger, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, ResourceQuota, SecurityContextDeny, ServiceAccount
+ --admission-control-config-file="": File with admission control configuration.
+ --advertise-address=<nil>: The IP address on which to advertise the apiserver to members of the cluster. This address must be reachable by the rest of the cluster. If blank, the --bind-address will be used. If --bind-address is unspecified, the host's default interface will be used.
+ --allow-privileged[=false]: If true, allow privileged containers.
+ --authorization-mode="AlwaysAllow": Ordered list of plug-ins to do authorization on secure port. Comma-delimited list of: AlwaysAllow,AlwaysDeny,ABAC
+ --authorization-policy-file="": File with authorization policy in csv format, used with --authorization-mode=ABAC, on the secure port.
+ --basic-auth-file="": If set, the file that will be used to admit requests to the secure port of the API server via http basic authentication.
+ --bind-address=0.0.0.0: The IP address on which to serve the --read-only-port and --secure-port ports. The associated interface(s) must be reachable by the rest of the cluster, and by CLI/web clients. If blank, all interfaces will be used (0.0.0.0).
+ --cert-dir="/var/run/kubernetes": The directory where the TLS certs are located (by default /var/run/kubernetes). If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.
+ --client-ca-file="": If set, any request presenting a client certificate signed by one of the authorities in the client-ca-file is authenticated with an identity corresponding to the CommonName of the client certificate.
+ --cloud-config="": The path to the cloud provider configuration file. Empty string for no configuration file.
+ --cloud-provider="": The provider for cloud services. Empty string for no provider.
+ --cluster-name="kubernetes": The instance prefix for the cluster
+ --cors-allowed-origins=[]: List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. If this list is empty CORS will not be enabled.
+ --etcd-config="": The config file for the etcd client. Mutually exclusive with -etcd-servers.
+ --etcd-prefix="/registry": The prefix for all resource paths in etcd.
+ --etcd-servers=[]: List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd-config
+ --etcd-servers-overrides=[]: Per-resource etcd servers overrides, comma separated. The individual override format: group/resource#servers, where servers are http://ip:port, semicolon separated.
+ --event-ttl=1h0m0s: Amount of time to retain events. Default 1 hour.
+ --experimental-keystone-url="": If passed, activates the keystone authentication plugin
+ --external-hostname="": The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs.)
+ --google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication.
+ --insecure-bind-address=127.0.0.1: The IP address on which to serve the --insecure-port (set to 0.0.0.0 for all interfaces). Defaults to localhost.
+ --insecure-port=8080: The port on which to serve unsecured, unauthenticated access. Default 8080. It is assumed that firewall rules are set up such that this port is not reachable from outside of the cluster and that port 443 on the cluster's public address is proxied to this port. This is performed by nginx in the default setup.
+ --kubelet-certificate-authority="": Path to a cert. file for the certificate authority.
+ --kubelet-client-certificate="": Path to a client cert file for TLS.
+ --kubelet-client-key="": Path to a client key file for TLS.
+ --kubelet-https[=true]: Use https for kubelet connections
+ --kubelet-port=10250: Kubelet port
+ --kubelet-timeout=5s: Timeout for kubelet operations
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --long-running-request-regexp="(/|^)((watch|proxy)(/|$)|(logs?|portforward|exec|attach)/?$)": A regular expression matching long running requests which should be excluded from maximum inflight request handling.
+ --master-service-namespace="default": The namespace from which the kubernetes master services should be injected into pods
+ --max-connection-bytes-per-sec=0: If non-zero, throttle each user connection to this number of bytes/sec. Currently only applies to long-running requests
+ --max-requests-inflight=400: The maximum number of requests in flight at a given time. When the server exceeds this, it rejects requests. Zero for no limit.
+ --min-request-timeout=1800: An optional field indicating the minimum number of seconds a handler must keep a request open before timing it out. Currently only honored by the watch request handler, which picks a randomized value above this number as the connection timeout, to spread out load.
+ --oidc-ca-file="": If set, the OpenID server's certificate will be verified by one of the authorities in the oidc-ca-file, otherwise the host's root CA set will be used
+ --oidc-client-id="": The client ID for the OpenID Connect client, must be set if oidc-issuer-url is set
+ --oidc-issuer-url="": The URL of the OpenID issuer, only HTTPS scheme will be accepted. If set, it will be used to verify the OIDC JSON Web Token (JWT)
+ --oidc-username-claim="sub": The OpenID claim to use as the user name. Note that claims other than the default ('sub') is not guaranteed to be unique and immutable. This flag is experimental, please see the authentication documentation for further details.
+ --profiling[=true]: Enable profiling via web interface host:port/debug/pprof/
+ --runtime-config=: A set of key=value pairs that describe runtime configuration that may be passed to apiserver. apis/<groupVersion> key can be used to turn on/off specific api versions. apis/<groupVersion>/<resource> can be used to turn on/off specific resources. api/all and api/legacy are special keys to control all and legacy api versions respectively.
+ --secure-port=6443: The port on which to serve HTTPS with authentication and authorization. If 0, don't serve HTTPS at all.
+ --service-account-key-file="": File containing PEM-encoded x509 RSA private or public key, used to verify ServiceAccount tokens. If unspecified, --tls-private-key-file is used.
+ --service-account-lookup[=false]: If true, validate ServiceAccount tokens exist in etcd as part of authentication.
+ --service-cluster-ip-range=<nil>: A CIDR notation IP range from which to assign service cluster IPs. This must not overlap with any IP ranges assigned to nodes for pods.
+ --service-node-port-range=: A port range to reserve for services with NodePort visibility. Example: '30000-32767'. Inclusive at both ends of the range.
+ --ssh-keyfile="": If non-empty, use secure SSH proxy to the nodes, using this user keyfile
+ --ssh-user="": If non-empty, use secure SSH proxy to the nodes, using this user name
+ --storage-versions="extensions/v1beta1,v1": The versions to store resources with. Different groups may be stored in different versions. Specified in the format "group1/version1,group2/version2...". This flag expects a complete list of storage versions of ALL groups registered in the server. It defaults to a list of preferred versions of all registered groups, which is derived from the KUBE_API_VERSIONS environment variable.
+ --tls-cert-file="": File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert). If HTTPS serving is enabled, and --tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate and key are generated for the public address and saved to /var/run/kubernetes.
+ --tls-private-key-file="": File containing x509 private key matching --tls-cert-file.
+ --token-auth-file="": If set, the file that will be used to secure the secure port of the API server via token authentication.
+ --watch-cache[=true]: Enable watch caching in the apiserver
+
+
+
+The Kubernetes controller manager is a daemon that embeds +the core control loops shipped with Kubernetes. In applications of robotics and +automation, a control loop is a non-terminating loop that regulates the state of +the system. In Kubernetes, a controller is a control loop that watches the shared +state of the cluster through the apiserver and makes changes attempting to move the +current state towards the desired state. Examples of controllers that ship with +Kubernetes today are the replication controller, endpoints controller, namespace +controller, and serviceaccounts controller.
+ +
+kube-controller-manager
+
+
+
+
+ --address=127.0.0.1: The IP address to serve on (set to 0.0.0.0 for all interfaces)
+ --allocate-node-cidrs[=false]: Should CIDRs for Pods be allocated and set on the cloud provider.
+ --cloud-config="": The path to the cloud provider configuration file. Empty string for no configuration file.
+ --cloud-provider="": The provider for cloud services. Empty string for no provider.
+ --cluster-cidr=<nil>: CIDR Range for Pods in cluster.
+ --cluster-name="kubernetes": The instance prefix for the cluster
+ --concurrent-endpoint-syncs=5: The number of endpoint syncing operations that will be done concurrently. Larger number = faster endpoint updating, but more CPU (and network) load
+ --concurrent_rc_syncs=5: The number of replication controllers that are allowed to sync concurrently. Larger number = more reponsive replica management, but more CPU (and network) load
+ --deleting-pods-burst=10: Number of nodes on which pods are bursty deleted in case of node failure. For more details look into RateLimiter.
+ --deleting-pods-qps=0.1: Number of nodes per second on which pods are deleted in case of node failure.
+ --deployment-controller-sync-period=30s: Period for syncing the deployments.
+ --google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication.
+ --horizontal-pod-autoscaler-sync-period=30s: The period for syncing the number of pods in horizontal pod autoscaler.
+ --kubeconfig="": Path to kubeconfig file with authorization and master location information.
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --master="": The address of the Kubernetes API server (overrides any value in kubeconfig)
+ --min-resync-period=12h0m0s: The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod
+ --namespace-sync-period=5m0s: The period for syncing namespace life-cycle updates
+ --node-monitor-grace-period=40s: Amount of time which we allow running Node to be unresponsive before marking it unhealty. Must be N times more than kubelet's nodeStatusUpdateFrequency, where N means number of retries allowed for kubelet to post node status.
+ --node-monitor-period=5s: The period for syncing NodeStatus in NodeController.
+ --node-startup-grace-period=1m0s: Amount of time which we allow starting Node to be unresponsive before marking it unhealty.
+ --node-sync-period=10s: The period for syncing nodes from cloudprovider. Longer periods will result in fewer calls to cloud provider, but may delay addition of new nodes to cluster.
+ --pod-eviction-timeout=5m0s: The grace period for deleting pods on failed nodes.
+ --port=10252: The port that the controller-manager's http service runs on
+ --profiling[=true]: Enable profiling via web interface host:port/debug/pprof/
+ --pv-recycler-increment-timeout-nfs=30: the increment of time added per Gi to ActiveDeadlineSeconds for an NFS scrubber pod
+ --pv-recycler-minimum-timeout-hostpath=60: The minimum ActiveDeadlineSeconds to use for a HostPath Recycler pod. This is for development and testing only and will not work in a multi-node cluster.
+ --pv-recycler-minimum-timeout-nfs=300: The minimum ActiveDeadlineSeconds to use for an NFS Recycler pod
+ --pv-recycler-pod-template-filepath-hostpath="": The file path to a pod definition used as a template for HostPath persistent volume recycling. This is for development and testing only and will not work in a multi-node cluster.
+ --pv-recycler-pod-template-filepath-nfs="": The file path to a pod definition used as a template for NFS persistent volume recycling
+ --pv-recycler-timeout-increment-hostpath=30: the increment of time added per Gi to ActiveDeadlineSeconds for a HostPath scrubber pod. This is for development and testing only and will not work in a multi-node cluster.
+ --pvclaimbinder-sync-period=10s: The period for syncing persistent volumes and persistent volume claims
+ --resource-quota-sync-period=10s: The period for syncing quota usage status in the system
+ --root-ca-file="": If set, this root certificate authority will be included in service account's token secret. This must be a valid PEM-encoded CA bundle.
+ --service-account-private-key-file="": Filename containing a PEM-encoded private RSA key used to sign service account tokens.
+ --service-sync-period=5m0s: The period for syncing services with their external load balancers
+ --terminated-pod-gc-threshold=12500: Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.
+
+
+
+The Kubernetes network proxy runs on each node. This +reflects services as defined in the Kubernetes API on each node and can do simple +TCP,UDP stream forwarding or round robin TCP,UDP forwarding across a set of backends. +Service cluster ips and ports are currently found through Docker-links-compatible +environment variables specifying ports opened by the service proxy. There is an optional +addon that provides cluster DNS for these cluster IPs. The user must create a service +with the apiserver API to configure the proxy.
+ +
+kube-proxy
+
+
+
+
+ --bind-address=0.0.0.0: The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)
+ --cleanup-iptables[=false]: If true cleanup iptables rules and exit.
+ --google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication.
+ --healthz-bind-address=127.0.0.1: The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)
+ --healthz-port=10249: The port to bind the health check server. Use 0 to disable.
+ --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname.
+ --iptables-sync-period=30s: How often iptables rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.
+ --kubeconfig="": Path to kubeconfig file with authorization information (the master location is set by the master flag).
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --masquerade-all[=false]: If using the pure iptables proxy, SNAT everything
+ --master="": The address of the Kubernetes API server (overrides any value in kubeconfig)
+ --oom-score-adj=-999: The oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]
+ --proxy-mode="": Which proxy mode to use: 'userspace' (older, stable) or 'iptables' (experimental). If blank, look at the Node object on the Kubernetes API and respect the 'net.experimental.kubernetes.io/proxy-mode' annotation if provided. Otherwise use the best-available proxy (currently userspace, but may change in future versions). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.
+ --proxy-port-range=: Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.
+ --resource-container="/kube-proxy": Absolute name of the resource-only container to create and run the Kube-proxy in (Default: /kube-proxy).
+ --udp-timeout=250ms: How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace
+
+
+
+The Kubernetes scheduler is a policy-rich, topology-aware, +workload-specific function that significantly impacts availability, performance, +and capacity. The scheduler needs to take into account individual and collective +resource requirements, quality of service requirements, hardware/software/policy +constraints, affinity and anti-affinity specifications, data locality, inter-workload +interference, deadlines, and so on. Workload-specific requirements will be exposed +through the API as necessary.
+ +
+kube-scheduler
+
+
+
+
+ --address=127.0.0.1: The IP address to serve on (set to 0.0.0.0 for all interfaces)
+ --algorithm-provider="DefaultProvider": The scheduling algorithm provider to use, one of: DefaultProvider
+ --bind-pods-burst=100: Number of bindings per second scheduler is allowed to make during bursts
+ --bind-pods-qps=50: Number of bindings per second scheduler is allowed to continuously make
+ --google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication.
+ --kubeconfig="": Path to kubeconfig file with authorization and master location information.
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --master="": The address of the Kubernetes API server (overrides any value in kubeconfig)
+ --policy-config-file="": File with scheduler policy configuration
+ --port=10251: The port that the scheduler's http service runs on
+ --profiling[=true]: Enable profiling via web interface host:port/debug/pprof/
+
+
+
+The kubelet is the primary “node agent” that runs on each +node. The kubelet works in terms of a PodSpec. A PodSpec is a YAML or JSON object +that describes a pod. The kubelet takes a set of PodSpecs that are provided through +various mechanisms (primarily through the apiserver) and ensures that the containers +described in those PodSpecs are running and healthy.
+ +Other than from an PodSpec from the apiserver, there are three ways that a container +manifest can be provided to the Kubelet.
+ +File: Path passed as a flag on the command line. This file is rechecked every 20 +seconds (configurable with a flag).
+ +HTTP endpoint: HTTP endpoint passed as a parameter on the command line. This endpoint +is checked every 20 seconds (also configurable with a flag).
+ +HTTP server: The kubelet can also listen for HTTP and respond to a simple API +(underspec’d currently) to submit a new manifest.
+ +
+kubelet
+
+
+
+
+ --address=0.0.0.0: The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces)
+ --allow-privileged[=false]: If true, allow containers to request privileged mode. [default=false]
+ --api-servers=[]: List of Kubernetes API servers for publishing events, and reading pods and services. (ip:port), comma separated.
+ --cadvisor-port=4194: The port of the localhost cAdvisor endpoint
+ --cert-dir="/var/run/kubernetes": The directory where the TLS certs are located (by default /var/run/kubernetes). If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.
+ --cgroup-root="": Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '', which means use the container runtime default.
+ --chaos-chance=0: If > 0.0, introduce random client errors and latency. Intended for testing. [default=0.0]
+ --cloud-config="": The path to the cloud provider configuration file. Empty string for no configuration file.
+ --cloud-provider="": The provider for cloud services. Empty string for no provider.
+ --cluster-dns=<nil>: IP address for a cluster DNS server. If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers
+ --cluster-domain="": Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains
+ --config="": Path to the config file or directory of files
+ --configure-cbr0[=false]: If true, kubelet will configure cbr0 based on Node.Spec.PodCIDR.
+ --container-runtime="docker": The container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'.
+ --containerized[=false]: Experimental support for running kubelet in a container. Intended for testing. [default=false]
+ --cpu-cfs-quota[=false]: Enable CPU CFS quota enforcement for containers that specify CPU limits
+ --docker-endpoint="": If non-empty, use this for the docker endpoint to communicate with
+ --docker-exec-handler="native": Handler to use when executing a command in a container. Valid values are 'native' and 'nsenter'. Defaults to 'native'.
+ --enable-debugging-handlers[=true]: Enables server endpoints for log collection and local running of containers and commands
+ --enable-server[=true]: Enable the Kubelet's server
+ --event-burst=0: Maximum size of a bursty event records, temporarily allows event records to burst to this number, while still not exceeding event-qps. Only used if --event-qps > 0
+ --event-qps=0: If > 0, limit event creations per second to this value. If 0, unlimited. [default=0.0]
+ --file-check-frequency=20s: Duration between checking config files for new data
+ --google-json-key="": The Google Cloud Platform Service Account JSON Key to use for authentication.
+ --healthz-bind-address=127.0.0.1: The IP address for the healthz server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)
+ --healthz-port=10248: The port of the localhost healthz endpoint
+ --host-ipc-sources="*": Comma-separated list of sources from which the Kubelet allows pods to use the host ipc namespace. [default="*"]
+ --host-network-sources="*": Comma-separated list of sources from which the Kubelet allows pods to use of host network. [default="*"]
+ --host-pid-sources="*": Comma-separated list of sources from which the Kubelet allows pods to use the host pid namespace. [default="*"]
+ --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname.
+ --http-check-frequency=20s: Duration between checking http for new data
+ --image-gc-high-threshold=90: The percent of disk usage after which image garbage collection is always run. Default: 90%%
+ --image-gc-low-threshold=80: The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. Default: 80%%
+ --kubeconfig="/var/lib/kubelet/kubeconfig": Path to a kubeconfig file, specifying how to authenticate to API server (the master location is set by the api-servers flag).
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --low-diskspace-threshold-mb=256: The absolute free disk space, in MB, to maintain. When disk space falls below this threshold, new pods would be rejected. Default: 256
+ --manifest-url="": URL for accessing the container manifest
+ --manifest-url-header="": HTTP header to use when accessing the manifest URL, with the key separated from the value with a ':', as in 'key:value'
+ --master-service-namespace="default": The namespace from which the kubernetes master services should be injected into pods
+ --max-open-files=1000000: Number of files that can be opened by Kubelet process. [default=1000000]
+ --max-pods=40: Number of Pods that can run on this Kubelet.
+ --maximum-dead-containers=100: Maximum number of old instances of containers to retain globally. Each container takes up some disk space. Default: 100.
+ --maximum-dead-containers-per-container=2: Maximum number of old instances to retain per container. Each container takes up some disk space. Default: 2.
+ --minimum-container-ttl-duration=1m0s: Minimum age for a finished container before it is garbage collected. Examples: '300ms', '10s' or '2h45m'
+ --network-plugin="": <Warning: Alpha feature> The name of the network plugin to be invoked for various events in kubelet/pod lifecycle
+ --network-plugin-dir="/usr/libexec/kubernetes/kubelet-plugins/net/exec/": <Warning: Alpha feature> The full path of the directory in which to search for network plugins
+ --node-status-update-frequency=10s: Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. Default: 10s
+ --oom-score-adj=-999: The oom-score-adj value for kubelet process. Values must be within the range [-1000, 1000]
+ --pod-cidr="": The CIDR to use for pod IP addresses, only used in standalone mode. In cluster mode, this is obtained from the master.
+ --pod-infra-container-image="gcr.io/google_containers/pause:0.8.0": The image whose network/ipc namespaces containers in each pod will use.
+ --port=10250: The port for the Kubelet to serve on. Note that "kubectl logs" will not work if you set this flag.
+ --read-only-port=10255: The read-only port for the Kubelet to serve on (set to 0 to disable)
+ --really-crash-for-testing[=false]: If true, when panics occur crash. Intended for testing.
+ --register-node[=true]: Register the node with the apiserver (defaults to true if --api-servers is set)
+ --registry-burst=10: Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry-qps. Only used if --registry-qps > 0
+ --registry-qps=0: If > 0, limit registry pull QPS to this value. If 0, unlimited. [default=0.0]
+ --resolv-conf="/etc/resolv.conf": Resolver configuration file used as the basis for the container DNS resolution configuration.
+ --resource-container="/kubelet": Absolute name of the resource-only container to create and run the Kubelet in (Default: /kubelet).
+ --rkt-path="": Path of rkt binary. Leave empty to use the first rkt in $PATH. Only used if --container-runtime='rkt'
+ --rkt-stage1-image="": image to use as stage1. Local paths and http/https URLs are supported. If empty, the 'stage1.aci' in the same directory as '--rkt-path' will be used
+ --root-dir="/var/lib/kubelet": Directory path for managing kubelet files (volume mounts,etc).
+ --runonce[=false]: If true, exit after spawning pods from local manifests or remote urls. Exclusive with --api-servers, and --enable-server
+ --serialize-image-pulls[=true]: Pull images one at a time. We recommend *not* changing the default value on nodes that run docker daemon with version < 1.9 or an Aufs storage backend. Issue #10959 has more details. [default=true]
+ --streaming-connection-idle-timeout=0: Maximum time a streaming connection can be idle before the connection is automatically closed. Example: '5m'
+ --sync-frequency=10s: Max period between synchronizing running containers and config
+ --system-container="": Optional resource-only container in which to place all non-kernel processes that are not already in a container. Empty for no container. Rolling back the flag requires a reboot. (Default: "").
+ --tls-cert-file="": File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert). If --tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate and key are generated for the public address and saved to the directory passed to --cert-dir.
+ --tls-private-key-file="": File containing x509 private key matching --tls-cert-file.
+
+
+
+Limit Range +======================================== +By default, pods run with unbounded CPU and memory limits. This means that any pod in the +system will be able to consume as much CPU and memory on the node that executes the pod.
+ +Users may want to impose restrictions on the amount of resource a single pod in the system may consume +for a variety of reasons.
+ +For example:
+ +This example demonstrates how limits can be applied to a Kubernetes namespace to control +min/max resource limits per pod. In addition, this example demonstrates how you can +apply default resource limits to pods in the absence of an end-user specified value.
+ +See LimitRange design doc for more information. For a detailed description of the Kubernetes resource model, see Resources
+ +This example requires a running Kubernetes cluster. See the Getting Started guides for how to get started.
+ +Change to the <kubernetes>
directory if you’re not already there.
This example will work in a custom namespace to demonstrate the concepts involved.
+ +Let’s create a new namespace called limit-example:
+ +$ kubectl create -f docs/admin/limitrange/namespace.yaml
+namespace "limit-example" created
+$ kubectl get namespaces
+NAME LABELS STATUS AGE
+default <none> Active 5m
+limit-example <none> Active 53s
+
+Let’s create a simple limit in our namespace.
+ +$ kubectl create -f docs/admin/limitrange/limits.yaml --namespace=limit-example
+limitrange "mylimits" created
+
+Let’s describe the limits that we have imposed in our namespace.
+ +$ kubectl describe limits mylimits --namespace=limit-example
+Name: mylimits
+Namespace: limit-example
+Type Resource Min Max Request Limit Limit/Request
+---- -------- --- --- ------- ----- -------------
+Pod cpu 200m 2 - - -
+Pod memory 6Mi 1Gi - - -
+Container cpu 100m 2 200m 300m -
+Container memory 3Mi 1Gi 100Mi 200Mi -
+
+In this scenario, we have said the following:
+ +limits.yaml
(300m CPU and 200Mi memory).limits.yaml
(200m CPU and 100Mi memory).The limits enumerated in a namespace are only enforced when a pod is created or updated in +the cluster. If you change the limits to a different value range, it does not affect pods that +were previously created in a namespace.
+ +If a resource (cpu or memory) is being restricted by a limit, the user will get an error at time +of creation explaining why.
+ +Let’s first spin up a replication controller that creates a single container pod to demonstrate +how default values are applied to each pod.
+ +$ kubectl run nginx --image=nginx --replicas=1 --namespace=limit-example
+replicationcontroller "nginx" created
+$ kubectl get pods --namespace=limit-example
+NAME READY STATUS RESTARTS AGE
+nginx-aq0mf 1/1 Running 0 35s
+$ kubectl get pods nginx-aq0mf --namespace=limit-example -o yaml | grep resources -C 8
+
+resourceVersion: "127"
+ selfLink: /api/v1/namespaces/limit-example/pods/nginx-aq0mf
+ uid: 51be42a7-7156-11e5-9921-286ed488f785
+spec:
+ containers:
+ - image: nginx
+ imagePullPolicy: IfNotPresent
+ name: nginx
+ resources:
+ limits:
+ cpu: 300m
+ memory: 200Mi
+ requests:
+ cpu: 200m
+ memory: 100Mi
+ terminationMessagePath: /dev/termination-log
+ volumeMounts:
+
+Note that our nginx container has picked up the namespace default cpu and memory resource limits and requests.
+ +Let’s create a pod that exceeds our allowed limits by having it have a container that requests 3 cpu cores.
+ +$ kubectl create -f docs/admin/limitrange/invalid-pod.yaml --namespace=limit-example
+Error from server: error when creating "docs/admin/limitrange/invalid-pod.yaml": Pod "invalid-pod" is forbidden: [Maximum cpu usage per Pod is 2, but limit is 3., Maximum cpu usage per Container is 2, but limit is 3.]
+
+Let’s create a pod that falls within the allowed limit boundaries.
+ +$ kubectl create -f docs/admin/limitrange/valid-pod.yaml --namespace=limit-example
+pod "valid-pod" created
+$ kubectl get pods valid-pod --namespace=limit-example -o yaml | grep -C 6 resources
+
+uid: 162a12aa-7157-11e5-9921-286ed488f785
+spec:
+ containers:
+ - image: gcr.io/google_containers/serve_hostname
+ imagePullPolicy: IfNotPresent
+ name: kubernetes-serve-hostname
+ resources:
+ limits:
+ cpu: "1"
+ memory: 512Mi
+ requests:
+ cpu: "1"
+ memory: 512Mi
+
+Note that this pod specifies explicit resource limits and requests so it did not pick up the namespace +default values.
+ +Note: The limits for CPU resource are not enforced in the default Kubernetes setup on the physical node +that runs the container unless the administrator deploys the kubelet with the folllowing flag:
+ +
+$ kubelet --help
+Usage of kubelet
+....
+ --cpu-cfs-quota[=false]: Enable CPU CFS quota enforcement for containers that specify CPU limits
+$ kubelet --cpu-cfs-quota=true ...
+
+
+
+To remove the resources used by this example, you can just delete the limit-example namespace.
+ +$ kubectl delete namespace limit-example
+namespace "limit-example" deleted
+$ kubectl get namespaces
+NAME LABELS STATUS AGE
+default <none> Active 20m
+
+Cluster operators that want to restrict the amount of resources a single container or pod may consume +are able to define allowable ranges per Kubernetes namespace. In the absence of any explicit assignments, +the Kubernetes system is able to apply default resource limits and requests if desired in order to +constrain the amount of resource a pod consumes on a node.
+ + + + + + + + + +Limit Range +======================================== +By default, pods run with unbounded CPU and memory limits. This means that any pod in the +system will be able to consume as much CPU and memory on the node that executes the pod.
+ +Users may want to impose restrictions on the amount of resource a single pod in the system may consume +for a variety of reasons.
+ +For example:
+ +This example demonstrates how limits can be applied to a Kubernetes namespace to control +min/max resource limits per pod. In addition, this example demonstrates how you can +apply default resource limits to pods in the absence of an end-user specified value.
+ +See LimitRange design doc for more information. For a detailed description of the Kubernetes resource model, see Resources
+ +This example requires a running Kubernetes cluster. See the Getting Started guides for how to get started.
+ +Change to the <kubernetes>
directory if you’re not already there.
This example will work in a custom namespace to demonstrate the concepts involved.
+ +Let’s create a new namespace called limit-example:
+ +$ kubectl create -f docs/admin/limitrange/namespace.yaml
+namespace "limit-example" created
+$ kubectl get namespaces
+NAME LABELS STATUS AGE
+default <none> Active 5m
+limit-example <none> Active 53s
+
+Let’s create a simple limit in our namespace.
+ +$ kubectl create -f docs/admin/limitrange/limits.yaml --namespace=limit-example
+limitrange "mylimits" created
+
+Let’s describe the limits that we have imposed in our namespace.
+ +$ kubectl describe limits mylimits --namespace=limit-example
+Name: mylimits
+Namespace: limit-example
+Type Resource Min Max Request Limit Limit/Request
+---- -------- --- --- ------- ----- -------------
+Pod cpu 200m 2 - - -
+Pod memory 6Mi 1Gi - - -
+Container cpu 100m 2 200m 300m -
+Container memory 3Mi 1Gi 100Mi 200Mi -
+
+In this scenario, we have said the following:
+ +limits.yaml
(300m CPU and 200Mi memory).limits.yaml
(200m CPU and 100Mi memory).The limits enumerated in a namespace are only enforced when a pod is created or updated in +the cluster. If you change the limits to a different value range, it does not affect pods that +were previously created in a namespace.
+ +If a resource (cpu or memory) is being restricted by a limit, the user will get an error at time +of creation explaining why.
+ +Let’s first spin up a replication controller that creates a single container pod to demonstrate +how default values are applied to each pod.
+ +$ kubectl run nginx --image=nginx --replicas=1 --namespace=limit-example
+replicationcontroller "nginx" created
+$ kubectl get pods --namespace=limit-example
+NAME READY STATUS RESTARTS AGE
+nginx-aq0mf 1/1 Running 0 35s
+$ kubectl get pods nginx-aq0mf --namespace=limit-example -o yaml | grep resources -C 8
+
+resourceVersion: "127"
+ selfLink: /api/v1/namespaces/limit-example/pods/nginx-aq0mf
+ uid: 51be42a7-7156-11e5-9921-286ed488f785
+spec:
+ containers:
+ - image: nginx
+ imagePullPolicy: IfNotPresent
+ name: nginx
+ resources:
+ limits:
+ cpu: 300m
+ memory: 200Mi
+ requests:
+ cpu: 200m
+ memory: 100Mi
+ terminationMessagePath: /dev/termination-log
+ volumeMounts:
+
+Note that our nginx container has picked up the namespace default cpu and memory resource limits and requests.
+ +Let’s create a pod that exceeds our allowed limits by having it have a container that requests 3 cpu cores.
+ +$ kubectl create -f docs/admin/limitrange/invalid-pod.yaml --namespace=limit-example
+Error from server: error when creating "docs/admin/limitrange/invalid-pod.yaml": Pod "invalid-pod" is forbidden: [Maximum cpu usage per Pod is 2, but limit is 3., Maximum cpu usage per Container is 2, but limit is 3.]
+
+Let’s create a pod that falls within the allowed limit boundaries.
+ +$ kubectl create -f docs/admin/limitrange/valid-pod.yaml --namespace=limit-example
+pod "valid-pod" created
+$ kubectl get pods valid-pod --namespace=limit-example -o yaml | grep -C 6 resources
+
+uid: 162a12aa-7157-11e5-9921-286ed488f785
+spec:
+ containers:
+ - image: gcr.io/google_containers/serve_hostname
+ imagePullPolicy: IfNotPresent
+ name: kubernetes-serve-hostname
+ resources:
+ limits:
+ cpu: "1"
+ memory: 512Mi
+ requests:
+ cpu: "1"
+ memory: 512Mi
+
+Note that this pod specifies explicit resource limits and requests so it did not pick up the namespace +default values.
+ +Note: The limits for CPU resource are not enforced in the default Kubernetes setup on the physical node +that runs the container unless the administrator deploys the kubelet with the folllowing flag:
+ +
+$ kubelet --help
+Usage of kubelet
+....
+ --cpu-cfs-quota[=false]: Enable CPU CFS quota enforcement for containers that specify CPU limits
+$ kubelet --cpu-cfs-quota=true ...
+
+
+
+To remove the resources used by this example, you can just delete the limit-example namespace.
+ +$ kubectl delete namespace limit-example
+namespace "limit-example" deleted
+$ kubectl get namespaces
+NAME LABELS STATUS AGE
+default <none> Active 20m
+
+Cluster operators that want to restrict the amount of resources a single container or pod may consume +are able to define allowable ranges per Kubernetes namespace. In the absence of any explicit assignments, +the Kubernetes system is able to apply default resource limits and requests if desired in order to +constrain the amount of resource a pod consumes on a node.
+ + + + + + + + + +You may want to set up multiple Kubernetes clusters, both to +have clusters in different regions to be nearer to your users, and to tolerate failures and/or invasive maintenance. +This document describes some of the issues to consider when making a decision about doing so.
+ +Note that at present, +Kubernetes does not offer a mechanism to aggregate multiple clusters into a single virtual cluster. However, +we plan to do this in the future.
+ +On IaaS providers such as Google Compute Engine or Amazon Web Services, a VM exists in a +zone or availability +zone. +We suggest that all the VMs in a Kubernetes cluster should be in the same availability zone, because: + - compared to having a single global Kubernetes cluster, there are fewer single-points of failure + - compared to a cluster that spans availability zones, it is easier to reason about the availability properties of a + single-zone cluster. + - when the Kubernetes developers are designing the system (e.g. making assumptions about latency, bandwidth, or + correlated failures) they are assuming all the machines are in a single data center, or otherwise closely connected.
+ +It is okay to have multiple clusters per availability zone, though on balance we think fewer is better. +Reasons to prefer fewer clusters are: + - improved bin packing of Pods in some cases with more nodes in one cluster (less resource fragmentation) + - reduced operational overhead (though the advantage is diminished as ops tooling and processes matures) + - reduced costs for per-cluster fixed resource costs, e.g. apiserver VMs (but small as a percentage + of overall cluster cost for medium to large clusters).
+ +Reasons to have multiple clusters include: + - strict security policies requiring isolation of one class of work from another (but, see Partitioning Clusters + below). + - test clusters to canary new Kubernetes releases or other cluster software.
+ +The selection of the number of Kubernetes clusters may be a relatively static choice, only revisited occasionally. +By contrast, the number of nodes in a cluster and the number of pods in a service may be change frequently according to +load and growth.
+ +To pick the number of clusters, first, decide which regions you need to be in to have adequate latency to all your end users, for services that will run
+on Kubernetes (if you use a Content Distribution Network, the latency requirements for the CDN-hosted content need not
+be considered). Legal issues might influence this as well. For example, a company with a global customer base might decide to have clusters in US, EU, AP, and SA regions.
+Call the number of regions to be in R
.
Second, decide how many clusters should be able to be unavailable at the same time, while still being available. Call
+the number that can be unavailable U
. If you are not sure, then 1 is a fine choice.
If it is allowable for load-balancing to direct traffic to any region in the event of a cluster failure, then
+you need R + U
clusters. If it is not (e.g you want to ensure low latency for all users in the event of a
+cluster failure), then you need to have R * U
clusters (U
in each of R
regions). In any case, try to put each cluster in a different zone.
Finally, if any of your clusters would need more than the maximum recommended number of nodes for a Kubernetes cluster, then +you may need even more clusters. Kubernetes v1.0 currently supports clusters up to 100 nodes in size, but we are targeting +1000-node clusters by early 2016.
+ +When you have multiple clusters, you would typically create services with the same config in each cluster and put each of those +service instances behind a load balancer (AWS Elastic Load Balancer, GCE Forwarding Rule or HTTP Load Balancer) spanning all of them, so that +failures of a single cluster are not visible to end users.
+ + + + + + + + + +Kubernetes namespaces help different projects, teams, or customers to share a Kubernetes cluster.
+ +It does this by providing the following:
+ +Use of multiple namespaces is optional.
+ +This example demonstrates how to use Kubernetes namespaces to subdivide your cluster.
+ +This example assumes the following:
+ +By default, a Kubernetes cluster will instantiate a default namespace when provisioning the cluster to hold the default set of pods, +services, and replication controllers used by the cluster.
+ +Assuming you have a fresh cluster, you can introspect the available namespace’s by doing the following:
+ +$ kubectl get namespaces
+NAME LABELS
+default <none>
+
+For this exercise, we will create two additional Kubernetes namespaces to hold our content.
+ +Let’s imagine a scenario where an organization is using a shared Kubernetes cluster for development and production use cases.
+ +The development team would like to maintain a space in the cluster where they can get a view on the list of pods, services, and replication controllers +they use to build and run their application. In this space, Kubernetes resources come and go, and the restrictions on who can or cannot modify resources +are relaxed to enable agile development.
+ +The operations team would like to maintain a space in the cluster where they can enforce strict procedures on who can or cannot manipulate the set of +pods, services, and replication controllers that run the production site.
+ +One pattern this organization could follow is to partition the Kubernetes cluster into two namespaces: development and production.
+ +Let’s create two new namespaces to hold our work.
+ +Use the file namespace-dev.json
which describes a development namespace:
{
+ "kind": "Namespace",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "development",
+ "labels": {
+ "name": "development"
+ }
+ }
+}
+
+Create the development namespace using kubectl.
+ +$ kubectl create -f docs/admin/namespaces/namespace-dev.json
+
+And then lets create the production namespace using kubectl.
+ +$ kubectl create -f docs/admin/namespaces/namespace-prod.json
+
+To be sure things are right, let’s list all of the namespaces in our cluster.
+ +$ kubectl get namespaces
+NAME LABELS STATUS
+default <none> Active
+development name=development Active
+production name=production Active
+
+A Kubernetes namespace provides the scope for pods, services, and replication controllers in the cluster.
+ +Users interacting with one namespace do not see the content in another namespace.
+ +To demonstrate this, let’s spin up a simple replication controller and pod in the development namespace.
+ +We first check what is the current context:
+ +apiVersion: v1
+clusters:
+- cluster:
+ certificate-authority-data: REDACTED
+ server: https://130.211.122.180
+ name: lithe-cocoa-92103_kubernetes
+contexts:
+- context:
+ cluster: lithe-cocoa-92103_kubernetes
+ user: lithe-cocoa-92103_kubernetes
+ name: lithe-cocoa-92103_kubernetes
+current-context: lithe-cocoa-92103_kubernetes
+kind: Config
+preferences: {}
+users:
+- name: lithe-cocoa-92103_kubernetes
+ user:
+ client-certificate-data: REDACTED
+ client-key-data: REDACTED
+ token: 65rZW78y8HbwXXtSXuUw9DbP4FLjHi4b
+- name: lithe-cocoa-92103_kubernetes-basic-auth
+ user:
+ password: h5M0FtUUIflBSdI7
+ username: admin
+
+The next step is to define a context for the kubectl client to work in each namespace. The value of “cluster” and “user” fields are copied from the current context.
+ +$ kubectl config set-context dev --namespace=development --cluster=lithe-cocoa-92103_kubernetes --user=lithe-cocoa-92103_kubernetes
+$ kubectl config set-context prod --namespace=production --cluster=lithe-cocoa-92103_kubernetes --user=lithe-cocoa-92103_kubernetes
+
+The above commands provided two request contexts you can alternate against depending on what namespace you +wish to work against.
+ +Let’s switch to operate in the development namespace.
+ +$ kubectl config use-context dev
+
+You can verify your current context by doing the following:
+ +$ kubectl config view
+
+apiVersion: v1
+clusters:
+- cluster:
+ certificate-authority-data: REDACTED
+ server: https://130.211.122.180
+ name: lithe-cocoa-92103_kubernetes
+contexts:
+- context:
+ cluster: lithe-cocoa-92103_kubernetes
+ namespace: development
+ user: lithe-cocoa-92103_kubernetes
+ name: dev
+- context:
+ cluster: lithe-cocoa-92103_kubernetes
+ user: lithe-cocoa-92103_kubernetes
+ name: lithe-cocoa-92103_kubernetes
+- context:
+ cluster: lithe-cocoa-92103_kubernetes
+ namespace: production
+ user: lithe-cocoa-92103_kubernetes
+ name: prod
+current-context: dev
+kind: Config
+preferences: {}
+users:
+- name: lithe-cocoa-92103_kubernetes
+ user:
+ client-certificate-data: REDACTED
+ client-key-data: REDACTED
+ token: 65rZW78y8HbwXXtSXuUw9DbP4FLjHi4b
+- name: lithe-cocoa-92103_kubernetes-basic-auth
+ user:
+ password: h5M0FtUUIflBSdI7
+ username: admin
+
+At this point, all requests we make to the Kubernetes cluster from the command line are scoped to the development namespace.
+ +Let’s create some content.
+ +$ kubectl run snowflake --image=kubernetes/serve_hostname --replicas=2
+
+We have just created a replication controller whose replica size is 2 that is running the pod called snowflake with a basic container that just serves the hostname.
+ +$ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+snowflake snowflake kubernetes/serve_hostname run=snowflake 2
+
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+snowflake-8w0qn 1/1 Running 0 22s
+snowflake-jrpzb 1/1 Running 0 22s
+
+And this is great, developers are able to do what they want, and they do not have to worry about affecting content in the production namespace.
+ +Let’s switch to the production namespace and show how resources in one namespace are hidden from the other.
+ +$ kubectl config use-context prod
+
+The production namespace should be empty.
+ +$ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+
+Production likes to run cattle, so let’s create some cattle pods.
+ +$ kubectl run cattle --image=kubernetes/serve_hostname --replicas=5
+
+$ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+cattle cattle kubernetes/serve_hostname run=cattle 5
+
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+cattle-97rva 1/1 Running 0 12s
+cattle-i9ojn 1/1 Running 0 12s
+cattle-qj3yv 1/1 Running 0 12s
+cattle-yc7vn 1/1 Running 0 12s
+cattle-zz7ea 1/1 Running 0 12s
+
+At this point, it should be clear that the resources users create in one namespace are hidden from the other namespace.
+ +As the policy support in Kubernetes evolves, we will extend this scenario to show how you can provide different +authorization rules for each namespace.
+ + + + + + + + + +A Namespace is a mechanism to partition resources created by users into +a logically named group.
+ +A single cluster should be able to satisfy the needs of multiple users or groups of users (henceforth a ‘user community’).
+ +Each user community wants to be able to work in isolation from other communities.
+ +Each user community has its own:
+ +A cluster operator may create a Namespace for each unique user community.
+ +The Namespace provides a unique scope for:
+ +Look here for an in depth example of namespaces.
+ +You can list the current namespaces in a cluster using:
+ +$ kubectl get namespaces
+NAME LABELS STATUS
+default <none> Active
+kube-system <none> Active
+
+Kubernetes starts with two initial namespaces:
+ * default
The default namespace for objects with no other namespace
+ * kube-system
The namespace for objects created by the Kubernetes system
You can also get the summary of a specific namespace using:
+ +$ kubectl get namespaces <name>
+
+Or you can get detailed information with:
+ +$ kubectl describe namespaces <name>
+Name: default
+Labels: <none>
+Status: Active
+
+No resource quota.
+
+Resource Limits
+ Type Resource Min Max Default
+ ---- -------- --- --- ---
+ Container cpu - - 100m
+
+Note that these details show both resource quota (if present) as well as resource limit ranges.
+ +Resource quota tracks aggregate usage of resources in the Namespace and allows cluster operators +to define Hard resource usage limits that a Namespace may consume.
+ +A limit range defines min/max constraints on the amount of resources a single entity can consume in +a Namespace.
+ +See Admission control: Limit Range
+ +A namespace can be in one of two phases:
+ * Active
the namespace is in use
+ * Terminating
the namespace is being deleted, and can not be used for new objects
See the design doc for more details.
+ +To create a new namespace, first create a new YAML file called my-namespace.yaml
with the contents:
apiVersion: v1
+kind: Namespace
+metadata:
+ name: <insert-namespace-name-here>
+
+Note that the name of your namespace must be a DNS compatible label.
+ +More information on the finalizers
field can be found in the namespace design doc.
Then run:
+ +$ kubectl create -f ./my-namespace.yaml
+
+See Setting the namespace for a request +and Setting the namespace preference.
+ +You can delete a namespace with
+ +$ kubectl delete namespaces <insert-some-namespace-name>
+
+WARNING, this deletes everything under the namespace!
+ +This delete is asynchronous, so for a time you will see the namespace in the Terminating
state.
When you create a Service, it creates a corresponding DNS entry.
+This entry is of the form <service-name>.<namespace-name>.svc.cluster.local
, which means
+that if a container just uses <service-name>
it will resolve to the service which
+is local to a namespace. This is useful for using the same configuration across
+multiple namespaces such as Development, Staging and Production. If you want to reach
+across namespaces, you need to use the fully qualified domain name (FQDN).
Details of the design of namespaces in Kubernetes, including a detailed example +can be found in the namespaces design doc
+ + + + + + + + + +Table of Contents +
+ +Kubernetes approaches networking somewhat differently than Docker does by
+default. There are 4 distinct networking problems to solve:
+1. Highly-coupled container-to-container communications: this is solved by
+ pods and localhost
communications.
+2. Pod-to-Pod communications: this is the primary focus of this document.
+3. Pod-to-Service communications: this is covered by services.
+4. External-to-Service communications: this is covered by services.
Kubernetes assumes that pods can communicate with other pods, regardless of +which host they land on. We give every pod its own IP address so you do not +need to explicitly create links between pods and you almost never need to deal +with mapping container ports to host ports. This creates a clean, +backwards-compatible model where pods can be treated much like VMs or physical +hosts from the perspectives of port allocation, naming, service discovery, load +balancing, application configuration, and migration.
+ +To achieve this we must impose some requirements on how you set up your cluster +networking.
+ +Before discussing the Kubernetes approach to networking, it is worthwhile to
+review the “normal” way that networking works with Docker. By default, Docker
+uses host-private networking. It creates a virtual bridge, called docker0
by
+default, and allocates a subnet from one of the private address blocks defined
+in RFC1918 for that bridge. For each
+container that Docker creates, it allocates a virtual ethernet device (called
+veth
) which is attached to the bridge. The veth is mapped to appear as eth0
+in the container, using Linux namespaces. The in-container eth0
interface is
+given an IP address from the bridge’s address range.
The result is that Docker containers can talk to other containers only if they +are on the same machine (and thus the same virtual bridge). Containers on +different machines can not reach each other - in fact they may end up with the +exact same network ranges and IP addresses.
+ +In order for Docker containers to communicate across nodes, they must be +allocated ports on the machine’s own IP address, which are then forwarded or +proxied to the containers. This obviously means that containers must either +coordinate which ports they use very carefully or else be allocated ports +dynamically.
+ +Coordinating ports across multiple developers is very difficult to do at +scale and exposes users to cluster-level issues outside of their control. +Dynamic port allocation brings a lot of complications to the system - every +application has to take ports as flags, the API servers have to know how to +insert dynamic port numbers into configuration blocks, services have to know +how to find each other, etc. Rather than deal with this, Kubernetes takes a +different approach.
+ +Kubernetes imposes the following fundamental requirements on any networking +implementation (barring any intentional network segmentation policies): + * all containers can communicate with all other containers without NAT + * all nodes can communicate with all containers (and vice-versa) without NAT + * the IP that a container sees itself as is the same IP that others see it as
+ +What this means in practice is that you can not just take two computers +running Docker and expect Kubernetes to work. You must ensure that the +fundamental requirements are met.
+ +This model is not only less complex overall, but it is principally compatible +with the desire for Kubernetes to enable low-friction porting of apps from VMs +to containers. If your job previously ran in a VM, your VM had an IP and could +talk to other VMs in your project. This is the same basic model.
+ +Until now this document has talked about containers. In reality, Kubernetes
+applies IP addresses at the Pod
scope - containers within a Pod
share their
+network namespaces - including their IP address. This means that containers
+within a Pod
can all reach each other’s ports on localhost
. This does imply
+that containers within a Pod
must coordinate port usage, but this is no
+different than processes in a VM. We call this the “IP-per-pod” model. This
+is implemented in Docker as a “pod container” which holds the network namespace
+open while “app containers” (the things the user specified) join that namespace
+with Docker’s --net=container:<id>
function.
As with Docker, it is possible to request host ports, but this is reduced to a
+very niche operation. In this case a port will be allocated on the host Node
+and traffic will be forwarded to the Pod
. The Pod
itself is blind to the
+existence or non-existence of host ports.
There are a number of ways that this network model can be implemented. This +document is not an exhaustive study of the various methods, but hopefully serves +as an introduction to various technologies and serves as a jumping-off point. +If some techniques become vastly preferable to others, we might detail them more +here.
+ +For the Google Compute Engine cluster configuration scripts, we use advanced
+routing to
+assign each VM a subnet (default is /24
- 254 IPs). Any traffic bound for that
+subnet will be routed directly to the VM by the GCE network fabric. This is in
+addition to the “main” IP address assigned to the VM, which is NAT’ed for
+outbound internet access. A linux bridge (called cbr0
) is configured to exist
+on that subnet, and is passed to docker’s --bridge
flag.
We start Docker with:
+ +DOCKER_OPTS="--bridge=cbr0 --iptables=false --ip-masq=false"
+
+This bridge is created by Kubelet (controlled by the --configure-cbr0=true
+flag) according to the Node
’s spec.podCIDR
.
Docker will now allocate IPs from the cbr-cidr
block. Containers can reach
+each other and Nodes
over the cbr0
bridge. Those IPs are all routable
+within the GCE project network.
GCE itself does not know anything about these IPs, though, so it will not NAT
+them for outbound internet traffic. To achieve that we use an iptables rule to
+masquerade (aka SNAT - to make it seem as if packets came from the Node
+itself) traffic that is bound for IPs outside the GCE project network
+(10.0.0.0/8).
iptables -t nat -A POSTROUTING ! -d 10.0.0.0/8 -o eth0 -j MASQUERADE
+
+Lastly we enable IP forwarding in the kernel (so the kernel will process +packets for bridged containers):
+ +sysctl net.ipv4.ip_forward=1
+
+The result of all this is that all Pods
can reach each other and can egress
+traffic to the internet.
If you have a “dumb” L2 network, such as a simple switch in a “bare-metal” +environment, you should be able to do something similar to the above GCE setup. +Note that these instructions have only been tried very casually - it seems to +work, but has not been thoroughly tested. If you use this technique and +perfect the process, please let us know.
+ +Follow the “With Linux Bridge devices” section of this very nice +tutorial from +Lars Kellogg-Stedman.
+ +Flannel is a very simple overlay +network that satisfies the Kubernetes requirements. It installs in minutes and +should get you up and running if the above techniques are not working. Many +people have reported success with Flannel and Kubernetes.
+ +OpenVSwitch is a somewhat more mature but also +complicated way to build an overlay network. This is endorsed by several of the +“Big Shops” for networking.
+ +Weave is yet another way to build an overlay +network, primarily aiming at Docker integration.
+ +Calico uses BGP to enable real container +IPs.
+ +The early design of the networking model and its rationale, and some future +plans are described in more detail in the networking design +document.
+ + + + + + + + + +Table of Contents +
+ +Node
is a worker machine in Kubernetes, previously known as Minion
. Node
+may be a VM or physical machine, depending on the cluster. Each node has
+the services necessary to run Pods and is managed by the master
+components. The services on a node include docker, kubelet and network proxy. See
+The Kubernetes Node section in the
+architecture design doc for more details.
Node status describes current status of a node. For now, there are the following +pieces of information:
+ +The usage of these fields varies depending on your cloud provider or bare metal configuration.
+ +HostName: Generally not used
+ExternalIP: Generally the IP address of the node that is externally routable (available from outside the cluster)
+InternalIP: Generally the IP address of the node that is routable only within the cluster
+Node Phase is the current lifecycle phase of node, one of Pending
,
+Running
and Terminated
.
Pending: New nodes are created in this state. A node stays in this state until it is configured.
+Running: Node has been configured and the Kubernetes components are running
+Terminated: Node has been removed from the cluster. It will not receive any scheduling requests, +and any running pods will be removed from the node.
+Node with Running
phase is necessary but not sufficient requirement for
+scheduling Pods. For a node to be considered a scheduling candidate, it
+must have appropriate conditions, see below.
Node Condition describes the conditions of Running
nodes. Currently the only
+node condition is Ready. The Status of this condition can be True, False, or
+Unknown. True means the Kubelet is healthy and ready to accept pods.
+False means the Kubelet is not healthy and is not accepting pods. Unknown
+means the Node Controller, which manages node lifecycle and is responsible for
+setting the Status of the condition, has not heard from the
+node recently (currently 40 seconds).
+Node condition is represented as a json object. For example,
+the following conditions mean the node is in sane state:
"conditions": [
+ {
+ "kind": "Ready",
+ "status": "True",
+ },
+]
+
+If the Status of the Ready condition +is Unknown or False for more than five minutes, then all of the Pods on the node are terminated by the Node Controller.
+ +Describes the resources available on the node: CPUs, memory and the maximum +number of pods that can be scheduled onto the node.
+ +General information about the node, for instance kernel version, Kubernetes version +(kubelet version, kube-proxy version), docker version (if used), OS name. +The information is gathered by Kubelet from the node.
+ +Unlike Pods and Services, a Node is not inherently +created by Kubernetes: it is either taken from cloud providers like Google Compute Engine, +or from your pool of physical or virtual machines. What this means is that when +Kubernetes creates a node, it is really just creating an object that represents the node in its internal state. +After creation, Kubernetes will check whether the node is valid or not. +For example, if you try to create a node from the following content:
+ +{
+ "kind": "Node",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "10.240.79.157",
+ "labels": {
+ "name": "my-first-k8s-node"
+ }
+ }
+}
+
+Kubernetes will create a Node object internally (the representation), and
+validate the node by health checking based on the metadata.name
field: we
+assume metadata.name
can be resolved. If the node is valid, i.e. all necessary
+services are running, it is eligible to run a Pod; otherwise, it will be
+ignored for any cluster activity, until it becomes valid. Note that Kubernetes
+will keep the object for the invalid node unless it is explicitly deleted by the client, and it will keep
+checking to see if it becomes valid.
Currently, there are three components that interact with the Kubernetes node interface: Node Controller, Kubelet, and kubectl.
+ +Node controller is a component in Kubernetes master which manages Node +objects. It performs two major functions: cluster-wide node synchronization +and single node life-cycle management.
+ +Node controller has a sync loop that creates/deletes Nodes from Kubernetes
+based on all matching VM instances listed from the cloud provider. The sync period
+can be controlled via flag --node-sync-period
. If a new VM instance
+gets created, Node Controller creates a representation for it. If an existing
+instance gets deleted, Node Controller deletes the representation. Note however,
+that Node Controller is unable to provision the node for you, i.e. it won’t install
+any binary; therefore, to
+join a node to a Kubernetes cluster, you as an admin need to make sure proper services are
+running in the node. In the future, we plan to automatically provision some node
+services.
When kubelet flag --register-node
is true (the default), the kubelet will attempt to
+register itself with the API server. This is the preferred pattern, used by most distros.
For self-registration, the kubelet is started with the following options:
+ - --api-servers=
tells the kubelet the location of the apiserver.
+ - --kubeconfig
tells kubelet where to find credentials to authenticate itself to the apiserver.
+ - --cloud-provider=
tells the kubelet how to talk to a cloud provider to read metadata about itself.
+ - --register-node
tells the kubelet to create its own node resource.
Currently, any kubelet is authorized to create/modify any node resource, but in practice it only creates/modifies +its own. (In the future, we plan to limit authorization to only allow a kubelet to modify its own Node resource.)
+ +A cluster administrator can create and modify Node objects.
+ +If the administrator wishes to create node objects manually, set kubelet flag
+--register-node=false
.
The administrator can modify Node resources (regardless of the setting of --register-node
).
+Modifications include setting labels on the Node, and marking it unschedulable.
Labels on nodes can be used in conjunction with node selectors on pods to control scheduling, +e.g. to constrain a Pod to only be eligible to run on a subset of the nodes.
+ +Making a node unscheduleable will prevent new pods from being scheduled to that +node, but will not affect any existing pods on the node. This is useful as a +preparatory step before a node reboot, etc. For example, to mark a node +unschedulable, run this command:
+ +kubectl replace nodes 10.1.2.3 --patch='{"apiVersion": "v1", "unschedulable": true}'
+
+Note that pods which are created by a daemonSet controller bypass the Kubernetes scheduler, +and do not respect the unschedulable attribute on a node. The assumption is that daemons belong on +the machine even if it is being drained of applications in preparation for a reboot.
+ +The capacity of the node (number of cpus and amount of memory) is part of the node resource. +Normally, nodes register themselves and report their capacity when creating the node resource. If +you are doing manual node administration, then you need to set node +capacity when adding a node.
+ +The Kubernetes scheduler ensures that there are enough resources for all the pods on a node. It +checks that the sum of the limits of containers on the node is no greater than the node capacity. It +includes all containers started by kubelet, but not containers started directly by docker, nor +processes not in containers.
+ +If you want to explicitly reserve resources for non-Pod processes, you can create a placeholder +pod. Use the following template:
+ +apiVersion: v1
+kind: Pod
+metadata:
+ name: resource-reserver
+spec:
+ containers:
+ - name: sleep-forever
+ image: gcr.io/google_containers/pause:0.8.0
+ resources:
+ limits:
+ cpu: 100m
+ memory: 100Mi
+
+Set the cpu
and memory
values to the amount of resources you want to reserve.
+Place the file in the manifest directory (--config=DIR
flag of kubelet). Do this
+on each kubelet where you want to reserve resources.
Node is a top-level resource in the kubernetes REST API. More details about the +API object can be found at: Node API +object.
+ + + + + + + + + +This document describes how OpenVSwitch is used to setup networking between pods across nodes. +The tunnel type could be GRE or VxLAN. VxLAN is preferable when large scale isolation needs to be performed within the network.
+ + + +The vagrant setup in Kubernetes does the following:
+ +The docker bridge is replaced with a brctl generated linux bridge (kbr0) with a 256 address space subnet. Basically, a node gets 10.244.x.0/24 subnet and docker is configured to use that bridge instead of the default docker0 bridge.
+ +Also, an OVS bridge is created(obr0) and added as a port to the kbr0 bridge. All OVS bridges across all nodes are linked with GRE tunnels. So, each node has an outgoing GRE tunnel to all other nodes. It does not need to be a complete mesh really, just meshier the better. STP (spanning tree) mode is enabled in the bridges to prevent loops.
+ +Routing rules enable any 10.244.0.0/16 target to become reachable via the OVS bridge connected with the tunnels.
+ + + + + + + + + +When several users or teams share a cluster with a fixed number of nodes, +there is a concern that one team could use more than its fair share of resources.
+ +Resource quotas are a tool for administrators to address this concern. Resource quotas
+work like this:
+- Different teams work in different namespaces. Currently this is voluntary, but
+ support for making this mandatory via ACLs is planned.
+- The administrator creates a Resource Quota for each namespace.
+- Users put compute resource requests on their pods. The sum of all resource requests across
+ all pods in the same namespace must not exceed any hard resource limit in any Resource Quota
+ document for the namespace. Note that we used to verify Resource Quota by taking the sum of
+ resource limits of the pods, but this was altered to use resource requests. Backwards compatibility
+ for those pods previously created is preserved because pods that only specify a resource limit have
+ their resource requests defaulted to match their defined limits. The user is only charged for the
+ resources they request in the Resource Quota versus their limits because the request is the minimum
+ amount of resource guaranteed by the cluster during scheduling. For more information on over commit,
+ see compute-resources.
+- If creating a pod would cause the namespace to exceed any of the limits specified in the
+ the Resource Quota for that namespace, then the request will fail with HTTP status
+ code 403 FORBIDDEN
.
+- If quota is enabled in a namespace and the user does not specify requests on the pod for each
+ of the resources for which quota is enabled, then the POST of the pod will fail with HTTP
+ status code 403 FORBIDDEN
. Hint: Use the LimitRange admission controller to force default
+ values of limits (then resource requests would be equal to limits by default, see
+ admission controller) before the quota is checked to avoid this problem.
Examples of policies that could be created using namespaces and quotas are: +- In a cluster with a capacity of 32 GiB RAM, and 16 cores, let team A use 20 Gib and 10 cores, + let B use 10GiB and 4 cores, and hold 2GiB and 2 cores in reserve for future allocation. +- Limit the “testing” namespace to using 1 core and 1GiB RAM. Let the “production” namespace + use any amount.
+ +In the case where the total capacity of the cluster is less than the sum of the quotas of the namespaces, +there may be contention for resources. This is handled on a first-come-first-served basis.
+ +Neither contention nor changes to quota will affect already-running pods.
+ +Resource Quota support is enabled by default for many Kubernetes distributions. It is
+enabled when the apiserver --admission-control=
flag has ResourceQuota
as
+one of its arguments.
Resource Quota is enforced in a particular namespace when there is a
+ResourceQuota
object in that namespace. There should be at most one
+ResourceQuota
object in a namespace.
The total sum of compute resources requested by pods +in a namespace can be limited. The following compute resource types are supported:
+ +ResourceName | +Description | +
---|---|
cpu | +Total cpu requests of containers | +
memory | +Total memory requests of containers | +
For example, cpu
quota sums up the resources.requests.cpu
fields of every
+container of every pod in the namespace, and enforces a maximum on that sum.
The number of objects of a given type can be restricted. The following types +are supported:
+ +ResourceName | +Description | +
---|---|
pods | +Total number of pods | +
services | +Total number of services | +
replicationcontrollers | +Total number of replication controllers | +
resourcequotas | +Total number of resource quotas | +
secrets | +Total number of secrets | +
persistentvolumeclaims | +Total number of persistent volume claims | +
For example, pods
quota counts and enforces a maximum on the number of pods
+created in a single namespace.
You might want to set a pods quota on a namespace +to avoid the case where a user creates many small pods and exhausts the cluster’s +supply of Pod IPs.
+ +Kubectl supports creating, updating, and viewing quotas:
+ +$ kubectl namespace myspace
+$ cat <<EOF > quota.json
+{
+ "apiVersion": "v1",
+ "kind": "ResourceQuota",
+ "metadata": {
+ "name": "quota",
+ },
+ "spec": {
+ "hard": {
+ "memory": "1Gi",
+ "cpu": "20",
+ "pods": "10",
+ "services": "5",
+ "replicationcontrollers":"20",
+ "resourcequotas":"1",
+ },
+ }
+}
+EOF
+$ kubectl create -f ./quota.json
+$ kubectl get quota
+NAME
+quota
+$ kubectl describe quota quota
+Name: quota
+Resource Used Hard
+-------- ---- ----
+cpu 0m 20
+memory 0 1Gi
+pods 5 10
+replicationcontrollers 5 20
+resourcequotas 1 1
+services 3 5
+
+Resource Quota objects are independent of the Cluster Capacity. They are +expressed in absolute units. So, if you add nodes to your cluster, this does not +automatically give each namespace the ability to consume more resources.
+ +Sometimes more complex policies may be desired, such as: + - proportionally divide total cluster resources among several teams. + - allow each tenant to grow resource usage as needed, but have a generous + limit to prevent accidental resource exhaustion. + - detect demand from one namespace, add nodes, and increase quota.
+ +Such policies could be implemented using ResourceQuota as a building-block, by +writing a ‘controller’ which watches the quota usage and adjusts the quota +hard limits of each namespace according to other signals.
+ +Note that resource quota divides up aggregate cluster resources, but it creates no +restrictions around nodes: pods from several namespaces may run on the same node.
+ +See a detailed example for how to use resource quota..
+ +See ResourceQuota design doc for more information.
+ + + + + + + + + +Resource Quota +======================================== +This example demonstrates how resource quota and +limitsranger can be applied to a Kubernetes namespace. +See ResourceQuota design doc for more information.
+ +This example assumes you have a functional Kubernetes setup.
+ +This example will work in a custom namespace to demonstrate the concepts involved.
+ +Let’s create a new namespace called quota-example:
+ +$ kubectl create -f docs/admin/resourcequota/namespace.yaml
+namespace "quota-example" created
+$ kubectl get namespaces
+NAME LABELS STATUS AGE
+default <none> Active 2m
+quota-example <none> Active 39s
+
+By default, a pod will run with unbounded CPU and memory requests/limits. This means that any pod in the +system will be able to consume as much CPU and memory on the node that executes the pod.
+ +Users may want to restrict how much of the cluster resources a given namespace may consume +across all of its pods in order to manage cluster usage. To do this, a user applies a quota to +a namespace. A quota lets the user set hard limits on the total amount of node resources (cpu, memory) +and API resources (pods, services, etc.) that a namespace may consume. In term of resources, Kubernetes +checks the total resource requests, not resource limits of all containers/pods in the namespace.
+ +Let’s create a simple quota in our namespace:
+ +$ kubectl create -f docs/admin/resourcequota/quota.yaml --namespace=quota-example
+resourcequota "quota" created
+
+Once your quota is applied to a namespace, the system will restrict any creation of content +in the namespace until the quota usage has been calculated. This should happen quickly.
+ +You can describe your current quota usage to see what resources are being consumed in your +namespace.
+ +$ kubectl describe quota quota --namespace=quota-example
+Name: quota
+Namespace: quota-example
+Resource Used Hard
+-------- ---- ----
+cpu 0 20
+memory 0 1Gi
+persistentvolumeclaims 0 10
+pods 0 10
+replicationcontrollers 0 20
+resourcequotas 1 1
+secrets 1 10
+services 0 5
+
+Pod authors rarely specify resource requests and limits for their pods.
+ +Since we applied a quota to our project, let’s see what happens when an end-user creates a pod that has unbounded +cpu and memory by creating an nginx container.
+ +To demonstrate, lets create a replication controller that runs nginx:
+ +$ kubectl run nginx --image=nginx --replicas=1 --namespace=quota-example
+replicationcontroller "nginx" created
+
+Now let’s look at the pods that were created.
+ +$ kubectl get pods --namespace=quota-example
+NAME READY STATUS RESTARTS AGE
+
+What happened? I have no pods! Let’s describe the replication controller to get a view of what is happening.
+ +kubectl describe rc nginx --namespace=quota-example
+Name: nginx
+Namespace: quota-example
+Image(s): nginx
+Selector: run=nginx
+Labels: run=nginx
+Replicas: 0 current / 1 desired
+Pods Status: 0 Running / 0 Waiting / 0 Succeeded / 0 Failed
+No volumes.
+Events:
+ FirstSeen LastSeen Count From SubobjectPath Reason Message
+ 42s 11s 3 {replication-controller } FailedCreate Error creating: Pod "nginx-" is forbidden: Must make a non-zero request for memory since it is tracked by quota.
+
+The Kubernetes API server is rejecting the replication controllers requests to create a pod because our pods +do not specify any memory usage request.
+ +So let’s set some default values for the amount of cpu and memory a pod can consume:
+ +$ kubectl create -f docs/admin/resourcequota/limits.yaml --namespace=quota-example
+limitrange "limits" created
+$ kubectl describe limits limits --namespace=quota-example
+Name: limits
+Namespace: quota-example
+Type Resource Min Max Request Limit Limit/Request
+---- -------- --- --- ------- ----- -------------
+Container memory - - 256Mi 512Mi -
+Container cpu - - 100m 200m -
+
+Now any time a pod is created in this namespace, if it has not specified any resource request/limit, the default +amount of cpu and memory per container will be applied, and the request will be used as part of admission control.
+ +Now that we have applied default resource request for our namespace, our replication controller should be able to +create its pods.
+ +$ kubectl get pods --namespace=quota-example
+NAME READY STATUS RESTARTS AGE
+nginx-fca65 1/1 Running 0 1m
+
+And if we print out our quota usage in the namespace:
+ +$ kubectl describe quota quota --namespace=quota-example
+Name: quota
+Namespace: quota-example
+Resource Used Hard
+-------- ---- ----
+cpu 100m 20
+memory 256Mi 1Gi
+persistentvolumeclaims 0 10
+pods 1 10
+replicationcontrollers 1 20
+resourcequotas 1 1
+secrets 1 10
+services 0 5
+
+You can now see the pod that was created is consuming explicit amounts of resources (specified by resource request), +and the usage is being tracked by the Kubernetes system properly.
+ +Actions that consume node resources for cpu and memory can be subject to hard quota limits defined +by the namespace quota. The resource consumption is measured by resource request in pod specification.
+ +Any action that consumes those resources can be tweaked, or can pick up namespace level defaults to +meet your end goal.
+ + + + + + + + + +Resource Quota +======================================== +This example demonstrates how resource quota and +limitsranger can be applied to a Kubernetes namespace. +See ResourceQuota design doc for more information.
+ +This example assumes you have a functional Kubernetes setup.
+ +This example will work in a custom namespace to demonstrate the concepts involved.
+ +Let’s create a new namespace called quota-example:
+ +$ kubectl create -f docs/admin/resourcequota/namespace.yaml
+namespace "quota-example" created
+$ kubectl get namespaces
+NAME LABELS STATUS AGE
+default <none> Active 2m
+quota-example <none> Active 39s
+
+By default, a pod will run with unbounded CPU and memory requests/limits. This means that any pod in the +system will be able to consume as much CPU and memory on the node that executes the pod.
+ +Users may want to restrict how much of the cluster resources a given namespace may consume +across all of its pods in order to manage cluster usage. To do this, a user applies a quota to +a namespace. A quota lets the user set hard limits on the total amount of node resources (cpu, memory) +and API resources (pods, services, etc.) that a namespace may consume. In term of resources, Kubernetes +checks the total resource requests, not resource limits of all containers/pods in the namespace.
+ +Let’s create a simple quota in our namespace:
+ +$ kubectl create -f docs/admin/resourcequota/quota.yaml --namespace=quota-example
+resourcequota "quota" created
+
+Once your quota is applied to a namespace, the system will restrict any creation of content +in the namespace until the quota usage has been calculated. This should happen quickly.
+ +You can describe your current quota usage to see what resources are being consumed in your +namespace.
+ +$ kubectl describe quota quota --namespace=quota-example
+Name: quota
+Namespace: quota-example
+Resource Used Hard
+-------- ---- ----
+cpu 0 20
+memory 0 1Gi
+persistentvolumeclaims 0 10
+pods 0 10
+replicationcontrollers 0 20
+resourcequotas 1 1
+secrets 1 10
+services 0 5
+
+Pod authors rarely specify resource requests and limits for their pods.
+ +Since we applied a quota to our project, let’s see what happens when an end-user creates a pod that has unbounded +cpu and memory by creating an nginx container.
+ +To demonstrate, lets create a replication controller that runs nginx:
+ +$ kubectl run nginx --image=nginx --replicas=1 --namespace=quota-example
+replicationcontroller "nginx" created
+
+Now let’s look at the pods that were created.
+ +$ kubectl get pods --namespace=quota-example
+NAME READY STATUS RESTARTS AGE
+
+What happened? I have no pods! Let’s describe the replication controller to get a view of what is happening.
+ +kubectl describe rc nginx --namespace=quota-example
+Name: nginx
+Namespace: quota-example
+Image(s): nginx
+Selector: run=nginx
+Labels: run=nginx
+Replicas: 0 current / 1 desired
+Pods Status: 0 Running / 0 Waiting / 0 Succeeded / 0 Failed
+No volumes.
+Events:
+ FirstSeen LastSeen Count From SubobjectPath Reason Message
+ 42s 11s 3 {replication-controller } FailedCreate Error creating: Pod "nginx-" is forbidden: Must make a non-zero request for memory since it is tracked by quota.
+
+The Kubernetes API server is rejecting the replication controllers requests to create a pod because our pods +do not specify any memory usage request.
+ +So let’s set some default values for the amount of cpu and memory a pod can consume:
+ +$ kubectl create -f docs/admin/resourcequota/limits.yaml --namespace=quota-example
+limitrange "limits" created
+$ kubectl describe limits limits --namespace=quota-example
+Name: limits
+Namespace: quota-example
+Type Resource Min Max Request Limit Limit/Request
+---- -------- --- --- ------- ----- -------------
+Container memory - - 256Mi 512Mi -
+Container cpu - - 100m 200m -
+
+Now any time a pod is created in this namespace, if it has not specified any resource request/limit, the default +amount of cpu and memory per container will be applied, and the request will be used as part of admission control.
+ +Now that we have applied default resource request for our namespace, our replication controller should be able to +create its pods.
+ +$ kubectl get pods --namespace=quota-example
+NAME READY STATUS RESTARTS AGE
+nginx-fca65 1/1 Running 0 1m
+
+And if we print out our quota usage in the namespace:
+ +$ kubectl describe quota quota --namespace=quota-example
+Name: quota
+Namespace: quota-example
+Resource Used Hard
+-------- ---- ----
+cpu 100m 20
+memory 256Mi 1Gi
+persistentvolumeclaims 0 10
+pods 1 10
+replicationcontrollers 1 20
+resourcequotas 1 1
+secrets 1 10
+services 0 5
+
+You can now see the pod that was created is consuming explicit amounts of resources (specified by resource request), +and the usage is being tracked by the Kubernetes system properly.
+ +Actions that consume node resources for cpu and memory can be subject to hard quota limits defined +by the namespace quota. The resource consumption is measured by resource request in pod specification.
+ +Any action that consumes those resources can be tweaked, or can pick up namespace level defaults to +meet your end goal.
+ + + + + + + + + +The Kubernetes cluster can be configured using Salt.
+ +The Salt scripts are shared across multiple hosting providers, so it’s important to understand some background information prior to making a modification to ensure your changes do not break hosting Kubernetes across multiple environments. Depending on where you host your Kubernetes cluster, you may be using different operating systems and different networking configurations. As a result, it’s important to understand some background information before making Salt changes in order to minimize introducing failures for other hosting providers.
+ +The salt-master service runs on the kubernetes-master (except on the default GCE setup).
+ +The salt-minion service runs on the kubernetes-master and each kubernetes-node in the cluster.
+ +Each salt-minion service is configured to interact with the salt-master service hosted on the kubernetes-master via the master.conf file (except on GCE).
+ +[root@kubernetes-master] $ cat /etc/salt/minion.d/master.conf
+master: kubernetes-master
+
+The salt-master is contacted by each salt-minion and depending upon the machine information presented, the salt-master will provision the machine as either a kubernetes-master or kubernetes-node with all the required capabilities needed to run Kubernetes.
+ +If you are running the Vagrant based environment, the salt-api service is running on the kubernetes-master. It is configured to enable the vagrant user to introspect the salt cluster in order to find out about machines in the Vagrant environment via a REST API.
+ +On GCE, the master and nodes are all configured as standalone minions. The configuration for each VM is derived from the VM’s instance metadata and then stored in Salt grains (/etc/salt/minion.d/grains.conf
) and pillars (/srv/salt-overlay/pillar/cluster-params.sls
) that local Salt uses to enforce state.
All remaining sections that refer to master/minion setups should be ignored for GCE. One fallout of the GCE setup is that the Salt mine doesn’t exist - there is no sharing of configuration amongst nodes.
+ +(Not applicable on default GCE setup.)
+ +Security is not enabled on the salt-master, and the salt-master is configured to auto-accept incoming requests from minions. It is not recommended to use this security configuration in production environments without deeper study. (In some environments this isn’t as bad as it might sound if the salt master port isn’t externally accessible and you trust everyone on your network.)
+ +[root@kubernetes-master] $ cat /etc/salt/master.d/auto-accept.conf
+open_mode: True
+auto_accept: True
+
+Each minion in the salt cluster has an associated configuration that instructs the salt-master how to provision the required resources on the machine.
+ +An example file is presented below using the Vagrant based environment.
+ +[root@kubernetes-master] $ cat /etc/salt/minion.d/grains.conf
+grains:
+ etcd_servers: $MASTER_IP
+ cloud_provider: vagrant
+ roles:
+ - kubernetes-master
+
+Each hosting environment has a slightly different grains.conf file that is used to build conditional logic where required in the Salt files.
+ +The following enumerates the set of defined key/value pairs that are supported today. If you add new ones, please make sure to update this list.
+ +Key | +Value | +
---|---|
api_servers |
+ (Optional) The IP address / host name where a kubelet can get read-only access to kube-apiserver | +
cbr-cidr |
+ (Optional) The minion IP address range used for the docker container bridge. | +
cloud |
+ (Optional) Which IaaS platform is used to host Kubernetes, gce, azure, aws, vagrant | +
etcd_servers |
+ (Optional) Comma-delimited list of IP addresses the kube-apiserver and kubelet use to reach etcd. Uses the IP of the first machine in the kubernetes_master role, or 127.0.0.1 on GCE. | +
hostnamef |
+ (Optional) The full host name of the machine, i.e. uname -n | +
node_ip |
+ (Optional) The IP address to use to address this node | +
hostname_override |
+ (Optional) Mapped to the kubelet hostname-override | +
network_mode |
+ (Optional) Networking model to use among nodes: openvswitch | +
networkInterfaceName |
+ (Optional) Networking interface to use to bind addresses, default value eth0 | +
publicAddressOverride |
+ (Optional) The IP address the kube-apiserver should use to bind against for external read-only access | +
roles |
+ (Required) 1. kubernetes-master means this machine is the master in the Kubernetes cluster. 2. kubernetes-pool means this machine is a kubernetes-node. Depending on the role, the Salt scripts will provision different resources on the machine. |
+
These keys may be leveraged by the Salt sls files to branch behavior.
+ +In addition, a cluster may be running a Debian based operating system or Red Hat based operating system (Centos, Fedora, RHEL, etc.). As a result, it’s important to sometimes distinguish behavior based on operating system using if branches like the following.
+ +{% if grains['os_family'] == 'RedHat' %}
+// something specific to a RedHat environment (Centos, Fedora, RHEL) where you may use yum, systemd, etc.
+{% else %}
+// something specific to Debian environment (apt-get, initd)
+{% endif %}
+
+Per pod IP configuration is provider-specific, so when making networking changes, it’s important to sandbox these as all providers may not use the same mechanisms (iptables, openvswitch, etc.)
+ +We should define a grains.conf key that captures more specifically what network configuration environment is being used to avoid future confusion across providers.
+ +The cluster/saltbase tree has more details on the current SaltStack configuration.
+ + + + + + + + + +This is a Cluster Administrator guide to service accounts. It assumes knowledge of +the User Guide to Service Accounts.
+ +Support for authorization and user accounts is planned but incomplete. Sometimes +incomplete features are referred to in order to better describe service accounts.
+ +Kubernetes distinguished between the concept of a user account and a service accounts +for a number of reasons: + - User accounts are for humans. Service accounts are for processes, which + run in pods. + - User accounts are intended to be global. Names must be unique across all + namespaces of a cluster, future user resource will not be namespaced). + Service accounts are namespaced. + - Typically, a cluster’s User accounts might be synced from a corporate + database, where new user account creation requires special privileges and + is tied to complex business processes. Service account creation is intended + to be more lightweight, allowing cluster users to create service accounts for + specific tasks (i.e. principle of least privilege). + - Auditing considerations for humans and service accounts may differ. + - A config bundle for a complex system may include definition of various service + accounts for components of that system. Because service accounts can be created + ad-hoc and have namespaced names, such config is portable.
+ +Three separate components cooperate to implement the automation around service accounts: + - A Service account admission controller + - A Token controller + - A Service account controller
+ +The modification of pods is implemented via a plugin
+called an Admission Controller. It is part of the apiserver.
+It acts synchronously to modify pods as they are created or updated. When this plugin is active
+(and it is by default on most distributions), then it does the following when a pod is created or modified:
+ 1. If the pod does not have a ServiceAccount
set, it sets the ServiceAccount
to default
.
+ 2. It ensures that the ServiceAccount
referenced by the pod exists, and otherwise rejects it.
+ 4. If the pod does not contain any ImagePullSecrets
, then ImagePullSecrets
of the
+ServiceAccount
are added to the pod.
+ 5. It adds a volume
to the pod which contains a token for API access.
+ 6. It adds a volumeSource
to each container of the pod mounted at /var/run/secrets/kubernetes.io/serviceaccount
.
TokenController runs as part of controller-manager. It acts asynchronously. It: +- observes serviceAccount creation and creates a corresponding Secret to allow API access. +- observes serviceAccount deletion and deletes all corresponding ServiceAccountToken Secrets +- observes secret addition, and ensures the referenced ServiceAccount exists, and adds a token to the secret if needed +- observes secret deletion and removes a reference from the corresponding ServiceAccount if needed
+ +A controller loop ensures a secret with an API token exists for each service
+account. To create additional API tokens for a service account, create a secret
+of type ServiceAccountToken
with an annotation referencing the service
+account, and the controller will update it with a generated token:
secret.json:
+{
+ "kind": "Secret",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "mysecretname",
+ "annotations": {
+ "kubernetes.io/service-account.name": "myserviceaccount"
+ }
+ },
+ "type": "kubernetes.io/service-account-token"
+}
+
+kubectl create -f ./secret.json
+kubectl describe secret mysecretname
+
+kubectl delete secret mysecretname
+
+Service Account Controller manages ServiceAccount inside namespaces, and ensures +a ServiceAccount named “default” exists in every active namespace.
+ + + + + + + + + +Static pods are to be deprecated and can be removed in any future Kubernetes release!
+ +Static pod are managed directly by kubelet daemon on a specific node, without API server observing it. It does not have associated any replication controller, kubelet daemon itself watches it and restarts it when it crashes. There is no health check though. Static pods are always bound to one kubelet daemon and always run on the same node with it.
+ +Kubelet automatically creates so-called mirror pod on Kubernetes API server for each static pod, so the pods are visible there, but they cannot be controlled from the API server.
+ +Static pod can be created in two ways: either by using configuration file(s) or by HTTP.
+ +The configuration files are just standard pod definition in json or yaml format in specific directory. Use kubelet --config=<the directory>
to start kubelet daemon, which periodically scans the directory and creates/deletes static pods as yaml/json files appear/disappear there.
For example, this is how to start a simple web server as a static pod:
+ +my-minion1
.[joe@host ~] $ ssh my-minion1
+
+/etc/kubelet.d
and place a web server pod definition there, e.g. /etc/kubernetes.d/static-web.yaml
:[root@my-minion1 ~] $ mkdir /etc/kubernetes.d/
+ [root@my-minion1 ~] $ cat <<EOF >/etc/kubernetes.d/static-web.yaml
+ apiVersion: v1
+ kind: Pod
+ metadata:
+ name: static-web
+ labels:
+ role: myrole
+ spec:
+ containers:
+ - name: web
+ image: nginx
+ ports:
+ - name: web
+ containerPort: 80
+ protocol: tcp
+ EOF
+
+Configure your kubelet daemon on the node to use this directory by running it with --config=/etc/kubelet.d/
argument. On Fedora Fedora 21 with Kubernetes 0.17 edit /etc/kubernetes/kubelet
to include this line:
```
+ +KUBELET_ARGS=”–cluster-dns=10.254.0.10 –cluster-domain=kube.local –config=/etc/kubelet.d/”
+ +```
+ +Instructions for other distributions or Kubernetes installations may vary.
+Restart kubelet. On Fedora 21, this is:
+[root@my-minion1 ~] $ systemctl restart kubelet
+
+Kubelet periodically downloads a file specified by --manifest-url=<URL>
argument and interprets it as a json/yaml file with a pod definition. It works the same as --config=<directory>
, i.e. it’s reloaded every now and then and changes are applied to running static pods (see below).
When kubelet starts, it automatically starts all pods defined in directory specified in --config=
or --manifest-url=
arguments, i.e. our static-web. (It may take some time to pull nginx image, be patient…):
[joe@my-minion1 ~] $ docker ps
+CONTAINER ID IMAGE COMMAND CREATED STATUS NAMES
+f6d05272b57e nginx:latest "nginx" 8 minutes ago Up 8 minutes k8s_web.6f802af4_static-web-fk-minion1_default_67e24ed9466ba55986d120c867395f3c_378e5f3c
+
+If we look at our Kubernetes API server (running on host my-master
), we see that a new mirror-pod was created there too:
[joe@host ~] $ ssh my-master
+[joe@my-master ~] $ kubectl get pods
+POD IP CONTAINER(S) IMAGE(S) HOST LABELS STATUS CREATED MESSAGE
+static-web-my-minion1 172.17.0.3 my-minion1/192.168.100.71 role=myrole Running 11 minutes
+ web nginx Running 11 minutes
+
+Labels from the static pod are propagated into the mirror-pod and can be used as usual for filtering.
+ +Notice we cannot delete the pod with the API server (e.g. via kubectl
command), kubelet simply won’t remove it.
[joe@my-master ~] $ kubectl delete pod static-web-my-minion1
+pods/static-web-my-minion1
+[joe@my-master ~] $ kubectl get pods
+POD IP CONTAINER(S) IMAGE(S) HOST ...
+static-web-my-minion1 172.17.0.3 my-minion1/192.168.100.71 ...
+
+Back to our my-minion1
host, we can try to stop the container manually and see, that kubelet automatically restarts it in a while:
[joe@host ~] $ ssh my-minion1
+[joe@my-minion1 ~] $ docker stop f6d05272b57e
+[joe@my-minion1 ~] $ sleep 20
+[joe@my-minion1 ~] $ docker ps
+CONTAINER ID IMAGE COMMAND CREATED ...
+5b920cbaf8b1 nginx:latest "nginx -g 'daemon of 2 seconds ago ...
+
+Running kubelet periodically scans the configured directory (/etc/kubelet.d
in our example) for changes and adds/removes pods as files appear/disappear in this directory.
[joe@my-minion1 ~] $ mv /etc/kubernetes.d/static-web.yaml /tmp
+[joe@my-minion1 ~] $ sleep 20
+[joe@my-minion1 ~] $ docker ps
+// no nginx container is running
+[joe@my-minion1 ~] $ mv /tmp/static-web.yaml /etc/kubernetes.d/
+[joe@my-minion1 ~] $ sleep 20
+[joe@my-minion1 ~] $ docker ps
+CONTAINER ID IMAGE COMMAND CREATED ...
+e7a62e3427f1 nginx:latest "nginx -g 'daemon of 27 seconds ago
+
+Primary system and API concepts are documented in the User guide.
+ +Overall API conventions are described in the API conventions doc.
+ +Complete API details are documented via Swagger. The Kubernetes apiserver (aka “master”) exports an API that can be used to retrieve the Swagger spec for the Kubernetes API, by default at /swaggerapi
, and a UI you can use to browse the API documentation at /swagger-ui
. We also periodically update a statically generated UI.
Remote access to the API is discussed in the access doc.
+ +The Kubernetes API also serves as the foundation for the declarative configuration schema for the system. The Kubectl command-line tool can be used to create, update, delete, and get API objects.
+ +Kubernetes also stores its serialized state (currently in etcd) in terms of the API resources.
+ +Kubernetes itself is decomposed into multiple components, which interact through its API.
+ +In our experience, any system that is successful needs to grow and change as new use cases emerge or existing ones change. Therefore, we expect the Kubernetes API to continuously change and grow. However, we intend to not break compatibility with existing clients, for an extended period of time. In general, new API resources and new resource fields can be expected to be added frequently. Elimination of resources or fields will require following a deprecation process. The precise deprecation policy for eliminating features is TBD, but once we reach our 1.0 milestone, there will be a specific policy.
+ +What constitutes a compatible change and how to change the API are detailed by the API change document.
+ +To make it easier to eliminate fields or restructure resource representations, Kubernetes supports
+multiple API versions, each at a different API path, such as /api/v1
or
+/apis/extensions/v1beta1
.
We chose to version at the API level rather than at the resource or field level to ensure that the API presents a clear, consistent view of system resources and behavior, and to enable controlling access to end-of-lifed and/or experimental APIs.
+ +Note that API versioning and Software versioning are only indirectly related. The API and release +versioning proposal describes the relationship between API versioning and +software versioning.
+ +Different API versions imply different levels of stability and support. The criteria for each level are described +in more detail in the API Changes documentation. They are summarized here:
+ +alpha
(e.g. v1alpha1
).beta
(e.g. v2beta3
).vX
where X
is an integer.To make it easier to extend the Kubernetes API, we are in the process of implementing API
+groups. These are simply different interfaces to read and/or modify the
+same underlying resources. The API group is specified in a REST path and in the apiVersion
field
+of a serialized object.
Currently there are two API groups in use:
+ +/api/v1
and is not specified as part of the apiVersion
field, e.g.
+apiVersion: v1
./apis/extensions/$VERSION
, and which uses
+ apiVersion: extensions/$VERSION
(e.g. currently apiVersion: extensions/v1beta1
).In the future we expect that there will be more API groups, all at REST path /apis/$API_GROUP
and
+using apiVersion: $API_GROUP/$VERSION
. We expect that there will be a way for (third parties to
+create their own API groups](design/extending-api.md), and to avoid naming collisions.
Jobs, Ingress and HorizontalPodAutoscalers are enabled by default.
+Other extensions resources can be enabled by setting runtime-config on
+apiserver. runtime-config accepts comma separated values. For ex: to enable deployments and disable jobs, set
+--runtime-config=extensions/v1beta1/deployments=true,extensions/v1beta1/jobs=false
As of June 4, 2015, the Kubernetes v1 API has been enabled by default. The v1beta1 and v1beta2 APIs were deleted on June 1, 2015. v1beta3 is planned to be deleted on July 6, 2015.
+ +We’re working to convert all documentation and examples to v1. A simple API conversion tool has been written to simplify the translation process. Use kubectl create --validate
in order to validate your json or yaml against our Swagger spec.
Changes to services are the most significant difference between v1beta3 and v1.
+ +service.spec.portalIP
property is renamed to service.spec.clusterIP
.service.spec.createExternalLoadBalancer
property is removed. Specify service.spec.type: "LoadBalancer"
to create an external load balancer instead.service.spec.publicIPs
property is deprecated and now called service.spec.deprecatedPublicIPs
. This property will be removed entirely when v1beta3 is removed. The vast majority of users of this field were using it to expose services on ports on the node. Those users should specify service.spec.type: "NodePort"
instead. Read External Services for more info. If this is not sufficient for your use case, please file an issue or contact @thockin.Some other difference between v1beta3 and v1:
+ +pod.spec.containers[*].privileged
and pod.spec.containers[*].capabilities
properties are now nested under the pod.spec.containers[*].securityContext
property. See Security Contexts.pod.spec.host
property is renamed to pod.spec.nodeName
.endpoints.subsets[*].addresses.IP
property is renamed to endpoints.subsets[*].addresses.ip
.pod.status.containerStatuses[*].state.termination
and pod.status.containerStatuses[*].lastState.termination
properties are renamed to pod.status.containerStatuses[*].state.terminated
and pod.status.containerStatuses[*].lastState.terminated
respectively.pod.status.Condition
property is renamed to pod.status.conditions
.status.details.id
property is renamed to status.details.name
.Some important differences between v1beta1/2 and v1beta3:
+ +id
is now called name
.name
, labels
, annotations
, and other metadata are now nested in a map called metadata
desiredState
is now called spec
, and currentState
is now called status
/minions
has been moved to /nodes
, and the resource has kind Node
/api/v1beta3/namespaces/{namespace}/{resource_collection}/{resource_name}
. If you were not using a namespace before, use default
here.replicationControllers
, use replicationcontrollers
.?watch=true
query parameter along with the desired resourceVersion
parameter to watch from.labels
query parameter has been renamed to labelSelector
.fields
query parameter has been renamed to fieldSelector
.entrypoint
has been renamed to command
, and command
has been renamed to args
.resources{cpu:1}
) rather than as individual fields, and resource values support scaling suffixes rather than fixed scales (e.g., milli-cores)."Always"
) rather than as a nested map (always{}
).PullAlways
, PullNever
, and PullIfNotPresent
to Always
, Never
, and IfNotPresent
.source
is inlined into volume
rather than nested.hostDir
to hostPath
to better reflect that they can be files or directories.Kubernetes is a system for managing containerized applications across multiple hosts, providing basic mechanisms for deployment, maintenance, and scaling of applications.
+ +Kubernetes establishes robust declarative primitives for maintaining the desired state requested by the user. We see these primitives as the main value added by Kubernetes. Self-healing mechanisms, such as auto-restarting, re-scheduling, and replicating containers require active controllers, not just imperative orchestration.
+ +Kubernetes is primarily targeted at applications composed of multiple containers, such as elastic, distributed micro-services. It is also designed to facilitate migration of non-containerized application stacks to Kubernetes. It therefore includes abstractions for grouping containers in both loosely coupled and tightly coupled formations, and provides ways for containers to find and communicate with each other in relatively familiar ways.
+ +Kubernetes enables users to ask a cluster to run a set of containers. The system automatically chooses hosts to run those containers on. While Kubernetes’s scheduler is currently very simple, we expect it to grow in sophistication over time. Scheduling is a policy-rich, topology-aware, workload-specific function that significantly impacts availability, performance, and capacity. The scheduler needs to take into account individual and collective resource requirements, quality of service requirements, hardware/software/policy constraints, affinity and anti-affinity specifications, data locality, inter-workload interference, deadlines, and so on. Workload-specific requirements will be exposed through the API as necessary.
+ +Kubernetes is intended to run on a number of cloud providers, as well as on physical hosts.
+ +A single Kubernetes cluster is not intended to span multiple availability zones. Instead, we recommend building a higher-level layer to replicate complete deployments of highly available applications across multiple zones (see the multi-cluster doc and cluster federation proposal for more details).
+ +Finally, Kubernetes aspires to be an extensible, pluggable, building-block OSS platform and toolkit. Therefore, architecturally, we want Kubernetes to be built as a collection of pluggable components and layers, with the ability to use alternative schedulers, controllers, storage systems, and distribution mechanisms, and we’re evolving its current code in that direction. Furthermore, we want others to be able to extend Kubernetes functionality, such as with higher-level PaaS functionality or multi-cluster layers, without modification of core Kubernetes source. Therefore, its API isn’t just (or even necessarily mainly) targeted at end users, but at tool and extension developers. Its APIs are intended to serve as the foundation for an open ecosystem of tools, automation systems, and higher-level API layers. Consequently, there are no “internal” inter-component APIs. All APIs are visible and available, including the APIs used by the scheduler, the node controller, the replication-controller manager, Kubelet’s API, etc. There’s no glass to break – in order to handle more complex use cases, one can just access the lower-level APIs in a fully transparent, composable manner.
+ +For more about the Kubernetes architecture, see architecture.
+ + + + + + + + + +This document suggests a direction for identity and access management in the Kubernetes system.
+ +High level goals are: + - Have a plan for how identity, authentication, and authorization will fit in to the API. + - Have a plan for partitioning resources within a cluster between independent organizational units. + - Ease integration with existing enterprise and hosted scenarios.
+ +Each of these can act as normal users or attackers. + - External Users: People who are accessing applications running on K8s (e.g. a web site served by webserver running in a container on K8s), but who do not have K8s API access. + - K8s Users : People who access the K8s API (e.g. create K8s API objects like Pods) + - K8s Project Admins: People who manage access for some K8s Users + - K8s Cluster Admins: People who control the machines, networks, or binaries that make up a K8s cluster. + - K8s Admin means K8s Cluster Admins and K8s Project Admins taken together.
+ +Both intentional attacks and accidental use of privilege are concerns.
+ +For both cases it may be useful to think about these categories differently: + - Application Path - attack by sending network messages from the internet to the IP/port of any application running on K8s. May exploit weakness in application or misconfiguration of K8s. + - K8s API Path - attack by sending network messages to any K8s API endpoint. + - Insider Path - attack on K8s system components. Attacker may have privileged access to networks, machines or K8s software and data. Software errors in K8s system components and administrator error are some types of threat in this category.
+ +This document is primarily concerned with K8s API paths, and secondarily with Internal paths. The Application path also needs to be secure, but is not the focus of this document.
+ +External User assets: + - Personal information like private messages, or images uploaded by External Users. + - web server logs.
+ +K8s User assets: + - External User assets of each K8s User. + - things private to the K8s app, like: + - credentials for accessing other services (docker private repos, storage services, facebook, etc) + - SSL certificates for web servers + - proprietary data and code
+ +K8s Cluster assets: + - Assets of each K8s User. + - Machine Certificates or secrets. + - The value of K8s cluster computing resources (cpu, memory, etc).
+ +This document is primarily about protecting K8s User assets and K8s cluster assets from other K8s Users and K8s Project and Cluster Admins.
+ +Cluster in Small organization: + - K8s Admins may be the same people as K8s Users. + - few K8s Admins. + - prefer ease of use to fine-grained access control/precise accounting, etc. + - Product requirement that it be easy for potential K8s Cluster Admin to try out setting up a simple cluster.
+ +Cluster in Large organization: + - K8s Admins typically distinct people from K8s Users. May need to divide K8s Cluster Admin access by roles. + - K8s Users need to be protected from each other. + - Auditing of K8s User and K8s Admin actions important. + - flexible accurate usage accounting and resource controls important. + - Lots of automated access to APIs. + - Need to integrate with existing enterprise directory, authentication, accounting, auditing, and security policy infrastructure.
+ +Org-run cluster: + - organization that runs K8s master components is same as the org that runs apps on K8s. + - Nodes may be on-premises VMs or physical machines; Cloud VMs; or a mix.
+ +Hosted cluster: + - Offering K8s API as a service, or offering a Paas or Saas built on K8s. + - May already offer web services, and need to integrate with existing customer account concept, and existing authentication, accounting, auditing, and security policy infrastructure. + - May want to leverage K8s User accounts and accounting to manage their User accounts (not a priority to support this use case.) + - Precise and accurate accounting of resources needed. Resource controls needed for hard limits (Users given limited slice of data) and soft limits (Users can grow up to some limit and then be expanded).
+ +K8s ecosystem services: + - There may be companies that want to offer their existing services (Build, CI, A/B-test, release automation, etc) for use with K8s. There should be some story for this case.
+ +Pods configs should be largely portable between Org-run and hosted configurations.
+ +Related discussion: +- http://issue.k8s.io/442 +- http://issue.k8s.io/443
+ +This doc describes two security profiles: + - Simple profile: like single-user mode. Make it easy to evaluate K8s without lots of configuring accounts and policies. Protects from unauthorized users, but does not partition authorized users. + - Enterprise profile: Provide mechanisms needed for large numbers of users. Defense in depth. Should integrate with existing enterprise security infrastructure.
+ +K8s distribution should include templates of config, and documentation, for simple and enterprise profiles. System should be flexible enough for knowledgeable users to create intermediate profiles, but K8s developers should only reason about those two Profiles, not a matrix.
+ +Features in this doc are divided into “Initial Feature”, and “Improvements”. Initial features would be candidates for version 1.00.
+ +K8s will have a userAccount
API object.
+- userAccount
has a UID which is immutable. This is used to associate users with objects and to record actions in audit logs.
+- userAccount
has a name which is a string and human readable and unique among userAccounts. It is used to refer to users in Policies, to ensure that the Policies are human readable. It can be changed only when there are no Policy objects or other objects which refer to that name. An email address is a suggested format for this field.
+- userAccount
is not related to the unix username of processes in Pods created by that userAccount.
+- userAccount
API objects can have labels.
The system may associate one or more Authentication Methods with a
+userAccount
(but they are not formally part of the userAccount object.)
+In a simple deployment, the authentication method for a
+user might be an authentication token which is verified by a K8s server. In a
+more complex deployment, the authentication might be delegated to
+another system which is trusted by the K8s API to authenticate users, but where
+the authentication details are unknown to K8s.
Initial Features:
+- there is no superuser userAccount
+- userAccount
objects are statically populated in the K8s API store by reading a config file. Only a K8s Cluster Admin can do this.
+- userAccount
can have a default namespace
. If API call does not specify a namespace
, the default namespace
for that caller is assumed.
+- userAccount
is global. A single human with access to multiple namespaces is recommended to only have one userAccount.
Improvements:
+- Make userAccount
part of a separate API group from core K8s objects like pod
. Facilitates plugging in alternate Access Management.
Simple Profile:
+ - single userAccount
, used by all K8s Users and Project Admins. One access token shared by all.
Enterprise Profile:
+ - every human user has own userAccount
.
+ - userAccount
s have labels that indicate both membership in groups, and ability to act in certain roles.
+ - each service using the API has own userAccount
too. (e.g. scheduler
, repcontroller
)
+ - automated jobs to denormalize the ldap group info into the local system list of users into the K8s userAccount file.
A userAccount
is not a Unix user account. The fact that a pod is started by a userAccount
does not mean that the processes in that pod’s containers run as a Unix user with a corresponding name or identity.
Initially:
+- The unix accounts available in a container, and used by the processes running in a container are those that are provided by the combination of the base operating system and the Docker manifest.
+- Kubernetes doesn’t enforce any relation between userAccount
and unix accounts.
Improvements: +- Kubelet allocates disjoint blocks of root-namespace uids for each container. This may provide some defense-in-depth against container escapes. (https://github.com/docker/docker/pull/4572) +- requires docker to integrate user namespace support, and deciding what getpwnam() does for these uids. +- any features that help users avoid use of privileged containers (http://issue.k8s.io/391)
+ +K8s will have a have a namespace
API object. It is similar to a Google Compute Engine project
. It provides a namespace for objects created by a group of people co-operating together, preventing name collisions with non-cooperating groups. It also serves as a reference point for authorization policies.
Namespaces are described in namespaces.md.
+ +In the Enterprise Profile:
+ - a userAccount
may have permission to access several namespace
s.
In the Simple Profile:
+ - There is a single namespace
used by the single user.
Namespaces versus userAccount vs Labels:
+- userAccount
s are intended for audit logging (both name and UID should be logged), and to define who has access to namespace
s.
+- labels
(see docs/user-guide/labels.md) should be used to distinguish pods, users, and other objects that cooperate towards a common goal but are different in some way, such as version, or responsibilities.
+- namespace
s prevent name collisions between uncoordinated groups of people, and provide a place to attach common policies for co-operating groups of people.
Goals for K8s authentication: +- Include a built-in authentication system with no configuration required to use in single-user mode, and little configuration required to add several user accounts, and no https proxy required. +- Allow for authentication to be handled by a system external to Kubernetes, to allow integration with existing to enterprise authorization systems. The Kubernetes namespace itself should avoid taking contributions of multiple authorization schemes. Instead, a trusted proxy in front of the apiserver can be used to authenticate users. + - For organizations whose security requirements only allow FIPS compliant implementations (e.g. apache) for authentication. + - So the proxy can terminate SSL, and isolate the CA-signed certificate from less trusted, higher-touch APIserver. + - For organizations that already have existing SaaS web services (e.g. storage, VMs) and want a common authentication portal. +- Avoid mixing authentication and authorization, so that authorization policies be centrally managed, and to allow changes in authentication methods without affecting authorization code.
+ +Initially:
+- Tokens used to authenticate a user.
+- Long lived tokens identify a particular userAccount
.
+- Administrator utility generates tokens at cluster setup.
+- OAuth2.0 Bearer tokens protocol, http://tools.ietf.org/html/rfc6750
+- No scopes for tokens. Authorization happens in the API server
+- Tokens dynamically generated by apiserver to identify pods which are making API calls.
+- Tokens checked in a module of the APIserver.
+- Authentication in apiserver can be disabled by flag, to allow testing without authorization enabled, and to allow use of an authenticating proxy. In this mode, a query parameter or header added by the proxy will identify the caller.
Improvements: +- Refresh of tokens. +- SSH keys to access inside containers.
+ +To be considered for subsequent versions: +- Fuller use of OAuth (http://tools.ietf.org/html/rfc6749) +- Scoped tokens. +- Tokens that are bound to the channel between the client and the api server + - http://www.ietf.org/proceedings/90/slides/slides-90-uta-0.pdf + - http://www.browserauth.net
+ +K8s authorization should: +- Allow for a range of maturity levels, from single-user for those test driving the system, to integration with existing to enterprise authorization systems. +- Allow for centralized management of users and policies. In some organizations, this will mean that the definition of users and access policies needs to reside on a system other than k8s and encompass other web services (such as a storage service). +- Allow processes running in K8s Pods to take on identity, and to allow narrow scoping of permissions for those identities in order to limit damage from software faults. +- Have Authorization Policies exposed as API objects so that a single config file can create or delete Pods, Replication Controllers, Services, and the identities and policies for those Pods and Replication Controllers. +- Be separate as much as practical from Authentication, to allow Authentication methods to change over time and space, without impacting Authorization policies.
+ +K8s will implement a relatively simple +Attribute-Based Access Control model. +The model will be described in more detail in a forthcoming document. The model will +- Be less complex than XACML +- Be easily recognizable to those familiar with Amazon IAM Policies. +- Have a subset/aliases/defaults which allow it to be used in a way comfortable to those users more familiar with Role-Based Access Control.
+ +Authorization policy is set by creating a set of Policy objects.
+ +The API Server will be the Enforcement Point for Policy. For each API call that it receives, it will construct the Attributes needed to evaluate the policy (what user is making the call, what resource they are accessing, what they are trying to do that resource, etc) and pass those attributes to a Decision Point. The Decision Point code evaluates the Attributes against all the Policies and allows or denies the API call. The system will be modular enough that the Decision Point code can either be linked into the APIserver binary, or be another service that the apiserver calls for each Decision (with appropriate time-limited caching as needed for performance).
+ +Policy objects may be applicable only to a single namespace or to all namespaces; K8s Project Admins would be able to create those as needed. Other Policy objects may be applicable to all namespaces; a K8s Cluster Admin might create those in order to authorize a new type of controller to be used by all namespaces, or to make a K8s User into a K8s Project Admin.)
+ +The API should have a quota
concept (see http://issue.k8s.io/442). A quota object relates a namespace (and optionally a label selector) to a maximum quantity of resources that may be used (see resources design doc).
Initially:
+- a quota
object is immutable.
+- for hosted K8s systems that do billing, Project is recommended level for billing accounts.
+- Every object that consumes resources should have a namespace
so that Resource usage stats are roll-up-able to namespace
.
+- K8s Cluster Admin sets quota objects by writing a config file.
Improvements: +- allow one namespace to charge the quota for one or more other namespaces. This would be controlled by a policy which allows changing a billing_namespace= label on an object. +- allow quota to be set by namespace owners for (namespace x label) combinations (e.g. let “webserver” namespace use 100 cores, but to prevent accidents, don’t allow “webserver” namespace and “instance=test” use more than 10 cores. +- tools to help write consistent quota config files based on number of nodes, historical namespace usages, QoS needs, etc. +- way for K8s Cluster Admin to incrementally adjust Quota objects.
+ +Simple profile:
+ - a single namespace
with infinite resource limits.
Enterprise profile: + - multiple namespaces each with their own limits.
+ +Issues: +- need for locking or “eventual consistency” when multiple apiserver goroutines are accessing the object store and handling pod creations.
+ +API actions can be logged.
+ +Initial implementation: +- All API calls logged to nginx logs.
+ +Improvements: +- API server does logging instead. +- Policies to drop logging for high rate trusted API calls, or by users performing audit or other sensitive functions.
+ + + + + + + + + +Related PR:
+ +Topic | +Link | +
---|---|
Separate validation from RESTStorage | +http://issue.k8s.io/2977 | +
High level goals:
+ +Authorization via policy is focused on answering if a user is authorized to perform an action.
+ +Admission Control is focused on if the system will accept an authorized action.
+ +Kubernetes may choose to dismiss an authorized action based on any number of admission control strategies.
+ +This proposal documents the basic design, and describes how any number of admission control plug-ins could be injected.
+ +Implementation of specific admission control strategies are handled in separate documents.
+ +The kube-apiserver takes the following OPTIONAL arguments to enable admission control
+ +Option | +Behavior | +
---|---|
admission-control | +Comma-delimited, ordered list of admission control choices to invoke prior to modifying or deleting an object. | +
admission-control-config-file | +File with admission control configuration parameters to boot-strap plug-in. | +
An AdmissionControl plug-in is an implementation of the following interface:
+ +package admission
+
+// Attributes is an interface used by a plug-in to make an admission decision on a individual request.
+type Attributes interface {
+ GetNamespace() string
+ GetKind() string
+ GetOperation() string
+ GetObject() runtime.Object
+}
+
+// Interface is an abstract, pluggable interface for Admission Control decisions.
+type Interface interface {
+ // Admit makes an admission decision based on the request attributes
+ // An error is returned if it denies the request.
+ Admit(a Attributes) (err error)
+}
+
+A plug-in must be compiled with the binary, and is registered as an available option by providing a name, and implementation +of admission.Interface.
+ +func init() {
+ admission.RegisterPlugin("AlwaysDeny", func(client client.Interface, config io.Reader) (admission.Interface, error) { return NewAlwaysDeny(), nil })
+}
+
+Invocation of admission control is handled by the APIServer and not individual RESTStorage implementations.
+ +This design assumes that Issue 297 is adopted, and as a consequence, the general framework of the APIServer request/response flow will ensure the following:
+ +If operation=create | +update | +delete | +connect, then admission.Admit(requestAttributes) | +
If operation=create | +update, then validate(object) and persist | +
If at any step, there is an error, the request is canceled.
+ + + + + + + + + +This document proposes a system for enforcing resource requirements constraints as part of admission control.
+ +The LimitRange resource is scoped to a Namespace.
+ +// LimitType is a type of object that is limited
+type LimitType string
+
+const (
+ // Limit that applies to all pods in a namespace
+ LimitTypePod LimitType = "Pod"
+ // Limit that applies to all containers in a namespace
+ LimitTypeContainer LimitType = "Container"
+)
+
+// LimitRangeItem defines a min/max usage limit for any resource that matches on kind.
+type LimitRangeItem struct {
+ // Type of resource that this limit applies to.
+ Type LimitType `json:"type,omitempty"`
+ // Max usage constraints on this kind by resource name.
+ Max ResourceList `json:"max,omitempty"`
+ // Min usage constraints on this kind by resource name.
+ Min ResourceList `json:"min,omitempty"`
+ // Default resource requirement limit value by resource name if resource limit is omitted.
+ Default ResourceList `json:"default,omitempty"`
+ // DefaultRequest is the default resource requirement request value by resource name if resource request is omitted.
+ DefaultRequest ResourceList `json:"defaultRequest,omitempty"`
+ // MaxLimitRequestRatio if specified, the named resource must have a request and limit that are both non-zero where limit divided by request is less than or equal to the enumerated value; this represents the max burst for the named resource.
+ MaxLimitRequestRatio ResourceList `json:"maxLimitRequestRatio,omitempty"`
+}
+
+// LimitRangeSpec defines a min/max usage limit for resources that match on kind.
+type LimitRangeSpec struct {
+ // Limits is the list of LimitRangeItem objects that are enforced.
+ Limits []LimitRangeItem `json:"limits"`
+}
+
+// LimitRange sets resource usage limits for each kind of resource in a Namespace.
+type LimitRange struct {
+ TypeMeta `json:",inline"`
+ // Standard object's metadata.
+ // More info: http://releases.k8s.io/release-1.1/docs/devel/api-conventions.md#metadata
+ ObjectMeta `json:"metadata,omitempty"`
+
+ // Spec defines the limits enforced.
+ // More info: http://releases.k8s.io/release-1.1/docs/devel/api-conventions.md#spec-and-status
+ Spec LimitRangeSpec `json:"spec,omitempty"`
+}
+
+// LimitRangeList is a list of LimitRange items.
+type LimitRangeList struct {
+ TypeMeta `json:",inline"`
+ // Standard list metadata.
+ // More info: http://releases.k8s.io/release-1.1/docs/devel/api-conventions.md#types-kinds
+ ListMeta `json:"metadata,omitempty"`
+
+ // Items is a list of LimitRange objects.
+ // More info: http://releases.k8s.io/release-1.1/docs/design/admission_control_limit_range.md
+ Items []LimitRange `json:"items"`
+}
+
+Validation of a LimitRange enforces that for a given named resource the following rules apply:
+ +Min (if specified) <= DefaultRequest (if specified) <= Default (if specified) <= Max (if specified)
+ +The following default value behaviors are applied to a LimitRange for a given named resource.
+ +
+if LimitRangeItem.Default[resourceName] is undefined
+ if LimitRangeItem.Max[resourceName] is defined
+ LimitRangeItem.Default[resourceName] = LimitRangeItem.Max[resourceName]
+
+
+
+
+if LimitRangeItem.DefaultRequest[resourceName] is undefined
+ if LimitRangeItem.Default[resourceName] is defined
+ LimitRangeItem.DefaultRequest[resourceName] = LimitRangeItem.Default[resourceName]
+ else if LimitRangeItem.Min[resourceName] is defined
+ LimitRangeItem.DefaultRequest[resourceName] = LimitRangeItem.Min[resourceName]
+
+
+
+The LimitRanger plug-in introspects all incoming pod requests and evaluates the constraints defined on a LimitRange.
+ +If a constraint is not specified for an enumerated resource, it is not enforced or tracked.
+ +To enable the plug-in and support for LimitRange, the kube-apiserver must be configured as follows:
+ +$ kube-apiserver --admission-control=LimitRanger
+
+Type: Container
+ +Supported Resources:
+ +Supported Constraints:
+ +Per container, the following must hold true
+ +Constraint | +Behavior | +
---|---|
Min | +Min <= Request (required) <= Limit (optional) | +
Max | +Limit (required) <= Max | +
LimitRequestRatio | +LimitRequestRatio <= ( Limit (required, non-zero) / Request (required, non-zero)) | +
Supported Defaults:
+ +Type: Pod
+ +Supported Resources:
+ +Supported Constraints:
+ +Across all containers in pod, the following must hold true
+ +Constraint | +Behavior | +
---|---|
Min | +Min <= Request (required) <= Limit (optional) | +
Max | +Limit (required) <= Max | +
LimitRequestRatio | +LimitRequestRatio <= ( Limit (required, non-zero) / Request (non-zero) ) | +
The default LimitRange
that is applied via Salt configuration will be updated as follows:
+apiVersion: "v1"
+kind: "LimitRange"
+metadata:
+ name: "limits"
+ namespace: default
+spec:
+ limits:
+ - type: "Container"
+ defaultRequests:
+ cpu: "100m"
+
+
+
+An example LimitRange configuration:
+ +Type | +Resource | +Min | +Max | +Default | +DefaultRequest | +LimitRequestRatio | +
---|---|---|---|---|---|---|
Container | +cpu | +.1 | +1 | +500m | +250m | +4 | +
Container | +memory | +250Mi | +1Gi | +500Mi | +250Mi | ++ |
Assuming an incoming container that specified no incoming resource requirements, +the following would happen.
+ +This document describes a system for enforcing hard resource usage limits per namespace as part of admission control.
+ +The ResourceQuota object is scoped to a Namespace.
+ +// The following identify resource constants for Kubernetes object types
+const (
+ // Pods, number
+ ResourcePods ResourceName = "pods"
+ // Services, number
+ ResourceServices ResourceName = "services"
+ // ReplicationControllers, number
+ ResourceReplicationControllers ResourceName = "replicationcontrollers"
+ // ResourceQuotas, number
+ ResourceQuotas ResourceName = "resourcequotas"
+ // ResourceSecrets, number
+ ResourceSecrets ResourceName = "secrets"
+ // ResourcePersistentVolumeClaims, number
+ ResourcePersistentVolumeClaims ResourceName = "persistentvolumeclaims"
+)
+
+// ResourceQuotaSpec defines the desired hard limits to enforce for Quota
+type ResourceQuotaSpec struct {
+ // Hard is the set of desired hard limits for each named resource
+ Hard ResourceList `json:"hard,omitempty" description:"hard is the set of desired hard limits for each named resource; see http://releases.k8s.io/release-1.1/docs/design/admission_control_resource_quota.md#admissioncontrol-plugin-resourcequota"`
+}
+
+// ResourceQuotaStatus defines the enforced hard limits and observed use
+type ResourceQuotaStatus struct {
+ // Hard is the set of enforced hard limits for each named resource
+ Hard ResourceList `json:"hard,omitempty" description:"hard is the set of enforced hard limits for each named resource; see http://releases.k8s.io/release-1.1/docs/design/admission_control_resource_quota.md#admissioncontrol-plugin-resourcequota"`
+ // Used is the current observed total usage of the resource in the namespace
+ Used ResourceList `json:"used,omitempty" description:"used is the current observed total usage of the resource in the namespace"`
+}
+
+// ResourceQuota sets aggregate quota restrictions enforced per namespace
+type ResourceQuota struct {
+ TypeMeta `json:",inline"`
+ ObjectMeta `json:"metadata,omitempty" description:"standard object metadata; see http://releases.k8s.io/release-1.1/docs/devel/api-conventions.md#metadata"`
+
+ // Spec defines the desired quota
+ Spec ResourceQuotaSpec `json:"spec,omitempty" description:"spec defines the desired quota; http://releases.k8s.io/release-1.1/docs/devel/api-conventions.md#spec-and-status"`
+
+ // Status defines the actual enforced quota and its current usage
+ Status ResourceQuotaStatus `json:"status,omitempty" description:"status defines the actual enforced quota and current usage; http://releases.k8s.io/release-1.1/docs/devel/api-conventions.md#spec-and-status"`
+}
+
+// ResourceQuotaList is a list of ResourceQuota items
+type ResourceQuotaList struct {
+ TypeMeta `json:",inline"`
+ ListMeta `json:"metadata,omitempty" description:"standard list metadata; see http://releases.k8s.io/release-1.1/docs/devel/api-conventions.md#metadata"`
+
+ // Items is a list of ResourceQuota objects
+ Items []ResourceQuota `json:"items" description:"items is a list of ResourceQuota objects; see http://releases.k8s.io/release-1.1/docs/design/admission_control_resource_quota.md#admissioncontrol-plugin-resourcequota"`
+}
+
+The following resources are supported by the quota system.
+ +Resource | +Description | +
---|---|
cpu | +Total requested cpu usage | +
memory | +Total requested memory usage | +
pods | +Total number of active pods where phase is pending or active. | +
services | +Total number of services | +
replicationcontrollers | +Total number of replication controllers | +
resourcequotas | +Total number of resource quotas | +
secrets | +Total number of secrets | +
persistentvolumeclaims | +Total number of persistent volume claims | +
If a third-party wants to track additional resources, it must follow the resource naming conventions prescribed +by Kubernetes. This means the resource must have a fully-qualified name (i.e. mycompany.org/shinynewresource)
+ +If a resource supports the ability to distinguish between a request and a limit for a resource, +the quota tracking system will only cost the request value against the quota usage. If a resource +is tracked by quota, and no request value is provided, the associated entity is rejected as part of admission.
+ +For an example, consider the following scenarios relative to tracking quota on CPU:
+ +Pod | +Container | +Request CPU | +Limit CPU | +Result | +
---|---|---|---|---|
X | +C1 | +100m | +500m | +The quota usage is incremented 100m | +
Y | +C2 | +100m | +none | +The quota usage is incremented 100m | +
Y | +C2 | +none | +500m | +The quota usage is incremented 500m since request will default to limit | +
Z | +C3 | +none | +none | +The pod is rejected since it does not enumerate a request. | +
The rationale for accounting for the requested amount of a resource versus the limit is the belief +that a user should only be charged for what they are scheduled against in the cluster. In addition, +attempting to track usage against actual usage, where request < actual < limit, is considered highly +volatile.
+ +As a consequence of this decision, the user is able to spread its usage of a resource across multiple tiers +of service. Let’s demonstrate this via an example with a 4 cpu quota.
+ +The quota may be allocated as follows:
+ +Pod | +Container | +Request CPU | +Limit CPU | +Tier | +Quota Usage | +
---|---|---|---|---|---|
X | +C1 | +1 | +4 | +Burstable | +1 | +
Y | +C2 | +2 | +2 | +Guaranteed | +2 | +
Z | +C3 | +1 | +3 | +Burstable | +1 | +
It is possible that the pods may consume 9 cpu over a given time period depending on the nodes available cpu +that held pod X and Z, but since we scheduled X and Z relative to the request, we only track the requesting +value against their allocated quota. If one wants to restrict the ratio between the request and limit, +it is encouraged that the user define a LimitRange with LimitRequestRatio to control burst out behavior. +This would in effect, let an administrator keep the difference between request and limit more in line with +tracked usage if desired.
+ +A REST API endpoint to update the status section of the ResourceQuota is exposed. It requires an atomic compare-and-swap +in order to keep resource usage tracking consistent.
+ +A resource quota controller monitors observed usage for tracked resources in the Namespace.
+ +If there is observed difference between the current usage stats versus the current ResourceQuota.Status, the controller +posts an update of the currently observed usage metrics to the ResourceQuota via the /status endpoint.
+ +The resource quota controller is the only component capable of monitoring and recording usage updates after a DELETE operation +since admission control is incapable of guaranteeing a DELETE request actually succeeded.
+ +The ResourceQuota plug-in introspects all incoming admission requests.
+ +To enable the plug-in and support for ResourceQuota, the kube-apiserver must be configured as follows:
+ +
+$ kube-apiserver --admission-control=ResourceQuota
+
+
+
+It makes decisions by evaluating the incoming object against all defined ResourceQuota.Status.Hard resource limits in the request +namespace. If acceptance of the resource would cause the total usage of a named resource to exceed its hard limit, the request is denied.
+ +If the incoming request does not cause the total usage to exceed any of the enumerated hard resource limits, the plug-in will post a +ResourceQuota.Status document to the server to atomically update the observed usage based on the previously read +ResourceQuota.ResourceVersion. This keeps incremental usage atomically consistent, but does introduce a bottleneck (intentionally) +into the system.
+ +To optimize system performance, it is encouraged that all resource quotas are tracked on the same ResourceQuota document in a Namespace. As a result, its encouraged to impose a cap on the total number of individual quotas that are tracked in the Namespace +to 1 in the ResourceQuota document.
+ +kubectl is modified to support the ResourceQuota resource.
+ +kubectl describe
provides a human-readable output of quota.
For example,
+ +$ kubectl create -f docs/admin/resourcequota/namespace.yaml
+namespace "quota-example" created
+$ kubectl create -f docs/admin/resourcequota/quota.yaml --namespace=quota-example
+resourcequota "quota" created
+$ kubectl describe quota quota --namespace=quota-example
+Name: quota
+Namespace: quota-example
+Resource Used Hard
+-------- ---- ----
+cpu 0 20
+memory 0 1Gi
+persistentvolumeclaims 0 10
+pods 0 10
+replicationcontrollers 0 20
+resourcequotas 1 1
+secrets 1 10
+services 0 5
+
+See resource quota document and the example of Resource Quota for more information.
+ + + + + + + + + +A running Kubernetes cluster contains node agents (kubelet
) and master components (APIs, scheduler, etc), on top of a distributed storage solution. This diagram shows our desired eventual state, though we’re still working on a few things, like making kubelet
itself (all our components, really) run within containers, and making the scheduler 100% pluggable.
When looking at the architecture of the system, we’ll break it down to services that run on the worker node and services that compose the cluster-level control plane.
+ +The Kubernetes node has the services necessary to run application containers and be managed from the master systems.
+ +Each node runs Docker, of course. Docker takes care of the details of downloading images and running containers.
+ +kubelet
The kubelet
manages pods and their containers, their images, their volumes, etc.
kube-proxy
Each node also runs a simple network proxy and load balancer (see the services FAQ for more details). This reflects services
(see the services doc for more details) as defined in the Kubernetes API on each node and can do simple TCP and UDP stream forwarding (round robin) across a set of backends.
Service endpoints are currently found via DNS or through environment variables (both Docker-links-compatible and Kubernetes {FOO}_SERVICE_HOST
and {FOO}_SERVICE_PORT
variables are supported). These variables resolve to ports managed by the service proxy.
The Kubernetes control plane is split into a set of components. Currently they all run on a single master node, but that is expected to change soon in order to support high-availability clusters. These components work together to provide a unified view of the cluster.
+ +etcd
All persistent master state is stored in an instance of etcd
. This provides a great way to store configuration data reliably. With watch
support, coordinating components can be notified very quickly of changes.
The apiserver serves up the Kubernetes API. It is intended to be a CRUD-y server, with most/all business logic implemented in separate components or in plug-ins. It mainly processes REST operations, validates them, and updates the corresponding objects in etcd
(and eventually other stores).
The scheduler binds unscheduled pods to nodes via the /binding
API. The scheduler is pluggable, and we expect to support multiple cluster schedulers and even user-provided schedulers in the future.
All other cluster-level functions are currently performed by the Controller Manager. For instance, Endpoints
objects are created and updated by the endpoints controller, and nodes are discovered, managed, and monitored by the node controller. These could eventually be split into separate components to make them independently pluggable.
The replicationcontroller
is a mechanism that is layered on top of the simple pod
API. We eventually plan to port it to a generic plug-in mechanism, once one is implemented.
This directory contains diagrams for the clustering design doc.
+ +This depends on the seqdiag
utility. Assuming you have a non-borked python install, this should be installable with
pip install seqdiag
+
+Just call make
to regenerate the diagrams.
If you are on a Mac or your pip install is messed up, you can easily build with docker.
+ +make docker
+
+The first run will be slow but things should be fast after that.
+ +To clean up the docker containers that are created (and other cruft that is left around) you can run make docker-clean
.
If you are using boot2docker and get warnings about clock skew (or if things aren’t building for some reason) then you can fix that up with make fix-clock-skew
.
If you have the fswatch utility installed, you can have it monitor the file system and automatically rebuild when files have changed. Just do a make watch
.
This directory contains diagrams for the clustering design doc.
+ +This depends on the seqdiag
utility. Assuming you have a non-borked python install, this should be installable with
pip install seqdiag
+
+Just call make
to regenerate the diagrams.
If you are on a Mac or your pip install is messed up, you can easily build with docker.
+ +make docker
+
+The first run will be slow but things should be fast after that.
+ +To clean up the docker containers that are created (and other cruft that is left around) you can run make docker-clean
.
If you are using boot2docker and get warnings about clock skew (or if things aren’t building for some reason) then you can fix that up with make fix-clock-skew
.
If you have the fswatch utility installed, you can have it monitor the file system and automatically rebuild when files have changed. Just do a make watch
.
This describes an approach for providing support for:
+ +There are several related issues/PRs:
+ +Users and administrators are accustomed to being able to access their systems +via SSH to run remote commands, get shell access, and do port forwarding.
+ +Supporting SSH to containers in Kubernetes is a difficult task. You must
+specify a “user” and a hostname to make an SSH connection, and sshd
requires
+real users (resolvable by NSS and PAM). Because a container belongs to a pod,
+and the pod belongs to a namespace, you need to specify namespace/pod/container
+to uniquely identify the target container. Unfortunately, a
+namespace/pod/container is not a real user as far as SSH is concerned. Also,
+most Linux systems limit user names to 32 characters, which is unlikely to be
+large enough to contain namespace/pod/container. We could devise some scheme to
+map each namespace/pod/container to a 32-character user name, adding entries to
+/etc/passwd
(or LDAP, etc.) and keeping those entries fully in sync all the
+time. Alternatively, we could write custom NSS and PAM modules that allow the
+host to resolve a namespace/pod/container to a user without needing to keep
+files or LDAP in sync.
As an alternative to SSH, we are using a multiplexed streaming protocol that +runs on top of HTTP. There are no requirements about users being real users, +nor is there any limitation on user name length, as the protocol is under our +control. The only downside is that standard tooling that expects to use SSH +won’t be able to work with this mechanism, unless adapters can be written.
+ +The current multiplexed streaming protocol used is SPDY. This is not the +long-term desire, however. As soon as there is viable support for HTTP/2 in Go, +we will switch to that.
+ +Clients should not be allowed to communicate directly with the Kubelet for +security reasons. Therefore, the Master is currently the only suggested entry +point to be used for remote command execution and port forwarding. This is not +necessarily desirable, as it means that all remote command execution and port +forwarding traffic must travel through the Master, potentially impacting other +API requests.
+ +In the future, it might make more sense to retrieve an authorization token from +the Master, and then use that token to initiate a remote command execution or +port forwarding request with a load balanced proxy service dedicated to this +functionality. This would keep the streaming traffic out of the Master.
+ +The kubelet is currently responsible for handling remote command execution and +port forwarding requests. Just like with the Master described above, this means +that all remote command execution and port forwarding streaming traffic must +travel through the Kubelet, which could result in a degraded ability to service +other requests.
+ +In the future, it might make more sense to use a separate service on the node.
+ +Alternatively, we could possibly inject a process into the container that only +listens for a single request, expose that process’s listening port on the node, +and then issue a redirect to the client such that it would connect to the first +level proxy, which would then proxy directly to the injected process’s exposed +port. This would minimize the amount of proxying that takes place.
+ +There are at least 2 different ways to execute a command in a container:
+docker exec
and nsenter
. While docker exec
might seem like an easier and
+more obvious choice, it has some drawbacks.
docker exec
We could expose docker exec
(i.e. have Docker listen on an exposed TCP port
+on the node), but this would require proxying from the edge and securing the
+Docker API. docker exec
calls go through the Docker daemon, meaning that all
+stdin/stdout/stderr traffic is proxied through the Daemon, adding an extra hop.
+Additionally, you can’t isolate 1 malicious docker exec
call from normal
+usage, meaning an attacker could initiate a denial of service or other attack
+and take down the Docker daemon, or the node itself.
We expect remote command execution and port forwarding requests to be long +running and/or high bandwidth operations, and routing all the streaming data +through the Docker daemon feels like a bottleneck we can avoid.
+ +nsenter
The implementation currently uses nsenter
to run commands in containers,
+joining the appropriate container namespaces. nsenter
runs directly on the
+node and is not proxied through any single daemon process.
Authentication and authorization hasn’t specifically been tested yet with this +functionality. We need to make sure that users are not allowed to execute +remote commands or do port forwarding to containers they aren’t allowed to +access.
+ +Additional work is required to ensure that multiple command execution or port forwarding connections from different clients are not able to see each other’s data. This can most likely be achieved via SELinux labeling and unique process contexts.
+ + + + + + + + + +Author: Ananya Kumar (@AnanyaKumar)
+ +Status: Implemented.
+ +This document presents the design of the Kubernetes DaemonSet, describes use cases, and gives an overview of the code.
+ +Many users have requested for a way to run a daemon on every node in a Kubernetes cluster, or on a certain set of nodes in a cluster. This is essential for use cases such as building a sharded datastore, or running a logger on every node. In comes the DaemonSet, a way to conveniently create and manage daemon-like workloads in Kubernetes.
+ +The DaemonSet can be used for user-specified system services, cluster-level applications with strong node ties, and Kubernetes node services. Below are example use cases in each category.
+ +Logging: Some users want a way to collect statistics about nodes in a cluster and send those logs to an external database. For example, system administrators might want to know if their machines are performing as expected, if they need to add more machines to the cluster, or if they should switch cloud providers. The DaemonSet can be used to run a data collection service (for example fluentd) on every node and send the data to a service like ElasticSearch for analysis.
+ +Datastore: Users might want to implement a sharded datastore in their cluster. A few nodes in the cluster, labeled ‘app=datastore’, might be responsible for storing data shards, and pods running on these nodes might serve data. This architecture requires a way to bind pods to specific nodes, so it cannot be achieved using a Replication Controller. A DaemonSet is a convenient way to implement such a datastore.
+ +For other uses, see the related feature request
+ +The DaemonSet supports standard API features: +- create + - The spec for DaemonSets has a pod template field. + - Using the pod’s nodeSelector field, DaemonSets can be restricted to operate over nodes that have a certain label. For example, suppose that in a cluster some nodes are labeled ‘app=database’. You can use a DaemonSet to launch a datastore pod on exactly those nodes labeled ‘app=database’. + - Using the pod’s nodeName field, DaemonSets can be restricted to operate on a specified node. + - The PodTemplateSpec used by the DaemonSet is the same as the PodTemplateSpec used by the Replication Controller. + - The initial implementation will not guarnatee that DaemonSet pods are created on nodes before other pods. + - The initial implementation of DaemonSet does not guarantee that DaemonSet pods show up on nodes (for example because of resource limitations of the node), but makes a best effort to launch DaemonSet pods (like Replication Controllers do with pods). Subsequent revisions might ensure that DaemonSet pods show up on nodes, preempting other pods if necessary. + - The DaemonSet controller adds an annotation “kubernetes.io/created-by: <json API object reference>” + - YAML example:
+ +apiVersion: v1
+ kind: DaemonSet
+ metadata:
+ labels:
+ app: datastore
+ name: datastore
+ spec:
+ template:
+ metadata:
+ labels:
+ app: datastore-shard
+ spec:
+ nodeSelector:
+ app: datastore-node
+ containers:
+ name: datastore-shard
+ image: kubernetes/sharded
+ ports:
+ - containerPort: 9042
+ name: main
+
+We considered several alternatives, that were deemed inferior to the approach of creating a new DaemonSet abstraction.
+ +One alternative is to include the daemon in the machine image. In this case it would run outside of Kubernetes proper, and thus not be monitored, health checked, usable as a service endpoint, easily upgradable, etc.
+ +A related alternative is to package daemons as static pods. This would address most of the problems described above, but they would still not be easily upgradable, and more generally could not be managed through the API server interface.
+ +A third alternative is to generalize the Replication Controller. We would do something like: if you set the replicas
field of the ReplicationConrollerSpec to -1, then it means “run exactly one replica on every node matching the nodeSelector in the pod template.” The ReplicationController would pretend replicas
had been set to some large number – larger than the largest number of nodes ever expected in the cluster – and would use some anti-affinity mechanism to ensure that no more than one Pod from the ReplicationController runs on any given node. There are two downsides to this approach. First, there would always be a large number of Pending pods in the scheduler (these will be scheduled onto new machines when they are added to the cluster). The second downside is more philosophical: DaemonSet and the Replication Controller are very different concepts. We believe that having small, targeted controllers for distinct purposes makes Kubernetes easier to understand and use, compared to having larger multi-functional controllers (see “Convert ReplicationController to a plugin” for some discussion of this topic).
This document captures the design of event compression.
+ +Kubernetes components can get into a state where they generate tons of events which are identical except for the timestamp. For example, when pulling a non-existing image, Kubelet will repeatedly generate image_not_existing
and container_is_waiting
events until upstream components correct the image. When this happens, the spam from the repeated events makes the entire event mechanism useless. It also appears to cause memory pressure in etcd (see #3853).
Each binary that generates events (for example, kubelet
) should keep track of previously generated events so that it can collapse recurring events into a single event instead of creating a new instance for each new event.
Event compression should be best effort (not guaranteed). Meaning, in the worst case, n
identical (minus timestamp) events may still result in n
event entries.
Instead of a single Timestamp, each event object contains the following fields:
+ * FirstTimestamp unversioned.Time
+ * The date/time of the first occurrence of the event.
+ * LastTimestamp unversioned.Time
+ * The date/time of the most recent occurrence of the event.
+ * On first occurrence, this is equal to the FirstTimestamp.
+ * Count int
+ * The number of occurrences of this event between FirstTimestamp and LastTimestamp
+ * On first occurrence, this is 1.
Each binary that generates events:
+ * Maintains a historical record of previously generated events:
+ * Implemented with “Least Recently Used Cache” in pkg/client/record/events_cache.go
.
+ * The key in the cache is generated from the event object minus timestamps/count/transient fields, specifically the following events fields are used to construct a unique key for an event:
+ * event.Source.Component
+ * event.Source.Host
+ * event.InvolvedObject.Kind
+ * event.InvolvedObject.Namespace
+ * event.InvolvedObject.Name
+ * event.InvolvedObject.UID
+ * event.InvolvedObject.APIVersion
+ * event.Reason
+ * event.Message
+ * The LRU cache is capped at 4096 events. That means if a component (e.g. kubelet) runs for a long period of time and generates tons of unique events, the previously generated events cache will not grow unchecked in memory. Instead, after 4096 unique events are generated, the oldest events are evicted from the cache.
+ * When an event is generated, the previously generated events cache is checked (see pkg/client/unversioned/record/event.go
).
+ * If the key for the new event matches the key for a previously generated event (meaning all of the above fields match between the new event and some previously generated event), then the event is considered to be a duplicate and the existing event entry is updated in etcd:
+ * The new PUT (update) event API is called to update the existing event entry in etcd with the new last seen timestamp and count.
+ * The event is also updated in the previously generated events cache with an incremented count, updated last seen timestamp, name, and new resource version (all required to issue a future event update).
+ * If the key for the new event does not match the key for any previously generated event (meaning none of the above fields match between the new event and any previously generated events), then the event is considered to be new/unique and a new event entry is created in etcd:
+ * The usual POST/create event API is called to create a new event entry in etcd.
+ * An entry for the event is also added to the previously generated events cache.
Sample kubectl output
+ +FIRSTSEEN LASTSEEN COUNT NAME KIND SUBOBJECT REASON SOURCE MESSAGE
+Thu, 12 Feb 2015 01:13:02 +0000 Thu, 12 Feb 2015 01:13:02 +0000 1 kubernetes-minion-4.c.saad-dev-vms.internal Minion starting {kubelet kubernetes-minion-4.c.saad-dev-vms.internal} Starting kubelet.
+Thu, 12 Feb 2015 01:13:09 +0000 Thu, 12 Feb 2015 01:13:09 +0000 1 kubernetes-minion-1.c.saad-dev-vms.internal Minion starting {kubelet kubernetes-minion-1.c.saad-dev-vms.internal} Starting kubelet.
+Thu, 12 Feb 2015 01:13:09 +0000 Thu, 12 Feb 2015 01:13:09 +0000 1 kubernetes-minion-3.c.saad-dev-vms.internal Minion starting {kubelet kubernetes-minion-3.c.saad-dev-vms.internal} Starting kubelet.
+Thu, 12 Feb 2015 01:13:09 +0000 Thu, 12 Feb 2015 01:13:09 +0000 1 kubernetes-minion-2.c.saad-dev-vms.internal Minion starting {kubelet kubernetes-minion-2.c.saad-dev-vms.internal} Starting kubelet.
+Thu, 12 Feb 2015 01:13:05 +0000 Thu, 12 Feb 2015 01:13:12 +0000 4 monitoring-influx-grafana-controller-0133o Pod failedScheduling {scheduler } Error scheduling: no nodes available to schedule pods
+Thu, 12 Feb 2015 01:13:05 +0000 Thu, 12 Feb 2015 01:13:12 +0000 4 elasticsearch-logging-controller-fplln Pod failedScheduling {scheduler } Error scheduling: no nodes available to schedule pods
+Thu, 12 Feb 2015 01:13:05 +0000 Thu, 12 Feb 2015 01:13:12 +0000 4 kibana-logging-controller-gziey Pod failedScheduling {scheduler } Error scheduling: no nodes available to schedule pods
+Thu, 12 Feb 2015 01:13:05 +0000 Thu, 12 Feb 2015 01:13:12 +0000 4 skydns-ls6k1 Pod failedScheduling {scheduler } Error scheduling: no nodes available to schedule pods
+Thu, 12 Feb 2015 01:13:05 +0000 Thu, 12 Feb 2015 01:13:12 +0000 4 monitoring-heapster-controller-oh43e Pod failedScheduling {scheduler } Error scheduling: no nodes available to schedule pods
+Thu, 12 Feb 2015 01:13:20 +0000 Thu, 12 Feb 2015 01:13:20 +0000 1 kibana-logging-controller-gziey BoundPod implicitly required container POD pulled {kubelet kubernetes-minion-4.c.saad-dev-vms.internal} Successfully pulled image "kubernetes/pause:latest"
+Thu, 12 Feb 2015 01:13:20 +0000 Thu, 12 Feb 2015 01:13:20 +0000 1 kibana-logging-controller-gziey Pod scheduled {scheduler } Successfully assigned kibana-logging-controller-gziey to kubernetes-minion-4.c.saad-dev-vms.internal
+
+This demonstrates what would have been 20 separate entries (indicating scheduling failure) collapsed/compressed down to 5 entries.
+ +A proposal for the expansion of environment variables using a simple $(var)
syntax.
It is extremely common for users to need to compose environment variables or pass arguments to +their commands using the values of environment variables. Kubernetes should provide a facility for +the 80% cases in order to decrease coupling and the use of workarounds.
+ +$(VARIABLE_NAME:"default")
), inline substitution, etc.Currently, containers are injected with docker-style environment variables for the services in +their pod’s namespace. There are several variables for each service, but users routinely need +to compose URLs based on these variables because there is not a variable for the exact format +they need. Users should be able to build new environment variables with the exact format they need. +Eventually, it should also be possible to turn off the automatic injection of the docker-style +variables into pods and let the users consume the exact information they need via the downward API +and composition.
+ +It should be possible to reference an variable which is itself the result of an expansion, if the +referenced variable is declared in the container’s environment prior to the one referencing it. +Put another way – a container’s environment is expanded in order, and expanded variables are +available to subsequent expansions.
+ +Users frequently need to pass the values of environment variables to a container’s command. +Currently, Kubernetes does not perform any expansion of variables. The workaround is to invoke a +shell in the container’s command and have the shell perform the substitution, or to write a wrapper +script that sets up the environment and runs the command. This has a number of drawbacks:
+ +Users should be able to do the 80% case of variable expansion in command without writing a wrapper +script or adding a shell invocation to their containers’ commands.
+ +The current workaround for variable expansion in a container’s command requires the container’s
+image to have a shell. This is unfriendly to images that do not contain a shell (scratch
images,
+for example). Users should be able to perform the other use-cases in this design without regard to
+the content of their images.
It is possible that a container with incorrect variable values or command line may continue to run +for a long period of time, and that the end-user would have no visual or obvious warning of the +incorrect configuration. If the kubelet creates an event when an expansion references a variable +that cannot be expanded, it will help users quickly detect problems with expansions.
+ +In order to limit complexity, we want to provide the right amount of functionality so that the 80% +cases can be realized and nothing more. We felt that the essentials boiled down to:
+ +The exact syntax for variable expansion has a large impact on how users perceive and relate to the
+feature. We considered implementing a very restrictive subset of the shell ${var}
syntax. This
+syntax is an attractive option on some level, because many people are familiar with it. However,
+this syntax also has a large number of lesser known features such as the ability to provide
+default values for unset variables, perform inline substitution, etc.
In the interest of preventing conflation of the expansion feature in Kubernetes with the shell
+feature, we chose a different syntax similar to the one in Makefiles, $(var)
. We also chose not
+to support the bar $var
format, since it is not required to implement the required use-cases.
Nested references, ie, variable expansion within variable names, are not supported.
+ +Ideally, it should be extremely clear when a variable reference couldn’t be expanded. We decided
+the best experience for unmatched variable references would be to have the entire reference, syntax
+included, show up in the output. As an example, if the reference $(VARIABLE_NAME)
cannot be
+expanded, then $(VARIABLE_NAME)
should be present in the output.
Although the $(var)
syntax does overlap with the $(command)
form of command substitution
+supported by many shells, because unexpanded variables are present verbatim in the output, we
+expect this will not present a problem to many users. If there is a collision between a variable
+name and command substitution syntax, the syntax can be escaped with the form $$(VARIABLE_NAME)
,
+which will evaluate to $(VARIABLE_NAME)
whether VARIABLE_NAME
can be expanded or not.
This design encompasses the variable expansion syntax and specification and the changes needed to +incorporate the expansion feature into the container’s environment and command.
+ +This section describes the expansion syntax, evaluation of variable values, and how unexpected or +malformed inputs are handled.
+ +The inputs to the expansion feature are:
+ +func(string) string
Variable references in the input string are indicated exclusively with the syntax
+$(<variable-name>)
. The syntax tokens are:
$
: the operator(
: the reference opener)
: the reference closerThe operator has no meaning unless accompanied by the reference opener and closer tokens. The
+operator can be escaped using $$
. One literal $
will be emitted for each $$
in the input.
The reference opener and closer characters have no meaning when not part of a variable reference.
+If a variable reference is malformed, viz: $(VARIABLE_NAME
without a closing expression, the
+operator and expression opening characters are treated as ordinary characters without special
+meanings.
The scope in which variable references are expanded is defined by the mapping function. Within the
+mapping function, any arbitrary strategy may be used to determine the value of a variable name.
+The most basic implementation of a mapping function is to use a map[string]string
to lookup the
+value of a variable.
In order to support default values for variables like service variables presented by the kubelet,
+which may not be bound because the service that provides them does not yet exist, there should be a
+mapping function that uses a list of map[string]string
like:
func MakeMappingFunc(maps ...map[string]string) func(string) string {
+ return func(input string) string {
+ for _, context := range maps {
+ val, ok := context[input]
+ if ok {
+ return val
+ }
+ }
+
+ return ""
+ }
+}
+
+// elsewhere
+containerEnv := map[string]string{
+ "FOO": "BAR",
+ "ZOO": "ZAB",
+ "SERVICE2_HOST": "some-host",
+}
+
+serviceEnv := map[string]string{
+ "SERVICE_HOST": "another-host",
+ "SERVICE_PORT": "8083",
+}
+
+// single-map variation
+mapping := MakeMappingFunc(containerEnv)
+
+// default variables not found in serviceEnv
+mappingWithDefaults := MakeMappingFunc(serviceEnv, containerEnv)
+
+The necessary changes to implement this functionality are:
+ +ObjectEventRecorder
, which is like the EventRecorder
interface, but
+scoped to a single object, and a function that returns an ObjectEventRecorder
given an
+ObjectReference
and an EventRecorder
third_party/golang/expansion
package that provides:
+ Expand(string, func(string) string) string
functionMappingFuncFor(ObjectEventRecorder, ...map[string]string) string
functionIn order to provide an event when an expansion references undefined variables, the mapping function
+must be able to create an event. In order to facilitate this, we should create a new interface in
+the api/client/record
package which is similar to EventRecorder
, but scoped to a single object:
// ObjectEventRecorder knows how to record events about a single object.
+type ObjectEventRecorder interface {
+ // Event constructs an event from the given information and puts it in the queue for sending.
+ // 'reason' is the reason this event is generated. 'reason' should be short and unique; it will
+ // be used to automate handling of events, so imagine people writing switch statements to
+ // handle them. You want to make that easy.
+ // 'message' is intended to be human readable.
+ //
+ // The resulting event will be created in the same namespace as the reference object.
+ Event(reason, message string)
+
+ // Eventf is just like Event, but with Sprintf for the message field.
+ Eventf(reason, messageFmt string, args ...interface{})
+
+ // PastEventf is just like Eventf, but with an option to specify the event's 'timestamp' field.
+ PastEventf(timestamp unversioned.Time, reason, messageFmt string, args ...interface{})
+}
+
+There should also be a function that can construct an ObjectEventRecorder
from a runtime.Object
+and an EventRecorder
:
type objectRecorderImpl struct {
+ object runtime.Object
+ recorder EventRecorder
+}
+
+func (r *objectRecorderImpl) Event(reason, message string) {
+ r.recorder.Event(r.object, reason, message)
+}
+
+func ObjectEventRecorderFor(object runtime.Object, recorder EventRecorder) ObjectEventRecorder {
+ return &objectRecorderImpl{object, recorder}
+}
+
+The expansion package should provide two methods:
+ +// MappingFuncFor returns a mapping function for use with Expand that
+// implements the expansion semantics defined in the expansion spec; it
+// returns the input string wrapped in the expansion syntax if no mapping
+// for the input is found. If no expansion is found for a key, an event
+// is raised on the given recorder.
+func MappingFuncFor(recorder record.ObjectEventRecorder, context ...map[string]string) func(string) string {
+ // ...
+}
+
+// Expand replaces variable references in the input string according to
+// the expansion spec using the given mapping function to resolve the
+// values of variables.
+func Expand(input string, mapping func(string) string) string {
+ // ...
+}
+
+The Kubelet should be made to correctly expand variables references in a container’s environment, +command, and args. Changes will need to be made to:
+ +makeEnvironmentVariables
function in the kubelet; this is used by
+GenerateRunContainerOptions
, which is used by both the docker and rkt container runtimessetEntrypointAndCommand
func has to be changed to perform variable
+expansionThese examples are in the context of the mapping:
+ +Name | +Value | +
---|---|
VAR_A |
+ "A" |
+
VAR_B |
+ "B" |
+
VAR_C |
+ "C" |
+
VAR_REF |
+ $(VAR_A) |
+
VAR_EMPTY |
+ "" |
+
No other variables are defined.
+ +Input | +Result | +
---|---|
"$(VAR_A)" |
+ "A" |
+
"___$(VAR_B)___" |
+ "___B___" |
+
"___$(VAR_C)" |
+ "___C" |
+
"$(VAR_A)-$(VAR_A)" |
+ "A-A" |
+
"$(VAR_A)-1" |
+ "A-1" |
+
"$(VAR_A)_$(VAR_B)_$(VAR_C)" |
+ "A_B_C" |
+
"$$(VAR_B)_$(VAR_A)" |
+ "$(VAR_B)_A" |
+
"$$(VAR_A)_$$(VAR_B)" |
+ "$(VAR_A)_$(VAR_B)" |
+
"f000-$$VAR_A" |
+ "f000-$VAR_A" |
+
"foo\\$(VAR_C)bar" |
+ "foo\Cbar" |
+
"foo\\\\$(VAR_C)bar" |
+ "foo\\Cbar" |
+
"foo\\\\\\\\$(VAR_A)bar" |
+ "foo\\\\Abar" |
+
"$(VAR_A$(VAR_B))" |
+ "$(VAR_A$(VAR_B))" |
+
"$(VAR_A$(VAR_B)" |
+ "$(VAR_A$(VAR_B)" |
+
"$(VAR_REF)" |
+ "$(VAR_A)" |
+
"%%$(VAR_REF)--$(VAR_REF)%%" |
+ "%%$(VAR_A)--$(VAR_A)%%" |
+
"foo$(VAR_EMPTY)bar" |
+ "foobar" |
+
"foo$(VAR_Awhoops!" |
+ "foo$(VAR_Awhoops!" |
+
"f00__(VAR_A)__" |
+ "f00__(VAR_A)__" |
+
"$?_boo_$!" |
+ "$?_boo_$!" |
+
"$VAR_A" |
+ "$VAR_A" |
+
"$(VAR_DNE)" |
+ "$(VAR_DNE)" |
+
"$$$$$$(BIG_MONEY)" |
+ "$$$(BIG_MONEY)" |
+
"$$$$$$(VAR_A)" |
+ "$$$(VAR_A)" |
+
"$$$$$$$(GOOD_ODDS)" |
+ "$$$$(GOOD_ODDS)" |
+
"$$$$$$$(VAR_A)" |
+ "$$$A" |
+
"$VAR_A)" |
+ "$VAR_A)" |
+
"${VAR_A}" |
+ "${VAR_A}" |
+
"$(VAR_B)_______$(A" |
+ "B_______$(A" |
+
"$(VAR_C)_______$(" |
+ "C_______$(" |
+
"$(VAR_A)foobarzab$" |
+ "Afoobarzab$" |
+
"foo-\\$(VAR_A" |
+ "foo-\$(VAR_A" |
+
"--$($($($($--" |
+ "--$($($($($--" |
+
"$($($($($--foo$(" |
+ "$($($($($--foo$(" |
+
"foo0--$($($($(" |
+ "foo0--$($($($(" |
+
"$(foo$$var) |
+ $(foo$$var) |
+
Notice the $(var)
syntax.
apiVersion: v1
+kind: Pod
+metadata:
+ name: expansion-pod
+spec:
+ containers:
+ - name: test-container
+ image: gcr.io/google_containers/busybox
+ command: [ "/bin/sh", "-c", "env" ]
+ env:
+ - name: PUBLIC_URL
+ value: "http://$(GITSERVER_SERVICE_HOST):$(GITSERVER_SERVICE_PORT)"
+ restartPolicy: Never
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: expansion-pod
+spec:
+ containers:
+ - name: test-container
+ image: gcr.io/google_containers/busybox
+ command: [ "/bin/sh", "-c", "env" ]
+ env:
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: "metadata.namespace"
+ - name: PUBLIC_URL
+ value: "http://gitserver.$(POD_NAMESPACE):$(SERVICE_PORT)"
+ restartPolicy: Never
+
+This document describes the design for implementing the storage of custom API types in the Kubernetes API Server.
+ +The ThirdPartyResource
resource describes the multiple versions of a custom resource that the user wants to add
+to the Kubernetes API. ThirdPartyResource
is a non-namespaced resource, attempting to place it in a resource
+will return an error.
Each ThirdPartyResource
resource has the following:
+ * Standard Kubernetes object metadata.
+ * ResourceKind - The kind of the resources described by this third party resource.
+ * Description - A free text description of the resource.
+ * APIGroup - An API group that this resource should be placed into.
+ * Versions - One or more Version
objects.
Version
ObjectThe Version
object describes a single concrete version of a custom resource. The Version
object currently
+only specifies:
+ * The Name
of the version.
+ * The APIGroup
this version should belong to.
Every object that is added to a third-party Kubernetes object store is expected to contain Kubernetes
+compatible object metadata. This requirement enables the
+Kubernetes API server to provide the following features:
+ * Filtering lists of objects via LabelQueries
+ * resourceVersion
-based optimistic concurrency via compare-and-swap
+ * Versioned storage
+ * Event recording
+ * Integration with basic kubectl
command line tooling.
+ * Watch for resource changes.
The Kind
for an instance of a third-party object (e.g. CronTab) below is expected to be
+programmatically convertible to the name of the resource using
+the following conversion. Kinds are expected to be of the form <CamelCaseKind>
, the
+APIVersion
for the object is expected to be <domain-name>/<api-group>/<api-version>
.
For example example.com/stable/v1
domain-name
is expected to be a fully qualified domain name.
‘CamelCaseKind’ is the specific type name.
+ +To convert this into the metadata.name
for the ThirdPartyResource
resource instance,
+the <domain-name>
is copied verbatim, the CamelCaseKind
is
+then converted
+using ‘-‘ instead of capitalization (‘camel-case’), with the first character being assumed to be
+capitalized. In pseudo code:
var result string
+for ix := range kindName {
+ if isCapital(kindName[ix]) {
+ result = append(result, '-')
+ }
+ result = append(result, toLowerCase(kindName[ix])
+}
+
+As a concrete example, the resource named camel-case-kind.example.com
defines resources of Kind CamelCaseKind
, in
+the APIGroup with the prefix example.com/...
.
The reason for this is to enable rapid lookup of a ThirdPartyResource
object given the kind information.
+This is also the reason why ThirdPartyResource
is not namespaced.
When a user creates a new ThirdPartyResource
, the Kubernetes API Server reacts by creating a new, namespaced
+RESTful resource path. For now, non-namespaced objects are not supported. As with existing built-in objects
+deleting a namespace, deletes all third party resources in that namespace.
For example, if a user creates:
+ +metadata:
+ name: cron-tab.example.com
+apiVersion: extensions/v1beta1
+kind: ThirdPartyResource
+description: "A specification of a Pod to run on a cron style schedule"
+versions:
+ - name: stable/v1
+ - name: experimental/v2
+
+Then the API server will program in two new RESTful resource paths:
+ * /thirdparty/example.com/stable/v1/namespaces/<namespace>/crontabs/...
+ * /thirdparty/example.com/experimental/v2/namespaces/<namespace>/crontabs/...
Now that this schema has been created, a user can POST
:
{
+ "metadata": {
+ "name": "my-new-cron-object"
+ },
+ "apiVersion": "example.com/stable/v1",
+ "kind": "CronTab",
+ "cronSpec": "* * * * /5",
+ "image": "my-awesome-chron-image"
+}
+
+to: /third-party/example.com/stable/v1/namespaces/default/crontabs/my-new-cron-object
and the corresponding data will be stored into etcd by the APIServer, so that when the user issues:
+ +
+GET /third-party/example.com/stable/v1/namespaces/default/crontabs/my-new-cron-object`
+
+
+
+And when they do that, they will get back the same data, but with additional Kubernetes metadata
+(e.g. resourceVersion
, createdTimestamp
) filled in.
Likewise, to list all resources, a user can issue:
+ +
+GET /third-party/example.com/stable/v1/namespaces/default/crontabs
+
+
+
+and get back:
+ +{
+ "apiVersion": "example.com/stable/v1",
+ "kind": "CronTabList",
+ "items": [
+ {
+ "metadata": {
+ "name": "my-new-cron-object"
+ },
+ "apiVersion": "example.com/stable/v1",
+ "kind": "CronTab",
+ "cronSpec": "* * * * /5",
+ "image": "my-awesome-chron-image"
+ }
+ ]
+}
+
+Because all objects are expected to contain standard Kubernetes metadata fields, these
+list operations can also use Label
queries to filter requests down to specific subsets.
Likewise, clients can use watch endpoints to watch for changes to stored objects.
+ +In order to store custom user data in a versioned fashion inside of etcd, we need to also introduce a
+Codec
-compatible object for persistent storage in etcd. This object is ThirdPartyResourceData
and it contains:
+ * Standard API Metadata
+ * Data
: The raw JSON data for this custom object.
Each custom object stored by the API server needs a custom key in storage, this is described below:
+ +resource-namespace
: the namespace of the particular resource that is being storedresource-name
: the name of the particular resource being storedthird-party-resource-namespace
: the namespace of the ThirdPartyResource
resource that represents the type for the specific instance being stored.third-party-resource-name
: the name of the ThirdPartyResource
resource that represents the type for the specific instance being stored.Given the definitions above, the key for a specific third-party object is:
+ +
+${standard-k8s-prefix}/third-party-resources/${third-party-resource-namespace}/${third-party-resource-name}/${resource-namespace}/${resource-name}
+
+
+
+Thus, listing a third-party resource can be achieved by listing the directory:
+ +
+${standard-k8s-prefix}/third-party-resources/${third-party-resource-namespace}/${third-party-resource-name}/${resource-namespace}/
+
+
+
+
+
+
+
+
+
+
+
+ This document briefly describes the design of the horizontal autoscaler for pods. +The autoscaler (implemented as a Kubernetes API resource and controller) is responsible for dynamically controlling +the number of replicas of some collection (e.g. the pods of a ReplicationController) to meet some objective(s), +for example a target per-pod CPU utilization.
+ +This design supersedes autoscaling.md.
+ +The resource usage of a serving application usually varies over time: sometimes the demand for the application rises, +and sometimes it drops. +In Kubernetes version 1.0, a user can only manually set the number of serving pods. +Our aim is to provide a mechanism for the automatic adjustment of the number of pods based on CPU utilization statistics +(a future version will allow autoscaling based on other resources/metrics).
+ +In Kubernetes version 1.1, we are introducing Scale subresource and implementing horizontal autoscaling of pods based on it. +Scale subresource is supported for replication controllers and deployments. +Scale subresource is a Virtual Resource (does not correspond to an object stored in etcd). +It is only present in the API as an interface that a controller (in this case the HorizontalPodAutoscaler) can use to dynamically scale +the number of replicas controlled by some other API object (currently ReplicationController and Deployment) and to learn the current number of replicas. +Scale is a subresource of the API object that it serves as the interface for. +The Scale subresource is useful because whenever we introduce another type we want to autoscale, we just need to implement the Scale subresource for it. +The wider discussion regarding Scale took place in #1629.
+ +Scale subresource is in API for replication controller or deployment under the following paths:
+ +apis/extensions/v1beta1/replicationcontrollers/myrc/scale
apis/extensions/v1beta1/deployments/mydeployment/scale
It has the following structure:
+ +// represents a scaling request for a resource.
+type Scale struct {
+ unversioned.TypeMeta
+ api.ObjectMeta
+
+ // defines the behavior of the scale.
+ Spec ScaleSpec
+
+ // current status of the scale.
+ Status ScaleStatus
+}
+
+// describes the attributes of a scale subresource
+type ScaleSpec struct {
+ // desired number of instances for the scaled object.
+ Replicas int `json:"replicas,omitempty"`
+}
+
+// represents the current status of a scale subresource.
+type ScaleStatus struct {
+ // actual number of observed instances of the scaled object.
+ Replicas int `json:"replicas"`
+
+ // label query over pods that should match the replicas count.
+ Selector map[string]string `json:"selector,omitempty"`
+}
+
+Writing to ScaleSpec.Replicas
resizes the replication controller/deployment associated with
+the given Scale subresource.
+ScaleStatus.Replicas
reports how many pods are currently running in the replication controller/deployment,
+and ScaleStatus.Selector
returns selector for the pods.
In Kubernetes version 1.1, we are introducing HorizontalPodAutoscaler object. It is accessible under:
+ +apis/extensions/v1beta1/horizontalpodautoscalers/myautoscaler
It has the following structure:
+ +// configuration of a horizontal pod autoscaler.
+type HorizontalPodAutoscaler struct {
+ unversioned.TypeMeta
+ api.ObjectMeta
+
+ // behavior of autoscaler.
+ Spec HorizontalPodAutoscalerSpec
+
+ // current information about the autoscaler.
+ Status HorizontalPodAutoscalerStatus
+}
+
+// specification of a horizontal pod autoscaler.
+type HorizontalPodAutoscalerSpec struct {
+ // reference to Scale subresource; horizontal pod autoscaler will learn the current resource
+ // consumption from its status,and will set the desired number of pods by modifying its spec.
+ ScaleRef SubresourceReference
+ // lower limit for the number of pods that can be set by the autoscaler, default 1.
+ MinReplicas *int
+ // upper limit for the number of pods that can be set by the autoscaler.
+ // It cannot be smaller than MinReplicas.
+ MaxReplicas int
+ // target average CPU utilization (represented as a percentage of requested CPU) over all the pods;
+ // if not specified it defaults to the target CPU utilization at 80% of the requested resources.
+ CPUUtilization *CPUTargetUtilization
+}
+
+type CPUTargetUtilization struct {
+ // fraction of the requested CPU that should be utilized/used,
+ // e.g. 70 means that 70% of the requested CPU should be in use.
+ TargetPercentage int
+}
+
+// current status of a horizontal pod autoscaler
+type HorizontalPodAutoscalerStatus struct {
+ // most recent generation observed by this autoscaler.
+ ObservedGeneration *int64
+
+ // last time the HorizontalPodAutoscaler scaled the number of pods;
+ // used by the autoscaler to control how often the number of pods is changed.
+ LastScaleTime *unversioned.Time
+
+ // current number of replicas of pods managed by this autoscaler.
+ CurrentReplicas int
+
+ // desired number of replicas of pods managed by this autoscaler.
+ DesiredReplicas int
+
+ // current average CPU utilization over all pods, represented as a percentage of requested CPU,
+ // e.g. 70 means that an average pod is using now 70% of its requested CPU.
+ CurrentCPUUtilizationPercentage *int
+}
+
+ScaleRef
is a reference to the Scale subresource.
+MinReplicas
, MaxReplicas
and CPUUtilization
define autoscaler configuration.
+We are also introducing HorizontalPodAutoscalerList object to enable listing all autoscalers in a namespace:
// list of horizontal pod autoscaler objects.
+type HorizontalPodAutoscalerList struct {
+ unversioned.TypeMeta
+ unversioned.ListMeta
+
+ // list of horizontal pod autoscaler objects.
+ Items []HorizontalPodAutoscaler
+}
+
+The autoscaler is implemented as a control loop. It periodically queries pods described by Status.PodSelector
of Scale subresource, and collects their CPU utilization.
+Then, it compares the arithmetic mean of the pods’ CPU utilization with the target defined in Spec.CPUUtilization
,
+and adjust the replicas of the Scale if needed to match the target
+(preserving condition: MinReplicas <= Replicas <= MaxReplicas).
The period of the autoscaler is controlled by --horizontal-pod-autoscaler-sync-period
flag of controller manager.
+The default value is 30 seconds.
CPU utilization is the recent CPU usage of a pod (average across the last 1 minute) divided by the CPU requested by the pod. +In Kubernetes version 1.1, CPU usage is taken directly from Heapster. +In future, there will be API on master for this purpose +(see #11951).
+ +The target number of pods is calculated from the following formula:
+ +
+TargetNumOfPods = ceil(sum(CurrentPodsCPUUtilization) / Target)
+
+
+
+Starting and stopping pods may introduce noise to the metric (for instance, starting may temporarily increase CPU).
+So, after each action, the autoscaler should wait some time for reliable data.
+Scale-up can only happen if there was no rescaling within the last 3 minutes.
+Scale-down will wait for 5 minutes from the last rescaling.
+Moreover any scaling will only be made if: avg(CurrentPodsConsumption) / Target
drops below 0.9 or increases above 1.1 (10% tolerance).
+Such approach has two benefits:
Autoscaler works in a conservative way. +If new user load appears, it is important for us to rapidly increase the number of pods, +so that user requests will not be rejected. +Lowering the number of pods is not that urgent.
+Autoscaler avoids thrashing, i.e.: prevents rapid execution of conflicting decision if the load is not stable.
+We chose values of the target metric to be relative (e.g. 90% of requested CPU resource) rather than absolute (e.g. 0.6 core) for the following reason. +If we choose absolute metric, user will need to guarantee that the target is lower than the request. +Otherwise, overloaded pods may not be able to consume more than the autoscaler’s absolute target utilization, +thereby preventing the autoscaler from seeing high enough utilization to trigger it to scale up. +This may be especially troublesome when user changes requested resources for a pod +because they would need to also change the autoscaler utilization threshold. +Therefore, we decided to choose relative metric. +For user, it is enough to set it to a value smaller than 100%, and further changes of requested resources will not invalidate it.
+ +To make manipulation of HorizontalPodAutoscaler object simpler, we added support for
+creating/updating/deleting/listing of HorizontalPodAutoscaler to kubectl.
+In addition, in future, we are planning to add kubectl support for the following use-cases:
+* When creating a replication controller or deployment with kubectl create [-f]
, there should be
+ a possibility to specify an additional autoscaler object.
+ (This should work out-of-the-box when creation of autoscaler is supported by kubectl as we may include
+ multiple objects in the same config file).
+* [future] When running an image with kubectl run
, there should be an additional option to create
+ an autoscaler for it.
+* [future] We will add a new command kubectl autoscale
that will allow for easy creation of an autoscaler object
+ for already existing replication controller/deployment.
We list here some features that are not supported in Kubernetes version 1.1. +However, we want to keep them in mind, as they will most probably be needed in future. +Our design is in general compatible with them. +* [future] Autoscale pods based on metrics different than CPU (e.g. memory, network traffic, qps). + This includes scaling based on a custom/application metric. +* [future] Autoscale pods base on an aggregate metric. + Autoscaler, instead of computing average for a target metric across pods, will use a single, external, metric (e.g. qps metric from load balancer). + The metric will be aggregated while the target will remain per-pod + (e.g. when observing 100 qps on load balancer while the target is 20 qps per pod, autoscaler will set the number of replicas to 5). +* [future] Autoscale pods based on multiple metrics. + If the target numbers of pods for different metrics are different, choose the largest target number of pods. +* [future] Scale the number of pods starting from 0. + All pods can be turned-off, and then turned-on when there is a demand for them. + When a request to service with no pods arrives, kube-proxy will generate an event for autoscaler + to create a new pod. + Discussed in #3247. +* [future] When scaling down, make more educated decision which pods to kill. + E.g.: if two or more pods from the same replication controller are on the same node, kill one of them. + Discussed in #4301.
+ + + + + + + + + +A summarization of the goals and recommendations for identifiers in Kubernetes. Described in GitHub issue #199.
+ +Uniquely identify (via a UID) an object across space and time
+Uniquely name (via a name) an object across space
+Provide human-friendly names in API operations and/or configuration files
+Allow idempotent creation of API resources (#148) and enforcement of space-uniqueness of singleton objects
+Allow DNS names to be automatically generated for some objects
+Pods can be placed onto a particular node in a number of ways. This case +study demonstrates how the above design can be applied to satisfy the +objectives.
+ +A user submits a pod with Namespace=”” and Name=”guestbook” to the apiserver.
+Kubelet validates the input.
+A config file is stored on the node, containing a pod with UID=””, Namespace=””, and Name=”cadvisor”.
+Kubernetes is a system for managing containerized applications across multiple hosts, providing basic mechanisms for deployment, maintenance, and scaling of applications.
+ +Kubernetes establishes robust declarative primitives for maintaining the desired state requested by the user. We see these primitives as the main value added by Kubernetes. Self-healing mechanisms, such as auto-restarting, re-scheduling, and replicating containers require active controllers, not just imperative orchestration.
+ +Kubernetes is primarily targeted at applications composed of multiple containers, such as elastic, distributed micro-services. It is also designed to facilitate migration of non-containerized application stacks to Kubernetes. It therefore includes abstractions for grouping containers in both loosely coupled and tightly coupled formations, and provides ways for containers to find and communicate with each other in relatively familiar ways.
+ +Kubernetes enables users to ask a cluster to run a set of containers. The system automatically chooses hosts to run those containers on. While Kubernetes’s scheduler is currently very simple, we expect it to grow in sophistication over time. Scheduling is a policy-rich, topology-aware, workload-specific function that significantly impacts availability, performance, and capacity. The scheduler needs to take into account individual and collective resource requirements, quality of service requirements, hardware/software/policy constraints, affinity and anti-affinity specifications, data locality, inter-workload interference, deadlines, and so on. Workload-specific requirements will be exposed through the API as necessary.
+ +Kubernetes is intended to run on a number of cloud providers, as well as on physical hosts.
+ +A single Kubernetes cluster is not intended to span multiple availability zones. Instead, we recommend building a higher-level layer to replicate complete deployments of highly available applications across multiple zones (see the multi-cluster doc and cluster federation proposal for more details).
+ +Finally, Kubernetes aspires to be an extensible, pluggable, building-block OSS platform and toolkit. Therefore, architecturally, we want Kubernetes to be built as a collection of pluggable components and layers, with the ability to use alternative schedulers, controllers, storage systems, and distribution mechanisms, and we’re evolving its current code in that direction. Furthermore, we want others to be able to extend Kubernetes functionality, such as with higher-level PaaS functionality or multi-cluster layers, without modification of core Kubernetes source. Therefore, its API isn’t just (or even necessarily mainly) targeted at end users, but at tool and extension developers. Its APIs are intended to serve as the foundation for an open ecosystem of tools, automation systems, and higher-level API layers. Consequently, there are no “internal” inter-component APIs. All APIs are visible and available, including the APIs used by the scheduler, the node controller, the replication-controller manager, Kubelet’s API, etc. There’s no glass to break – in order to handle more complex use cases, one can just access the lower-level APIs in a fully transparent, composable manner.
+ +For more about the Kubernetes architecture, see architecture.
+ + + + + + + + + +A Namespace is a mechanism to partition resources created by users into +a logically named group.
+ +A single cluster should be able to satisfy the needs of multiple user communities.
+ +Each user community wants to be able to work in isolation from other communities.
+ +Each user community has its own:
+ +A cluster operator may create a Namespace for each unique user community.
+ +The Namespace provides a unique scope for:
+ +A Namespace defines a logically named group for multiple Kinds of resources.
+ +type Namespace struct {
+ TypeMeta `json:",inline"`
+ ObjectMeta `json:"metadata,omitempty"`
+
+ Spec NamespaceSpec `json:"spec,omitempty"`
+ Status NamespaceStatus `json:"status,omitempty"`
+}
+
+A Namespace name is a DNS compatible label.
+ +A Namespace must exist prior to associating content with it.
+ +A Namespace must not be deleted if there is content associated with it.
+ +To associate a resource with a Namespace the following conditions must be satisfied:
+ +The Name of a resource associated with a Namespace is unique to that Kind in that Namespace.
+ +It is intended to be used in resource URLs; provided by clients at creation time, and encouraged to be +human friendly; intended to facilitate idempotent creation, space-uniqueness of singleton objects, +distinguish distinct entities, and reference particular entities across operations.
+ +A Namespace provides an authorization scope for accessing content associated with the Namespace.
+ + + +A Namespace provides a scope to limit resource consumption.
+ +A LimitRange defines min/max constraints on the amount of resources a single entity can consume in +a Namespace.
+ +See Admission control: Limit Range
+ +A ResourceQuota tracks aggregate usage of resources in the Namespace and allows cluster operators +to define Hard resource usage limits that a Namespace may consume.
+ +See Admission control: Resource Quota
+ +Upon creation of a Namespace, the creator may provide a list of Finalizer objects.
+ +type FinalizerName string
+
+// These are internal finalizers to Kubernetes, must be qualified name unless defined here
+const (
+ FinalizerKubernetes FinalizerName = "kubernetes"
+)
+
+// NamespaceSpec describes the attributes on a Namespace
+type NamespaceSpec struct {
+ // Finalizers is an opaque list of values that must be empty to permanently remove object from storage
+ Finalizers []FinalizerName
+}
+
+A FinalizerName is a qualified name.
+ +The API Server enforces that a Namespace can only be deleted from storage if and only if +it’s Namespace.Spec.Finalizers is empty.
+ +A finalize operation is the only mechanism to modify the Namespace.Spec.Finalizers field post creation.
+ +Each Namespace created has kubernetes as an item in its list of initial Namespace.Spec.Finalizers +set by default.
+ +A Namespace may exist in the following phases.
+ +type NamespacePhase string
+const(
+ NamespaceActive NamespacePhase = "Active"
+ NamespaceTerminating NamespaceTerminating = "Terminating"
+)
+
+type NamespaceStatus struct {
+ ...
+ Phase NamespacePhase
+}
+
+A Namespace is in the Active phase if it does not have a ObjectMeta.DeletionTimestamp.
+ +A Namespace is in the Terminating phase if it has a ObjectMeta.DeletionTimestamp.
+ +Active
+ +Upon creation, a Namespace goes in the Active phase. This means that content may be associated with +a namespace, and all normal interactions with the namespace are allowed to occur in the cluster.
+ +If a DELETE request occurs for a Namespace, the Namespace.ObjectMeta.DeletionTimestamp is set +to the current server time. A namespace controller observes the change, and sets the Namespace.Status.Phase +to Terminating.
+ +Terminating
+ +A namespace controller watches for Namespace objects that have a Namespace.ObjectMeta.DeletionTimestamp +value set in order to know when to initiate graceful termination of the Namespace associated content that +are known to the cluster.
+ +The namespace controller enumerates each known resource type in that namespace and deletes it one by one.
+ +Admission control blocks creation of new resources in that namespace in order to prevent a race-condition +where the controller could believe all of a given resource type had been deleted from the namespace, +when in fact some other rogue client agent had created new objects. Using admission control in this +scenario allows each of registry implementations for the individual objects to not need to take into account Namespace life-cycle.
+ +Once all objects known to the namespace controller have been deleted, the namespace controller +executes a finalize operation on the namespace that removes the kubernetes value from +the Namespace.Spec.Finalizers list.
+ +If the namespace controller sees a Namespace whose ObjectMeta.DeletionTimestamp is set, and +whose Namespace.Spec.Finalizers list is empty, it will signal the server to permanently remove +the Namespace from storage by sending a final DELETE action to the API server.
+ +To interact with the Namespace API:
+ +Action | +HTTP Verb | +Path | +Description | +
---|---|---|---|
CREATE | +POST | +/api/{version}/namespaces | +Create a namespace | +
LIST | +GET | +/api/{version}/namespaces | +List all namespaces | +
UPDATE | +PUT | +/api/{version}/namespaces/{namespace} | +Update namespace {namespace} | +
DELETE | +DELETE | +/api/{version}/namespaces/{namespace} | +Delete namespace {namespace} | +
FINALIZE | +POST | +/api/{version}/namespaces/{namespace}/finalize | +Finalize namespace {namespace} | +
WATCH | +GET | +/api/{version}/watch/namespaces | +Watch all namespaces | +
This specification reserves the name finalize as a sub-resource to namespace.
+ +As a consequence, it is invalid to have a resourceType managed by a namespace whose kind is finalize.
+ +To interact with content associated with a Namespace:
+ +Action | +HTTP Verb | +Path | +Description | +
---|---|---|---|
CREATE | +POST | +/api/{version}/namespaces/{namespace}/{resourceType}/ | +Create instance of {resourceType} in namespace {namespace} | +
GET | +GET | +/api/{version}/namespaces/{namespace}/{resourceType}/{name} | +Get instance of {resourceType} in namespace {namespace} with {name} | +
UPDATE | +PUT | +/api/{version}/namespaces/{namespace}/{resourceType}/{name} | +Update instance of {resourceType} in namespace {namespace} with {name} | +
DELETE | +DELETE | +/api/{version}/namespaces/{namespace}/{resourceType}/{name} | +Delete instance of {resourceType} in namespace {namespace} with {name} | +
LIST | +GET | +/api/{version}/namespaces/{namespace}/{resourceType} | +List instances of {resourceType} in namespace {namespace} | +
WATCH | +GET | +/api/{version}/watch/namespaces/{namespace}/{resourceType} | +Watch for changes to a {resourceType} in namespace {namespace} | +
WATCH | +GET | +/api/{version}/watch/{resourceType} | +Watch for changes to a {resourceType} across all namespaces | +
LIST | +GET | +/api/{version}/list/{resourceType} | +List instances of {resourceType} across all namespaces | +
The API server verifies the Namespace on resource creation matches the {namespace} on the path.
+ +The API server will associate a resource with a Namespace if not populated by the end-user based on the Namespace context +of the incoming request. If the Namespace of the resource being created, or updated does not match the Namespace on the request, +then the API server will reject the request.
+ +A namespace provides a unique identifier space and therefore must be in the storage path of a resource.
+ +In etcd, we want to continue to still support efficient WATCH across namespaces.
+ +Resources that persist content in etcd will have storage paths as follows:
+ +/{k8s_storage_prefix}/{resourceType}/{resource.Namespace}/{resource.Name}
+ +This enables consumers to WATCH /registry/{resourceType} for changes across namespace of a particular {resourceType}.
+ +The kubelet will register pod’s it sources from a file or http source with a namespace associated with the +cluster-id
+ +In this example, we demonstrate how the design allows for agents built on-top of +Kubernetes that manage their own set of resource types associated with a Namespace +to take part in Namespace termination.
+ +OpenShift creates a Namespace in Kubernetes
+ +{
+ "apiVersion":"v1",
+ "kind": "Namespace",
+ "metadata": {
+ "name": "development",
+ "labels": {
+ "name": "development"
+ }
+ },
+ "spec": {
+ "finalizers": ["openshift.com/origin", "kubernetes"]
+ },
+ "status": {
+ "phase": "Active"
+ }
+}
+
+OpenShift then goes and creates a set of resources (pods, services, etc) associated +with the “development” namespace. It also creates its own set of resources in its +own storage associated with the “development” namespace unknown to Kubernetes.
+ +User deletes the Namespace in Kubernetes, and Namespace now has following state:
+ +{
+ "apiVersion":"v1",
+ "kind": "Namespace",
+ "metadata": {
+ "name": "development",
+ "deletionTimestamp": "..."
+ "labels": {
+ "name": "development"
+ }
+ },
+ "spec": {
+ "finalizers": ["openshift.com/origin", "kubernetes"]
+ },
+ "status": {
+ "phase": "Terminating"
+ }
+}
+
+The Kubernetes namespace controller observes the namespace has a deletionTimestamp +and begins to terminate all of the content in the namespace that it knows about. Upon +success, it executes a finalize action that modifies the Namespace by +removing kubernetes from the list of finalizers:
+ +{
+ "apiVersion":"v1",
+ "kind": "Namespace",
+ "metadata": {
+ "name": "development",
+ "deletionTimestamp": "..."
+ "labels": {
+ "name": "development"
+ }
+ },
+ "spec": {
+ "finalizers": ["openshift.com/origin"]
+ },
+ "status": {
+ "phase": "Terminating"
+ }
+}
+
+OpenShift Origin has its own namespace controller that is observing cluster state, and +it observes the same namespace had a deletionTimestamp assigned to it. It too will go +and purge resources from its own storage that it manages associated with that namespace. +Upon completion, it executes a finalize action and removes the reference to “openshift.com/origin” +from the list of finalizers.
+ +This results in the following state:
+ +{
+ "apiVersion":"v1",
+ "kind": "Namespace",
+ "metadata": {
+ "name": "development",
+ "deletionTimestamp": "..."
+ "labels": {
+ "name": "development"
+ }
+ },
+ "spec": {
+ "finalizers": []
+ },
+ "status": {
+ "phase": "Terminating"
+ }
+}
+
+At this point, the Kubernetes namespace controller in its sync loop will see that the namespace +has a deletion timestamp and that its list of finalizers is empty. As a result, it knows all +content associated from that namespace has been purged. It performs a final DELETE action +to remove that Namespace from the storage.
+ +At this point, all content associated with that Namespace, and the Namespace itself are gone.
+ + + + + + + + + +There are 4 distinct networking problems to solve:
+ +Kubernetes deviates from the default Docker networking model (though as of +Docker 1.8 their network plugins are getting closer). The goal is for each pod +to have an IP in a flat shared networking namespace that has full communication +with other physical computers and containers across the network. IP-per-pod +creates a clean, backward-compatible model where pods can be treated much like +VMs or physical hosts from the perspectives of port allocation, networking, +naming, service discovery, load balancing, application configuration, and +migration.
+ +Dynamic port allocation, on the other hand, requires supporting both static +ports (e.g., for externally accessible services) and dynamically allocated +ports, requires partitioning centrally allocated and locally acquired dynamic +ports, complicates scheduling (since ports are a scarce resource), is +inconvenient for users, complicates application configuration, is plagued by +port conflicts and reuse and exhaustion, requires non-standard approaches to +naming (e.g. consul or etcd rather than DNS), requires proxies and/or +redirection for programs using standard naming/addressing mechanisms (e.g. web +browsers), requires watching and cache invalidation for address/port changes +for instances in addition to watching group membership changes, and obstructs +container/pod migration (e.g. using CRIU). NAT introduces additional complexity +by fragmenting the addressing space, which breaks self-registration mechanisms, +among other problems.
+ +All containers within a pod behave as if they are on the same host with regard +to networking. They can all reach each other’s ports on localhost. This offers +simplicity (static ports know a priori), security (ports bound to localhost +are visible within the pod but never outside it), and performance. This also +reduces friction for applications moving from the world of uncontainerized apps +on physical or virtual hosts. People running application stacks together on +the same host have already figured out how to make ports not conflict and have +arranged for clients to find them.
+ +The approach does reduce isolation between containers within a pod — +ports could conflict, and there can be no container-private ports, but these +seem to be relatively minor issues with plausible future workarounds. Besides, +the premise of pods is that containers within a pod share some resources +(volumes, cpu, ram, etc.) and therefore expect and tolerate reduced isolation. +Additionally, the user can control what containers belong to the same pod +whereas, in general, they don’t control what pods land together on a host.
+ +Because every pod gets a “real” (not machine-private) IP address, pods can +communicate without proxies or translations. The pod can use well-known port +numbers and can avoid the use of higher-level service discovery systems like +DNS-SD, Consul, or Etcd.
+ +When any container calls ioctl(SIOCGIFADDR) (get the address of an interface), +it sees the same IP that any peer container would see them coming from — +each pod has its own IP address that other pods can know. By making IP addresses +and ports the same both inside and outside the pods, we create a NAT-less, flat +address space. Running “ip addr show” should work as expected. This would enable +all existing naming/discovery mechanisms to work out of the box, including +self-registration mechanisms and applications that distribute IP addresses. We +should be optimizing for inter-pod network communication. Within a pod, +containers are more likely to use communication through volumes (e.g., tmpfs) or +IPC.
+ +This is different from the standard Docker model. In that mode, each container +gets an IP in the 172-dot space and would only see that 172-dot address from +SIOCGIFADDR. If these containers connect to another container the peer would see +the connect coming from a different IP than the container itself knows. In short +— you can never self-register anything from a container, because a +container can not be reached on its private IP.
+ +An alternative we considered was an additional layer of addressing: pod-centric +IP per container. Each container would have its own local IP address, visible +only within that pod. This would perhaps make it easier for containerized +applications to move from physical/virtual hosts to pods, but would be more +complex to implement (e.g., requiring a bridge per pod, split-horizon/VP DNS) +and to reason about, due to the additional layer of address translation, and +would break self-registration and IP distribution mechanisms.
+ +Like Docker, ports can still be published to the host node’s interface(s), but +the need for this is radically diminished.
+ +For the Google Compute Engine cluster configuration scripts, we use advanced
+routing rules
+and ip-forwarding-enabled VMs so that each VM has an extra 256 IP addresses that
+get routed to it. This is in addition to the ‘main’ IP address assigned to the
+VM that is NAT-ed for Internet access. The container bridge (called cbr0
to
+differentiate it from docker0
) is set up outside of Docker proper.
Example of GCE’s advanced routing rules:
+ +gcloud compute routes add "${MINION_NAMES[$i]}" \
+ --project "${PROJECT}" \
+ --destination-range "${MINION_IP_RANGES[$i]}" \
+ --network "${NETWORK}" \
+ --next-hop-instance "${MINION_NAMES[$i]}" \
+ --next-hop-instance-zone "${ZONE}" &
+
+GCE itself does not know anything about these IPs, though. This means that when +a pod tries to egress beyond GCE’s project the packets must be SNAT’ed +(masqueraded) to the VM’s IP, which GCE recognizes and allows.
+ +With the primary aim of providing IP-per-pod-model, other implementations exist +to serve the purpose outside of GCE. + - OpenVSwitch with GRE/VxLAN + - Flannel + - L2 networks + (“With Linux Bridge devices” section) + - Weave is yet another way to build an + overlay network, primarily aiming at Docker integration. + - Calico uses BGP to enable real + container IPs.
+ +The service abstraction provides a way to group pods under a
+common access policy (e.g. load-balanced). The implementation of this creates a
+virtual IP which clients can access and which is transparently proxied to the
+pods in a Service. Each node runs a kube-proxy process which programs
+iptables
rules to trap access to service IPs and redirect them to the correct
+backends. This provides a highly-available load-balancing solution with low
+performance overhead by balancing client traffic from a node on that same node.
So far the discussion has been about how to access a pod or service from within +the cluster. Accessing a pod from outside the cluster is a bit more tricky. We +want to offer highly-available, high-performance load balancing to target +Kubernetes Services. Most public cloud providers are simply not flexible enough +yet.
+ +The way this is generally implemented is to set up external load balancers (e.g. +GCE’s ForwardingRules or AWS’s ELB) which target all nodes in a cluster. When +traffic arrives at a node it is recognized as being part of a particular Service +and routed to an appropriate backend Pod. This does mean that some traffic will +get double-bounced on the network. Once cloud providers have better offerings +we can take advantage of those.
+ +Right now, docker inspect doesn’t show the networking configuration of the +containers, since they derive it from another container. That information should +be exposed somehow.
+ +We want to be able to assign IP addresses externally from Docker +#6743 so that we don’t need +to statically allocate fixed-size IP ranges to each node, so that IP addresses +can be made stable across pod infra container restarts +(#2801), and to facilitate +pod migration. Right now, if the pod infra container dies, all the user +containers must be stopped and restarted because the netns of the pod infra +container will change on restart, and any subsequent user container restart +will join that new netns, thereby not being able to see its peers. +Additionally, a change in IP address would encounter DNS caching/TTL problems. +External IP assignment would also simplify DNS support (see below).
+ +IPv6 would be a nice option, also, but we can’t depend on it yet. Docker support is in progress: Docker issue #2974, Docker issue #6923, Docker issue #6975. Additionally, direct ipv6 assignment to instances doesn’t appear to be supported by major cloud providers (e.g., AWS EC2, GCE) yet. We’d happily take pull requests from people running Kubernetes on bare metal, though. :-)
+ + + + + + + + + +This document proposes a model for managing persistent, cluster-scoped storage for applications requiring long lived data.
+ +Two new API kinds:
+ +A PersistentVolume
(PV) is a storage resource provisioned by an administrator. It is analogous to a node. See Persistent Volume Guide for how to use it.
A PersistentVolumeClaim
(PVC) is a user’s request for a persistent volume to use in a pod. It is analogous to a pod.
One new system component:
+ +PersistentVolumeClaimBinder
is a singleton running in master that watches all PersistentVolumeClaims in the system and binds them to the closest matching available PersistentVolume. The volume manager watches the API for newly created volumes to manage.
One new volume:
+ +PersistentVolumeClaimVolumeSource
references the user’s PVC in the same namespace. This volume finds the bound PV and mounts that volume for the pod. A PersistentVolumeClaimVolumeSource
is, essentially, a wrapper around another type of volume that is owned by someone else (the system).
Kubernetes makes no guarantees at runtime that the underlying storage exists or is available. High availability is left to the storage provider.
+ +Cluster administrators use the API to manage PersistentVolumes. A custom store NewPersistentVolumeOrderedIndex
will index volumes by access modes and sort by storage capacity. The PersistentVolumeClaimBinder
watches for new claims for storage and binds them to an available volume by matching the volume’s characteristics (AccessModes and storage size) to the user’s request.
PVs are system objects and, thus, have no namespace.
+ +Many means of dynamic provisioning will be eventually be implemented for various storage types.
+ +Action | +HTTP Verb | +Path | +Description | +
---|---|---|---|
CREATE | +POST | +/api/{version}/persistentvolumes/ | +Create instance of PersistentVolume | +
GET | +GET | +/api/{version}persistentvolumes/{name} | +Get instance of PersistentVolume with {name} | +
UPDATE | +PUT | +/api/{version}/persistentvolumes/{name} | +Update instance of PersistentVolume with {name} | +
DELETE | +DELETE | +/api/{version}/persistentvolumes/{name} | +Delete instance of PersistentVolume with {name} | +
LIST | +GET | +/api/{version}/persistentvolumes | +List instances of PersistentVolume | +
WATCH | +GET | +/api/{version}/watch/persistentvolumes | +Watch for changes to a PersistentVolume | +
Kubernetes users request persistent storage for their pod by creating a PersistentVolumeClaim
. Their request for storage is described by their requirements for resources and mount capabilities.
Requests for volumes are bound to available volumes by the volume manager, if a suitable match is found. Requests for resources can go unfulfilled.
+ +Users attach their claim to their pod using a new PersistentVolumeClaimVolumeSource
volume source.
Action | +HTTP Verb | +Path | +Description | +
---|---|---|---|
CREATE | +POST | +/api/{version}/namespaces/{ns}/persistentvolumeclaims/ | +Create instance of PersistentVolumeClaim in namespace {ns} | +
GET | +GET | +/api/{version}/namespaces/{ns}/persistentvolumeclaims/{name} | +Get instance of PersistentVolumeClaim in namespace {ns} with {name} | +
UPDATE | +PUT | +/api/{version}/namespaces/{ns}/persistentvolumeclaims/{name} | +Update instance of PersistentVolumeClaim in namespace {ns} with {name} | +
DELETE | +DELETE | +/api/{version}/namespaces/{ns}/persistentvolumeclaims/{name} | +Delete instance of PersistentVolumeClaim in namespace {ns} with {name} | +
LIST | +GET | +/api/{version}/namespaces/{ns}/persistentvolumeclaims | +List instances of PersistentVolumeClaim in namespace {ns} | +
WATCH | +GET | +/api/{version}/watch/namespaces/{ns}/persistentvolumeclaims | +Watch for changes to PersistentVolumeClaim in namespace {ns} | +
Scheduling constraints are to be handled similar to pod resource constraints. Pods will need to be annotated or decorated with the number of resources it requires on a node. Similarly, a node will need to list how many it has used or available.
+ +TBD
+ +The implementation of persistent storage will not require events to communicate to the user the state of their claim. The CLI for bound claims contains a reference to the backing persistent volume. This is always present in the API and CLI, making an event to communicate the same unnecessary.
+ +Events that communicate the state of a mounted volume are left to the volume plugins.
+ +An administrator provisions storage by posting PVs to the API. Various way to automate this task can be scripted. Dynamic provisioning is a future feature that can maintain levels of PVs.
+ +POST:
+
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+ name: pv0001
+spec:
+ capacity:
+ storage: 10
+ persistentDisk:
+ pdName: "abc123"
+ fsType: "ext4"
+
+$ kubectl get pv
+
+NAME LABELS CAPACITY ACCESSMODES STATUS CLAIM REASON
+pv0001 map[] 10737418240 RWO Pending
+
+A user requests storage by posting a PVC to the API. Their request contains the AccessModes they wish their volume to have and the minimum size needed.
+ +The user must be within a namespace to create PVCs.
+ +POST:
+
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+ name: myclaim-1
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 3
+
+$ kubectl get pvc
+
+NAME LABELS STATUS VOLUME
+myclaim-1 map[] pending
+
+The PersistentVolumeClaimBinder
attempts to find an available volume that most closely matches the user’s request. If one exists, they are bound by putting a reference on the PV to the PVC. Requests can go unfulfilled if a suitable match is not found.
$ kubectl get pv
+
+NAME LABELS CAPACITY ACCESSMODES STATUS CLAIM REASON
+pv0001 map[] 10737418240 RWO Bound myclaim-1 / f4b3d283-c0ef-11e4-8be4-80e6500a981e
+
+
+kubectl get pvc
+
+NAME LABELS STATUS VOLUME
+myclaim-1 map[] Bound b16e91d6-c0ef-11e4-8be4-80e6500a981e
+
+The claim holder can use their claim as a volume. The PersistentVolumeClaimVolumeSource
knows to fetch the PV backing the claim and mount its volume for a pod.
The claim holder owns the claim and its data for as long as the claim exists. The pod using the claim can be deleted, but the claim remains in the user’s namespace. It can be used again and again by many pods.
+ +POST:
+
+kind: Pod
+apiVersion: v1
+metadata:
+ name: mypod
+spec:
+ containers:
+ - image: nginx
+ name: myfrontend
+ volumeMounts:
+ - mountPath: "/var/www/html"
+ name: mypd
+ volumes:
+ - name: mypd
+ source:
+ persistentVolumeClaim:
+ accessMode: ReadWriteOnce
+ claimRef:
+ name: myclaim-1
+
+When a claim holder is finished with their data, they can delete their claim.
+ +$ kubectl delete pvc myclaim-1
+
+The PersistentVolumeClaimBinder
will reconcile this by removing the claim reference from the PV and change the PVs status to ‘Released’.
Admins can script the recycling of released volumes. Future dynamic provisioners will understand how a volume should be recycled.
+ + + + + + + + + +Principles to follow when extending Kubernetes.
+ +See also the API conventions.
+ +TODO: pluggability
+ +TODO
+ +Note: this is a design doc, which describes features that have not been completely implemented. +User documentation of the current state is here. The tracking issue for +implementation of this model is +#168. Currently, both limits and requests of memory and +cpu on containers (not pods) are supported. “memory” is in bytes and “cpu” is in +milli-cores.
+ +To do good pod placement, Kubernetes needs to know how big pods are, as well as the sizes of the nodes onto which they are being placed. The definition of “how big” is given by the Kubernetes resource model — the subject of this document.
+ +The resource model aims to be: +* simple, for common cases; +* extensible, to accommodate future growth; +* regular, with few special cases; and +* precise, to avoid misunderstandings and promote pod portability.
+ +A Kubernetes resource is something that can be requested by, allocated to, or consumed by a pod or container. Examples include memory (RAM), CPU, disk-time, and network bandwidth.
+ +Once resources on a node have been allocated to one pod, they should not be allocated to another until that pod is removed or exits. This means that Kubernetes schedulers should ensure that the sum of the resources allocated (requested and granted) to its pods never exceeds the usable capacity of the node. Testing whether a pod will fit on a node is called feasibility checking.
+ +Note that the resource model currently prohibits over-committing resources; we will want to relax that restriction later.
+ +All resources have a type that is identified by their typename (a string, e.g., “memory”). Several resource types are predefined by Kubernetes (a full list is below), although only two will be supported at first: CPU and memory. Users and system administrators can define their own resource types if they wish (e.g., Hadoop slots).
+ +A fully-qualified resource typename is constructed from a DNS-style subdomain, followed by a slash /
, followed by a name.
+* The subdomain must conform to RFC 1123 (e.g., kubernetes.io
, example.com
).
+* The name must be not more than 63 characters, consisting of upper- or lower-case alphanumeric characters, with the -
, _
, and .
characters allowed anywhere except the first or last character.
+* As a shorthand, any resource typename that does not start with a subdomain and a slash will automatically be prefixed with the built-in Kubernetes namespace, kubernetes.io/
in order to fully-qualify it. This namespace is reserved for code in the open source Kubernetes repository; as a result, all user typenames MUST be fully qualified, and cannot be created in this namespace.
Some example typenames include memory
(which will be fully-qualified as kubernetes.io/memory
), and example.com/Shiny_New-Resource.Type
.
For future reference, note that some resources, such as CPU and network bandwidth, are compressible, which means that their usage can potentially be throttled in a relatively benign manner. All other resources are incompressible, which means that any attempt to throttle them is likely to cause grief. This distinction will be important if a Kubernetes implementation supports over-committing of resources.
+ +Initially, all Kubernetes resource types are quantitative, and have an associated unit for quantities of the associated resource (e.g., bytes for memory, bytes per seconds for bandwidth, instances for software licences). The units will always be a resource type’s natural base units (e.g., bytes, not MB), to avoid confusion between binary and decimal multipliers and the underlying unit multiplier (e.g., is memory measured in MiB, MB, or GB?).
+ +Resource quantities can be added and subtracted: for example, a node has a fixed quantity of each resource type that can be allocated to pods/containers; once such an allocation has been made, the allocated resources cannot be made available to other pods/containers without over-committing the resources.
+ +To make life easier for people, quantities can be represented externally as unadorned integers, or as fixed-point integers with one of these SI suffices (E, P, T, G, M, K, m) or their power-of-two equivalents (Ei, Pi, Ti, Gi, Mi, Ki). For example, the following represent roughly the same value: 128974848, “129e6”, “129M” , “123Mi”. Small quantities can be represented directly as decimals (e.g., 0.3), or using milli-units (e.g., “300m”). + * “Externally” means in user interfaces, reports, graphs, and in JSON or YAML resource specifications that might be generated or read by people. + * Case is significant: “m” and “M” are not the same, so “k” is not a valid SI suffix. There are no power-of-two equivalents for SI suffixes that represent multipliers less than 1. + * These conventions only apply to resource quantities, not arbitrary values.
+ +Internally (i.e., everywhere else), Kubernetes will represent resource quantities as integers so it can avoid problems with rounding errors, and will not use strings to represent numeric values. To achieve this, quantities that naturally have fractional parts (e.g., CPU seconds/second) will be scaled to integral numbers of milli-units (e.g., milli-CPUs) as soon as they are read in. Internal APIs, data structures, and protobufs will use these scaled integer units. Raw measurement data such as usage may still need to be tracked and calculated using floating point values, but internally they should be rescaled to avoid some values being in milli-units and some not. + * Note that reading in a resource quantity and writing it out again may change the way its values are represented, and truncate precision (e.g., 1.0001 may become 1.000), so comparison and difference operations (e.g., by an updater) must be done on the internal representations. + * Avoiding milli-units in external representations has advantages for people who will use Kubernetes, but runs the risk of developers forgetting to rescale or accidentally using floating-point representations. That seems like the right choice. We will try to reduce the risk by providing libraries that automatically do the quantization for JSON/YAML inputs.
+ +Both users and a number of system components, such as schedulers, (horizontal) auto-scalers, (vertical) auto-sizers, load balancers, and worker-pool managers need to reason about resource requirements of workloads, resource capacities of nodes, and resource usage. Kubernetes divides specifications of desired state, aka the Spec, and representations of current state, aka the Status. Resource requirements and total node capacity fall into the specification category, while resource usage, characterizations derived from usage (e.g., maximum usage, histograms), and other resource demand signals (e.g., CPU load) clearly fall into the status category and are discussed in the Appendix for now.
+ +Resource requirements for a container or pod should have the following form:
+ +resourceRequirementSpec: [
+ request: [ cpu: 2.5, memory: "40Mi" ],
+ limit: [ cpu: 4.0, memory: "99Mi" ],
+]
+
+Where: +* request [optional]: the amount of resources being requested, or that were requested and have been allocated. Scheduler algorithms will use these quantities to test feasibility (whether a pod will fit onto a node). If a container (or pod) tries to use more resources than its request, any associated SLOs are voided — e.g., the program it is running may be throttled (compressible resource types), or the attempt may be denied. If request is omitted for a container, it defaults to limit if that is explicitly specified, otherwise to an implementation-defined value; this will always be 0 for a user-defined resource type. If request is omitted for a pod, it defaults to the sum of the (explicit or implicit) request values for the containers it encloses.
+ +Total capacity for a node should have a similar structure:
+ +resourceCapacitySpec: [
+ total: [ cpu: 12, memory: "128Gi" ]
+]
+
+Where: +* total: the total allocatable resources of a node. Initially, the resources at a given scope will bound the resources of the sum of inner scopes.
+ +It is an error to specify the same resource type more than once in each list.
+It is an error for the request or limit values for a pod to be less than the sum of the (explicit or defaulted) values for the containers it encloses. (We may relax this later.)
+If multiple pods are running on the same node and attempting to use more resources than they have requested, the result is implementation-defined. For example: unallocated or unused resources might be spread equally across claimants, or the assignment might be weighted by the size of the original request, or as a function of limits, or priority, or the phase of the moon, perhaps modulated by the direction of the tide. Thus, although it’s not mandatory to provide a request, it’s probably a good idea. (Note that the request could be filled in by an automated system that is observing actual usage and/or historical data.)
+Internally, the Kubernetes master can decide the defaulting behavior and the kubelet implementation may expected an absolute specification. For example, if the master decided that “the default is unbounded” it would pass 2^64 to the kubelet.
+The following resource types are predefined (“reserved”) by Kubernetes in the kubernetes.io
namespace, and so cannot be used for user-defined resources. Note that the syntax of all resource types in the resource spec is deliberately similar, but some resource types (e.g., CPU) may receive significantly more support than simply tracking quantities in the schedulers and/or the Kubelet.
cpu
(or kubernetes.io/cpu
)schedulingLatency
: as per lmctfycpuConversionFactor
: property of a node: the speed of a CPU core on the node’s processor divided by the speed of the canonical Kubernetes CPU (a floating point value; default = 1.0).To reduce performance portability problems for pods, and to avoid worse-case provisioning behavior, the units of CPU will be normalized to a canonical “Kubernetes Compute Unit” (KCU, pronounced ˈko͝oko͞o), which will roughly be equivalent to a single CPU hyperthreaded core for some recent x86 processor. The normalization may be implementation-defined, although some reasonable defaults will be provided in the open-source Kubernetes code.
+ +Note that requesting 2 KCU won’t guarantee that precisely 2 physical cores will be allocated — control of aspects like this will be handled by resource qualities (a future feature).
+ +memory
(or kubernetes.io/memory
)The precise meaning of what “memory” means is implementation dependent, but the basic idea is to rely on the underlying memcg
mechanisms, support, and definitions.
Note that most people will want to use power-of-two suffixes (Mi, Gi) for memory quantities +rather than decimal ones: “64MiB” rather than “64MB”.
+ +A resource type may have an associated read-only ResourceType structure, that contains metadata about the type. For example:
+ +resourceTypes: [
+ "kubernetes.io/memory": [
+ isCompressible: false, ...
+ ]
+ "kubernetes.io/cpu": [
+ isCompressible: true,
+ internalScaleExponent: 3, ...
+ ]
+ "kubernetes.io/disk-space": [ ... ]
+]
+
+Kubernetes will provide ResourceType metadata for its predefined types. If no resource metadata can be found for a resource type, Kubernetes will assume that it is a quantified, incompressible resource that is not specified in milli-units, and has no default value.
+ +The defined properties are as follows:
+ +field name | +type | +contents | +
---|---|---|
name | +string, required | +the typename, as a fully-qualified string (e.g., kubernetes.io/cpu ) |
+
internalScaleExponent | +int, default=0 | +external values are multiplied by 10 to this power for internal storage (e.g., 3 for milli-units) | +
units | +string, required | +format: unit* [per unit+] (e.g., second , byte per second ). An empty unit field means “dimensionless”. |
+
isCompressible | +bool, default=false | +true if the resource type is compressible | +
defaultRequest | +string, default=none | +in the same format as a user-supplied value | +
[future] quantization | +number, default=1 | +smallest granularity of allocation: requests may be rounded up to a multiple of this unit; implementation-defined unit (e.g., the page size for RAM). | +
The following are planned future extensions to the resource model, included here to encourage comments.
+ +Because resource usage and related metrics change continuously, need to be tracked over time (i.e., historically), can be characterized in a variety of ways, and are fairly voluminous, we will not include usage in core API objects, such as Pods and Nodes, but will provide separate APIs for accessing and managing that data. See the Appendix for possible representations of usage data, but the representation we’ll use is TBD.
+ +Singleton values for observed and predicted future usage will rapidly prove inadequate, so we will support the following structure for extended usage information:
+ +resourceStatus: [
+ usage: [ cpu: <CPU-info>, memory: <memory-info> ],
+ maxusage: [ cpu: <CPU-info>, memory: <memory-info> ],
+ predicted: [ cpu: <CPU-info>, memory: <memory-info> ],
+]
+
+where a <CPU-info>
or <memory-info>
structure looks like this:
{
+ mean: <value> # arithmetic mean
+ max: <value> # minimum value
+ min: <value> # maximum value
+ count: <value> # number of data points
+ percentiles: [ # map from %iles to values
+ "10": <10th-percentile-value>,
+ "50": <median-value>,
+ "99": <99th-percentile-value>,
+ "99.9": <99.9th-percentile-value>,
+ ...
+ ]
+}
+
+All parts of this structure are optional, although we strongly encourage including quantities for 50, 90, 95, 99, 99.5, and 99.9 percentiles. [In practice, it will be important to include additional info such as the length of the time window over which the averages are calculated, the confidence level, and information-quality metrics such as the number of dropped or discarded data points.] +and predicted
+ +kubernetes.io/network-bandwidth
)kubernetes.io/network-iops
)kubernetes.io/storage-space
)The amount of secondary storage space available to a container. The main target is local disk drives and SSDs, although this could also be used to qualify remotely-mounted volumes. Specifying whether a resource is a raw disk, an SSD, a disk array, or a file system fronting any of these, is left for future work.
+ +kubernetes.io/storage-time
)This is the amount of time a container spends accessing disk, including actuator and transfer time. A standard disk drive provides 1.0 diskTime seconds per second.
+ +kubernetes.io/storage-iops
)A proposal for the distribution of secrets (passwords, keys, etc) to the Kubelet and to +containers inside Kubernetes using a custom volume type. See the secrets example for more information.
+ +Secrets are needed in containers to access internal resources like the Kubernetes master or +external resources such as git repositories, databases, etc. Users may also want behaviors in the +kubelet that depend on secret data (credentials for image pull from a docker registry) associated +with pods.
+ +Goals of this design:
+ +.kubeconfig
file, so that I can securely reach the master.dockercfg
file, so that containers can push imagesServiceAccount
that consumes a secret and have
+the kubelet implement some reserved behaviors based on the types of secrets the service account
+consumes:
+ Many configuration files contain secrets intermixed with other configuration information. For +example, a user’s application may contain a properties file than contains database credentials, +SaaS API tokens, etc. Users should be able to consume configuration artifacts in their containers +and be able to control the path on the container’s filesystems where the artifact will be +presented.
+ +Most pieces of information about how to use a service are secrets. For example, a service that +provides a MySQL database needs to provide the username, password, and database name to consumers +so that they can authenticate and use the correct database. Containers in pods consuming the MySQL +service would also consume the secrets associated with the MySQL service.
+ +Service Accounts are proposed as a
+mechanism to decouple capabilities and security contexts from individual human users. A
+ServiceAccount
contains references to some number of secrets. A Pod
can specify that it is
+associated with a ServiceAccount
. Secrets should have a Type
field to allow the Kubelet and
+other system components to take action based on the secret’s type.
As an example, the service account proposal discusses service accounts consuming secrets which +contain Kubernetes auth tokens. When a Kubelet starts a pod associated with a service account +which consumes this type of secret, the Kubelet may take a number of actions:
+ +.kubernetes_auth
file in a well-known location in the container’s
+file systemkube-proxy
to decorate HTTP requests from that pod to the
+kubernetes-master
service with the auth token, e. g. by adding a header to the request
+(see the LOAS Daemon proposal)Another example use case is where a pod is associated with a secret containing docker registry +credentials. The Kubelet could use these credentials for the docker pull to retrieve the image.
+ +Rotation is considered a good practice for many types of secret data. It should be possible to +express that a secret has an expiry date; this would make it possible to implement a system +component that could regenerate expired secrets. As an example, consider a component that rotates +expired secrets. The rotator could periodically regenerate the values for expired secrets of +common types and update their expiry dates.
+ +Some images will expect to receive configuration items as environment variables instead of files. +We should consider what the best way to allow this is; there are a few different options:
+ +Force the user to adapt files into environment variables. Users can store secrets that need to +be presented as environment variables in a format that is easy to consume from a shell:
+ +$ cat /etc/secrets/my-secret.txt
+export MY_SECRET_ENV=MY_SECRET_VALUE
+
+
+ The user could source
the file at /etc/secrets/my-secret
prior to executing the command for
+the image either inline in the command or in an init script,
Give secrets an attribute that allows users to express the intent that the platform should +generate the above syntax in the file used to present a secret. The user could consume these +files in the same manner as the above option.
+Give secrets attributes that allow the user to express that the secret should be presented to +the container as an environment variable. The container’s environment would contain the +desired values and the software in the container could use them without accommodation the +command or setup script.
+For our initial work, we will treat all secrets as files to narrow the problem space. There will +be a future proposal that handles exposing secrets as environment variables.
+ +There are two fundamentally different use-cases for access to secrets:
+ +In use cases for CRUD operations, the user experience for secrets should be no different than for +other API resources.
+ +The data store backing the REST API should be pluggable because different cluster operators will +have different preferences for the central store of secret data. Some possibilities for storage:
+ +There should be a size limit for secrets in order to:
+ +The size limit should satisfy the following conditions:
+ +To begin discussion, we propose an initial value for this size limit of 1MB.
+ +Defining a policy for limitations on how a secret may be referenced by another API resource and how +constraints should be applied throughout the cluster is tricky due to the number of variables +involved:
+ +For now, we will not implement validations around these limits. Cluster operators will decide how +much node storage is allocated to secrets. It will be the operator’s responsibility to ensure that +the allocated storage is sufficient for the workload scheduled onto a node.
+ +For now, kubelets will only attach secrets to api-sourced pods, and not file- or http-sourced +ones. Doing so would: + - confuse the secrets admission controller in the case of mirror pods. + - create an apiserver-liveness dependency – avoiding this dependency is a main reason to use non-api-source pods.
+ +The use-case where the kubelet reads secrets has several additional requirements:
+ +The Kubelet should only be allowed to read secrets which are consumed by pods scheduled onto that +Kubelet’s node and their associated service accounts. Authorization of the Kubelet to read this +data would be delegated to an authorization plugin and associated policy rule.
+ +Consideration must be given to whether secret data should be allowed to be at rest on the node:
+ +For the sake of limiting complexity, we propose that initially secret data should not be allowed +to be at rest on a node; secret data should be stored on a node-level tmpfs filesystem. This +filesystem can be subdivided into directories for use by the kubelet and by the volume plugin.
+ +The Kubelet will be responsible for creating the per-node tmpfs file system for secret storage. +It is hard to make a prescriptive declaration about how much storage is appropriate to reserve for +secrets because different installations will vary widely in available resources, desired pod to +node density, overcommit policy, and other operation dimensions. That being the case, we propose +for simplicity that the amount of secret storage be controlled by a new parameter to the kubelet +with a default value of 64MB. It is the cluster operator’s responsibility to handle choosing +the right storage size for their installation and configuring their Kubelets correctly.
+ +Configuring each Kubelet is not the ideal story for operator experience; it is more intuitive that +the cluster-wide storage size be readable from a central configuration store like the one proposed +in #1553. When such a store +exists, the Kubelet could be modified to read this configuration item from the store.
+ +When the Kubelet is modified to advertise node resources (as proposed in +#4441), the capacity calculation +for available memory should factor in the potential size of the node-level tmpfs in order to avoid +memory overcommit on the node.
+ +Every pod will have a security context. +Secret data on the node should be isolated according to the security context of the container. The +Kubelet volume plugin API will be changed so that a volume plugin receives the security context of +a volume along with the volume spec. This will allow volume plugins to implement setting the +security context of volumes they manage.
+ +Several proposals / upstream patches are notable as background for this proposal:
+ +We propose a new Secret
resource which is mounted into containers with a new volume type. Secret
+volumes will be handled by a volume plugin that does the actual work of fetching the secret and
+storing it. Secrets contain multiple pieces of data that are presented as different files within
+the secret volume (example: SSH key pair).
In order to remove the burden from the end user in specifying every file that a secret consists of,
+it should be possible to mount all files provided by a secret with a single VolumeMount
entry
+in the container specification.
A new resource for secrets will be added to the API:
+ +type Secret struct {
+ TypeMeta
+ ObjectMeta
+
+ // Data contains the secret data. Each key must be a valid DNS_SUBDOMAIN.
+ // The serialized form of the secret data is a base64 encoded string,
+ // representing the arbitrary (possibly non-string) data value here.
+ Data map[string][]byte `json:"data,omitempty"`
+
+ // Used to facilitate programmatic handling of secret data.
+ Type SecretType `json:"type,omitempty"`
+}
+
+type SecretType string
+
+const (
+ SecretTypeOpaque SecretType = "Opaque" // Opaque (arbitrary data; default)
+ SecretTypeServiceAccountToken SecretType = "kubernetes.io/service-account-token" // Kubernetes auth token
+ SecretTypeDockercfg SecretType = "kubernetes.io/dockercfg" // Docker registry auth
+ // FUTURE: other type values
+)
+
+const MaxSecretSize = 1 * 1024 * 1024
+
+A Secret can declare a type in order to provide type information to system components that work
+with secrets. The default type is opaque
, which represents arbitrary user-owned data.
Secrets are validated against MaxSecretSize
. The keys in the Data
field must be valid DNS
+subdomains.
A new REST API and registry interface will be added to accompany the Secret
resource. The
+default implementation of the registry will store Secret
information in etcd. Future registry
+implementations could store the TypeMeta
and ObjectMeta
fields in etcd and store the secret
+data in another data store entirely, or store the whole object in another data store.
Initially there will be no validations for the number of secrets a pod references, or the number of +secrets that can be associated with a service account. These may be added in the future as the +finer points of secrets and resource allocation are fleshed out.
+ +A new SecretSource
type of volume source will be added to the VolumeSource
struct in the
+API:
type VolumeSource struct {
+ // Other fields omitted
+
+ // SecretSource represents a secret that should be presented in a volume
+ SecretSource *SecretSource `json:"secret"`
+}
+
+type SecretSource struct {
+ Target ObjectReference
+}
+
+Secret volume sources are validated to ensure that the specified object reference actually points
+to an object of type Secret
.
In the future, the SecretSource
will be extended to allow:
A new Kubelet volume plugin will be added to handle volumes with a secret source. This plugin will
+require access to the API server to retrieve secret data and therefore the volume Host
interface
+will have to change to expose a client interface:
type Host interface {
+ // Other methods omitted
+
+ // GetKubeClient returns a client interface
+ GetKubeClient() client.Interface
+}
+
+The secret volume plugin will be responsible for:
+ +volume.Builder
implementation from NewBuilder
that:
+ SecurityContext
volume.Cleaner
implementation from NewClear
that cleans the volume from the
+container’s filesystemThe Kubelet must be modified to accept a new parameter for the secret storage size and to create +a tmpfs file system of that size to store secret data. Rough accounting of specific changes:
+ +secretStorageSize
; units are megabytesNewMainKubelet
should accept a value for secret storage sizesetupDataDirs
method should be changed to create the secret storageFor use-cases where the Kubelet’s behavior is affected by the secrets associated with a pod’s
+ServiceAccount
, the Kubelet will need to be changed. For example, if secrets of type
+docker-reg-auth
affect how the pod’s images are pulled, the Kubelet will need to be changed
+to accommodate this. Subsequent proposals can address this on a type-by-type basis.
For clarity, let’s examine some detailed examples of some common use-cases in terms of the
+suggested changes. All of these examples are assumed to be created in a namespace called
+example
.
To create a pod that uses an ssh key stored as a secret, we first need to create a secret:
+ +{
+ "kind": "Secret",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "ssh-key-secret"
+ },
+ "data": {
+ "id-rsa": "dmFsdWUtMg0KDQo=",
+ "id-rsa.pub": "dmFsdWUtMQ0K"
+ }
+}
+
+Note: The serialized JSON and YAML values of secret data are encoded as +base64 strings. Newlines are not valid within these strings and must be +omitted.
+ +Now we can create a pod which references the secret with the ssh key and consumes it in a volume:
+ +{
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "secret-test-pod",
+ "labels": {
+ "name": "secret-test"
+ }
+ },
+ "spec": {
+ "volumes": [
+ {
+ "name": "secret-volume",
+ "secret": {
+ "secretName": "ssh-key-secret"
+ }
+ }
+ ],
+ "containers": [
+ {
+ "name": "ssh-test-container",
+ "image": "mySshImage",
+ "volumeMounts": [
+ {
+ "name": "secret-volume",
+ "readOnly": true,
+ "mountPath": "/etc/secret-volume"
+ }
+ ]
+ }
+ ]
+ }
+}
+
+When the container’s command runs, the pieces of the key will be available in:
+ +/etc/secret-volume/id-rsa.pub
+/etc/secret-volume/id-rsa
+
+
+The container is then free to use the secret data to establish an ssh connection.
+ +This example illustrates a pod which consumes a secret containing prod +credentials and another pod which consumes a secret with test environment +credentials.
+ +The secrets:
+ +{
+ "apiVersion": "v1",
+ "kind": "List",
+ "items":
+ [{
+ "kind": "Secret",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "prod-db-secret"
+ },
+ "data": {
+ "password": "dmFsdWUtMg0KDQo=",
+ "username": "dmFsdWUtMQ0K"
+ }
+ },
+ {
+ "kind": "Secret",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "test-db-secret"
+ },
+ "data": {
+ "password": "dmFsdWUtMg0KDQo=",
+ "username": "dmFsdWUtMQ0K"
+ }
+ }]
+}
+
+The pods:
+ +{
+ "apiVersion": "v1",
+ "kind": "List",
+ "items":
+ [{
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "prod-db-client-pod",
+ "labels": {
+ "name": "prod-db-client"
+ }
+ },
+ "spec": {
+ "volumes": [
+ {
+ "name": "secret-volume",
+ "secret": {
+ "secretName": "prod-db-secret"
+ }
+ }
+ ],
+ "containers": [
+ {
+ "name": "db-client-container",
+ "image": "myClientImage",
+ "volumeMounts": [
+ {
+ "name": "secret-volume",
+ "readOnly": true,
+ "mountPath": "/etc/secret-volume"
+ }
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "test-db-client-pod",
+ "labels": {
+ "name": "test-db-client"
+ }
+ },
+ "spec": {
+ "volumes": [
+ {
+ "name": "secret-volume",
+ "secret": {
+ "secretName": "test-db-secret"
+ }
+ }
+ ],
+ "containers": [
+ {
+ "name": "db-client-container",
+ "image": "myClientImage",
+ "volumeMounts": [
+ {
+ "name": "secret-volume",
+ "readOnly": true,
+ "mountPath": "/etc/secret-volume"
+ }
+ ]
+ }
+ ]
+ }
+ }]
+}
+
+The specs for the two pods differ only in the value of the object referred to by the secret volume +source. Both containers will have the following files present on their filesystems:
+ +/etc/secret-volume/username
+/etc/secret-volume/password
+
+
+
+
+
+
+
+
+
+
+ Kubernetes should define a reasonable set of security best practices that allows processes to be isolated from each other, from the cluster infrastructure, and which preserves important boundaries between those who manage the cluster, and those who use the cluster.
+ +While Kubernetes today is not primarily a multi-tenant system, the long term evolution of Kubernetes will increasingly rely on proper boundaries between users and administrators. The code running on the cluster must be appropriately isolated and secured to prevent malicious parties from affecting the entire cluster.
+ +We define “user” as a unique identity accessing the Kubernetes API server, which may be a human or an automated process. Human users fall into the following categories:
+ +Automated process users fall into the following categories:
+ +A pod runs in a security context under a service account that is defined by an administrator or project administrator, and the secrets a pod has access to is limited by that service account.
+ +Access to the central data store (etcd) in Kubernetes allows an attacker to run arbitrary containers on hosts, to gain access to any protected information stored in either volumes or in pods (such as access tokens or shared secrets provided as environment variables), to intercept and redirect traffic from running services by inserting middlemen, or to simply delete the entire history of the custer.
+ +As a general principle, access to the central data store should be restricted to the components that need full control over the system and which can apply appropriate authorization and authentication of change requests. In the future, etcd may offer granular access control, but that granularity will require an administrator to understand the schema of the data to properly apply security. An administrator must be able to properly secure Kubernetes at a policy level, rather than at an implementation level, and schema changes over time should not risk unintended security leaks.
+ +Both the Kubelet and Kube Proxy need information related to their specific roles - for the Kubelet, the set of pods it should be running, and for the Proxy, the set of services and endpoints to load balance. The Kubelet also needs to provide information about running pods and historical termination data. The access pattern for both Kubelet and Proxy to load their configuration is an efficient “wait for changes” request over HTTP. It should be possible to limit the Kubelet and Proxy to only access the information they need to perform their roles and no more.
+ +The controller manager for Replication Controllers and other future controllers act on behalf of a user via delegation to perform automated maintenance on Kubernetes resources. Their ability to access or modify resource state should be strictly limited to their intended duties and they should be prevented from accessing information not pertinent to their role. For example, a replication controller needs only to create a copy of a known pod configuration, to determine the running state of an existing pod, or to delete an existing pod that it created - it does not need to know the contents or current state of a pod, nor have access to any data in the pods attached volumes.
+ +The Kubernetes pod scheduler is responsible for reading data from the pod to fit it onto a node in the cluster. At a minimum, it needs access to view the ID of a pod (to craft the binding), its current state, any resource information necessary to identify placement, and other data relevant to concerns like anti-affinity, zone or region preference, or custom logic. It does not need the ability to modify pods or see other resources, only to create bindings. It should not need the ability to delete bindings unless the scheduler takes control of relocating components on failed hosts (which could be implemented by a separate component that can delete bindings but not create them). The scheduler may need read access to user or project-container information to determine preferential location (underspecified at this time).
+ + + + + + + + + +A security context is a set of constraints that are applied to a container in order to achieve the following goals (from security design):
+ +The problem of securing containers in Kubernetes has come up before and the potential problems with container security are well known. Although it is not possible to completely isolate Docker containers from their hosts, new features like user namespaces make it possible to greatly reduce the attack surface.
+ +In order to improve container isolation from host and other containers running on the host, containers should only be +granted the access they need to perform their work. To this end it should be possible to take advantage of Docker +features such as the ability to add or remove capabilities and assign MCS labels +to the container process.
+ +Support for user namespaces has recently been merged into Docker’s libcontainer project and should soon surface in Docker itself. It will make it possible to assign a range of unprivileged uids and gids from the host to each container, improving the isolation between host and container and between containers.
+ +In order to support external integration with shared storage, processes running in a Kubernetes cluster +should be able to be uniquely identified by their Unix UID, such that a chain of ownership can be established. +Processes in pods will need to have consistent UID/GID/SELinux category labels in order to access shared disks.
+ +In order of increasing complexity, following are example use cases that would +be addressed with security contexts:
+ +A security context consists of a set of constraints that determine how a container +is secured before getting created and run. A security context resides on the container and represents the runtime parameters that will +be used to create and run the container via container APIs. A security context provider is passed to the Kubelet so it can have a chance +to mutate Docker API calls in order to apply the security context.
+ +It is recommended that this design be implemented in two phases:
+ +The Kubelet will have an interface that points to a SecurityContextProvider
. The SecurityContextProvider
is invoked before creating and running a given container:
type SecurityContextProvider interface {
+ // ModifyContainerConfig is called before the Docker createContainer call.
+ // The security context provider can make changes to the Config with which
+ // the container is created.
+ // An error is returned if it's not possible to secure the container as
+ // requested with a security context.
+ ModifyContainerConfig(pod *api.Pod, container *api.Container, config *docker.Config)
+
+ // ModifyHostConfig is called before the Docker runContainer call.
+ // The security context provider can make changes to the HostConfig, affecting
+ // security options, whether the container is privileged, volume binds, etc.
+ // An error is returned if it's not possible to secure the container as requested
+ // with a security context.
+ ModifyHostConfig(pod *api.Pod, container *api.Container, hostConfig *docker.HostConfig)
+}
+
+If the value of the SecurityContextProvider field on the Kubelet is nil, the kubelet will create and run the container as it does today.
+ +A security context resides on the container and represents the runtime parameters that will +be used to create and run the container via container APIs. Following is an example of an initial implementation:
+ +type Container struct {
+ ... other fields omitted ...
+ // Optional: SecurityContext defines the security options the pod should be run with
+ SecurityContext *SecurityContext
+}
+
+// SecurityContext holds security configuration that will be applied to a container. SecurityContext
+// contains duplication of some existing fields from the Container resource. These duplicate fields
+// will be populated based on the Container configuration if they are not set. Defining them on
+// both the Container AND the SecurityContext will result in an error.
+type SecurityContext struct {
+ // Capabilities are the capabilities to add/drop when running the container
+ Capabilities *Capabilities
+
+ // Run the container in privileged mode
+ Privileged *bool
+
+ // SELinuxOptions are the labels to be applied to the container
+ // and volumes
+ SELinuxOptions *SELinuxOptions
+
+ // RunAsUser is the UID to run the entrypoint of the container process.
+ RunAsUser *int64
+}
+
+// SELinuxOptions are the labels to be applied to the container.
+type SELinuxOptions struct {
+ // SELinux user label
+ User string
+
+ // SELinux role label
+ Role string
+
+ // SELinux type label
+ Type string
+
+ // SELinux level label.
+ Level string
+}
+
+It is up to an admission plugin to determine if the security context is acceptable or not. At the +time of writing, the admission control plugin for security contexts will only allow a context that +has defined capabilities or privileged. Contexts that attempt to define a UID or SELinux options +will be denied by default. In the future the admission plugin will base this decision upon +configurable policies that reside within the service account.
+ + + + + + + + + +Processes in Pods may need to call the Kubernetes API. For example: + - scheduler + - replication controller + - node controller + - a map-reduce type framework which has a controller that then tries to make a dynamically determined number of workers and watch them + - continuous build and push system + - monitoring system
+ +They also may interact with services other than the Kubernetes API, such as: + - an image repository, such as docker – both when the images are pulled to start the containers, and for writing + images in the case of pods that generate images. + - accessing other cloud services, such as blob storage, in the context of a large, integrated, cloud offering (hosted + or private). + - accessing files in an NFS volume attached to the pod
+ +A service account binds together several things: + - a name, understood by users, and perhaps by peripheral systems, for an identity + - a principal that can be authenticated and authorized + - a security context, which defines the Linux Capabilities, User IDs, Groups IDs, and other + capabilities and controls on interaction with the file system and OS. + - a set of secrets, which a container may use to + access various networked resources.
+ +A new object Kind is added:
+ +type ServiceAccount struct {
+ TypeMeta `json:",inline" yaml:",inline"`
+ ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"`
+
+ username string
+ securityContext ObjectReference // (reference to a securityContext object)
+ secrets []ObjectReference // (references to secret objects
+}
+
+The name ServiceAccount is chosen because it is widely used already (e.g. by Kerberos and LDAP) +to refer to this type of account. Note that it has no relation to Kubernetes Service objects.
+ +The ServiceAccount object does not include any information that could not be defined separately: + - username can be defined however users are defined. + - securityContext and secrets are only referenced and are created using the REST API.
+ +The purpose of the serviceAccount object is twofold: + - to bind usernames to securityContexts and secrets, so that the username can be used to refer succinctly + in contexts where explicitly naming securityContexts and secrets would be inconvenient + - to provide an interface to simplify allocation of new securityContexts and secrets. +These features are explained later.
+ +From the standpoint of the Kubernetes API, a user
is any principal which can authenticate to Kubernetes API.
+This includes a human running kubectl
on her desktop and a container in a Pod on a Node making API calls.
There is already a notion of a username in Kubernetes, which is populated into a request context after authentication. +However, there is no API object representing a user. While this may evolve, it is expected that in mature installations, +the canonical storage of user identifiers will be handled by a system external to Kubernetes.
+ +Kubernetes does not dictate how to divide up the space of user identifier strings. User names can be
+simple Unix-style short usernames, (e.g. alice
), or may be qualified to allow for federated identity (
+alice@example.com
vs alice@example.org
.) Naming convention may distinguish service accounts from user
+accounts (e.g. alice@example.com
vs build-service-account-a3b7f0@foo-namespace.service-accounts.example.com
),
+but Kubernetes does not require this.
Kubernetes also does not require that there be a distinction between human and Pod users. It will be possible
+to setup a cluster where Alice the human talks to the Kubernetes API as username alice
and starts pods that
+also talk to the API as user alice
and write files to NFS as user alice
. But, this is not recommended.
Instead, it is recommended that Pods and Humans have distinct identities, and reference implementations will +make this distinction.
+ +The distinction is useful for a number of reasons: + - the requirements for humans and automated processes are different: + - Humans need a wide range of capabilities to do their daily activities. Automated processes often have more narrowly-defined activities. + - Humans may better tolerate the exceptional conditions created by expiration of a token. Remembering to handle + this in a program is more annoying. So, either long-lasting credentials or automated rotation of credentials is + needed. + - A Human typically keeps credentials on a machine that is not part of the cluster and so not subject to automatic + management. A VM with a role/service-account can have its credentials automatically managed. + - the identity of a Pod cannot in general be mapped to a single human. + - If policy allows, it may be created by one human, and then updated by another, and another, until its behavior cannot be attributed to a single human.
+ +TODO: consider getting rid of separate serviceAccount object and just rolling its parts into the SecurityContext or +Pod Object.
+ +The secrets
field is a list of references to /secret objects that an process started as that service account should
+have access to be able to assert that role.
The secrets are not inline with the serviceAccount object. This way, most or all users can have permission to GET /serviceAccounts
so they can remind themselves
+what serviceAccounts are available for use.
Nothing will prevent creation of a serviceAccount with two secrets of type SecretTypeKubernetesAuth
, or secrets of two
+different types. Kubelet and client libraries will have some behavior, TBD, to handle the case of multiple secrets of a
+given type (pick first or provide all and try each in order, etc).
When a serviceAccount and a matching secret exist, then a User.Info
for the serviceAccount and a BearerToken
from the secret
+are added to the map of tokens used by the authentication process in the apiserver, and similarly for other types. (We
+might have some types that do not do anything on apiserver but just get pushed to the kubelet.)
The PodSpec
is extended to have a Pods.Spec.ServiceAccountUsername
field. If this is unset, then a
+default value is chosen. If it is set, then the corresponding value of Pods.Spec.SecurityContext
is set by the
+Service Account Finalizer (see below).
TBD: how policy limits which users can make pods with which service accounts.
+ +Kubernetes API Authorization Policies refer to users. Pods created with a Pods.Spec.ServiceAccountUsername
typically
+get a Secret
which allows them to authenticate to the Kubernetes APIserver as a particular user. So any
+policy that is desired can be applied to them.
A higher level workflow is needed to coordinate creation of serviceAccounts, secrets and relevant policy objects. +Users are free to extend Kubernetes to put this business logic wherever is convenient for them, though the +Service Account Finalizer is one place where this can happen (see below).
+ +The kubelet will treat as “not ready to run” (needing a finalizer to act on it) any Pod which has an empty +SecurityContext.
+ +The kubelet will set a default, restrictive, security context for any pods created from non-Apiserver config +sources (http, file).
+ +Kubelet watches apiserver for secrets which are needed by pods bound to it.
+ +TODO: how to only let kubelet see secrets it needs to know.
+ +There are several ways to use Pods with SecurityContexts and Secrets.
+ +One way is to explicitly specify the securityContext and all secrets of a Pod when the pod is initially created, +like this:
+ +TODO: example of pod with explicit refs.
+ +Another way is with the Service Account Finalizer, a plugin process which is optional, and which handles +business logic around service accounts.
+ +The Service Account Finalizer watches Pods, Namespaces, and ServiceAccount definitions.
+ +First, if it finds pods which have a Pod.Spec.ServiceAccountUsername
but no Pod.Spec.SecurityContext
set,
+then it copies in the referenced securityContext and secrets references for the corresponding serviceAccount
.
Second, if ServiceAccount definitions change, it may take some actions. +TODO: decide what actions it takes when a serviceAccount definition changes. Does it stop pods, or just +allow someone to list ones that are out of spec? In general, people may want to customize this?
+ +Third, if a new namespace is created, it may create a new serviceAccount for that namespace. This may include
+a new username (e.g. NAMESPACE-default-service-account@serviceaccounts.$CLUSTERID.kubernetes.io
), a new
+securityContext, a newly generated secret to authenticate that serviceAccount to the Kubernetes API, and default
+policies for that service account.
+TODO: more concrete example. What are typical default permissions for default service account (e.g. readonly access
+to services in the same namespace and read-write access to events in that namespace?)
Finally, it may provide an interface to automate creation of new serviceAccounts. In that case, the user may want +to GET serviceAccounts to see what has been created.
+ + + + + + + + + +This is a lightweight design document for simple rolling update in kubectl
.
Complete execution flow can be found here. See the example of rolling update for more information.
+ +Assume that we have a current replication controller named foo
and it is running image image:v1
kubectl rolling-update foo [foo-v2] --image=myimage:v2
If the user doesn’t specify a name for the ‘next’ replication controller, then the ‘next’ replication controller is renamed to +the name of the original replication controller.
+ +Obviously there is a race here, where if you kill the client between delete foo, and creating the new version of ‘foo’ you might be surprised about what is there, but I think that’s ok. +See Recovery below
+ +If the user does specify a name for the ‘next’ replication controller, then the ‘next’ replication controller is retained with its existing name,
+and the old ‘foo’ replication controller is deleted. For the purposes of the rollout, we add a unique-ifying label kubernetes.io/deployment
to both the foo
and foo-next
replication controllers.
+The value of that label is the hash of the complete JSON representation of thefoo-next
orfoo
replication controller. The name of this label can be overridden by the user with the --deployment-label-key
flag.
If a rollout fails or is terminated in the middle, it is important that the user be able to resume the roll out.
+To facilitate recovery in the case of a crash of the updating process itself, we add the following annotations to each replication controller in the kubernetes.io/
annotation namespace:
+ * desired-replicas
The desired number of replicas for this replication controller (either N or zero)
+ * update-partner
A pointer to the replication controller resource that is the other half of this update (syntax <name>
the namespace is assumed to be identical to the namespace of this replication controller.)
Recovery is achieved by issuing the same command again:
+ +kubectl rolling-update foo [foo-v2] --image=myimage:v2
+
+Whenever the rolling update command executes, the kubectl client looks for replication controllers called foo
and foo-next
, if they exist, an attempt is
+made to roll foo
to foo-next
. If foo-next
does not exist, then it is created, and the rollout is a new rollout. If foo
doesn’t exist, then
+it is assumed that the rollout is nearly completed, and foo-next
is renamed to foo
. Details of the execution flow are given below.
Abort is assumed to want to reverse a rollout in progress.
+ +kubectl rolling-update foo [foo-v2] --rollback
This is really just semantic sugar for:
+ +kubectl rolling-update foo-v2 foo
With the added detail that it moves the desired-replicas
annotation from foo-v2
to foo
For the purposes of this example, assume that we are rolling from foo
to foo-next
where the only change is an image update from v1
to v2
If the user doesn’t specify a foo-next
name, then it is either discovered from the update-partner
annotation on foo
. If that annotation doesn’t exist,
+then foo-next
is synthesized using the pattern <controller-name>-<hash-of-next-controller-JSON>
foo
and foo-next
do not exist:
+ foo
exists, but foo-next
does not:
+ foo-next
populate it with the v2
image, set desired-replicas
to foo.Spec.Replicas
foo-next
exists, but foo
does not:
+ foo
and foo-next
exist:
+ foo-next
is missing the desired-replicas
annotation
+ desired-replicas
annotation to foo-next
using the current size of foo
foo-next
< desired-replicas
annotation on foo-next
+ foo-next
foo
> 0
+ decrease size of foo
foo
foo
that is identical to foo-next
foo-next
foo-next
doesn’t exist
+ foo
doesn’t exist
+ foo-next
and foo
both exist
+ desired-replicas
annotation on foo
to match the annotation on foo-next
foo
and foo-next
trading places.Legend:
+ +There is no mandated timeline for major versions. They only occur when we need to start the clock on deprecating features. A given major version should be the latest major version for at least one year from its original release date.
+ +Here is an example major release cycle:
+ +It may seem a bit strange to complete the v2 API before v2.0 is released, but adding a v2 API is not a breaking change. Removing the v2beta* APIs is a breaking change, which is what necessitates the major version bump. There are other ways to do this, but having the major release be the fresh start of that release’s API without the baggage of its beta versions seems most intuitive out of the available options.
+ +Patch releases are intended for critical bug fixes to the latest minor version, such as addressing security vulnerabilities, fixes to problems affecting a large number of users, severe problems with no workaround, and blockers for products based on Kubernetes.
+ +They should not contain miscellaneous feature additions or improvements, and especially no incompatibilities should be introduced between patch versions of the same minor version (or even major version).
+ +Dependencies, such as Docker or Etcd, should also not be changed unless absolutely necessary, and also just to fix critical bugs (so, at most patch version changes, not new major nor minor versions).
+ +There is a separate question of how to track the capabilities of a kubelet to facilitate rolling upgrades. That is not addressed here.
+ + + + + + + + + +The developer guide is for anyone wanting to either write code which directly accesses the +Kubernetes API, or to contribute directly to the Kubernetes project. +It assumes some familiarity with concepts in the User Guide and the Cluster Admin +Guide.
+ +On Collaborative Development (collab.md): Info on pull requests and code reviews.
+GitHub Issues (issues.md): How incoming issues are reviewed and prioritized.
+Pull Request Process (pull-requests.md): When and why pull requests are closed.
+Faster PR reviews (faster_reviews.md): How to get faster PR reviews.
+Getting Recent Builds (getting-builds.md): How to get recent builds including the latest builds that pass CI.
+Automated Tools (automation.md): Descriptions of the automation that is running on our github repository.
+Development Guide (development.md): Setting up your development environment.
+Hunting flaky tests (flaky-tests.md): We have a goal of 99.9% flake free tests. +Here’s how to run your tests many times.
+Logging Conventions (logging.md]: Glog levels.
+Profiling Kubernetes (profiling.md): How to plug in go pprof profiler to Kubernetes.
+Instrumenting Kubernetes with a new metric +(instrumentation.md): How to add a new metrics to the +Kubernetes code base.
+Coding Conventions (coding-conventions.md): +Coding style advice for contributors.
+API objects are explained at http://kubernetes.io/third_party/swagger-ui/.
+Annotations (docs/user-guide/annotations.md): are for attaching arbitrary non-identifying metadata to objects. +Programs that automate Kubernetes objects may use annotations to store small amounts of their state.
+API Conventions (api-conventions.md): +Defining the verbs and resources used in the Kubernetes API.
+API Client Libraries (client-libraries.md): +A list of existing client libraries, both supported and user-contributed.
+Authentication Plugins (docs/admin/authentication.md): +The current and planned states of authentication tokens.
+Authorization Plugins (docs/admin/authorization.md): +Authorization applies to all HTTP requests on the main apiserver port. +This doc explains the available authorization implementations.
+Admission Control Plugins (admission_control)
+Making release notes (making-release-notes.md): Generating release nodes for a new release.
+Releasing Kubernetes (releasing.md): How to create a Kubernetes release (as in version) +and how the version information gets embedded into the built binaries.
+API Conventions +===============
+ +Updated: 9/20/2015
+ +This document is oriented at users who want a deeper understanding of the Kubernetes +API structure, and developers wanting to extend the Kubernetes API. An introduction to +using resources with kubectl can be found in Working with resources.
+ +Table of Contents +
+ +The conventions of the Kubernetes API (and related APIs in the ecosystem) are intended to ease client development and ensure that configuration mechanisms can be implemented that work across a diverse set of use cases consistently.
+ +The general style of the Kubernetes API is RESTful - clients create, update, delete, or retrieve a description of an object via the standard HTTP verbs (POST, PUT, DELETE, and GET) - and those APIs preferentially accept and return JSON. Kubernetes also exposes additional endpoints for non-standard verbs and allows alternative content types. All of the JSON accepted and returned by the server has a schema, identified by the “kind” and “apiVersion” fields. Where relevant HTTP header fields exist, they should mirror the content of JSON fields, but the information should not be represented only in the HTTP header.
+ +The following terms are defined:
+ +Each resource typically accepts and returns data of a single kind. A kind may be accepted or returned by multiple resources that reflect specific use cases. For instance, the kind “Pod” is exposed as a “pods” resource that allows end users to create, update, and delete pods, while a separate “pod status” resource (that acts on “Pod” kind) allows automated processes to update a subset of the fields in that resource.
+ +Resource collections should be all lowercase and plural, whereas kinds are CamelCase and singular.
+ +Kinds are grouped into three categories:
+ +Objects represent a persistent entity in the system.
+ +Creating an API object is a record of intent - once created, the system will work to ensure that resource exists. All API objects have common metadata.
+ +An object may have multiple resources that clients can use to perform specific actions that create, update, delete, or get.
+ +Examples: Pod
, ReplicationController
, Service
, Namespace
, Node
.
Lists are collections of resources of one (usually) or more (occasionally) kinds.
+ +Lists have a limited set of common metadata. All lists use the “items” field to contain the array of objects they return.
+ +Most objects defined in the system should have an endpoint that returns the full set of resources, as well as zero or more endpoints that return subsets of the full list. Some objects may be singletons (the current user, the system defaults) and may not have lists.
+ +In addition, all lists that return objects with labels should support label filtering (see docs/user-guide/labels.md, and most lists should support filtering by fields.
+ +Examples: PodLists, ServiceLists, NodeLists
+ +TODO: Describe field filtering below or in a separate doc.
+Simple kinds are used for specific actions on objects and for non-persistent entities.
+ +Given their limited scope, they have the same set of limited common metadata as lists.
+ +For instance, the “Status” kind is returned when errors occur and is not persisted in the system.
+ +Many simple resources are “subresources”, which are rooted at API paths of specific resources. When resources wish to expose alternative actions or views that are closely coupled to a single resource, they should do so using new sub-resources. Common subresources include:
+ +/binding
: Used to bind a resource representing a user request (e.g., Pod, PersistentVolumeClaim) to a cluster infrastructure resource (e.g., Node, PersistentVolume)./status
: Used to write just the status portion of a resource. For example, the /pods
endpoint only allows updates to metadata
and spec
, since those reflect end-user intent. An automated process should be able to modify status for users to see by sending an updated Pod kind to the server to the “/pods/<name>/status” endpoint - the alternate endpoint allows different rules to be applied to the update, and access to be appropriately restricted./scale
: Used to read and write the count of a resource in a manner that is independent of the specific resource schema.Two additional subresources, proxy
and portforward
, provide access to cluster resources as described in docs/user-guide/accessing-the-cluster.md.
The standard REST verbs (defined below) MUST return singular JSON objects. Some API endpoints may deviate from the strict REST pattern and return resources that are not singular JSON objects, such as streams of JSON objects or unstructured text log data.
+ +The term “kind” is reserved for these “top-level” API types. The term “type” should be used for distinguishing sub-categories within objects or subobjects.
+ +All JSON objects returned by an API MUST have the following fields:
+ +These fields are required for proper decoding of the object. They may be populated by the server by default from the specified URL path, but the client likely needs to know the values in order to construct the URL path.
+ +Every object kind MUST have the following metadata in a nested object field called “metadata”:
+ +Every object SHOULD have the following metadata in a nested object field called “metadata”:
+ +Labels are intended for organizational purposes by end users (select the pods that match this label query). Annotations enable third-party automation and tooling to decorate objects with additional metadata for their own use.
+ +By convention, the Kubernetes API makes a distinction between the specification of the desired state of an object (a nested object field called “spec”) and the status of the object at the current time (a nested object field called “status”). The specification is a complete description of the desired state, including configuration settings provided by the user, default values expanded by the system, and properties initialized or otherwise changed after creation by other ecosystem components (e.g., schedulers, auto-scalers), and is persisted in stable storage with the API object. If the specification is deleted, the object will be purged from the system. The status summarizes the current state of the object in the system, and is usually persisted with the object by an automated processes but may be generated on the fly. At some cost and perhaps some temporary degradation in behavior, the status could be reconstructed by observation if it were lost.
+ +When a new version of an object is POSTed or PUT, the “spec” is updated and available immediately. Over time the system will work to bring the “status” into line with the “spec”. The system will drive toward the most recent “spec” regardless of previous versions of that stanza. In other words, if a value is changed from 2 to 5 in one PUT and then back down to 3 in another PUT the system is not required to ‘touch base’ at 5 before changing the “status” to 3. In other words, the system’s behavior is level-based rather than edge-based. This enables robust behavior in the presence of missed intermediate state changes.
+ +The Kubernetes API also serves as the foundation for the declarative configuration schema for the system. In order to facilitate level-based operation and expression of declarative configuration, fields in the specification should have declarative rather than imperative names and semantics – they represent the desired state, not actions intended to yield the desired state.
+ +The PUT and POST verbs on objects will ignore the “status” values. A /status
subresource is provided to enable system components to update statuses of resources they manage.
Otherwise, PUT expects the whole object to be specified. Therefore, if a field is omitted it is assumed that the client wants to clear that field’s value. The PUT verb does not accept partial updates. Modification of just part of an object may be achieved by GETting the resource, modifying part of the spec, labels, or annotations, and then PUTting it back. See concurrency control, below, regarding read-modify-write consistency when using this pattern. Some objects may expose alternative resource representations that allow mutation of the status, or performing custom actions on the object.
+ +All objects that represent a physical resource whose state may vary from the user’s desired intent SHOULD have a “spec” and a “status”. Objects whose state cannot vary from the user’s desired intent MAY have only “spec”, and MAY rename “spec” to a more appropriate name.
+ +Objects that contain both spec and status should not contain additional top-level fields other than the standard metadata fields.
+ +Conditions represent the latest available observations of an object’s current state. Objects may report multiple conditions, and new types of conditions may be added in the future. Therefore, conditions are represented using a list/slice, where all have similar structure.
+ +The FooCondition
type for some resource type Foo
may include a subset of the following fields, but must contain at least type
and status
fields:
Type FooConditionType `json:"type" description:"type of Foo condition"`
+ Status ConditionStatus `json:"status" description:"status of the condition, one of True, False, Unknown"`
+ LastHeartbeatTime unversioned.Time `json:"lastHeartbeatTime,omitempty" description:"last time we got an update on a given condition"`
+ LastTransitionTime unversioned.Time `json:"lastTransitionTime,omitempty" description:"last time the condition transit from one status to another"`
+ Reason string `json:"reason,omitempty" description:"one-word CamelCase reason for the condition's last transition"`
+ Message string `json:"message,omitempty" description:"human-readable message indicating details about last transition"`
+
+Additional fields may be added in the future.
+ +Conditions should be added to explicitly convey properties that users and components care about rather than requiring those properties to be inferred from other observations.
+ +Condition status values may be True
, False
, or Unknown
. The absence of a condition should be interpreted the same as Unknown
.
In general, condition values may change back and forth, but some condition transitions may be monotonic, depending on the resource and condition type. However, conditions are observations and not, themselves, state machines, nor do we define comprehensive state machines for objects, nor behaviors associated with state transitions. The system is level-based rather than edge-triggered, and should assume an Open World.
+ +A typical oscillating condition type is Ready
, which indicates the object was believed to be fully operational at the time it was last probed. A possible monotonic condition could be Succeeded
. A False
status for Succeeded
would imply failure. An object that was still active would not have a Succeeded
condition, or its status would be Unknown
.
Some resources in the v1 API contain fields called phase
, and associated message
, reason
, and other status fields. The pattern of using phase
is deprecated. Newer API types should use conditions instead. Phase was essentially a state-machine enumeration field, that contradicted system-design principles and hampered evolution, since adding new enum values breaks backward compatibility. Rather than encouraging clients to infer implicit properties from phases, we intend to explicitly expose the conditions that clients need to monitor. Conditions also have the benefit that it is possible to create some conditions with uniform meaning across all resource types, while still exposing others that are unique to specific resource types. See #7856 for more details and discussion.
In condition types, and everywhere else they appear in the API, Reason
is intended to be a one-word, CamelCase representation of the category of cause of the current status, and Message
is intended to be a human-readable phrase or sentence, which may contain specific details of the individual occurrence. Reason
is intended to be used in concise output, such as one-line kubectl get
output, and in summarizing occurrences of causes, whereas Message
is intended to be presented to users in detailed status explanations, such as kubectl describe
output.
Historical information status (e.g., last transition time, failure counts) is only provided with reasonable effort, and is not guaranteed to not be lost.
+ +Status information that may be large (especially proportional in size to collections of other resources, such as lists of references to other objects – see below) and/or rapidly changing, such as resource usage, should be put into separate objects, with possibly a reference from the original object. This helps to ensure that GETs and watch remain reasonably efficient for the majority of clients, which may not need that data.
+ +Some resources report the observedGeneration
, which is the generation
most recently observed by the component responsible for acting upon changes to the desired state of the resource. This can be used, for instance, to ensure that the reported status reflects the most recent desired status.
References to loosely coupled sets of objects, such as pods overseen by a replication controller, are usually best referred to using a label selector. In order to ensure that GETs of individual objects remain bounded in time and space, these sets may be queried via separate API queries, but will not be expanded in the referring object’s status.
+ +References to specific objects, especially specific resource versions and/or specific fields of those objects, are specified using the ObjectReference
type (or other types representing strict subsets of it). Unlike partial URLs, the ObjectReference type facilitates flexible defaulting of fields from the referring object or other contextual information.
References in the status of the referee to the referrer may be permitted, when the references are one-to-one and do not need to be frequently updated, particularly in an edge-based manner.
+ +Discussed in #2004 and elsewhere. There are no maps of subobjects in any API objects. Instead, the convention is to use a list of subobjects containing name fields.
+ +For example:
+ +ports:
+ - name: www
+ containerPort: 80
+
+vs.
+ +ports:
+ www:
+ containerPort: 80
+
+This rule maintains the invariant that all JSON/YAML keys are fields in API objects. The only exceptions are pure maps in the API (currently, labels, selectors, annotations, data), as opposed to sets of subobjects.
+ +Some fields will have a list of allowed values (enumerations). These values will be strings, and they will be in CamelCase, with an initial uppercase letter. Examples: “ClusterFirst”, “Pending”, “ClientIP”.
+ +Every list or simple kind SHOULD have the following metadata in a nested object field called “metadata”:
+ +Every simple kind returned by the server, and any simple kind sent to the server that must support idempotency or optimistic concurrency should return this value.Since simple resources are often used as input alternate actions that modify objects, the resource version of the simple resource should correspond to the resource version of the object.
+ +An API may represent a single entity in different ways for different clients, or transform an object after certain transitions in the system occur. In these cases, one request object may have two representations available as different resources, or different kinds.
+ +An example is a Service, which represents the intent of the user to group a set of pods with common behavior on common ports. When Kubernetes detects a pod matches the service selector, the IP address and port of the pod are added to an Endpoints resource for that Service. The Endpoints resource exists only if the Service exists, but exposes only the IPs and ports of the selected pods. The full service is represented by two distinct resources - under the original Service resource the user created, as well as in the Endpoints resource.
+ +As another example, a “pod status” resource may accept a PUT with the “pod” kind, with different rules about what fields may be changed.
+ +Future versions of Kubernetes may allow alternative encodings of objects beyond JSON.
+ +API resources should use the traditional REST pattern:
+ +The API supports three different PATCH operations, determined by their corresponding Content-Type header:
+ +Content-Type: application/json-patch+json
{"op": "add", "path": "/a/b/c", "value": [ "foo", "bar" ]}
. For more details on how to use JSON Patch, see the RFC.Content-Type: application/merge-patch+json
Content-Type: application/strategic-merge-patch+json
In the standard JSON merge patch, JSON objects are always merged but lists are always replaced. Often that isn’t what we want. Let’s say we start with the following Pod:
+ +spec:
+ containers:
+ - name: nginx
+ image: nginx-1.0
+
+…and we POST that to the server (as JSON). Then let’s say we want to add a container to this Pod.
+ +PATCH /api/v1/namespaces/default/pods/pod-name
+spec:
+ containers:
+ - name: log-tailer
+ image: log-tailer-1.0
+
+If we were to use standard Merge Patch, the entire container list would be replaced with the single log-tailer container. However, our intent is for the container lists to merge together based on the name
field.
To solve this problem, Strategic Merge Patch uses metadata attached to the API objects to determine what lists should be merged and which ones should not. Currently the metadata is available as struct tags on the API objects themselves, but will become available to clients as Swagger annotations in the future. In the above example, the patchStrategy
metadata for the containers
field would be merge
and the patchMergeKey
would be name
.
Note: If the patch results in merging two lists of scalars, the scalars are first deduplicated and then merged.
+ +Strategic Merge Patch also supports special operations as listed below.
+ +To override the container list to be strictly replaced, regardless of the default:
+ +containers:
+ - name: nginx
+ image: nginx-1.0
+ - $patch: replace # any further $patch operations nested in this list will be ignored
+
+To delete an element of a list that should be merged:
+ +containers:
+ - name: nginx
+ image: nginx-1.0
+ - $patch: delete
+ name: log-tailer # merge key and value goes here
+
+To indicate that a map should not be merged and instead should be taken literally:
+ +$patch: replace # recursive and applies to all fields of the map it's in
+containers:
+- name: nginx
+ image: nginx-1.0
+
+To delete a field of a map:
+ +name: nginx
+image: nginx-1.0
+labels:
+ live: null # set the value of the map key to null
+
+All compatible Kubernetes APIs MUST support “name idempotency” and respond with an HTTP status code 409 when a request is made to POST an object that has the same name as an existing object in the system. See docs/user-guide/identifiers.md for details.
+ +Names generated by the system may be requested using metadata.generateName
. GenerateName indicates that the name should be made unique by the server prior to persisting it. A non-empty value for the field indicates the name will be made unique (and the name returned to the client will be different than the name passed). The value of this field will be combined with a unique suffix on the server if the Name field has not been provided. The provided value must be valid within the rules for Name, and may be truncated by the length of the suffix required to make the value unique on the server. If this field is specified, and Name is not present, the server will NOT return a 409 if the generated name exists - instead, it will either return 201 Created or 504 with Reason ServerTimeout
indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).
Default resource values are API version-specific, and they are applied during
+the conversion from API-versioned declarative configuration to internal objects
+representing the desired state (Spec
) of the resource. Subsequent GETs of the
+resource will include the default values explicitly.
Incorporating the default values into the Spec
ensures that Spec
depicts the
+full desired state so that it is easier for the system to determine how to
+achieve the state, and for the user to know what to anticipate.
API version-specific default values are set by the API server.
+ +Late initialization is when resource fields are set by a system controller +after an object is created/updated.
+ +For example, the scheduler sets the pod.spec.nodeName
field after the pod is created.
Late-initializers should only make the following types of modifications:
+ - Setting previously unset fields
+ - Adding keys to maps
+ - Adding values to arrays which have mergeable semantics (patchStrategy:"merge"
attribute in
+ the type definition).
These conventions: + 1. allow a user (with sufficient privilege) to override any system-default behaviors by setting + the fields that would otherwise have been defaulted. + 1. enables updates from users to be merged with changes made during late initialization, using + strategic merge patch, as opposed to clobbering the change. + 1. allow the component which does the late-initialization to use strategic merge patch, which + facilitates composition and concurrency of such components.
+ +Although the apiserver Admission Control stage acts prior to object creation, +Admission Control plugins should follow the Late Initialization conventions +too, to allow their implementation to be later moved to a ‘controller’, or to client libraries.
+ +Kubernetes leverages the concept of resource versions to achieve optimistic concurrency. All Kubernetes resources have a “resourceVersion” field as part of their metadata. This resourceVersion is a string that identifies the internal version of an object that can be used by clients to determine when objects have changed. When a record is about to be updated, it’s version is checked against a pre-saved value, and if it doesn’t match, the update fails with a StatusConflict (HTTP status code 409).
+ +The resourceVersion is changed by the server every time an object is modified. If resourceVersion is included with the PUT operation the system will verify that there have not been other successful mutations to the resource during a read/modify/write cycle, by verifying that the current value of resourceVersion matches the specified value.
+ +The resourceVersion is currently backed by etcd’s modifiedIndex. However, it’s important to note that the application should not rely on the implementation details of the versioning system maintained by Kubernetes. We may change the implementation of resourceVersion in the future, such as to change it to a timestamp or per-object counter.
+ +The only way for a client to know the expected value of resourceVersion is to have received it from the server in response to a prior operation, typically a GET. This value MUST be treated as opaque by clients and passed unmodified back to the server. Clients should not assume that the resource version has meaning across namespaces, different kinds of resources, or different servers. Currently, the value of resourceVersion is set to match etcd’s sequencer. You could think of it as a logical clock the API server can use to order requests. However, we expect the implementation of resourceVersion to change in the future, such as in the case we shard the state by kind and/or namespace, or port to another storage system.
+ +In the case of a conflict, the correct client action at this point is to GET the resource again, apply the changes afresh, and try submitting again. This mechanism can be used to prevent races like the following:
+ +
+Client #1 Client #2
+GET Foo GET Foo
+Set Foo.Bar = "one" Set Foo.Baz = "two"
+PUT Foo PUT Foo
+
+
+
+When these sequences occur in parallel, either the change to Foo.Bar or the change to Foo.Baz can be lost.
+ +On the other hand, when specifying the resourceVersion, one of the PUTs will fail, since whichever write succeeds changes the resourceVersion for Foo.
+ +resourceVersion may be used as a precondition for other operations (e.g., GET, DELETE) in the future, such as for read-after-write consistency in the presence of caching.
+ +“Watch” operations specify resourceVersion using a query parameter. It is used to specify the point at which to begin watching the specified resources. This may be used to ensure that no mutations are missed between a GET of a resource (or list of resources) and a subsequent Watch, even if the current version of the resource is more recent. This is currently the main reason that list operations (GET on a collection) return resourceVersion.
+ +APIs may return alternative representations of any resource in response to an Accept header or under alternative endpoints, but the default serialization for input and output of API responses MUST be JSON.
+ +All dates should be serialized as RFC3339 strings.
+ +Units must either be explicit in the field name (e.g., timeoutSeconds
), or must be specified as part of the value (e.g., resource.Quantity
). Which approach is preferred is TBD, though currently we use the fooSeconds
convention for durations.
Some APIs may need to identify which field in a JSON object is invalid, or to reference a value to extract from a separate resource. The current recommendation is to use standard JavaScript syntax for accessing that field, assuming the JSON object was transformed into a JavaScript object, without the leading dot, such as metadata.name
.
Examples:
+ +fields[1].state.current
Object references should either be called fooName
if referring to an object of kind Foo
by just the name (within the current namespace, if a namespaced resource), or should be called fooRef
, and should contain a subset of the fields of the ObjectReference
type.
TODO: Plugins, extensions, nested kinds, headers
+ +The server will respond with HTTP status codes that match the HTTP spec. See the section below for a breakdown of the types of status codes the server will send.
+ +The following HTTP status codes may be returned by the API.
+ +200 StatusOK
+ 201 StatusCreated
+ 204 StatusNoContent
+ 307 StatusTemporaryRedirect
+ 400 StatusBadRequest
+ 401 StatusUnauthorized
+ 403 StatusForbidden
+ 404 StatusNotFound
+ 405 StatusMethodNotAllowed
+ 409 StatusConflict
+ Conflict
from the status
response section below on how to retrieve more information about the nature of the conflict.ResourceVersion
).422 StatusUnprocessableEntity
+ 429 StatusTooManyRequests
+ Retry-After
HTTP header from the response, and wait at least that long before retrying.500 StatusInternalServerError
+ 503 StatusServiceUnavailable
+ 504 StatusServerTimeout
+ Kubernetes will always return the Status
kind from any API endpoint when an error occurs.
+Clients SHOULD handle these types of objects when appropriate.
A Status
kind will be returned by the API in two cases:
+ * When an operation is not successful (i.e. when the server would return a non 2xx HTTP status code).
+ * When a HTTP DELETE
call is successful.
The status object is encoded as JSON and provided as the body of the response. The status object contains fields for humans and machine consumers of the API to get more detailed information for the cause of the failure. The information in the status object supplements, but does not override, the HTTP status code’s meaning. When fields in the status object have the same meaning as generally defined HTTP headers and that header is returned with the response, the header should be considered as having higher priority.
+ +Example:
+ +$ curl -v -k -H "Authorization: Bearer WhCDvq4VPpYhrcfmF6ei7V9qlbqTubUc" https://10.240.122.184:443/api/v1/namespaces/default/pods/grafana
+
+> GET /api/v1/namespaces/default/pods/grafana HTTP/1.1
+> User-Agent: curl/7.26.0
+> Host: 10.240.122.184
+> Accept: */*
+> Authorization: Bearer WhCDvq4VPpYhrcfmF6ei7V9qlbqTubUc
+>
+
+< HTTP/1.1 404 Not Found
+< Content-Type: application/json
+< Date: Wed, 20 May 2015 18:10:42 GMT
+< Content-Length: 232
+<
+{
+ "kind": "Status",
+ "apiVersion": "v1",
+ "metadata": {},
+ "status": "Failure",
+ "message": "pods \"grafana\" not found",
+ "reason": "NotFound",
+ "details": {
+ "name": "grafana",
+ "kind": "pods"
+ },
+ "code": 404
+}
+
+status
field contains one of two possible values:
+* Success
+* Failure
message
may contain human-readable description of the error
reason
may contain a machine-readable, one-word, CamelCase description of why this operation is in the Failure
status. If this value is empty there is no information available. The reason
clarifies an HTTP status code but does not override it.
details
may contain extended data associated with the reason. Each reason may define its own extended details. This field is optional and the data returned is not guaranteed to conform to any schema except that defined by the reason type.
Possible values for the reason
and details
fields:
+* BadRequest
+ * Indicates that the request itself was invalid, because the request doesn’t make any sense, for example deleting a read-only object.
+ * This is different than status reason
Invalid
above which indicates that the API call could possibly succeed, but the data was invalid.
+ * API calls that return BadRequest can never succeed.
+ * Http status code: 400 StatusBadRequest
+* Unauthorized
+ * Indicates that the server can be reached and understood the request, but refuses to take any further action without the client providing appropriate authorization. If the client has provided authorization, this error indicates the provided credentials are insufficient or invalid.
+ * Details (optional):
+ * kind string
+ * The kind attribute of the unauthorized resource (on some operations may differ from the requested resource).
+ * name string
+ * The identifier of the unauthorized resource.
+ * HTTP status code: 401 StatusUnauthorized
+* Forbidden
+ * Indicates that the server can be reached and understood the request, but refuses to take any further action, because it is configured to deny access for some reason to the requested resource by the client.
+ * Details (optional):
+ * kind string
+ * The kind attribute of the forbidden resource (on some operations may differ from the requested resource).
+ * name string
+ * The identifier of the forbidden resource.
+ * HTTP status code: 403 StatusForbidden
+* NotFound
+ * Indicates that one or more resources required for this operation could not be found.
+ * Details (optional):
+ * kind string
+ * The kind attribute of the missing resource (on some operations may differ from the requested resource).
+ * name string
+ * The identifier of the missing resource.
+ * HTTP status code: 404 StatusNotFound
+* AlreadyExists
+ * Indicates that the resource you are creating already exists.
+ * Details (optional):
+ * kind string
+ * The kind attribute of the conflicting resource.
+ * name string
+ * The identifier of the conflicting resource.
+ * HTTP status code: 409 StatusConflict
+* Conflict
+ * Indicates that the requested update operation cannot be completed due to a conflict. The client may need to alter the request. Each resource may define custom details that indicate the nature of the conflict.
+ * HTTP status code: 409 StatusConflict
+* Invalid
+ * Indicates that the requested create or update operation cannot be completed due to invalid data provided as part of the request.
+ * Details (optional):
+ * kind string
+ * the kind attribute of the invalid resource
+ * name string
+ * the identifier of the invalid resource
+ * causes
+ * One or more StatusCause
entries indicating the data in the provided resource that was invalid. The reason
, message
, and field
attributes will be set.
+ * HTTP status code: 422 StatusUnprocessableEntity
+* Timeout
+ * Indicates that the request could not be completed within the given time. Clients may receive this response if the server has decided to rate limit the client, or if the server is overloaded and cannot process the request at this time.
+ * Http status code: 429 TooManyRequests
+ * The server should set the Retry-After
HTTP header and return retryAfterSeconds
in the details field of the object. A value of 0
is the default.
+* ServerTimeout
+ * Indicates that the server can be reached and understood the request, but cannot complete the action in a reasonable time. This maybe due to temporary server load or a transient communication issue with another server.
+ * Details (optional):
+ * kind string
+ * The kind attribute of the resource being acted on.
+ * name string
+ * The operation that is being attempted.
+ * The server should set the Retry-After
HTTP header and return retryAfterSeconds
in the details field of the object. A value of 0
is the default.
+ * Http status code: 504 StatusServerTimeout
+* MethodNotAllowed
+ * Indicates that the action the client attempted to perform on the resource was not supported by the code.
+ * For instance, attempting to delete a resource that can only be created.
+ * API calls that return MethodNotAllowed can never succeed.
+ * Http status code: 405 StatusMethodNotAllowed
+* InternalError
+ * Indicates that an internal error occurred, it is unexpected and the outcome of the call is unknown.
+ * Details (optional):
+ * causes
+ * The original error.
+ * Http status code: 500 StatusInternalServerError
code
may contain the suggested HTTP return code for this status.
Events are complementary to status information, since they can provide some historical information about status and occurrences in addition to current or previous status. Generate events for situations users or administrators should be alerted about.
+ +Choose a unique, specific, short, CamelCase reason for each event category. For example, FreeDiskSpaceInvalid
is a good event reason because it is likely to refer to just one situation, but Started
is not a good reason because it doesn’t sufficiently indicate what started, even when combined with other event fields.
Error creating foo
or Error creating foo %s
would be appropriate for an event message, with the latter being preferable, since it is more informational.
Accumulate repeated events in the client, especially for frequent events, to reduce data volume, load on the system, and noise exposed to users.
+ +Minion
has been deprecated in favor of Node
. Use Node
where referring to the node resource in the context of the cluster. Use Host
where referring to properties of the individual physical/virtual system, such as hostname
, hostPath
, hostNetwork
, etc.FooController
is a deprecated kind naming convention. Name the kind after the thing being controlled instead (e.g., Job
rather than JobController
).something
occurs should be called somethingTime
. Do not use stamp
(e.g., creationTimestamp
).fooSeconds
convention for durations, as discussed in the units subsection.
+ fooPeriodSeconds
is preferred for periodic intervals and other waiting periods (e.g., over fooIntervalSeconds
).fooTimeoutSeconds
is preferred for inactivity/unresponsiveness deadlines.fooDeadlineSeconds
is preferred for activity completion deadlines.Foo
by name should be called fooName
. The name of a field referring to another resource of kind Foo
by ObjectReference (or subset thereof) should be called fooRef
.Labels are the domain of users. They are intended to facilitate organization and management of API resources using attributes that are meaningful to users, as opposed to meaningful to the system. Think of them as user-created mp3 or email inbox labels, as opposed to the directory structure used by a program to store its data. The former is enables the user to apply an arbitrary ontology, whereas the latter is implementation-centric and inflexible. Users will use labels to select resources to operate on, display label values in CLI/UI columns, etc. Users should always retain full power and flexibility over the label schemas they apply to labels in their namespaces.
+ +However, we should support conveniences for common cases by default. For example, what we now do in ReplicationController is automatically set the RC’s selector and labels to the labels in the pod template by default, if they are not already set. That ensures that the selector will match the template, and that the RC can be managed using the same labels as the pods it creates. Note that once we generalize selectors, it won’t necessarily be possible to unambiguously generate labels that match an arbitrary selector.
+ +If the user wants to apply additional labels to the pods that it doesn’t select upon, such as to facilitate adoption of pods or in the expectation that some label values will change, they can set the selector to a subset of the pod labels. Similarly, the RC’s labels could be initialized to a subset of the pod template’s labels, or could include additional/different labels.
+ +For disciplined users managing resources within their own namespaces, it’s not that hard to consistently apply schemas that ensure uniqueness. One just needs to ensure that at least one value of some label key in common differs compared to all other comparable resources. We could/should provide a verification tool to check that. However, development of conventions similar to the examples in Labels make uniqueness straightforward. Furthermore, relatively narrowly used namespaces (e.g., per environment, per application) can be used to reduce the set of resources that could potentially cause overlap.
+ +In cases where users could be running misc. examples with inconsistent schemas, or where tooling or components need to programmatically generate new objects to be selected, there needs to be a straightforward way to generate unique label sets. A simple way to ensure uniqueness of the set is to ensure uniqueness of a single label value, such as by using a resource name, uid, resource hash, or generation number.
+ +Problems with uids and hashes, however, include that they have no semantic meaning to the user, are not memorable nor readily recognizable, and are not predictable. Lack of predictability obstructs use cases such as creation of a replication controller from a pod, such as people want to do when exploring the system, bootstrapping a self-hosted cluster, or deletion and re-creation of a new RC that adopts the pods of the previous one, such as to rename it. Generation numbers are more predictable and much clearer, assuming there is a logical sequence. Fortunately, for deployments that’s the case. For jobs, use of creation timestamps is common internally. Users should always be able to turn off auto-generation, in order to permit some of the scenarios described above. Note that auto-generated labels will also become one more field that needs to be stripped out when cloning a resource, within a namespace, in a new namespace, in a new cluster, etc., and will need to be ignored around when updating a resource via patch or read-modify-write sequence.
+ +Inclusion of a system prefix in a label key is fairly hostile to UX. A prefix is only necessary in the case that the user cannot choose the label key, in order to avoid collisions with user-defined labels. However, I firmly believe that the user should always be allowed to select the label keys to use on their resources, so it should always be possible to override default label keys.
+ +Therefore, resources supporting auto-generation of unique labels should have a uniqueLabelKey
field, so that the user could specify the key if they wanted to, but if unspecified, it could be set by default, such as to the resource type, like job, deployment, or replicationController. The value would need to be at least spatially unique, and perhaps temporally unique in the case of job.
Annotations have very different intended usage from labels. We expect them to be primarily generated and consumed by tooling and system extensions. I’m inclined to generalize annotations to permit them to directly store arbitrary json. Rigid names and name prefixes make sense, since they are analogous to API fields.
+ +In fact, in-development API fields, including those used to represent fields of newer alpha/beta API versions in the older stable storage version, may be represented as annotations with the form something.alpha.kubernetes.io/name
or something.beta.kubernetes.io/name
(depending on our confidence in it). For example net.alpha.kubernetes.io/policy
might represent an experimental network policy field.
Other advice regarding use of labels, annotations, and other generic map keys by Kubernetes components and tools:
+ - Key names should be all lowercase, with words separated by dashes, such as desired-replicas
+ - Prefix the key with kubernetes.io/
or foo.kubernetes.io/
, preferably the latter if the label/annotation is specific to foo
+ - For instance, prefer service-account.kubernetes.io/name
over kubernetes.io/service-account.name
+ - Use annotations to store API extensions that the controller responsible for the resource doesn’t need to know about, experimental fields that aren’t intended to be generally used API fields, etc. Beware that annotations aren’t automatically handled by the API conversion machinery.
Before attempting a change to the API, you should familiarize yourself +with a number of existing API types and with the API +conventions. If creating a new API +type/resource, we also recommend that you first send a PR containing +just a proposal for the new API types, and that you initially target +the extensions API (pkg/apis/extensions).
+ +The Kubernetes API has two major components - the internal structures and +the versioned APIs. The versioned APIs are intended to be stable, while the +internal structures are implemented to best reflect the needs of the Kubernetes +code itself.
+ +What this means for API changes is that you have to be somewhat thoughtful in +how you approach changes, and that you have to touch a number of pieces to make +a complete change. This document aims to guide you through the process, though +not all API changes will need all of these steps.
+ +It is important to have a high level understanding of the API system used in +Kubernetes in order to navigate the rest of this document.
+ +As mentioned above, the internal representation of an API object is decoupled +from any one API version. This provides a lot of freedom to evolve the code, +but it requires robust infrastructure to convert between representations. There +are multiple steps in processing an API operation - even something as simple as +a GET involves a great deal of machinery.
+ +The conversion process is logically a “star” with the internal form at the +center. Every versioned API can be converted to the internal form (and +vice-versa), but versioned APIs do not convert to other versioned APIs directly. +This sounds like a heavy process, but in reality we do not intend to keep more +than a small number of versions alive at once. While all of the Kubernetes code +operates on the internal structures, they are always converted to a versioned +form before being written to storage (disk or etcd) or being sent over a wire. +Clients should consume and operate on the versioned APIs exclusively.
+ +To demonstrate the general process, here is a (hypothetical) example:
+ +Pod
object to /api/v7beta1/...
v7beta1.Pod
structurev7beta1.Pod
v7beta1.Pod
is converted to an api.Pod
structureapi.Pod
is validated, and any errors are returned to the userapi.Pod
is converted to a v6.Pod
(because v6 is the latest stable
+version)v6.Pod
is marshalled into JSON and written to etcdNow that we have the Pod
object stored, a user can GET that object in any
+supported api version. For example:
Pod
from /api/v5/...
v6.Pod
structurev6.Pod
v6.Pod
is converted to an api.Pod
structureapi.Pod
is converted to a v5.Pod
structurev5.Pod
is marshalled into JSON and sent to the userThe implication of this process is that API changes must be done carefully and +backward-compatibly.
+ +Before talking about how to make API changes, it is worthwhile to clarify what +we mean by API compatibility. An API change is considered backward-compatible +if it: + * adds new functionality that is not required for correct behavior (e.g., + does not add a new required field) + * does not change existing semantics, including: + * default values and behavior + * interpretation of existing API types, fields, and values + * which fields are required and which are not
+ +Put another way:
+ +If your change does not meet these criteria, it is not considered strictly +compatible.
+ +Let’s consider some examples. In a hypothetical API (assume we’re at version
+v6), the Frobber
struct looks something like this:
// API v6.
+type Frobber struct {
+ Height int `json:"height"`
+ Param string `json:"param"`
+}
+
+You want to add a new Width
field. It is generally safe to add new fields
+without changing the API version, so you can simply change it to:
// Still API v6.
+type Frobber struct {
+ Height int `json:"height"`
+ Width int `json:"width"`
+ Param string `json:"param"`
+}
+
+The onus is on you to define a sane default value for Width
such that rule #1
+above is true - API calls and stored objects that used to work must continue to
+work.
For your next change you want to allow multiple Param
values. You can not
+simply change Param string
to Params []string
(without creating a whole new
+API version) - that fails rules #1 and #2. You can instead do something like:
// Still API v6, but kind of clumsy.
+type Frobber struct {
+ Height int `json:"height"`
+ Width int `json:"width"`
+ Param string `json:"param"` // the first param
+ ExtraParams []string `json:"params"` // additional params
+}
+
+Now you can satisfy the rules: API calls that provide the old style Param
+will still work, while servers that don’t understand ExtraParams
can ignore
+it. This is somewhat unsatisfying as an API, but it is strictly compatible.
Part of the reason for versioning APIs and for using internal structs that are +distinct from any one version is to handle growth like this. The internal +representation can be implemented as:
+ +// Internal, soon to be v7beta1.
+type Frobber struct {
+ Height int
+ Width int
+ Params []string
+}
+
+The code that converts to/from versioned APIs can decode this into the somewhat +uglier (but compatible!) structures. Eventually, a new API version, let’s call +it v7beta1, will be forked and it can use the clean internal structure.
+ +We’ve seen how to satisfy rules #1 and #2. Rule #3 means that you can not
+extend one versioned API without also extending the others. For example, an
+API call might POST an object in API v7beta1 format, which uses the cleaner
+Params
field, but the API server might store that object in trusty old v6
+form (since v7beta1 is “beta”). When the user reads the object back in the
+v7beta1 API it would be unacceptable to have lost all but Params[0]
. This
+means that, even though it is ugly, a compatible change must be made to the v6
+API.
However, this is very challenging to do correctly. It often requires
+multiple representations of the same information in the same API resource, which
+need to be kept in sync in the event that either is changed. For example,
+let’s say you decide to rename a field within the same API version. In this case,
+you add units to height
and width
. You implement this by adding duplicate
+fields:
type Frobber struct {
+ Height *int `json:"height"`
+ Width *int `json:"width"`
+ HeightInInches *int `json:"heightInInches"`
+ WidthInInches *int `json:"widthInInches"`
+}
+
+You convert all of the fields to pointers in order to distinguish between unset and
+set to 0, and then set each corresponding field from the other in the defaulting
+pass (e.g., heightInInches
from height
, and vice versa), which runs just prior
+to conversion. That works fine when the user creates a resource from a hand-written
+configuration – clients can write either field and read either field, but what about
+creation or update from the output of GET, or update via PATCH (see
+In-place updates)?
+In this case, the two fields will conflict, because only one field would be updated
+in the case of an old client that was only aware of the old field (e.g., height
).
Say the client creates:
+ +{
+ "height": 10,
+ "width": 5
+}
+
+and GETs:
+ +{
+ "height": 10,
+ "heightInInches": 10,
+ "width": 5,
+ "widthInInches": 5
+}
+
+then PUTs back:
+ +{
+ "height": 13,
+ "heightInInches": 10,
+ "width": 5,
+ "widthInInches": 5
+}
+
+The update should not fail, because it would have worked before heightInInches
was added.
Therefore, when there are duplicate fields, the old field MUST take precedence +over the new, and the new field should be set to match by the server upon write. +A new client would be aware of the old field as well as the new, and so can ensure +that the old field is either unset or is set consistently with the new field. However, +older clients would be unaware of the new field. Please avoid introducing duplicate +fields due to the complexity they incur in the API.
+ +A new representation, even in a new API version, that is more expressive than an old one +breaks backward compatibility, since clients that only understood the old representation +would not be aware of the new representation nor its semantics. Examples of +proposals that have run into this challenge include generalized label +selectors and pod-level security +context.
+ +As another interesting example, enumerated values cause similar challenges. +Adding a new value to an enumerated set is not a compatible change. Clients +which assume they know how to handle all possible values of a given field will +not be able to handle the new values. However, removing value from an +enumerated set can be a compatible change, if handled properly (treat the +removed value as deprecated but allowed). This is actually a special case of +a new representation, discussed above.
+ +There are times when this might be OK, but mostly we want changes that +meet this definition. If you think you need to break compatibility, +you should talk to the Kubernetes team first.
+ +Breaking compatibility of a beta or stable API version, such as v1, is unacceptable. +Compatibility for experimental or alpha APIs is not strictly required, but +breaking compatibility should not be done lightly, as it disrupts all users of the +feature. Experimental APIs may be removed. Alpha and beta API versions may be deprecated +and eventually removed wholesale, as described in the versioning document. +Document incompatible changes across API versions under the conversion tips.
+ +If your change is going to be backward incompatible or might be a breaking change for API
+consumers, please send an announcement to kubernetes-dev@googlegroups.com
before
+the change gets in. If you are unsure, ask. Also make sure that the change gets documented in
+the release notes for the next release by labeling the PR with the “release-note” github label.
If you found that your change accidentally broke clients, it should be reverted.
+ +In short, the expected API evolution is as follows:
+* extensions/v1alpha1
->
+* newapigroup/v1alpha1
-> … -> newapigroup/v1alphaN
->
+* newapigroup/v1beta1
-> … -> newapigroup/v1betaN
->
+* newapigroup/v1
->
+* newapigroup/v2alpha1
-> …
While in extensions we have no obligation to move forward with the API at all and may delete or break it at any time.
+ +While in alpha we expect to move forward with it, but may break it.
+ +Once in beta we will preserve forward compatibility, but may introduce new versions and delete old ones.
+ +v1 must be backward-compatible for an extended length of time.
+ +For most changes, you will probably find it easiest to change the versioned +APIs first. This forces you to think about how to make your change in a +compatible way. Rather than doing each step in every version, it’s usually +easier to do each versioned API one at a time, or to do all of one version +before starting “all the rest”.
+ +The struct definitions for each API are in pkg/api/<version>/types.go
. Edit
+those files to reflect the change you want to make. Note that all types and non-inline
+fields in versioned APIs must be preceded by descriptive comments - these are used to generate
+documentation.
Optional fields should have the ,omitempty
json tag; fields are interpreted as being
+required otherwise.
If your change includes new fields for which you will need default values, you
+need to add cases to pkg/api/<version>/defaults.go
. Of course, since you
+have added code, you have to add a test: pkg/api/<version>/defaults_test.go
.
Do use pointers to scalars when you need to distinguish between an unset value
+and an automatic zero value. For example,
+PodSpec.TerminationGracePeriodSeconds
is defined as *int64
the go type
+definition. A zero value means 0 seconds, and a nil value asks the system to
+pick a default.
Don’t forget to run the tests!
+ +Given that you have not yet changed the internal structs, this might feel
+premature, and that’s because it is. You don’t yet have anything to convert to
+or from. We will revisit this in the “internal” section. If you’re doing this
+all in a different order (i.e. you started with the internal structs), then you
+should jump to that topic below. In the very rare case that you are making an
+incompatible change you might or might not want to do this now, but you will
+have to do more later. The files you want are
+pkg/api/<version>/conversion.go
and pkg/api/<version>/conversion_test.go
.
Note that the conversion machinery doesn’t generically handle conversion of values, +such as various kinds of field references and API constants. The client +library has custom conversion code for +field references. You also need to add a call to api.Scheme.AddFieldLabelConversionFunc +with a mapping function that understands supported translations.
+ +Now it is time to change the internal structs so your versioned changes can be +used.
+ +Similar to the versioned APIs, the definitions for the internal structs are in
+pkg/api/types.go
. Edit those files to reflect the change you want to make.
+Keep in mind that the internal structs must be able to express all of the
+versioned APIs.
Most changes made to the internal structs need some form of input validation.
+Validation is currently done on internal objects in
+pkg/api/validation/validation.go
. This validation is the one of the first
+opportunities we have to make a great user experience - good error messages and
+thorough validation help ensure that users are giving you what you expect and,
+when they don’t, that they know why and how to fix it. Think hard about the
+contents of string
fields, the bounds of int
fields and the
+requiredness/optionalness of fields.
Of course, code needs tests - pkg/api/validation/validation_test.go
.
At this point you have both the versioned API changes and the internal
+structure changes done. If there are any notable differences - field names,
+types, structural change in particular - you must add some logic to convert
+versioned APIs to and from the internal representation. If you see errors from
+the serialization_test
, it may indicate the need for explicit conversions.
Performance of conversions very heavily influence performance of apiserver. +Thus, we are auto-generating conversion functions that are much more efficient +than the generic ones (which are based on reflections and thus are highly +inefficient).
+ +The conversion code resides with each versioned API. There are two files:
+ - pkg/api/<version>/conversion.go
containing manually written conversion
+ functions
+ - pkg/api/<version>/conversion_generated.go
containing auto-generated
+ conversion functions
+ - pkg/apis/extensions/<version>/conversion.go
containing manually written
+ conversion functions
+ - pkg/apis/extensions/<version>/conversion_generated.go
containing
+ auto-generated conversion functions
Since auto-generated conversion functions are using manually written ones,
+those manually written should be named with a defined convention, i.e. a function
+converting type X in pkg a to type Y in pkg b, should be named:
+convert_a_X_To_b_Y
.
Also note that you can (and for efficiency reasons should) use auto-generated +conversion functions when writing your conversion functions.
+ +Once all the necessary manually written conversions are added, you need to +regenerate auto-generated ones. To regenerate them: + - run
+ +hack/update-generated-conversions.sh
+
+If running the above script is impossible due to compile errors, the easiest +workaround is to comment out the code causing errors and let the script to +regenerate it. If the auto-generated conversion methods are not used by the +manually-written ones, it’s fine to just remove the whole file and let the +generator to create it from scratch.
+ +Unsurprisingly, adding manually written conversion also requires you to add tests to
+pkg/api/<version>/conversion_test.go
.
At this point you have both the versioned API changes and the internal +structure changes done. You now need to generate code to handle deep copy +of your versioned api objects.
+ +The deep copy code resides with each versioned API:
+ - pkg/api/<version>/deep_copy_generated.go
containing auto-generated copy functions
+ - pkg/apis/extensions/<version>/deep_copy_generated.go
containing auto-generated copy functions
To regenerate them: + - run
+ +hack/update-generated-deep-copies.sh
+
+We are auto-generating code for marshaling and unmarshaling json representation +of api objects - this is to improve the overall system performance.
+ +The auto-generated code resides with each versioned API:
+ - pkg/api/<version>/types.generated.go
+ - pkg/apis/extensions/<version>/types.generated.go
To regenerate them: + - run
+ +hack/update-codecgen.sh
+
+This section is under construction, as we make the tooling completely generic.
+ +At the moment, you’ll have to make a new directory under pkg/apis/; copy the +directory structure from pkg/apis/extensions. Add the new group/version to all +of the hack/{verify,update}-generated-{deep-copy,conversions,swagger}.sh files +in the appropriate places–it should just require adding your new group/version +to a bash array. You will also need to make sure your new types are imported by +the generation commands (cmd/gendeepcopy/ & cmd/genconversion). These +instructions may not be complete and will be updated as we gain experience.
+ +Adding API groups outside of the pkg/apis/ directory is not currently supported, +but is clearly desirable. The deep copy & conversion generators need to work by +parsing go files instead of by reflection; then they will be easy to point at +arbitrary directories: see issue #13775.
+ +Part of our testing regimen for APIs is to “fuzz” (fill with random values) API
+objects and then convert them to and from the different API versions. This is
+a great way of exposing places where you lost information or made bad
+assumptions. If you have added any fields which need very careful formatting
+(the test does not run validation) or if you have made assumptions such as
+“this slice will always have at least 1 element”, you may get an error or even
+a panic from the serialization_test
. If so, look at the diff it produces (or
+the backtrace in case of a panic) and figure out what you forgot. Encode that
+into the fuzzer’s custom fuzz functions. Hint: if you added defaults for a field,
+that field will need to have a custom fuzz function that ensures that the field is
+fuzzed to a non-empty value.
The fuzzer can be found in pkg/api/testing/fuzzer.go
.
VERY VERY rarely is this needed, but when it hits, it hurts. In some rare +cases we end up with objects (e.g. resource quantities) that have morally +equivalent values with different bitwise representations (e.g. value 10 with a +base-2 formatter is the same as value 0 with a base-10 formatter). The only way +Go knows how to do deep-equality is through field-by-field bitwise comparisons. +This is a problem for us.
+ +The first thing you should do is try not to do that. If you really can’t avoid
+this, I’d like to introduce you to our semantic DeepEqual routine. It supports
+custom overrides for specific types - you can find that in pkg/api/helpers.go
.
There’s one other time when you might have to touch this: unexported fields.
+You see, while Go’s reflect
package is allowed to touch unexported fields, us
+mere mortals are not - this includes semantic DeepEqual. Fortunately, most of
+our API objects are “dumb structs” all the way down - all fields are exported
+(start with a capital letter) and there are no unexported fields. But sometimes
+you want to include an object in our API that does have unexported fields
+somewhere in it (for example, time.Time
has unexported fields). If this hits
+you, you may have to touch the semantic DeepEqual customization functions.
Now you have the API all changed - go implement whatever it is that you’re +doing!
+ +Check out the E2E docs for detailed information about how to write end-to-end +tests for your feature.
+ +At last, your change is done, all unit tests pass, e2e passes, you’re done,
+right? Actually, no. You just changed the API. If you are touching an
+existing facet of the API, you have to try really hard to make sure that
+all the examples and docs are updated. There’s no easy way to do this, due
+in part to JSON and YAML silently dropping unknown fields. You’re clever -
+you’ll figure it out. Put grep
or ack
to good use.
If you added functionality, you should consider documenting it and/or writing +an example to illustrate your change.
+ +Make sure you update the swagger API spec by running:
+ +hack/update-swagger-spec.sh
+
+The API spec changes should be in a commit separate from your other changes.
+ +TODO(smarterclayton): write this.
+ +New feature development proceeds through a series of stages of increasing maturity:
+ +alpha
(e.g. v1alpha1
)beta
(e.g. v2beta3
)v1beta1
and v1beta2
; or v1beta2
and
+v1
) for at least one minor release cycle (typically 3 months) so that users have enough time
+to upgrade and migrate objectsvX
where X
is an integer (e.g. v1
)Kubernetes uses a variety of automated tools in an attempt to relieve developers of repeptitive, low +brain power work. This document attempts to describe these processes.
+ +In an effort to + * reduce load on core developers + * maintain e2e stability + * load test githubs label feature
+ +We have added an automated submit-queue +for kubernetes.
+ +The submit-queue does the following:
+ +for _, pr := range readyToMergePRs() {
+ if testsAreStable() {
+ mergePR(pr)
+ }
+}
+
+The status of the submit-queue is online.
+ +A PR is considered “ready for merging” if it matches the following:
+ * it has the lgtm
label, and that lgtm
is newer than the latest commit
+ * it has passed the cla pre-submit and has the cla:yes
label
+ * it has passed the travis and shippable pre-submit tests
+ * one (or all) of
+ * its author is in kubernetes/contrib/submit-queue/whitelist.txt
+ * its author is in contributors.txt via the github API.
+ * the PR has the ok-to-merge
label
+ * One (or both of)
+ * it has passed the Jenkins e2e test
+ * it has the e2e-not-required
label
Note that the combined whitelist/committer list is available at submit-queue.k8s.io
+ +Merges only occur when the critical builds
(Jenkins e2e for gce, gke, scalability, upgrade) are passing.
+We’re open to including more builds here, let us know…
Merges are serialized, so only a single PR is merged at a time, to ensure against races.
+ +If the PR has the e2e-not-required
label, it is simply merged.
+If the PR does not have this label, e2e tests are re-run, if these new tests pass, the PR is merged.
If e2e flakes or is currently buggy, the PR will not be merged, but it will be re-run on the following +pass.
+ +We also run a github “munger”
+ +This runs repeatedly over github pulls and issues and runs modular “mungers” similar to “mungedocs”
+ +Currently this runs:
+ * blunderbuss - Tries to automatically find an owner for a PR without an owner, uses mapping file here:
+ https://github.com/kubernetes/contrib/blob/master/mungegithub/blunderbuss.yml
+ * needs-rebase - Adds needs-rebase
to PRs that aren’t currently mergeable, and removes it from those that are.
+ * size - Adds size/xs
- size/xxl
labels to PRs
+ * ok-to-test - Adds the ok-to-test
message to PRs that have an lgtm
but the e2e-builder would otherwise not test due to whitelist
+ * ping-ci - Attempts to ping the ci systems (Travis/Shippable) if they are missing from a PR.
+ * lgtm-after-commit - Removes the lgtm
label from PRs where there are commits that are newer than the lgtm
label
In the works:
+ * issue-detector - machine learning for determining if an issue that has been filed is a support
issue, bug
or feature
Please feel free to unleash your creativity on this tool, send us new mungers that you think will help support the Kubernetes development process.
+ +We also run a robotic PR builder that attempts to run e2e tests for each PR.
+ +Before a PR from an unknown user is run, the PR builder bot (k8s-bot
) asks to a message from a
+contributor that a PR is “ok to test”, the contributor replies with that message. Contributors can also
+add users to the whitelist by replying with the message “add to whitelist” (“please” is optional, but
+remember to treat your robots with kindness…)
If a PR is approved for testing, and tests either haven’t run, or need to be re-run, you can ask the
+PR builder to re-run the tests. To do this, reply to the PR with a message that begins with @k8s-bot test this
, this should trigger a re-build/re-test.
Right now you have to ask a contributor (this may be you!) to re-run the test with “@k8s-bot test this”
+ +Right now the easiest way is to close and then immediately re-open the PR.
+ + + + + + + + + +This document explains cherry picks are managed on release branches within the +Kubernetes projects.
+ +Any contributor can propose a cherry pick of any pull request, like so:
+ +hack/cherry_pick_pull.sh upstream/release-3.14 98765
+
+This will walk you through the steps to propose an automated cherry pick of pull
+ #98765 for remote branch upstream/release-3.14
.
Cherry pick pull requests are reviewed differently than normal pull requests. In +particular, they may be self-merged by the release branch owner without fanfare, +in the case the release branch owner knows the cherry pick was already +requested - this should not be the norm, but it may happen.
+ +Contributor License Agreements is considered implicit +for all code within cherry-pick pull requests, unless there is a large +conflict.
+ +Now that we’ve structured cherry picks as PRs, searching for all cherry-picks +against a release is a GitHub query: For example, +this query is all of the v0.21.x cherry-picks
+ + + + + + + + + +See github issues with the following labels: +* area/app-config-deployment +* component/kubectl +* component/clientlib
+ + + + + + + + + +Note: Libraries provided by outside parties are supported by their authors, not the core Kubernetes team
+ +Code conventions
+ - Bash
+ - https://google-styleguide.googlecode.com/svn/trunk/shell.xml
+ - Ensure that build, release, test, and cluster-management scripts run on OS X
+ - Go
+ - Ensure your code passes the presubmit checks
+ - Go Code Review Comments
+ - Effective Go
+ - Comment your code.
+ - Go’s commenting conventions
+ - If reviewers ask questions about why the code is the way it is, that’s a sign that comments might be helpful.
+ - Command-line flags should use dashes, not underscores
+ - Naming
+ - Please consider package name when selecting an interface name, and avoid redundancy.
+ - e.g.: storage.Interface
is better than storage.StorageInterface
.
+ - Do not use uppercase characters, underscores, or dashes in package names.
+ - Please consider parent directory name when choosing a package name.
+ - so pkg/controllers/autoscaler/foo.go should say package autoscaler
not package autoscalercontroller
.
+ - Unless there’s a good reason, the package foo
line should match the name of the directory in which the .go file exists.
+ - Importers can use a different name if they need to disambiguate.
+ - Locks should be called lock
and should never be embedded (always lock sync.Mutex
). When multiple locks are present, give each lock a distinct name following Go conventions - stateLock
, mapLock
etc.
+ - API conventions
+ - API changes
+ - API conventions
+ - Kubectl conventions
+ - Logging conventions
Testing conventions + - All new packages and most new significant functionality must come with unit tests + - Table-driven tests are preferred for testing multiple scenarios/inputs; for example, see TestNamespaceAuthorization + - Significant features should come with integration (test/integration) and/or end-to-end (test/e2e) tests + - Including new kubectl commands and major features of existing commands + - Unit tests must pass on OS X and Windows platforms - if you use Linux specific features, your test case must either be skipped on windows or compiled out (skipped is better when running Linux specific commands, compiled out is required when your code does not compile on Windows).
+ +Directory and file conventions + - Avoid package sprawl. Find an appropriate subdirectory for new packages. (See #4851 for discussion.) + - Libraries with no more appropriate home belong in new package subdirectories of pkg/util + - Avoid general utility packages. Packages called “util” are suspect. Instead, derive a name that describes your desired function. For example, the utility functions dealing with waiting for operations are in the “wait” package and include functionality like Poll. So the full name is wait.Poll + - Go source files and directories use underscores, not dashes + - Package directories should generally avoid using separators as much as possible (when packages are multiple words, they usually should be in nested subdirectories). + - Document directories and filenames should use dashes rather than underscores + - Contrived examples that illustrate system features belong in /docs/user-guide or /docs/admin, depending on whether it is a feature primarily intended for users that deploy applications or cluster administrators, respectively. Actual application examples belong in /examples. + - Examples should also illustrate best practices for using the system + - Third-party code + - Third-party Go code is managed using Godeps + - Other third-party code belongs in /third_party + - Third-party code must include licenses + - This includes modified third-party code and excerpts, as well
+ +Coding advice + - Go + - Go landmines
+ + + + + + + + + +Kubernetes is open source, but many of the people working on it do so as their day job. In order to avoid forcing people to be “at work” effectively 24/7, we want to establish some semi-formal protocols around development. Hopefully these rules make things go more smoothly. If you find that this is not the case, please complain loudly.
+ +First and foremost: as a potential contributor, your changes and ideas are welcome at any hour of the day or night, weekdays, weekends, and holidays. Please do not ever hesitate to ask a question or send a PR.
+ +All changes must be code reviewed. For non-maintainers this is obvious, since you can’t commit anyway. But even for maintainers, we want all changes to get at least one review, preferably (for non-trivial changes obligatorily) from someone who knows the areas the change touches. For non-trivial changes we may want two reviewers. The primary reviewer will make this decision and nominate a second reviewer, if needed. Except for trivial changes, PRs should not be committed until relevant parties (e.g. owners of the subsystem affected by the PR) have had a reasonable chance to look at PR in their local business hours.
+ +Most PRs will find reviewers organically. If a maintainer intends to be the primary reviewer of a PR they should set themselves as the assignee on GitHub and say so in a reply to the PR. Only the primary reviewer of a change should actually do the merge, except in rare cases (e.g. they are unavailable in a reasonable timeframe).
+ +If a PR has gone 2 work days without an owner emerging, please poke the PR thread and ask for a reviewer to be assigned.
+ +Except for rare cases, such as trivial changes (e.g. typos, comments) or emergencies (e.g. broken builds), maintainers should not merge their own changes.
+ +Expect reviewers to request that you avoid common go style mistakes in your PRs.
+ +Maintainers can assign reviews to other maintainers, when appropriate. The assignee becomes the shepherd for that PR and is responsible for merging the PR once they are satisfied with it or else closing it. The assignee might request reviews from non-maintainers.
+ +Maintainers will do merges of appropriately reviewed-and-approved changes during their local “business hours” (typically 7:00 am Monday to 5:00 pm (17:00h) Friday). PRs that arrive over the weekend or on holidays will only be merged if there is a very good reason for it and if the code review requirements have been met. Concretely this means that nobody should merge changes immediately before going to bed for the night.
+ +There may be discussion an even approvals granted outside of the above hours, but merges will generally be deferred.
+ +If a PR is considered complex or controversial, the merge of that PR should be delayed to give all interested parties in all timezones the opportunity to provide feedback. Concretely, this means that such PRs should be held for 24 +hours before merging. Of course “complex” and “controversial” are left to the judgment of the people involved, but we trust that part of being a committer is the judgment required to evaluate such things honestly, and not be +motivated by your desire (or your cube-mate’s desire) to get their code merged. Also see “Holds” below, any reviewer can issue a “hold” to indicate that the PR is in fact complicated or complex and deserves further review.
+ +PRs that are incorrectly judged to be merge-able, may be reverted and subject to re-review, if subsequent reviewers believe that they in fact are controversial or complex.
+ +Any maintainer or core contributor who wants to review a PR but does not have time immediately may put a hold on a PR simply by saying so on the PR discussion and offering an ETA measured in single-digit days at most. Any PR that has a hold shall not be merged until the person who requested the hold acks the review, withdraws their hold, or is overruled by a preponderance of maintainers.
+ + + + + + + + + +Running kubernetes with Vagrant (and VirtualBox) is an easy way to run/test/develop on your local machine (Linux, Mac OS X).
+ +By default, the Vagrant setup will create a single master VM (called kubernetes-master) and one node (called kubernetes-minion-1). Each VM will take 1 GB, so make sure you have at least 2GB to 4GB of free memory (plus appropriate free disk space). To start your local cluster, open a shell and run:
+ +cd kubernetes
+
+export KUBERNETES_PROVIDER=vagrant
+./cluster/kube-up.sh
+
+The KUBERNETES_PROVIDER
environment variable tells all of the various cluster management scripts which variant to use. If you forget to set this, the assumption is you are running on Google Compute Engine.
If you installed more than one Vagrant provider, Kubernetes will usually pick the appropriate one. However, you can override which one Kubernetes will use by setting the VAGRANT_DEFAULT_PROVIDER
environment variable:
export VAGRANT_DEFAULT_PROVIDER=parallels
+export KUBERNETES_PROVIDER=vagrant
+./cluster/kube-up.sh
+
+Vagrant will provision each machine in the cluster with all the necessary components to run Kubernetes. The initial setup can take a few minutes to complete on each machine.
+ +By default, each VM in the cluster is running Fedora, and all of the Kubernetes services are installed into systemd.
+ +To access the master or any node:
+ +vagrant ssh master
+vagrant ssh minion-1
+
+If you are running more than one nodes, you can access the others by:
+ +vagrant ssh minion-2
+vagrant ssh minion-3
+
+To view the service status and/or logs on the kubernetes-master:
+ +$ vagrant ssh master
+[vagrant@kubernetes-master ~] $ sudo systemctl status kube-apiserver
+[vagrant@kubernetes-master ~] $ sudo journalctl -r -u kube-apiserver
+
+[vagrant@kubernetes-master ~] $ sudo systemctl status kube-controller-manager
+[vagrant@kubernetes-master ~] $ sudo journalctl -r -u kube-controller-manager
+
+[vagrant@kubernetes-master ~] $ sudo systemctl status etcd
+[vagrant@kubernetes-master ~] $ sudo systemctl status nginx
+
+To view the services on any of the nodes:
+ +$ vagrant ssh minion-1
+[vagrant@kubernetes-minion-1] $ sudo systemctl status docker
+[vagrant@kubernetes-minion-1] $ sudo journalctl -r -u docker
+[vagrant@kubernetes-minion-1] $ sudo systemctl status kubelet
+[vagrant@kubernetes-minion-1] $ sudo journalctl -r -u kubelet
+
+With your Kubernetes cluster up, you can manage the nodes in your cluster with the regular Vagrant commands.
+ +To push updates to new Kubernetes code after making source changes:
+ +./cluster/kube-push.sh
+
+To stop and then restart the cluster:
+ +vagrant halt
+./cluster/kube-up.sh
+
+To destroy the cluster:
+ +vagrant destroy
+
+Once your Vagrant machines are up and provisioned, the first thing to do is to check that you can use the kubectl.sh
script.
You may need to build the binaries first, you can do this with make
$ ./cluster/kubectl.sh get nodes
+
+NAME LABELS STATUS
+kubernetes-minion-0whl kubernetes.io/hostname=kubernetes-minion-0whl Ready
+kubernetes-minion-4jdf kubernetes.io/hostname=kubernetes-minion-4jdf Ready
+kubernetes-minion-epbe kubernetes.io/hostname=kubernetes-minion-epbe Ready
+
+kube-*
scripts.Alternatively to using the vagrant commands, you can also use the cluster/kube-*.sh
scripts to interact with the vagrant based provider just like any other hosting platform for kubernetes.
All of these commands assume you have set KUBERNETES_PROVIDER
appropriately:
export KUBERNETES_PROVIDER=vagrant
+
+Bring up a vagrant cluster
+ +./cluster/kube-up.sh
+
+Destroy the vagrant cluster
+ +./cluster/kube-down.sh
+
+Update the vagrant cluster after you make changes (only works when building your own releases locally):
+ +./cluster/kube-push.sh
+
+Interact with the cluster
+ +./cluster/kubectl.sh
+
+When using the vagrant provider in Kubernetes, the cluster/kubectl.sh
script will cache your credentials in a ~/.kubernetes_vagrant_auth
file so you will not be prompted for them in the future.
$ cat ~/.kubernetes_vagrant_auth
+{ "User": "vagrant",
+ "Password": "vagrant"
+ "CAFile": "/home/k8s_user/.kubernetes.vagrant.ca.crt",
+ "CertFile": "/home/k8s_user/.kubecfg.vagrant.crt",
+ "KeyFile": "/home/k8s_user/.kubecfg.vagrant.key"
+}
+
+You should now be set to use the cluster/kubectl.sh
script. For example try to list the nodes that you have started with:
./cluster/kubectl.sh get nodes
+
+Your cluster is running, you can list the nodes in your cluster:
+ +$ ./cluster/kubectl.sh get nodes
+
+NAME LABELS STATUS
+kubernetes-minion-0whl kubernetes.io/hostname=kubernetes-minion-0whl Ready
+kubernetes-minion-4jdf kubernetes.io/hostname=kubernetes-minion-4jdf Ready
+kubernetes-minion-epbe kubernetes.io/hostname=kubernetes-minion-epbe Ready
+
+Now start running some containers!
+ +You can now use any of the cluster/kube-*.sh commands to interact with your VM machines. +Before starting a container there will be no pods, services and replication controllers.
+ +$ cluster/kubectl.sh get pods
+NAME READY STATUS RESTARTS AGE
+
+$ cluster/kubectl.sh get services
+NAME LABELS SELECTOR IP(S) PORT(S)
+
+$ cluster/kubectl.sh get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+
+Start a container running nginx with a replication controller and three replicas
+ +$ cluster/kubectl.sh run my-nginx --image=nginx --replicas=3 --port=80
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+my-nginx my-nginx nginx run=my-nginx 3
+
+When listing the pods, you will see that three containers have been started and are in Waiting state:
+ +$ cluster/kubectl.sh get pods
+NAME READY STATUS RESTARTS AGE
+my-nginx-389da 1/1 Waiting 0 33s
+my-nginx-kqdjk 1/1 Waiting 0 33s
+my-nginx-nyj3x 1/1 Waiting 0 33s
+
+You need to wait for the provisioning to complete, you can monitor the minions by doing:
+ +$ sudo salt '*minion-1' cmd.run 'docker images'
+kubernetes-minion-1:
+ REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE
+ <none> <none> 96864a7d2df3 26 hours ago 204.4 MB
+ kubernetes/pause latest 6c4579af347b 8 weeks ago 239.8 kB
+
+Once the docker image for nginx has been downloaded, the container will start and you can list it:
+ +$ sudo salt '*minion-1' cmd.run 'docker ps'
+kubernetes-minion-1:
+ CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
+ dbe79bf6e25b nginx:latest "nginx" 21 seconds ago Up 19 seconds k8s--mynginx.8c5b8a3a--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1.etcd--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1--fcfa837f
+ fa0e29c94501 kubernetes/pause:latest "/pause" 8 minutes ago Up 8 minutes 0.0.0.0:8080->80/tcp k8s--net.a90e7ce4--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1.etcd--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1--baf5b21b
+
+Going back to listing the pods, services and replicationcontrollers, you now have:
+ +$ cluster/kubectl.sh get pods
+NAME READY STATUS RESTARTS AGE
+my-nginx-389da 1/1 Running 0 33s
+my-nginx-kqdjk 1/1 Running 0 33s
+my-nginx-nyj3x 1/1 Running 0 33s
+
+$ cluster/kubectl.sh get services
+NAME LABELS SELECTOR IP(S) PORT(S)
+
+$ cluster/kubectl.sh get rc
+NAME IMAGE(S) SELECTOR REPLICAS
+my-nginx nginx run=my-nginx 3
+
+We did not start any services, hence there are none listed. But we see three replicas displayed properly. +Check the guestbook application to learn how to create a service. +You can already play with scaling the replicas with:
+ +$ ./cluster/kubectl.sh scale rc my-nginx --replicas=2
+$ ./cluster/kubectl.sh get pods
+NAME READY STATUS RESTARTS AGE
+my-nginx-kqdjk 1/1 Running 0 13m
+my-nginx-nyj3x 1/1 Running 0 13m
+
+Congratulations!
+ +The following will run all of the end-to-end testing scenarios assuming you set your environment in cluster/kube-env.sh
:
NUM_MINIONS=3 hack/e2e-test.sh
+
+By default the Vagrantfile will download the box from S3. You can change this (and cache the box locally) by providing a name and an alternate URL when calling kube-up.sh
export KUBERNETES_BOX_NAME=choose_your_own_name_for_your_kuber_box
+export KUBERNETES_BOX_URL=path_of_your_kuber_box
+export KUBERNETES_PROVIDER=vagrant
+./cluster/kube-up.sh
+
+You probably have an incorrect ~/.kubernetes_vagrant_auth file for the cluster you are attempting to contact.
+ +rm ~/.kubernetes_vagrant_auth
+
+After using kubectl.sh make sure that the correct credentials are set:
+ +$ cat ~/.kubernetes_vagrant_auth
+{
+ "User": "vagrant",
+ "Password": "vagrant"
+}
+
+If this is your first time creating the cluster, the kubelet on each node schedules a number of docker pull requests to fetch prerequisite images. This can take some time and as a result may delay your initial pod getting provisioned.
+ +Are you sure there was no build error? After running $ vagrant provision
, scroll up and ensure that each Salt state was completed successfully on each box in the cluster.
+It’s very likely you see a build error due to an error in your source files!
Are you sure you built a release first? Did you install net-tools
? For more clues, login to one of the nodes (vagrant ssh minion-1
) and inspect the salt minion log (sudo cat /var/log/salt/minion
).
You can control the number of nodes that are instantiated via the environment variable NUM_MINIONS
on your host machine. If you plan to work with replicas, we strongly encourage you to work with enough nodes to satisfy your largest intended replica size. If you do not plan to work with replicas, you can save some system resources by running with a single node. You do this, by setting NUM_MINIONS
to 1 like so:
export NUM_MINIONS=1
+
+You can control the memory allotted to virtual machines with the KUBERNETES_MEMORY
environment variable.
+Just set it to the number of megabytes you would like the machines to have. For example:
export KUBERNETES_MEMORY=2048
+
+If you need more granular control, you can set the amount of memory for the master and nodes independently. For example:
+ +export KUBERNETES_MASTER_MEMORY=1536
+export KUBERNETES_MINION_MEMORY=2048
+
+vagrant suspend
seems to mess up the network. It’s not supported at this time.
Official releases are built in Docker containers. Details are here. You can do simple builds and development with just a local Docker installation. If want to build go locally outside of docker, please continue below.
+ +Kubernetes is written in Go programming language. If you haven’t set up Go development environment, please follow this instruction to install go tool and set up GOPATH. Ensure your version of Go is at least 1.3.
+ +Below, we outline one of the more common git workflows that core developers use. Other git workflows are also valid.
+ +The commands below require that you have $GOPATH set ($GOPATH docs). We highly recommend you put Kubernetes’ code into your GOPATH. Note: the commands below will not work if there is more than one directory in your $GOPATH
.
mkdir -p $GOPATH/src/k8s.io
+cd $GOPATH/src/k8s.io
+# Replace "$YOUR_GITHUB_USERNAME" below with your github username
+git clone https://github.com/$YOUR_GITHUB_USERNAME/kubernetes.git
+cd kubernetes
+git remote add upstream 'https://github.com/kubernetes/kubernetes.git'
+
+git checkout -b myfeature
+# Make your code changes
+
+git fetch upstream
+git rebase upstream/master
+
+Note: If you have write access to the main repository at github.com/kubernetes/kubernetes, you should modify your git configuration so that you can’t accidentally push to upstream:
+ +git remote set-url --push upstream no_push
+
+git commit
+git push -f origin myfeature
+
+Upon merge, all git commits should represent meaningful milestones or units of +work. Use commits to add clarity to the development and review process.
+ +Before merging a PR, squash any “fix review feedback”, “typo”, and “rebased” +sorts of commits. It is not imperative that every commit in a PR compile and +pass tests independently, but it is worth striving for. For mass automated +fixups (e.g. automated doc formatting), use one or more commits for the +changes to tooling and a final commit to apply the fixup en masse. This makes +reviews much easier.
+ +See Faster Reviews for more details.
+ +Kubernetes uses godep to manage dependencies. It is not strictly required for building Kubernetes but it is required when managing dependencies under the Godeps/ tree, and is required by a number of the build and test scripts. Please make sure that godep
is installed and in your $PATH
.
There are many ways to build and host go binaries. Here is an easy way to get utilities like godep
installed:
1) Ensure that mercurial is installed on your system. (some of godep’s dependencies use the mercurial
+source control system). Use apt-get install mercurial
or yum install mercurial
on Linux, or brew.sh on OS X, or download
+directly from mercurial.
2) Create a new GOPATH for your tools and install godep:
+ +export GOPATH=$HOME/go-tools
+mkdir -p $GOPATH
+go get github.com/tools/godep
+
+3) Add $GOPATH/bin to your path. Typically you’d add this to your ~/.profile:
+ +export GOPATH=$HOME/go-tools
+export PATH=$PATH:$GOPATH/bin
+
+Here’s a quick walkthrough of one way to use godeps to add or update a Kubernetes dependency into Godeps/_workspace. For more details, please see the instructions in godep’s documentation.
+ +1) Devote a directory to this endeavor:
+ +Devoting a separate directory is not required, but it is helpful to separate dependency updates from other changes.
+ +export KPATH=$HOME/code/kubernetes
+mkdir -p $KPATH/src/k8s.io/kubernetes
+cd $KPATH/src/k8s.io/kubernetes
+git clone https://path/to/your/fork .
+# Or copy your existing local repo here. IMPORTANT: making a symlink doesn't work.
+
+2) Set up your GOPATH.
+ +# Option A: this will let your builds see packages that exist elsewhere on your system.
+export GOPATH=$KPATH:$GOPATH
+# Option B: This will *not* let your local builds see packages that exist elsewhere on your system.
+export GOPATH=$KPATH
+# Option B is recommended if you're going to mess with the dependencies.
+
+3) Populate your new GOPATH.
+ +cd $KPATH/src/k8s.io/kubernetes
+godep restore
+
+4) Next, you can either add a new dependency or update an existing one.
+ +# To add a new dependency, do:
+cd $KPATH/src/k8s.io/kubernetes
+go get path/to/dependency
+# Change code in Kubernetes to use the dependency.
+godep save ./...
+
+# To update an existing dependency, do:
+cd $KPATH/src/k8s.io/kubernetes
+go get -u path/to/dependency
+# Change code in Kubernetes accordingly if necessary.
+godep update path/to/dependency/...
+
+If go get -u path/to/dependency
fails with compilation errors, instead try go get -d -u path/to/dependency
+to fetch the dependencies without compiling them. This can happen when updating the cadvisor dependency.
5) Before sending your PR, it’s a good idea to sanity check that your Godeps.json file is ok by running hack/verify-godeps.sh
+ +If hack/verify-godeps.sh fails after a godep update
, it is possible that a transitive dependency was added or removed but not
+updated by godeps. It then may be necessary to perform a godep save ./...
to pick up the transitive dependency changes.
It is sometimes expedient to manually fix the /Godeps/godeps.json file to minimize the changes.
+ +Please send dependency updates in separate commits within your PR, for easier reviewing.
+ +Before committing any changes, please link/copy these hooks into your .git +directory. This will keep you from accidentally committing non-gofmt’d go code.
+ +cd kubernetes/.git/hooks/
+ln -s ../../hooks/pre-commit .
+
+cd kubernetes
+hack/test-go.sh
+
+Alternatively, you could also run:
+ +cd kubernetes
+godep go test ./...
+
+If you only want to run unit tests in one package, you could run godep go test
under the package directory. For example, the following commands will run all unit tests in package kubelet:
$ cd kubernetes # step into the kubernetes directory.
+$ cd pkg/kubelet
+$ godep go test
+# some output from unit tests
+PASS
+ok k8s.io/kubernetes/pkg/kubelet 0.317s
+
+Currently, collecting coverage is only supported for the Go unit tests.
+ +To run all unit tests and generate an HTML coverage report, run the following:
+ +cd kubernetes
+KUBE_COVER=y hack/test-go.sh
+
+At the end of the run, an the HTML report will be generated with the path printed to stdout.
+ +To run tests and collect coverage in only one package, pass its relative path under the kubernetes
directory as an argument, for example:
cd kubernetes
+KUBE_COVER=y hack/test-go.sh pkg/kubectl
+
+Multiple arguments can be passed, in which case the coverage results will be combined for all tests run.
+ +Coverage results for the project can also be viewed on Coveralls, and are continuously updated as commits are merged. Additionally, all pull requests which spawn a Travis build will report unit test coverage results to Coveralls. Coverage reports from before the Kubernetes Github organization was created can be found here.
+ +You need an etcd in your path, please make sure it is installed and in your $PATH
.
cd kubernetes
+hack/test-integration.sh
+
+You can run an end-to-end test which will bring up a master and two nodes, perform some tests, and then tear everything down. Make sure you have followed the getting started steps for your chosen cloud platform (which might involve changing the KUBERNETES_PROVIDER
environment variable to something other than “gce”.
cd kubernetes
+hack/e2e-test.sh
+
+Pressing control-C should result in an orderly shutdown but if something goes wrong and you still have some VMs running you can force a cleanup with this command:
+ +go run hack/e2e.go --down
+
+See the flag definitions in hack/e2e.go
for more options, such as reusing an existing cluster, here is an overview:
# Build binaries for testing
+go run hack/e2e.go --build
+
+# Create a fresh cluster. Deletes a cluster first, if it exists
+go run hack/e2e.go --up
+
+# Create a fresh cluster at a specific release version.
+go run hack/e2e.go --up --version=0.7.0
+
+# Test if a cluster is up.
+go run hack/e2e.go --isup
+
+# Push code to an existing cluster
+go run hack/e2e.go --push
+
+# Push to an existing cluster, or bring up a cluster if it's down.
+go run hack/e2e.go --pushup
+
+# Run all tests
+go run hack/e2e.go --test
+
+# Run tests matching the regex "Pods.*env"
+go run hack/e2e.go -v -test --test_args="--ginkgo.focus=Pods.*env"
+
+# Alternately, if you have the e2e cluster up and no desire to see the event stream, you can run ginkgo-e2e.sh directly:
+hack/ginkgo-e2e.sh --ginkgo.focus=Pods.*env
+
+# Flags can be combined, and their actions will take place in this order:
+# -build, -push|-up|-pushup, -test|-tests=..., -down
+# e.g.:
+go run hack/e2e.go -build -pushup -test -down
+
+# -v (verbose) can be added if you want streaming output instead of only
+# seeing the output of failed commands.
+
+# -ctl can be used to quickly call kubectl against your e2e cluster. Useful for
+# cleaning up after a failed test or viewing logs. Use -v to avoid suppressing
+# kubectl output.
+go run hack/e2e.go -v -ctl='get events'
+go run hack/e2e.go -v -ctl='delete pod foobar'
+
+End-to-end testing, as described above, is for development +distributions. A conformance test is used on +a versioned distro.
+ +The conformance test runs a subset of the e2e-tests against a manually-created cluster. It does not +require support for up/push/down and other operations. To run a conformance test, you need to know the +IP of the master for your cluster and the authorization arguments to use. The conformance test is +intended to run against a cluster at a specific binary release of Kubernetes. +See conformance-test.sh.
+ +hack/update-generated-docs.sh
+
+The end-2-end tests for kubernetes provide a mechanism to test behavior of the system, and to ensure end user operations match developer specifications. In distributed systems it is not uncommon that a minor change may pass all unit tests, but cause unforseen changes at the system level. Thus, the primary objectives of the end-2-end tests are to ensure a consistent and reliable behavior of the kubernetes code base, and to catch bugs early.
+ +The end-2-end tests in kubernetes are built atop of [ginkgo] (http://onsi.github.io/ginkgo/) and [gomega] (http://onsi.github.io/gomega/). There are a host of features that this BDD testing framework provides, and it is recommended that the developer read the documentation prior to diving into the tests.
+ +The purpose of this document is to serve as a primer for developers who are looking to execute, or add tests, using a local development environment.
+ +NOTE: The tests have an array of options. For simplicity, the examples will focus on leveraging the tests on a local cluster using sudo ./hack/local-up-cluster.sh
The tests are built into a single binary which can be run against any deployed kubernetes system. To build the tests, navigate to your source directory and execute:
+ +$ make all
The output for the end-2-end tests will be a single binary called e2e.test
under the default output directory, which is typically _output/local/bin/linux/amd64/
. Within the repository there are scripts that are provided under the ./hack
directory that are helpful for automation, but may not apply for a local development purposes. Instead, we recommend familiarizing yourself with the executable options. To obtain the full list of options, run the following:
$ ./e2e.test --help
For the purposes of brevity, we will look at a subset of the options, which are listed below:
+ +
+-ginkgo.dryRun=false: If set, ginkgo will walk the test hierarchy without actually running anything. Best paired with -v.
+-ginkgo.failFast=false: If set, ginkgo will stop running a test suite after a failure occurs.
+-ginkgo.failOnPending=false: If set, ginkgo will mark the test suite as failed if any specs are pending.
+-ginkgo.focus="": If set, ginkgo will only run specs that match this regular expression.
+-ginkgo.skip="": If set, ginkgo will only run specs that do not match this regular expression.
+-ginkgo.trace=false: If set, default reporter prints out the full stack trace when a failure occurs
+-ginkgo.v=false: If set, default reporter print out all specs as they begin.
+-host="": The host, or api-server, to connect to
+-kubeconfig="": Path to kubeconfig containing embedded authinfo.
+-prom-push-gateway="": The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.
+-provider="": The name of the Kubernetes provider (gce, gke, local, vagrant, etc.)
+-repo-root="../../": Root directory of kubernetes repository, for finding test files.
+
+
+
+Prior to running the tests, it is recommended that you first create a simple auth file in your home directory, e.g. $HOME/.kubernetes_auth
, with the following:
+{
+ "User": "root",
+ "Password": ""
+}
+
+
+
+Next, you will need a cluster that you can test against. As mentioned earlier, you will want to execute sudo ./hack/local-up-cluster.sh
. To get a sense of what tests exist, you may want to run:
e2e.test --host="127.0.0.1:8080" --provider="local" --ginkgo.v=true -ginkgo.dryRun=true --kubeconfig="$HOME/.kubernetes_auth" --repo-root="$KUBERNETES_SRC_PATH"
If you wish to execute a specific set of tests you can use the -ginkgo.focus=
regex, e.g.:
e2e.test ... --ginkgo.focus="DNS|(?i)nodeport(?-i)|kubectl guestbook"
Conversely, if you wish to exclude a set of tests, you can run:
+ +e2e.test ... --ginkgo.skip="Density|Scale"
As mentioned earlier there are a host of other options that are available, but are left to the developer
+ +NOTE: If you are running tests on a local cluster repeatedly, you may need to periodically perform some manual cleanup.
+- rm -rf /var/run/kubernetes
, clear kube generated credentials, sometimes stale permissions can cause problems.
+- sudo iptables -F
, clear ip tables rules left by the kube-proxy.
As mentioned above, prior to adding a new test, it is a good idea to perform a -ginkgo.dryRun=true
on the system, in order to see if a behavior is already being tested, or to determine if it may be possible to augment an existing set of tests for a specific use case.
If a behavior does not currently have coverage and a developer wishes to add a new e2e test, navigate to the ./test/e2e directory and create a new test using the existing suite as a guide.
+ +TODO: Create a self-documented example which has been disabled, but can be copied to create new tests and outlines the capabilities and libraries used.
+ +Another benefit of the end-2-end tests is the ability to create reproducible loads on the system, which can then be used to determine the responsiveness, or analyze other characteristics of the system. For example, the density tests load the system to 30,50,100 pods per/node and measures the different characteristics of the system, such as throughput, api-latency, etc.
+ +For a good overview of how we analyze performance data, please read the following post
+ +For developers who are interested in doing their own performance analysis, we recommend setting up prometheus for data collection, and using promdash to visualize the data. There also exists the option of pushing your own metrics in from the tests using a prom-push-gateway. Containers for all of these components can be found here.
+ +For more accurate measurements, you may wish to set up prometheus external to kubernetes in an environment where it can access the major system components (api-server, controller-manager, scheduler). This is especially useful when attempting to gather metrics in a load-balanced api-server environment, because all api-servers can be analyzed independently as well as collectively. On startup, configuration file is passed to prometheus that specifies the endpoints that prometheus will scrape, as well as the sampling interval.
+ +
+#prometheus.conf
+job: {
+ name: "kubernetes"
+ scrape_interval: "1s"
+ target_group: {
+ # apiserver(s)
+ target: "http://localhost:8080/metrics"
+ # scheduler
+ target: "http://localhost:10251/metrics"
+ # controller-manager
+ target: "http://localhost:10252/metrics"
+ }
+
+
+
+Once prometheus is scraping the kubernetes endpoints, that data can then be plotted using promdash, and alerts can be created against the assortment of metrics that kubernetes provides.
+ +HAPPY TESTING!
+ + + + + + + + + +Most of what is written here is not at all specific to Kubernetes, but it bears +being written down in the hope that it will occasionally remind people of “best +practices” around code reviews.
+ +You’ve just had a brilliant idea on how to make Kubernetes better. Let’s call +that idea “FeatureX”. Feature X is not even that complicated. You have a +pretty good idea of how to implement it. You jump in and implement it, fixing a +bunch of stuff along the way. You send your PR - this is awesome! And it sits. +And sits. A week goes by and nobody reviews it. Finally someone offers a few +comments, which you fix up and wait for more review. And you wait. Another +week or two goes by. This is horrible.
+ +What went wrong? One particular problem that comes up frequently is this - your +PR is too big to review. You’ve touched 39 files and have 8657 insertions. +When your would-be reviewers pull up the diffs they run away - this PR is going +to take 4 hours to review and they don’t have 4 hours right now. They’ll get to it +later, just as soon as they have more free time (ha!).
+ +Let’s talk about how to avoid this.
+ +Are you sure FeatureX is something the Kubernetes team wants or will accept, or +that it is implemented to fit with other changes in flight? Are you willing to +bet a few days or weeks of work on it? If you have any doubt at all about the +usefulness of your feature or the design - make a proposal doc (in docs/proposals; +for example the QoS proposal) or a sketch PR (e.g., just +the API or Go interface) or both. Write or code up just enough to express the idea +and the design and why you made those choices, then get feedback on this. Be clear +about what type of feedback you are asking for. Now, if we ask you to change a +bunch of facets of the design, you won’t have to re-write it all.
+ +Small PRs get reviewed faster and are more likely to be correct than big ones. +Let’s face it - attention wanes over time. If your PR takes 60 minutes to +review, I almost guarantee that the reviewer’s eye for details is not as keen in +the last 30 minutes as it was in the first. This leads to multiple rounds of +review when one might have sufficed. In some cases the review is delayed in its +entirety by the need for a large contiguous block of time to sit and read your +code.
+ +Whenever possible, break up your PRs into multiple commits. Making a series of +discrete commits is a powerful way to express the evolution of an idea or the +different ideas that make up a single feature. There’s a balance to be struck, +obviously. If your commits are too small they become more cumbersome to deal +with. Strive to group logically distinct ideas into commits.
+ +For example, if you found that FeatureX needed some “prefactoring” to fit in, +make a commit that JUST does that prefactoring. Then make a new commit for +FeatureX. Don’t lump unrelated things together just because you didn’t think +about prefactoring. If you need to, fork a new branch, do the prefactoring +there and send a PR for that. If you can explain why you are doing seemingly +no-op work (“it makes the FeatureX change easier, I promise”) we’ll probably be +OK with it.
+ +Obviously, a PR with 25 commits is still very cumbersome to review, so use +common sense.
+ +If you can extract whole ideas from your PR and send those as PRs of their own, +you can avoid the painful problem of continually rebasing. Kubernetes is a +fast-moving codebase - lock in your changes ASAP, and make merges be someone +else’s problem.
+ +Obviously, we want every PR to be useful on its own, so you’ll have to use +common sense in deciding what can be a PR vs what should be a commit in a larger +PR. Rule of thumb - if this commit or set of commits is directly related to +FeatureX and nothing else, it should probably be part of the FeatureX PR. If +you can plausibly imagine someone finding value in this commit outside of +FeatureX, try it as a PR.
+ +Don’t worry about flooding us with PRs. We’d rather have 100 small, obvious PRs +than 10 unreviewable monoliths.
+ +Often, as you are implementing FeatureX, you find things that are just wrong. +Bad comments, poorly named functions, bad structure, weak type-safety. You +should absolutely fix those things (or at least file issues, please) - but not +in this PR. See the above points - break unrelated changes out into different +PRs or commits. Otherwise your diff will have WAY too many changes, and your +reviewer won’t see the forest because of all the trees.
+ +Read up on GoDoc - follow those general rules. If you’re writing code and you +think there is any possible chance that someone might not understand why you did +something (or that you won’t remember what you yourself did), comment it. If +you think there’s something pretty obvious that we could follow up on, add a +TODO. Many code-review comments are about this exact issue.
+ +Nothing is more frustrating than doing a review, only to find that the tests are +inadequate or even entirely absent. Very few PRs can touch code and NOT touch +tests. If you don’t know how to test FeatureX - ask! We’ll be happy to help +you design things for easy testing or to suggest appropriate test cases.
+ +If you find yourself writing something that touches a lot of modules, think hard +about the dependencies you are introducing between packages. Can some of what +you’re doing be made more generic and moved up and out of the FeatureX package? +Do you need to use a function or type from an otherwise unrelated package? If +so, promote! We have places specifically for hosting more generic code.
+ +Likewise if FeatureX is similar in form to FeatureW which was checked in last +month and it happens to exactly duplicate some tricky stuff from FeatureW, +consider prefactoring core logic out and using it in both FeatureW and FeatureX. +But do that in a different commit or PR, please.
+ +Your reviewer has finally sent you some feedback on FeatureX. You make a bunch +of changes and … what? You could patch those into your commits with git +“squash” or “fixup” logic. But that makes your changes hard to verify. Unless +your whole PR is pretty trivial, you should instead put your fixups into a new +commit and re-push. Your reviewer can then look at that commit on its own - so +much faster to review than starting over.
+ +We might still ask you to clean up your commits at the very end, for the sake +of a more readable history, but don’t do this until asked, typically at the point +where the PR would otherwise be tagged LGTM.
+ +General squashing guidelines:
+ +Sausage => squash
+ +When there are several commits to fix bugs in the original commit(s), address reviewer feedback, etc. Really we only want to see the end state and commit message for the whole PR.
+Layers => don’t squash
+ +When there are independent changes layered upon each other to achieve a single goal. For instance, writing a code munger could be one commit, applying it could be another, and adding a precommit check could be a third. One could argue they should be separate PRs, but there’s really no way to test/review the munger without seeing it applied, and there needs to be a precommit check to ensure the munged output doesn’t immediately get out of date.
+A commit, as much as possible, should be a single logical change. Each commit should always have a good title line (<70 characters) and include an additional description paragraph describing in more detail the change intended. Do not link pull requests by #
in a commit description, because GitHub creates lots of spam. Instead, reference other PRs via the PR your commit is in.
Sometimes we need to remind each other of core tenets of software design - Keep +It Simple, You Aren’t Gonna Need It, Minimum Viable Product, and so on. Adding +features “because we might need it later” is antithetical to software that +ships. Add the things you need NOW and (ideally) leave room for things you +might need later - but don’t implement them now.
+ +We understand that it is hard to imagine, but sometimes we make mistakes. It’s +OK to push back on changes requested during a review. If you have a good reason +for doing something a certain way, you are absolutely allowed to debate the +merits of a requested change. You might be overruled, but you might also +prevail. We’re mostly pretty reasonable people. Mostly.
+ +So, you’ve done all that and you still aren’t getting any PR love? Here’s some +things you can do that might help kick a stalled process along:
+ +Make sure that your PR has an assigned reviewer (assignee in GitHub). If +this is not the case, reply to the PR comment stream asking for one to be +assigned.
+Ping the assignee (@username) on the PR comment stream asking for an +estimate of when they can get to it.
+Ping the assignee by email (many of us have email addresses that are well +published or are the same as our GitHub handle @google.com or @redhat.com).
+Ping the team (via @team-name) +that works in the area you’re submitting code.
+If you think you have fixed all the issues in a round of review, and you haven’t +heard back, you should ping the reviewer (assignee) on the comment stream with a +“please take another look” (PTAL) or similar comment indicating you are done and +you think it is ready for re-review. In fact, this is probably a good habit for +all PRs.
+ +One phenomenon of open-source projects (where anyone can comment on any issue) +is the dog-pile - your PR gets so many comments from so many people it becomes +hard to follow. In this situation you can ask the primary reviewer +(assignee) whether they want you to fork a new PR to clear out all the comments. +Remember: you don’t HAVE to fix every issue raised by every person who feels +like commenting, but you should at least answer reasonable comments with an +explanation.
+ +Obviously, none of these points are hard rules. There is no document that can +take the place of common sense and good taste. Use your best judgment, but put +a bit of thought into how your work can be made easier to review. If you do +these things your PRs will flow much more easily.
+ + + + + + + + + +Sometimes unit tests are flaky. This means that due to (usually) race conditions, they will occasionally fail, even though most of the time they pass.
+ +We have a goal of 99.9% flake free tests. This means that there is only one flake in one thousand runs of a test.
+ +Running a test 1000 times on your own machine can be tedious and time consuming. Fortunately, there is a better way to achieve this using Kubernetes.
+ +Note: these instructions are mildly hacky for now, as we get run once semantics and logging they will get better
+ +There is a testing image brendanburns/flake
up on the docker hub. We will use this image to test our fix.
Create a replication controller with the following config:
+ +apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: flakecontroller
+spec:
+ replicas: 24
+ template:
+ metadata:
+ labels:
+ name: flake
+ spec:
+ containers:
+ - name: flake
+ image: brendanburns/flake
+ env:
+ - name: TEST_PACKAGE
+ value: pkg/tools
+ - name: REPO_SPEC
+ value: https://github.com/kubernetes/kubernetes
+
+Note that we omit the labels and the selector fields of the replication controller, because they will be populated from the labels field of the pod template by default.
+ +kubectl create -f ./controller.yaml
+
+This will spin up 24 instances of the test. They will run to completion, then exit, and the kubelet will restart them, accumulating more and more runs of the test.
+You can examine the recent runs of the test by calling docker ps -a
and looking for tasks that exited with non-zero exit codes. Unfortunately, docker ps -a only keeps around the exit status of the last 15-20 containers with the same image, so you have to check them frequently.
+You can use this script to automate checking for failures, assuming your cluster is running on GCE and has four nodes:
echo "" > output.txt
+for i in {1..4}; do
+ echo "Checking kubernetes-minion-${i}"
+ echo "kubernetes-minion-${i}:" >> output.txt
+ gcloud compute ssh "kubernetes-minion-${i}" --command="sudo docker ps -a" >> output.txt
+done
+grep "Exited ([^0])" output.txt
+
+Eventually you will have sufficient runs for your purposes. At that point you can stop and delete the replication controller by running:
+ +kubectl stop replicationcontroller flakecontroller
+
+If you do a final check for flakes with docker ps -a
, ignore tasks that exited -1, since that’s what happens when you stop the replication controller.
Happy flake hunting!
+ + + + + + + + + +You can use hack/get-build.sh to or use as a reference on how to get the most recent builds with curl. With get-build.sh
you can grab the most recent stable build, the most recent release candidate, or the most recent build to pass our ci and gce e2e tests (essentially a nightly build).
Run ./hack/get-build.sh -h
for its usage.
For example, to get a build at a specific version (v1.0.2):
+ +./hack/get-build.sh v1.0.2
+
+Alternatively, to get the latest stable release:
+ +./hack/get-build.sh release/stable
+
+Finally, you can just print the latest or stable version:
+ +./hack/get-build.sh -v ci/latest
+
+You can also use the gsutil tool to explore the Google Cloud Storage release buckets. Here are some examples:
+ +gsutil cat gs://kubernetes-release/ci/latest.txt # output the latest ci version number
+gsutil cat gs://kubernetes-release/ci/latest-green.txt # output the latest ci version number that passed gce e2e
+gsutil ls gs://kubernetes-release/ci/v0.20.0-29-g29a55cc/ # list the contents of a ci release
+gsutil ls gs://kubernetes-release/release # list all official releases and rcs
+
+The developer guide is for anyone wanting to either write code which directly accesses the +Kubernetes API, or to contribute directly to the Kubernetes project. +It assumes some familiarity with concepts in the User Guide and the Cluster Admin +Guide.
+ +On Collaborative Development (collab.md): Info on pull requests and code reviews.
+GitHub Issues (issues.md): How incoming issues are reviewed and prioritized.
+Pull Request Process (pull-requests.md): When and why pull requests are closed.
+Faster PR reviews (faster_reviews.md): How to get faster PR reviews.
+Getting Recent Builds (getting-builds.md): How to get recent builds including the latest builds that pass CI.
+Automated Tools (automation.md): Descriptions of the automation that is running on our github repository.
+Development Guide (development.md): Setting up your development environment.
+Hunting flaky tests (flaky-tests.md): We have a goal of 99.9% flake free tests. +Here’s how to run your tests many times.
+Logging Conventions (logging.md]: Glog levels.
+Profiling Kubernetes (profiling.md): How to plug in go pprof profiler to Kubernetes.
+Instrumenting Kubernetes with a new metric +(instrumentation.md): How to add a new metrics to the +Kubernetes code base.
+Coding Conventions (coding-conventions.md): +Coding style advice for contributors.
+API objects are explained at http://kubernetes.io/third_party/swagger-ui/.
+Annotations (docs/user-guide/annotations.md): are for attaching arbitrary non-identifying metadata to objects. +Programs that automate Kubernetes objects may use annotations to store small amounts of their state.
+API Conventions (api-conventions.md): +Defining the verbs and resources used in the Kubernetes API.
+API Client Libraries (client-libraries.md): +A list of existing client libraries, both supported and user-contributed.
+Authentication Plugins (docs/admin/authentication.md): +The current and planned states of authentication tokens.
+Authorization Plugins (docs/admin/authorization.md): +Authorization applies to all HTTP requests on the main apiserver port. +This doc explains the available authorization implementations.
+Admission Control Plugins (admission_control)
+Making release notes (making-release-notes.md): Generating release nodes for a new release.
+Releasing Kubernetes (releasing.md): How to create a Kubernetes release (as in version) +and how the version information gets embedded into the built binaries.
+Instrumenting Kubernetes with a new metric +===================
+ +The following is a step-by-step guide for adding a new metric to the Kubernetes code base.
+ +We use the Prometheus monitoring system’s golang client library for instrumenting our code. Once you’ve picked out a file that you want to add a metric to, you should:
+ +Import “github.com/prometheus/client_golang/prometheus”.
+https://github.com/kubernetes/kubernetes/blob/cd3299307d44665564e1a5c77d0daa0286603ff5/pkg/apiserver/apiserver.go#L53 + https://github.com/kubernetes/kubernetes/blob/cd3299307d44665564e1a5c77d0daa0286603ff5/pkg/kubelet/metrics/metrics.go#L31
+ +Register the metric so that prometheus will know to export it.
+ +https://github.com/kubernetes/kubernetes/blob/cd3299307d44665564e1a5c77d0daa0286603ff5/pkg/kubelet/metrics/metrics.go#L74 +https://github.com/kubernetes/kubernetes/blob/cd3299307d44665564e1a5c77d0daa0286603ff5/pkg/apiserver/apiserver.go#L78
+Use the metric by calling the appropriate method for your metric type (Set, Inc/Add, or Observe, respectively for Gauge, Counter, or Histogram/Summary), first calling WithLabelValues if your metric has any labels
+ +https://github.com/kubernetes/kubernetes/blob/3ce7fe8310ff081dbbd3d95490193e1d5250d2c9/pkg/kubelet/kubelet.go#L1384 +https://github.com/kubernetes/kubernetes/blob/cd3299307d44665564e1a5c77d0daa0286603ff5/pkg/apiserver/apiserver.go#L87
+These are the metric type definitions if you’re curious to learn about them or need more information: +https://github.com/prometheus/client_golang/blob/master/prometheus/gauge.go +https://github.com/prometheus/client_golang/blob/master/prometheus/counter.go +https://github.com/prometheus/client_golang/blob/master/prometheus/histogram.go +https://github.com/prometheus/client_golang/blob/master/prometheus/summary.go
+ + + + + + + + + +GitHub Issues for the Kubernetes Project +========================================
+ +A list quick overview of how we will review and prioritize incoming issues at https://github.com/kubernetes/kubernetes/issues
+ +We will use GitHub issue labels for prioritization. The absence of a priority label means the bug has not been reviewed and prioritized yet.
+ +Updated: 8/27/2015
+ +Table of Contents +
+ +run
or expose
, should obey the following conventions:
+ --generator=
--dry-run
should output the resource that would be created, without creating itkubectl config
) may be used to group related non-standard commands, such as custom generators, mutations, and computations"Flags()"
before adding a new flag-f
: Resource file
+ --follow
in logs
, but should be deprecated in favor of -F
-l
: Label selector
+ --labels
in expose
, but should be deprecated-L
: Label columns-c
: Container
+ --client
in version
, but should be deprecated-i
: Attach stdin-t
: Allocate TTY
+ --template
, but deprecated-w
: Watch (currently also used for --www
in proxy
, but should be deprecated)-p
: Previous
+ --pod
in exec
, but deprecated--patch
in patch
, but should be deprecated--port
in proxy
, but should be deprecated-P
: Static file prefix in proxy
, but should be deprecated-r
: Replicas-u
: Unix socket-v
: Verbose logging level--dry-run
: Don’t modify the live state; simulate the mutation and display the output--local
: Don’t contact the server; just do local read, transformation, generation, etc. and display the output--output-version=...
: Convert the output to a different API group/version--validate
: Validate the resource schemaget
outputget
commands should output one row per resource, and one resource per row
+ get
output should fit within about 80 columns
+ -o wide
may be used to display additional columnsNAME
(may change this to an abbreviation of resource type)AGE
-Lkey
should append a column containing the value of label with key key
, with <none>
if not presentdescribe
commands may output on multiple lines and may include information from related resources, such as events. Describe should add additional information from related resources that a normal user may need to know - if a user would always run “describe resource1” and the immediately want to run a “get type2” or “describe resource2”, consider including that info. Examples, persistent volume claims for pods that reference claims, events for most resources, nodes and the pods scheduled on them. When fetching related resources, a targeted field selector should be used in favor of client side filtering of related resources.-o name
may be used to just display TYPE/name, which may be used to specify resources in other commandshack/update-generated-docs.sh
.
+ $
#
Logging Conventions +===================
+ +The following conventions for the glog levels to use. glog is globally preferred to log for better runtime control.
+ +As per the comments, the practical default level is V(2). Developers and QE environments may wish to run at V(3) or V(4). If you wish to change the log level, you can pass in -v=X
where X is the desired maximum level to log.
This documents the process for making release notes for a release.
+ +Find the most-recent PR that was merged with the previous .0 release. Remember this as $LASTPR. +TODO: Figure out a way to record this somewhere to save the next release engineer time.
+ +Find the most-recent PR that was merged with the current .0 release. Remember this as $CURRENTPR.
+ +${KUBERNETES_ROOT}/build/make-release-notes.sh $LASTPR $CURRENTPR
+
+This generates a list of the entire set of PRs merged since the last minor +release. It is likely long and many PRs aren’t worth mentioning. If any of the +PRs were cherrypicked into patches on the last minor release, you should exclude +them from the current release’s notes.
+ +Open up candidate-notes.md
in your favorite editor.
Remove, regroup, organize to your hearts content.
+ +With the final markdown all set, cut and paste it to the top of CHANGELOG.md
This document explain how to plug in profiler and how to profile Kubernetes services.
+ +Go comes with inbuilt ‘net/http/pprof’ profiling library and profiling web service. The way service works is binding debug/pprof/ subtree on a running webserver to the profiler. Reading from subpages of debug/pprof returns pprof-formatted profiles of the running binary. The output can be processed offline by the tool of choice, or used as an input to handy ‘go tool pprof’, which can graphically represent the result.
+ +TL;DR: Add lines:
+ +m.mux.HandleFunc("/debug/pprof/", pprof.Index)
+m.mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+m.mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+
+to the init(c *Config) method in ‘pkg/master/master.go’ and import ‘net/http/pprof’ package.
+ +In most use cases to use profiler service it’s enough to do ‘import _ net/http/pprof’, which automatically registers a handler in the default http.Server. Slight inconvenience is that APIserver uses default server for intra-cluster communication, so plugging profiler to it is not really useful. In ‘pkg/master/server/server.go’ more servers are created and started as separate goroutines. The one that is usually serving external traffic is secureServer. The handler for this traffic is defined in ‘pkg/master/master.go’ and stored in Handler variable. It is created from HTTP multiplexer, so the only thing that needs to be done is adding profiler handler functions to this multiplexer. This is exactly what lines after TL;DR do.
+ +Even when running profiler I found not really straightforward to use ‘go tool pprof’ with it. The problem is that at least for dev purposes certificates generated for APIserver are not signed by anyone trusted and because secureServer serves only secure traffic it isn’t straightforward to connect to the service. The best workaround I found is by creating an ssh tunnel from the kubernetes_master open unsecured port to some external server, and use this server as a proxy. To save everyone looking for correct ssh flags, it is done by running:
+ +ssh kubernetes_master -L<local_port>:localhost:8080
+
+or analogous one for you Cloud provider. Afterwards you can e.g. run
+ +go tool pprof http://localhost:<local_port>/debug/pprof/profile
+
+to get 30 sec. CPU profile.
+ +To enable contention profiling you need to add line rt.SetBlockProfileRate(1)
in addition to m.mux.HandleFunc(...)
added before (rt
stands for runtime
in master.go
). This enables ‘debug/pprof/block’ subpage, which can be used as an input to go tool pprof
.
Pull Request Process +====================
+ +An overview of how we will manage old or out-of-date pull requests.
+ +We will close any pull requests older than two weeks.
+ +Exceptions can be made for PRs that have active review comments, or that are awaiting other dependent PRs. Closed pull requests are easy to recreate, and little work is lost by closing a pull request that subsequently needs to be reopened.
+ +We want to limit the total number of PRs in flight to: +* Maintain a clean project +* Remove old PRs that would be difficult to rebase as the underlying code has changed over time +* Encourage code velocity
+ +Unless in the last few weeks of a milestone when we need to reduce churn and stabilize, we aim to be always accepting pull requests.
+ +Either the on call manually or the submit queue automatically will manage merging PRs.
+ +There are several requirements for the submit queue to work: +* Author must have signed CLA (“cla: yes” label added to PR) +* No changes can be made since last lgtm label was applied +* k8s-bot must have reported the GCE E2E build and test steps passed (Travis, Shippable and Jenkins build)
+ +Additionally, for infrequent or new contributors, we require the on call to apply the “ok-to-merge” label manually. This is gated by the whitelist.
+ +We use a variety of automation to manage pull requests. This automation is described in detail +elsewhere.
+ + + + + + + + + +This document explains how to cut a release, and the theory behind it. If you +just want to cut a release and move on with your life, you can stop reading +after the first section.
+ +Regardless of whether you are cutting a major or minor version, cutting a +release breaks down into four pieces:
+ +You should progress in this strict order.
+ +vX.Y.0
)When cutting a major/minor release, your first job is to find the branch
+point. We cut vX.Y.0
releases directly from master
, which is also the
+branch that we have most continuous validation on. Go first to the main GCE
+Jenkins end-to-end job and next to the
+Critical Builds page and hopefully find a
+recent Git hash that looks stable across at least kubernetes-e2e-gce
and
+kubernetes-e2e-gke-ci
. First glance through builds and look for nice solid
+rows of green builds, and then check temporally with the other Critical Builds
+to make sure they’re solid around then as well. Once you find some greens, you
+can find the Git hash for a build by looking at the “Console Log”, then look for
+githash=
. You should see a line line:
+ githash=v0.20.2-322-g974377b
+
+Because Jenkins builds frequently, if you’re looking between jobs
+(e.g. kubernetes-e2e-gke-ci
and kubernetes-e2e-gce
), there may be no single
+githash
that’s been run on both jobs. In that case, take the a green
+kubernetes-e2e-gce
build (but please check that it corresponds to a temporally
+similar build that’s green on kubernetes-e2e-gke-ci
). Lastly, if you’re having
+trouble understanding why the GKE continuous integration clusters are failing
+and you’re trying to cut a release, don’t hesitate to contact the GKE
+oncall.
Before proceeding to the next step:
+ +export BRANCHPOINT=v0.20.2-322-g974377b
+
+Where v0.20.2-322-g974377b
is the git hash you decided on. This will become
+our (retroactive) branch point.
Do the following:
+ +export VER=x.y
(e.g. 0.20
for v0.20)git fetch upstream && git checkout -b release-${VER} ${BRANCHPOINT}
(you did set ${BRANCHPOINT}
, right?)make clean && git reset --hard HEAD && git clean -xdf
make
(TBD: you really shouldn’t have to do this, but the swagger output step requires it right now)./build/mark-new-version.sh v${VER}.0
to mark the new release and get further
+instructions. This creates a series of commits on the branch you’re working
+on (release-${VER}
), including forking our documentation for the release,
+the release version commit (which is then tagged), and the post-release
+version commit.TODO: how to fix tags, etc., if you have to shift the release branchpoint.
+ +In your git repo (you still have ${VER}
set from above right?):
git checkout upstream/master && build/build-official-release.sh v${VER}.0
(the build-official-release.sh
script is version agnostic, so it’s best to run it off master
directly).This helpful guide describes how to write release
+notes for a major/minor release. In the release template on GitHub, leave the
+last PR number that the tool finds for the .0
release, so the next releaser
+doesn’t have to hunt.
vX.Y.Z
for Z > 0
)We cut vX.Y.Z
releases from the release-vX.Y
branch after all cherry picks
+to the branch have been resolved. You should ensure all outstanding cherry picks
+have been reviewed and merged and the branch validated on Jenkins (validation
+TBD). See the Cherry Picks for more information on how to
+manage cherry picks prior to cutting the release.
export VER=x.y
(e.g. 0.20
for v0.20)export PATCH=Z
where Z
is the patch level of vX.Y.Z
git fetch upstream && git checkout -b upstream/release-${VER} release-${VER}
make clean && git reset --hard HEAD && git clean -xdf
make
(TBD: you really shouldn’t have to do this, but the swagger output step requires it right now)./build/mark-new-version.sh v${VER}.${PATCH}
to mark the new release and get further
+instructions. This creates a series of commits on the branch you’re working
+on (release-${VER}
), including forking our documentation for the release,
+the release version commit (which is then tagged), and the post-release
+version commit.hack/cherry_pick_list.sh upstream/release-${VER}
to inform people of what
+is already on the branch.TODO: how to fix tags, etc., if the release is changed.
+ +In your git repo (you still have ${VER}
and ${PATCH}
set from above right?):
git checkout upstream/master && build/build-official-release.sh
+v${VER}.${PATCH}
(the build-official-release.sh
script is version
+agnostic, so it’s best to run it off master
directly).Run hack/cherry_pick_list.sh ${VER}.${PATCH}~1
to get the release notes for
+the patch release you just created. Feel free to prune anything internal, like
+you would for a major release, but typically for patch releases we tend to
+include everything in the release notes.
Kubernetes may be built from either a git tree (using hack/build-go.sh
) or
+from a tarball (using either hack/build-go.sh
or go install
) or directly by
+the Go native build system (using go get
).
When building from git, we want to be able to insert specific information about
+the build tree at build time. In particular, we want to use the output of git
+describe
to generate the version of Kubernetes and the status of the build
+tree (add a -dirty
prefix if the tree was modified.)
When building from a tarball or using the Go build system, we will not have +access to the information about the git tree, but we still want to be able to +tell whether this build corresponds to an exact release (e.g. v0.3) or is +between releases (e.g. at some point in development between v0.3 and v0.4).
+ +In order to account for these use cases, there are some specific formats that +may end up representing the Kubernetes version. Here are a few examples:
+ +git describe
output and it indicates
+that we are 15 commits past the v0.5 release and that the SHA1 of the commit
+where the binaries were built was 0123abcd4567
. It is only possible to have
+this level of detail in the version information when building from git, not
+when building from a tarball.-dirty
prefix
+means that the tree had local modifications or untracked files at the time of
+the build, so there’s no guarantee that the source code matches exactly the
+state of the tree at the 0123abcd4567
commit or at the v0.5
git tag
+(resp.)go get
or,
+if we have a git tree, we are using go install
directly, so it is not
+possible to inject the git version into the build information. Additionally,
+this is not an official release, so the -dev
prefix indicates that the
+version we are building is after v0.5
but before v0.6
. (There is actually
+an exception where a commit with v0.5-dev
is not present on v0.6
, see
+later for details.)In order to cover the different build cases, we start by providing information +that can be used when using only Go build tools or when we do not have the git +version information available.
+ +To be able to provide a meaningful version in those cases, we set the contents +of variables in a Go source file that will be used when no overrides are +present.
+ +We are using pkg/version/base.go
as the source of versioning in absence of
+information from git. Here is a sample of that file’s contents:
var (
+ gitVersion string = "v0.4-dev" // version from git, output of $(git describe)
+ gitCommit string = "" // sha1 from git, output of $(git rev-parse HEAD)
+)
+
+This means a build with go install
or go get
or a build from a tarball will
+yield binaries that will identify themselves as v0.4-dev
and will not be able
+to provide you with a SHA1.
To add the extra versioning information when building from git, the
+hack/build-go.sh
script will gather that information (using git describe
and
+git rev-parse
) and then create a -ldflags
string to pass to go install
and
+tell the Go linker to override the contents of those variables at build time. It
+can, for instance, tell it to override gitVersion
and set it to
+v0.4-13-g4567bcdef6789-dirty
and set gitCommit
to 4567bcdef6789...
which
+is the complete SHA1 of the (dirty) tree used at build time.
Handling official versions from git is easy, as long as there is an annotated
+git tag pointing to a specific version then git describe
will return that tag
+exactly which will match the idea of an official version (e.g. v0.5
).
Handling it on tarballs is a bit harder since the exact version string must be
+present in pkg/version/base.go
for it to get embedded into the binaries. But
+simply creating a commit with v0.5
on its own would mean that the commits
+coming after it would also get the v0.5
version when built from tarball or go
+get
while in fact they do not match v0.5
(the one that was tagged) exactly.
To handle that case, creating a new release should involve creating two adjacent
+commits where the first of them will set the version to v0.5
and the second
+will set it to v0.5-dev
. In that case, even in the presence of merges, there
+will be a single commit where the exact v0.5
version will be used and all
+others around it will either have v0.4-dev
or v0.5-dev
.
The diagram below illustrates it.
+ + + +After working on v0.4-dev
and merging PR 99 we decide it is time to release
+v0.5
. So we start a new branch, create one commit to update
+pkg/version/base.go
to include gitVersion = "v0.5"
and git commit
it.
We test it and make sure everything is working as expected.
+ +Before sending a PR for it, we create a second commit on that same branch,
+updating pkg/version/base.go
to include gitVersion = "v0.5-dev"
. That will
+ensure that further builds (from tarball or go install
) on that tree will
+always include the -dev
prefix and will not have a v0.5
version (since they
+do not match the official v0.5
exactly.)
We then send PR 100 with both commits in it.
+ +Once the PR is accepted, we can use git tag -a
to create an annotated tag
+pointing to the one commit that has v0.5
in pkg/version/base.go
and push
+it to GitHub. (Unfortunately GitHub tags/releases are not annotated tags, so
+this needs to be done from a git client and pushed to GitHub using SSH or
+HTTPS.)
While we are working on releasing v0.5
, other development takes place and
+other PRs get merged. For instance, in the example above, PRs 101 and 102 get
+merged to the master branch before the versioning PR gets merged.
This is not a problem, it is only slightly inaccurate that checking out the tree
+at commit 012abc
or commit 345cde
or at the commit of the merges of PR 101
+or 102 will yield a version of v0.4-dev
but those commits are not present in
+v0.5
.
In that sense, there is a small window in which commits will get a
+v0.4-dev
or v0.4-N-gXXX
label and while they’re indeed later than v0.4
+but they are not really before v0.5
in that v0.5
does not contain those
+commits.
Unfortunately, there is not much we can do about it. On the other hand, other +projects seem to live with that and it does not really become a large problem.
+ +As an example, Docker commit a327d9b91edf has a v1.1.1-N-gXXX
label but it is
+not present in Docker v1.2.0
:
$ git describe a327d9b91edf
+v1.1.1-822-ga327d9b91edf
+
+$ git log --oneline v1.2.0..a327d9b91edf
+a327d9b91edf Fix data space reporting from Kb/Mb to KB/MB
+
+(Non-empty output here means the commit is not present on v1.2.0.)
+
+No official release should be made final without properly matching release notes.
+ +There should be made available, per release, a small summary, preamble, of the +major changes, both in terms of feature improvements/bug fixes and notes about +functional feature changes (if any) regarding the previous released version so +that the BOM regarding updating to it gets as obvious and trouble free as possible.
+ +After this summary, preamble, all the relevant PRs/issues that got in that +version should be listed and linked together with a small summary understandable +by plain mortals (in a perfect world PR/issue’s title would be enough but often +it is just too cryptic/geeky/domain-specific that it isn’t).
+ + + + + + + + + +The Kubernetes scheduler runs as a process alongside the other master +components such as the API server. Its interface to the API server is to watch +for Pods with an empty PodSpec.NodeName, and for each Pod, it posts a Binding +indicating where the Pod should be scheduled.
+ +The scheduler tries to find a node for each Pod, one at a time, as it notices
+these Pods via watch. There are three steps. First it applies a set of “predicates” that filter out
+inappropriate nodes. For example, if the PodSpec specifies resource requests, then the scheduler
+will filter out nodes that don’t have at least that much resources available (computed
+as the capacity of the node minus the sum of the resource requests of the containers that
+are already running on the node). Second, it applies a set of “priority functions”
+that rank the nodes that weren’t filtered out by the predicate check. For example,
+it tries to spread Pods across nodes while at the same time favoring the least-loaded
+nodes (where “load” here is sum of the resource requests of the containers running on the node,
+divided by the node’s capacity).
+Finally, the node with the highest priority is chosen
+(or, if there are multiple such nodes, then one of them is chosen at random). The code
+for this main scheduling loop is in the function Schedule()
in
+plugin/pkg/scheduler/generic_scheduler.go
The scheduler is extensible: the cluster administrator can choose which of the pre-defined
+scheduling policies to apply, and can add new ones. The built-in predicates and priorities are
+defined in plugin/pkg/scheduler/algorithm/predicates/predicates.go and
+plugin/pkg/scheduler/algorithm/priorities/priorities.go, respectively.
+The policies that are applied when scheduling can be chosen in one of two ways. Normally,
+the policies used are selected by the functions defaultPredicates()
and defaultPriorities()
in
+plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go.
+However, the choice of policies
+can be overridden by passing the command-line flag --policy-config-file
to the scheduler, pointing to a JSON
+file specifying which scheduling policies to use. See
+examples/scheduler-policy-config.json for an example
+config file. (Note that the config file format is versioned; the API is defined in
+plugin/pkg/scheduler/api).
+Thus to add a new scheduling policy, you should modify predicates.go or priorities.go,
+and either register the policy in defaultPredicates()
or defaultPriorities()
, or use a policy config file.
If you want to get a global picture of how the scheduler works, you can start in +plugin/cmd/kube-scheduler/app/server.go
+ + + + + + + + + +For each unscheduled Pod, the Kubernetes scheduler tries to find a node across the cluster according to a set of rules. A general introduction to the Kubernetes scheduler can be found at scheduler.md. In this document, the algorithm of how to select a node for the Pod is explained. There are two steps before a destination node of a Pod is chosen. The first step is filtering all the nodes and the second is ranking the remaining nodes to find a best fit for the Pod.
+ +The purpose of filtering the nodes is to filter out the nodes that do not meet certain requirements of the Pod. For example, if the free resource on a node (measured by the capacity minus the sum of the resource requests of all the Pods that already run on the node) is less than the Pod’s required resource, the node should not be considered in the ranking phase so it is filtered out. Currently, there are several “predicates” implementing different filtering policies, including:
+ +NoDiskConflict
: Evaluate if a pod can fit due to the volumes it requests, and those that are already mounted.PodFitsResources
: Check if the free resource (CPU and Memory) meets the requirement of the Pod. The free resource is measured by the capacity minus the sum of requests of all Pods on the node. To learn more about the resource QoS in Kubernetes, please check QoS proposal.PodFitsHostPorts
: Check if any HostPort required by the Pod is already occupied on the node.PodFitsHost
: Filter out all nodes except the one specified in the PodSpec’s NodeName field.PodSelectorMatches
: Check if the labels of the node match the labels specified in the Pod’s nodeSelector
field (Here is an example of how to use nodeSelector
field).CheckNodeLabelPresence
: Check if all the specified labels exist on a node or not, regardless of the value.The details of the above predicates can be found in plugin/pkg/scheduler/algorithm/predicates/predicates.go. All predicates mentioned above can be used in combination to perform a sophisticated filtering policy. Kubernetes uses some, but not all, of these predicates by default. You can see which ones are used by default in plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go.
+ +The filtered nodes are considered suitable to host the Pod, and it is often that there are more than one nodes remaining. Kubernetes prioritizes the remaining nodes to find the “best” one for the Pod. The prioritization is performed by a set of priority functions. For each remaining node, a priority function gives a score which scales from 0-10 with 10 representing for “most preferred” and 0 for “least preferred”. Each priority function is weighted by a positive number and the final score of each node is calculated by adding up all the weighted scores. For example, suppose there are two priority functions, priorityFunc1
and priorityFunc2
with weighting factors weight1
and weight2
respectively, the final score of some NodeA is:
finalScoreNodeA = (weight1 * priorityFunc1) + (weight2 * priorityFunc2)
+
+
+After the scores of all nodes are calculated, the node with highest score is chosen as the host of the Pod. If there are more than one nodes with equal highest scores, a random one among them is chosen.
+ +Currently, Kubernetes scheduler provides some practical priority functions, including:
+ +LeastRequestedPriority
: The node is prioritized based on the fraction of the node that would be free if the new Pod were scheduled onto the node. (In other words, (capacity - sum of requests of all Pods already on the node - request of Pod that is being scheduled) / capacity). CPU and memory are equally weighted. The node with the highest free fraction is the most preferred. Note that this priority function has the effect of spreading Pods across the nodes with respect to resource consumption.CalculateNodeLabelPriority
: Prefer nodes that have the specified label.BalancedResourceAllocation
: This priority function tries to put the Pod on a node such that the CPU and Memory utilization rate is balanced after the Pod is deployed.CalculateSpreadPriority
: Spread Pods by minimizing the number of Pods belonging to the same service on the same node.CalculateAntiAffinityPriority
: Spread Pods by minimizing the number of Pods belonging to the same service on nodes with the same value for a particular label.The details of the above priority functions can be found in plugin/pkg/scheduler/algorithm/priorities. Kubernetes uses some, but not all, of these priority functions by default. You can see which ones are used by default in plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go. Similar as predicates, you can combine the above priority functions and assign weight factors (positive number) to them as you want (check scheduler.md for how to customize).
+ + + + + + + + + +This page gives some advice for anyone planning to write or update a Getting Started Guide for Kubernetes. +It also gives some guidelines which reviewers should follow when reviewing a pull request for a +guide.
+ +A Getting Started Guide is instructions on how to create a Kubernetes cluster on top of a particular +type(s) of infrastructure. Infrastructure includes: the IaaS provider for VMs; +the node OS; inter-node networking; and node Configuration Management system. +A guide refers to scripts, Configuration Management files, and/or binary assets such as RPMs. We call +the combination of all these things needed to run on a particular type of infrastructure a +distro.
+ +The Matrix lists the distros. If there is already a guide +which is similar to the one you have planned, consider improving that one.
+ +Distros fall into two categories: + - versioned distros are tested to work with a particular binary release of Kubernetes. These + come in a wide variety, reflecting a wide range of ideas and preferences in how to run a cluster. + - development distros are tested work with the latest Kubernetes source code. But, there are + relatively few of these and the bar is much higher for creating one. They must support + fully automated cluster creation, deletion, and upgrade.
+ +There are different guidelines for each.
+ +These guidelines say what to do. See the Rationale section for why.
+ - Send us a PR.
+ - Put the instructions in docs/getting-started-guides/...
. Scripts go there too. This helps devs easily
+ search for uses of flags by guides.
+ - We may ask that you host binary assets or large amounts of code in our contrib
directory or on your
+ own repo.
+ - Add or update a row in The Matrix.
+ - State the binary version of Kubernetes that you tested clearly in your Guide doc.
+ - Setup a cluster and run the conformance test against it, and report the
+ results in your PR.
+ - Versioned distros should typically not modify or add code in cluster/
. That is just scripts for developer
+ distros.
+ - When a new major or minor release of Kubernetes comes out, we may also release a new
+ conformance test, and require a new conformance test run to earn a conformance checkmark.
If you have a cluster partially working, but doing all the above steps seems like too much work, +we still want to hear from you. We suggest you write a blog post or a Gist, and we will link to it on our wiki page. +Just file an issue or chat us on Slack and one of the committers will link to it from the wiki.
+ +These guidelines say what to do. See the Rationale section for why.
+ - the main reason to add a new development distro is to support a new IaaS provider (VM and
+ network management). This means implementing a new pkg/cloudprovider/providers/$IAAS_NAME
.
+ - Development distros should use Saltstack for Configuration Management.
+ - development distros need to support automated cluster creation, deletion, upgrading, etc.
+ This mean writing scripts in cluster/$IAAS_NAME
.
+ - all commits to the tip of this repo need to not break any of the development distros
+ - the author of the change is responsible for making changes necessary on all the cloud-providers if the
+ change affects any of them, and reverting the change if it breaks any of the CIs.
+ - a development distro needs to have an organization which owns it. This organization needs to:
+ - Setting up and maintaining Continuous Integration that runs e2e frequently (multiple times per day) against the
+ Distro at head, and which notifies all devs of breakage.
+ - being reasonably available for questions and assisting with
+ refactoring and feature additions that affect code for their IaaS.
Kubernetes can run on a range of platforms, from your laptop, to VMs on a cloud provider, to rack of +bare metal servers. The effort required to set up a cluster varies from running a single command to +crafting your own customized cluster. We’ll guide you in picking a solution that fits for your needs.
+ +Table of Contents +
+ +If you just want to “kick the tires” on Kubernetes, we recommend the local Docker-based solution.
+ +The local Docker-based solution is one of several Local cluster solutions +that are quick to set up, but are limited to running on one machine.
+ +When you are ready to scale up to more machines and higher availability, a Hosted +solution is the easiest to create and maintain.
+ +Turn-key cloud solutions require only a few commands to create +and cover a wider range of cloud providers.
+ +Custom solutions require more effort to setup but cover and even +they vary from step-by-step instructions to general advice for setting up +a Kubernetes cluster from scratch.
+ +Local-machine solutions create a single cluster with one or more Kubernetes nodes on a single +physical machine. Setup is completely automated and doesn’t require a cloud provider account. +But their size and availability is limited to that of a single machine.
+ +The local-machine solutions are:
+ +Google Container Engine offers managed Kubernetes +clusters.
+ +These solutions allow you to create Kubernetes clusters on a range of Cloud IaaS providers with only a +few commands, and have active community support.
+ + + +Kubernetes can run on a wide range of Cloud providers and bare-metal environments, and with many +base operating systems.
+ +If you can find a guide below that matches your needs, use it. It may be a little out of date, but +it will be easier than starting from scratch. If you do want to start from scratch because you +have special requirements or just because you want to understand what is underneath a Kubernetes +cluster, try the Getting Started from Scratch guide.
+ +If you are interested in supporting Kubernetes on a new platform, check out our advice for +writing a new solution.
+ +These solutions are combinations of cloud provider and OS not covered by the above solutions.
+ + + +These solutions provide integration with 3rd party schedulers, resource managers, and/or lower level platforms.
+ +Here are all the solutions mentioned above in table form.
+ +IaaS Provider | +Config. Mgmt | +OS | +Networking | +Docs | +Conforms | +Support Level | +
---|---|---|---|---|---|---|
GKE | ++ | + | GCE | +docs | +✓ | +Commercial | +
Vagrant | +Saltstack | +Fedora | +flannel | +docs | +✓ | +Project | +
GCE | +Saltstack | +Debian | +GCE | +docs | +✓ | +Project | +
Azure | +CoreOS | +CoreOS | +Weave | +docs | ++ | Community (@errordeveloper, @squillace, @chanezon, @crossorigin) | +
Docker Single Node | +custom | +N/A | +local | +docs | ++ | Project (@brendandburns) | +
Docker Multi Node | +Flannel | +N/A | +local | +docs | ++ | Project (@brendandburns) | +
Bare-metal | +Ansible | +Fedora | +flannel | +docs | ++ | Project | +
Digital Ocean | +custom | +Fedora | +Calico | +docs | ++ | Community (@djosborne) | +
Bare-metal | +custom | +Fedora | +none | +docs | ++ | Project | +
Bare-metal | +custom | +Fedora | +flannel | +docs | ++ | Community (@aveshagarwal) | +
libvirt | +custom | +Fedora | +flannel | +docs | ++ | Community (@aveshagarwal) | +
KVM | +custom | +Fedora | +flannel | +docs | ++ | Community (@aveshagarwal) | +
Mesos/Docker | +custom | +Ubuntu | +Docker | +docs | ++ | Community (Kubernetes-Mesos Authors) | +
Mesos/GCE | ++ | + | + | docs | ++ | Community (Kubernetes-Mesos Authors) | +
DCOS | +Marathon | +CoreOS/Alpine | +custom | +docs | ++ | Community (Kubernetes-Mesos Authors) | +
AWS | +CoreOS | +CoreOS | +flannel | +docs | ++ | Community | +
GCE | +CoreOS | +CoreOS | +flannel | +docs | ++ | Community (@pires) | +
Vagrant | +CoreOS | +CoreOS | +flannel | +docs | ++ | Community (@pires, @AntonioMeireles) | +
Bare-metal (Offline) | +CoreOS | +CoreOS | +flannel | +docs | ++ | Community (@jeffbean) | +
Bare-metal | +CoreOS | +CoreOS | +Calico | +docs | ++ | Community (@caseydavenport) | +
CloudStack | +Ansible | +CoreOS | +flannel | +docs | ++ | Community (@runseb) | +
Vmware | ++ | Debian | +OVS | +docs | ++ | Community (@pietern) | +
Bare-metal | +custom | +CentOS | +none | +docs | ++ | Community (@coolsvap) | +
AWS | +Juju | +Ubuntu | +flannel | +docs | ++ | Community ( @whit, @matt, @chuck ) | +
OpenStack/HPCloud | +Juju | +Ubuntu | +flannel | +docs | ++ | Community ( @whit, @matt, @chuck ) | +
Joyent | +Juju | +Ubuntu | +flannel | +docs | ++ | Community ( @whit, @matt, @chuck ) | +
AWS | +Saltstack | +Ubuntu | +OVS | +docs | ++ | Community (@justinsb) | +
Bare-metal | +custom | +Ubuntu | +Calico | +docs | ++ | Community (@djosborne) | +
Bare-metal | +custom | +Ubuntu | +flannel | +docs | ++ | Community (@resouer, @WIZARD-CXY) | +
Local | ++ | + | none | +docs | ++ | Community (@preillyme) | +
libvirt/KVM | +CoreOS | +CoreOS | +libvirt/KVM | +docs | ++ | Community (@lhuard1A) | +
oVirt | ++ | + | + | docs | ++ | Community (@simon3z) | +
Rackspace | +CoreOS | +CoreOS | +flannel | +docs | ++ | Community (@doublerr) | +
any | +any | +any | +any | +docs | ++ | Community (@erictune) | +
Note: The above table is ordered by version test/used in notes followed by support level.
+ +Definition of columns:
+ +Getting started on AWS EC2 +————————–
+ +Table of Contents
+ +NOTE: This script use the ‘default’ AWS profile by default.
+You may explicitly set AWS profile to use using the AWS_DEFAULT_PROFILE
environment variable:
export AWS_DEFAULT_PROFILE=myawsprofile
+
+get-kube
#Using wget
+export KUBERNETES_PROVIDER=aws; wget -q -O - https://get.k8s.io | bash
+
+#Using cURL
+export KUBERNETES_PROVIDER=aws; curl -sS https://get.k8s.io | bash
+
+NOTE: This script calls cluster/kube-up.sh +which in turn calls cluster/aws/util.sh +using cluster/aws/config-default.sh.
+ +This process takes about 5 to 10 minutes. Once the cluster is up, the IP addresses of your master and node(s) will be printed,
+as well as information about the default services running in the cluster (monitoring, logging, dns). User credentials and security
+tokens are written in ~/.kube/config
, they will be necessary to use the CLI or the HTTP Basic Auth.
By default, the script will provision a new VPC and a 4 node k8s cluster in us-west-2a (Oregon) with t2.micro
instances running on Ubuntu.
+You can override the variables defined in config-default.sh to change this behavior as follows:
export KUBE_AWS_ZONE=eu-west-1c
+export NUM_MINIONS=2
+export MINION_SIZE=m3.medium
+export AWS_S3_REGION=eu-west-1
+export AWS_S3_BUCKET=mycompany-kubernetes-artifacts
+export INSTANCE_PREFIX=k8s
+...
+
+It will also try to create or reuse a keypair called “kubernetes”, and IAM profiles called “kubernetes-master” and “kubernetes-minion”. +If these already exist, make sure you want them to be used here.
+ +NOTE: If using an existing keypair named “kubernetes” then you must set the AWS_SSH_KEY
key to point to your private key.
A contributed example allows you to setup a Kubernetes cluster based on CoreOS, using +EC2 with user data (cloud-config).
+ +kubectl
The cluster startup script will leave you with a kubernetes
directory on your workstation.
+Alternately, you can download the latest Kubernetes release from this page.
Next, add the appropriate binary folder to your PATH
to access kubectl:
# OS X
+export PATH=<path/to/kubernetes-directory>/platforms/darwin/amd64:$PATH
+
+# Linux
+export PATH=<path/to/kubernetes-directory>/platforms/linux/amd64:$PATH
+
+An up-to-date documentation page for this tool is available here: kubectl manual
+ +By default, kubectl
will use the kubeconfig
file generated during the cluster startup for authenticating against the API.
+For more information, please read kubeconfig files
See a simple nginx example to try out your new cluster.
+ +The “Guestbook” application is another popular example to get started with Kubernetes: guestbook example
+ +For more complete applications, please look in the examples directory
+ +Make sure the environment variables you used to provision your cluster are still exported, then call the following script inside the
+kubernetes
directory:
cluster/kube-down.sh
+
+Please see the Kubernetes docs for more details on administering +and using a Kubernetes cluster.
+ + + + + + + + + +Checkout the coreos azure getting started guide
+ + + + + + + + + +You can either build a release from sources or download a pre-built release. If you do not plan on developing Kubernetes itself, we suggest a pre-built release.
+ +The list of binary releases is available for download from the GitHub Kubernetes repo release page.
+ +Download the latest release and unpack this tar file on Linux or OS X, cd to the created kubernetes/
directory, and then follow the getting started guide for your cloud.
Get the Kubernetes source. If you are simply building a release from source there is no need to set up a full golang environment as all building happens in a Docker container.
+ +Building a release is simple.
+ +git clone https://github.com/kubernetes/kubernetes.git
+cd kubernetes
+make release
+
+For more details on the release process see the build/
directory
Getting started on CentOS +———————————————-
+ +Table of Contents
+ + + +You need two machines with CentOS installed on them.
+ +This is a getting started guide for CentOS. It is a manual configuration so you understand all the underlying packages / services / ports, etc…
+ +This guide will only get ONE node working. Multiple nodes requires a functional networking configuration done outside of kubernetes. Although the additional Kubernetes configuration requirements should be obvious.
+ +The Kubernetes package provides a few services: kube-apiserver, kube-scheduler, kube-controller-manager, kubelet, kube-proxy. These services are managed by systemd and the configuration resides in a central location: /etc/kubernetes. We will break the services up between the hosts. The first host, centos-master, will be the Kubernetes master. This host will run the kube-apiserver, kube-controller-manager, and kube-scheduler. In addition, the master will also run etcd. The remaining host, centos-minion will be the node and run kubelet, proxy, cadvisor and docker.
+ +System Information:
+ +Hosts:
+ +
+centos-master = 192.168.121.9
+centos-minion = 192.168.121.65
+
+
+
+Prepare the hosts:
+ +
+[virt7-testing]
+name=virt7-testing
+baseurl=http://cbs.centos.org/repos/virt7-testing/x86_64/os/
+gpgcheck=0
+
+
+
+yum -y install --enablerepo=virt7-testing kubernetes
+
+If you do not get etcd-0.4.6-7 installed with virt7-testing repo,
+ +In the current virt7-testing repo, the etcd package is updated which causes service failure. To avoid this,
+ +yum erase etcd
+
+It will uninstall the current available etcd package
+ +yum install http://cbs.centos.org/kojifiles/packages/etcd/0.4.6/7.el7.centos/x86_64/etcd-0.4.6-7.el7.centos.x86_64.rpm
+yum -y install --enablerepo=virt7-testing kubernetes
+
+echo "192.168.121.9 centos-master
+192.168.121.65 centos-minion" >> /etc/hosts
+
+# Comma separated list of nodes in the etcd cluster
+KUBE_ETCD_SERVERS="--etcd-servers=http://centos-master:4001"
+
+# logging to stderr means we get it in the systemd journal
+KUBE_LOGTOSTDERR="--logtostderr=true"
+
+# journal message level, 0 is debug
+KUBE_LOG_LEVEL="--v=0"
+
+# Should this cluster be allowed to run privileged docker containers
+KUBE_ALLOW_PRIV="--allow-privileged=false"
+
+systemctl disable iptables-services firewalld
+systemctl stop iptables-services firewalld
+
+Configure the Kubernetes services on the master.
+ +# The address on the local server to listen to.
+KUBE_API_ADDRESS="--address=0.0.0.0"
+
+# The port on the local server to listen on.
+KUBE_API_PORT="--port=8080"
+
+# How the replication controller and scheduler find the kube-apiserver
+KUBE_MASTER="--master=http://centos-master:8080"
+
+# Port kubelets listen on
+KUBELET_PORT="--kubelet-port=10250"
+
+# Address range to use for services
+KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
+
+# Add your own!
+KUBE_API_ARGS=""
+
+for SERVICES in etcd kube-apiserver kube-controller-manager kube-scheduler; do
+ systemctl restart $SERVICES
+ systemctl enable $SERVICES
+ systemctl status $SERVICES
+done
+
+Configure the Kubernetes services on the node.
+ +We need to configure the kubelet and start the kubelet and proxy
+ +# The address for the info server to serve on
+KUBELET_ADDRESS="--address=0.0.0.0"
+
+# The port for the info server to serve on
+KUBELET_PORT="--port=10250"
+
+# You may leave this blank to use the actual hostname
+KUBELET_HOSTNAME="--hostname-override=centos-minion"
+
+# Location of the api-server
+KUBELET_API_SERVER="--api-servers=http://centos-master:8080"
+
+# Add your own!
+KUBELET_ARGS=""
+
+for SERVICES in kube-proxy kubelet docker; do
+ systemctl restart $SERVICES
+ systemctl enable $SERVICES
+ systemctl status $SERVICES
+done
+
+You should be finished!
+ +$ kubectl get nodes
+NAME LABELS STATUS
+centos-minion <none> Ready
+
+The cluster should be running! Launch a test pod.
+ +You should have a functional cluster, check out 101!
+ + + + + + + + + +Getting started on CloudStack +————————————————————
+ +Table of Contents
+ + + +CloudStack is a software to build public and private clouds based on hardware virtualization principles (traditional IaaS). To deploy Kubernetes on CloudStack there are several possibilities depending on the Cloud being used and what images are made available. Exoscale for instance makes a CoreOS template available, therefore instructions to deploy Kubernetes on coreOS can be used. CloudStack also has a vagrant plugin available, hence Vagrant could be used to deploy Kubernetes either using the existing shell provisioner or using new Salt based recipes.
+ +CoreOS templates for CloudStack are built nightly. CloudStack operators need to register this template in their cloud before proceeding with these Kubernetes deployment instructions.
+ +This guide uses an Ansible playbook. +This is a completely automated, a single playbook deploys Kubernetes based on the coreOS instructions.
+ +This Ansible playbook deploys Kubernetes on a CloudStack based Cloud using CoreOS images. The playbook, creates an ssh key pair, creates a security group and associated rules and finally starts coreOS instances configured via cloud-init.
+ +$ sudo apt-get install -y python-pip
+$ sudo pip install ansible
+$ sudo pip install cs
+
+
+cs is a python module for the CloudStack API.
+ +Set your CloudStack endpoint, API keys and HTTP method used.
+ +You can define them as environment variables: CLOUDSTACK_ENDPOINT
, CLOUDSTACK_KEY
, CLOUDSTACK_SECRET
and CLOUDSTACK_METHOD
.
Or create a ~/.cloudstack.ini
file:
[cloudstack]
+endpoint = <your cloudstack api endpoint>
+key = <your api access key>
+secret = <your api secret key>
+method = post
+
+
+We need to use the http POST method to pass the large userdata to the coreOS instances.
+ +$ git clone --recursive https://github.com/runseb/ansible-kubernetes.git
+$ cd ansible-kubernetes
+
+
+The ansible-cloudstack module is setup in this repository as a submodule, hence the --recursive
.
You simply need to run the playbook.
+ +$ ansible-playbook k8s.yml
+
+
+Some variables can be edited in the k8s.yml
file.
vars:
+ ssh_key: k8s
+ k8s_num_nodes: 2
+ k8s_security_group_name: k8s
+ k8s_node_prefix: k8s2
+ k8s_template: Linux CoreOS alpha 435 64-bit 10GB Disk
+ k8s_instance_type: Tiny
+
+
+This will start a Kubernetes master node and a number of compute nodes (by default 2).
+The instance_type
and template
by default are specific to exoscale, edit them to specify your CloudStack cloud specific template and instance type (i.e service offering).
Check the tasks and templates in roles/k8s
if you want to modify anything.
Once the playbook as finished, it will print out the IP of the Kubernetes master:
+ +TASK: [k8s | debug msg='k8s master IP is '] ********
+
+
+SSH to it using the key that was created and using the core user and you can list the machines in your cluster:
+ +$ ssh -i ~/.ssh/id_rsa_k8s core@<master IP>
+$ fleetctl list-machines
+MACHINE IP METADATA
+a017c422... <node #1 IP> role=node
+ad13bf84... <master IP> role=master
+e9af8293... <node #2 IP> role=node
+
+
+
+
+
+
+
+
+
+
+ Kubernetes on Azure with CoreOS and Weave +—————————————————————
+ +Table of Contents
+ +In this guide I will demonstrate how to deploy a Kubernetes cluster to Azure cloud. You will be using CoreOS with Weave, which implements simple and secure networking, in a transparent, yet robust way. The purpose of this guide is to provide an out-of-the-box implementation that can ultimately be taken into production with little change. It will demonstrate how to provision a dedicated Kubernetes master and etcd nodes, and show how to scale the cluster with ease.
+ +To get started, you need to checkout the code:
+ +git clone https://github.com/kubernetes/kubernetes
+cd kubernetes/docs/getting-started-guides/coreos/azure/
+
+You will need to have Node.js installed on you machine. If you have previously used Azure CLI, you should have it already.
+ +First, you need to install some of the dependencies with
+ +npm install
+
+Now, all you need to do is:
+ +./azure-login.js -u <your_username>
+./create-kubernetes-cluster.js
+
+This script will provision a cluster suitable for production use, where there is a ring of 3 dedicated etcd nodes: 1 kubernetes master and 2 kubernetes nodes. The kube-00
VM will be the master, your work loads are only to be deployed on the nodes, kube-01
and kube-02
. Initially, all VMs are single-core, to ensure a user of the free tier can reproduce it without paying extra. I will show how to add more bigger VMs later.
Once the creation of Azure VMs has finished, you should see the following:
+ +...
+azure_wrapper/info: Saved SSH config, you can use it like so: `ssh -F ./output/kube_1c1496016083b4_ssh_conf <hostname>`
+azure_wrapper/info: The hosts in this deployment are:
+ [ 'etcd-00', 'etcd-01', 'etcd-02', 'kube-00', 'kube-01', 'kube-02' ]
+azure_wrapper/info: Saved state into `./output/kube_1c1496016083b4_deployment.yml`
+
+Let’s login to the master node like so:
+ +ssh -F ./output/kube_1c1496016083b4_ssh_conf kube-00
+
+++ +Note: config file name will be different, make sure to use the one you see.
+
Check there are 2 nodes in the cluster:
+ +core@kube-00 ~ $ kubectl get nodes
+NAME LABELS STATUS
+kube-01 kubernetes.io/hostname=kube-01 Ready
+kube-02 kubernetes.io/hostname=kube-02 Ready
+
+Let’s follow the Guestbook example now:
+ +kubectl create -f ~/guestbook-example
+
+You need to wait for the pods to get deployed, run the following and wait for STATUS
to change from Pending
to Running
.
kubectl get pods --watch
+
+++ +Note: the most time it will spend downloading Docker container images on each of the nodes.
+
Eventually you should see:
+ +NAME READY STATUS RESTARTS AGE
+frontend-0a9xi 1/1 Running 0 4m
+frontend-4wahe 1/1 Running 0 4m
+frontend-6l36j 1/1 Running 0 4m
+redis-master-talmr 1/1 Running 0 4m
+redis-slave-12zfd 1/1 Running 0 4m
+redis-slave-3nbce 1/1 Running 0 4m
+
+Two single-core nodes are certainly not enough for a production system of today. Let’s scale the cluster by adding a couple of bigger nodes.
+ +You will need to open another terminal window on your machine and go to the same working directory (e.g. ~/Workspace/kubernetes/docs/getting-started-guides/coreos/azure/
).
First, lets set the size of new VMs:
+ +export AZ_VM_SIZE=Large
+
+Now, run scale script with state file of the previous deployment and number of nodes to add:
+ +core@kube-00 ~ $ ./scale-kubernetes-cluster.js ./output/kube_1c1496016083b4_deployment.yml 2
+...
+azure_wrapper/info: Saved SSH config, you can use it like so: `ssh -F ./output/kube_8f984af944f572_ssh_conf <hostname>`
+azure_wrapper/info: The hosts in this deployment are:
+ [ 'etcd-00',
+ 'etcd-01',
+ 'etcd-02',
+ 'kube-00',
+ 'kube-01',
+ 'kube-02',
+ 'kube-03',
+ 'kube-04' ]
+azure_wrapper/info: Saved state into `./output/kube_8f984af944f572_deployment.yml`
+
+++ +Note: this step has created new files in
+./output
.
Back on kube-00
:
core@kube-00 ~ $ kubectl get nodes
+NAME LABELS STATUS
+kube-01 kubernetes.io/hostname=kube-01 Ready
+kube-02 kubernetes.io/hostname=kube-02 Ready
+kube-03 kubernetes.io/hostname=kube-03 Ready
+kube-04 kubernetes.io/hostname=kube-04 Ready
+
+You can see that two more nodes joined happily. Let’s scale the number of Guestbook instances now.
+ +First, double-check how many replication controllers there are:
+ +core@kube-00 ~ $ kubectl get rc
+ONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+frontend php-redis kubernetes/example-guestbook-php-redis:v2 name=frontend 3
+redis-master master redis name=redis-master 1
+redis-slave worker kubernetes/redis-slave:v2 name=redis-slave 2
+
+As there are 4 nodes, let’s scale proportionally:
+ +core@kube-00 ~ $ kubectl scale --replicas=4 rc redis-slave
+>>>>>>> coreos/azure: Updates for 1.0
+scaled
+core@kube-00 ~ $ kubectl scale --replicas=4 rc frontend
+scaled
+
+Check what you have now:
+ +core@kube-00 ~ $ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+frontend php-redis kubernetes/example-guestbook-php-redis:v2 name=frontend 4
+redis-master master redis name=redis-master 1
+redis-slave worker kubernetes/redis-slave:v2 name=redis-slave 4
+
+You now will have more instances of front-end Guestbook apps and Redis slaves; and, if you look up all pods labeled name=frontend
, you should see one running on each node.
core@kube-00 ~/guestbook-example $ kubectl get pods -l name=frontend
+NAME READY STATUS RESTARTS AGE
+frontend-0a9xi 1/1 Running 0 22m
+frontend-4wahe 1/1 Running 0 22m
+frontend-6l36j 1/1 Running 0 22m
+frontend-z9oxo 1/1 Running 0 41s
+
+There is no native Azure load-balancer support in Kubernetes 1.0, however here is how you can expose the Guestbook app to the Internet.
+ +
+./expose_guestbook_app_port.sh ./output/kube_1c1496016083b4_ssh_conf
+Guestbook app is on port 31605, will map it to port 80 on kube-00
+info: Executing command vm endpoint create
++ Getting virtual machines
++ Reading network configuration
++ Updating network configuration
+info: vm endpoint create command OK
+info: Executing command vm endpoint show
++ Getting virtual machines
+data: Name : tcp-80-31605
+data: Local port : 31605
+data: Protcol : tcp
+data: Virtual IP Address : 137.117.156.164
+data: Direct server return : Disabled
+info: vm endpoint show command OK
+
+
+
+You then should be able to access it from anywhere via the Azure virtual IP for kube-00
displayed above, i.e. http://137.117.156.164/
in my case.
You now have a full-blow cluster running in Azure, congrats!
+ +You should probably try deploy other example apps or write your own ;)
+ +If you don’t wish care about the Azure bill, you can tear down the cluster. It’s easy to redeploy it, as you can see.
+ +./destroy-cluster.js ./output/kube_8f984af944f572_deployment.yml
+
+++ +Note: make sure to use the latest state file, as after scaling there is a new one.
+
By the way, with the scripts shown, you can deploy multiple clusters, if you like :)
+ + + + + + + + + +Kubernetes on Azure with CoreOS and Weave +—————————————————————
+ +Table of Contents
+ +In this guide I will demonstrate how to deploy a Kubernetes cluster to Azure cloud. You will be using CoreOS with Weave, which implements simple and secure networking, in a transparent, yet robust way. The purpose of this guide is to provide an out-of-the-box implementation that can ultimately be taken into production with little change. It will demonstrate how to provision a dedicated Kubernetes master and etcd nodes, and show how to scale the cluster with ease.
+ +To get started, you need to checkout the code:
+ +git clone https://github.com/kubernetes/kubernetes
+cd kubernetes/docs/getting-started-guides/coreos/azure/
+
+You will need to have Node.js installed on you machine. If you have previously used Azure CLI, you should have it already.
+ +First, you need to install some of the dependencies with
+ +npm install
+
+Now, all you need to do is:
+ +./azure-login.js -u <your_username>
+./create-kubernetes-cluster.js
+
+This script will provision a cluster suitable for production use, where there is a ring of 3 dedicated etcd nodes: 1 kubernetes master and 2 kubernetes nodes. The kube-00
VM will be the master, your work loads are only to be deployed on the nodes, kube-01
and kube-02
. Initially, all VMs are single-core, to ensure a user of the free tier can reproduce it without paying extra. I will show how to add more bigger VMs later.
Once the creation of Azure VMs has finished, you should see the following:
+ +...
+azure_wrapper/info: Saved SSH config, you can use it like so: `ssh -F ./output/kube_1c1496016083b4_ssh_conf <hostname>`
+azure_wrapper/info: The hosts in this deployment are:
+ [ 'etcd-00', 'etcd-01', 'etcd-02', 'kube-00', 'kube-01', 'kube-02' ]
+azure_wrapper/info: Saved state into `./output/kube_1c1496016083b4_deployment.yml`
+
+Let’s login to the master node like so:
+ +ssh -F ./output/kube_1c1496016083b4_ssh_conf kube-00
+
+++ +Note: config file name will be different, make sure to use the one you see.
+
Check there are 2 nodes in the cluster:
+ +core@kube-00 ~ $ kubectl get nodes
+NAME LABELS STATUS
+kube-01 kubernetes.io/hostname=kube-01 Ready
+kube-02 kubernetes.io/hostname=kube-02 Ready
+
+Let’s follow the Guestbook example now:
+ +kubectl create -f ~/guestbook-example
+
+You need to wait for the pods to get deployed, run the following and wait for STATUS
to change from Pending
to Running
.
kubectl get pods --watch
+
+++ +Note: the most time it will spend downloading Docker container images on each of the nodes.
+
Eventually you should see:
+ +NAME READY STATUS RESTARTS AGE
+frontend-0a9xi 1/1 Running 0 4m
+frontend-4wahe 1/1 Running 0 4m
+frontend-6l36j 1/1 Running 0 4m
+redis-master-talmr 1/1 Running 0 4m
+redis-slave-12zfd 1/1 Running 0 4m
+redis-slave-3nbce 1/1 Running 0 4m
+
+Two single-core nodes are certainly not enough for a production system of today. Let’s scale the cluster by adding a couple of bigger nodes.
+ +You will need to open another terminal window on your machine and go to the same working directory (e.g. ~/Workspace/kubernetes/docs/getting-started-guides/coreos/azure/
).
First, lets set the size of new VMs:
+ +export AZ_VM_SIZE=Large
+
+Now, run scale script with state file of the previous deployment and number of nodes to add:
+ +core@kube-00 ~ $ ./scale-kubernetes-cluster.js ./output/kube_1c1496016083b4_deployment.yml 2
+...
+azure_wrapper/info: Saved SSH config, you can use it like so: `ssh -F ./output/kube_8f984af944f572_ssh_conf <hostname>`
+azure_wrapper/info: The hosts in this deployment are:
+ [ 'etcd-00',
+ 'etcd-01',
+ 'etcd-02',
+ 'kube-00',
+ 'kube-01',
+ 'kube-02',
+ 'kube-03',
+ 'kube-04' ]
+azure_wrapper/info: Saved state into `./output/kube_8f984af944f572_deployment.yml`
+
+++ +Note: this step has created new files in
+./output
.
Back on kube-00
:
core@kube-00 ~ $ kubectl get nodes
+NAME LABELS STATUS
+kube-01 kubernetes.io/hostname=kube-01 Ready
+kube-02 kubernetes.io/hostname=kube-02 Ready
+kube-03 kubernetes.io/hostname=kube-03 Ready
+kube-04 kubernetes.io/hostname=kube-04 Ready
+
+You can see that two more nodes joined happily. Let’s scale the number of Guestbook instances now.
+ +First, double-check how many replication controllers there are:
+ +core@kube-00 ~ $ kubectl get rc
+ONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+frontend php-redis kubernetes/example-guestbook-php-redis:v2 name=frontend 3
+redis-master master redis name=redis-master 1
+redis-slave worker kubernetes/redis-slave:v2 name=redis-slave 2
+
+As there are 4 nodes, let’s scale proportionally:
+ +core@kube-00 ~ $ kubectl scale --replicas=4 rc redis-slave
+>>>>>>> coreos/azure: Updates for 1.0
+scaled
+core@kube-00 ~ $ kubectl scale --replicas=4 rc frontend
+scaled
+
+Check what you have now:
+ +core@kube-00 ~ $ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+frontend php-redis kubernetes/example-guestbook-php-redis:v2 name=frontend 4
+redis-master master redis name=redis-master 1
+redis-slave worker kubernetes/redis-slave:v2 name=redis-slave 4
+
+You now will have more instances of front-end Guestbook apps and Redis slaves; and, if you look up all pods labeled name=frontend
, you should see one running on each node.
core@kube-00 ~/guestbook-example $ kubectl get pods -l name=frontend
+NAME READY STATUS RESTARTS AGE
+frontend-0a9xi 1/1 Running 0 22m
+frontend-4wahe 1/1 Running 0 22m
+frontend-6l36j 1/1 Running 0 22m
+frontend-z9oxo 1/1 Running 0 41s
+
+There is no native Azure load-balancer support in Kubernetes 1.0, however here is how you can expose the Guestbook app to the Internet.
+ +
+./expose_guestbook_app_port.sh ./output/kube_1c1496016083b4_ssh_conf
+Guestbook app is on port 31605, will map it to port 80 on kube-00
+info: Executing command vm endpoint create
++ Getting virtual machines
++ Reading network configuration
++ Updating network configuration
+info: vm endpoint create command OK
+info: Executing command vm endpoint show
++ Getting virtual machines
+data: Name : tcp-80-31605
+data: Local port : 31605
+data: Protcol : tcp
+data: Virtual IP Address : 137.117.156.164
+data: Direct server return : Disabled
+info: vm endpoint show command OK
+
+
+
+You then should be able to access it from anywhere via the Azure virtual IP for kube-00
displayed above, i.e. http://137.117.156.164/
in my case.
You now have a full-blow cluster running in Azure, congrats!
+ +You should probably try deploy other example apps or write your own ;)
+ +If you don’t wish care about the Azure bill, you can tear down the cluster. It’s easy to redeploy it, as you can see.
+ +./destroy-cluster.js ./output/kube_8f984af944f572_deployment.yml
+
+++ +Note: make sure to use the latest state file, as after scaling there is a new one.
+
By the way, with the scripts shown, you can deploy multiple clusters, if you like :)
+ + + + + + + + + +This guide explains how to deploy a bare-metal Kubernetes cluster on CoreOS using Calico networking.
+ +Specifically, this guide will have you do the following: +- Deploy a Kubernetes master node on CoreOS using cloud-config +- Deploy two Kubernetes compute nodes with Calico Networking using cloud-config
+ +This guide will use cloud-config to configure each of the nodes in our Kubernetes cluster.
+ +For ease of distribution, the cloud-config files required for this demonstration can be found on GitHub.
+ +This repo includes two cloud config files:
+- master-config.yaml
: Cloud-config for the Kubernetes master
+- node-config.yaml
: Cloud-config for each Kubernetes compute host
In the next few steps you will be asked to configure these files and host them on an HTTP server where your cluster can access them.
+ +To get the Kubernetes source, clone the GitHub repo, and build the binaries.
+ +
+git clone https://github.com/kubernetes/kubernetes.git
+cd kubernetes
+./build/release.sh
+
+
+
+Once the binaries are built, host the entire <kubernetes>/_output/dockerized/bin/<OS>/<ARCHITECTURE>/
folder on an accessible HTTP server so they can be accessed by the cloud-config. You’ll point your cloud-config files at this HTTP server later.
Let’s download the CoreOS bootable ISO. We’ll use this image to boot and install CoreOS on each server.
+ +
+wget http://stable.release.core-os.net/amd64-usr/current/coreos_production_iso_image.iso
+
+
+
+You can also download the ISO from the CoreOS website.
+ +Once you’ve downloaded the image, use it to boot your Kubernetes Master server. Once booted, you should be automatically logged in as the core
user.
Let’s get the master-config.yaml and fill in the necessary variables. Run the following commands on your HTTP server to get the cloud-config files.
+ +
+git clone https://github.com/Metaswitch/calico-kubernetes-demo.git
+cd calico-kubernetes-demo/coreos
+
+
+
+You’ll need to replace the following variables in the master-config.yaml
file to match your deployment.
+- <SSH_PUBLIC_KEY>
: The public key you will use for SSH access to this server.
+- <KUBERNETES_LOC>
: The address used to get the kubernetes binaries over HTTP.
++ +Note: The config will prepend
+"http://"
and append"/(kubernetes | kubectl | ...)"
to yourKUBERNETES_LOC
variable:, format accordingly
Host the modified master-config.yaml
file and pull it on to your Kubernetes Master server.
The CoreOS bootable ISO comes with a tool called coreos-install
which will allow us to install CoreOS to disk and configure the install using cloud-config. The following command will download and install stable CoreOS, using the master-config.yaml file for configuration.
+sudo coreos-install -d /dev/sda -C stable -c master-config.yaml
+
+
+
+Once complete, eject the bootable ISO and restart the server. When it comes back up, you should have SSH access as the core
user using the public key provided in the master-config.yaml file.
++ +The following steps will set up a Kubernetes node for use as a compute host. This demo uses two compute hosts, so you should run the following steps on each.
+
First, boot up your node using the bootable ISO we downloaded earlier. You should be automatically logged in as the core
user.
Let’s modify the node-config.yaml
cloud-config file on your HTTP server. Make a copy for this node, and fill in the necessary variables.
You’ll need to replace the following variables in the node-config.yaml
file to match your deployment.
+- <HOSTNAME>
: Hostname for this node (e.g. kube-node1, kube-node2)
+- <SSH_PUBLIC_KEY>
: The public key you will use for SSH access to this server.
+- <KUBERNETES_MASTER>
: The IPv4 address of the Kubernetes master.
+- <KUBERNETES_LOC>
: The address to use in order to get the kubernetes binaries over HTTP.
+- <DOCKER_BRIDGE_IP>
: The IP and subnet to use for pods on this node. By default, this should fall within the 192.168.0.0/16 subnet.
++ +Note: The DOCKER_BRIDGE_IP is the range used by this Kubernetes node to assign IP addresses to pods on this node. This subnet must not overlap with the subnets assigned to the other Kubernetes nodes in your cluster. Calico expects each DOCKER_BRIDGE_IP subnet to fall within 192.168.0.0/16 by default (e.g. 192.168.1.1/24 for node 1), but if you’d like to use pod IPs within a different subnet, simply run
+calicoctl pool add <your_subnet>
and select DOCKER_BRIDGE_IP accordingly.
Host the modified node-config.yaml
file and pull it on to your Kubernetes node.
+wget http://<http_server_ip>/node-config.yaml
+
+
+
+Install and configure CoreOS on the node using the following command.
+ +
+sudo coreos-install -d /dev/sda -C stable -c node-config.yaml
+
+
+
+Once complete, restart the server. When it comes back up, you should have SSH access as the core
user using the public key provided in the node-config.yaml
file. It will take some time for the node to be fully configured. Once fully configured, you can check that the node is running with the following command on the Kubernetes master.
+/home/core/kubectl get nodes
+
+
+
+You should now have a functional bare-metal Kubernetes cluster with one master and two compute hosts. +Try running the guestbook demo to test out your new cluster!
+ + + + + + + + + +Bare Metal CoreOS with Kubernetes (OFFLINE) +—————————————— +Deploy a CoreOS running Kubernetes environment. This particular guild is made to help those in an OFFLINE system, wither for testing a POC before the real deal, or you are restricted to be totally offline for your applications.
+ +Table of Contents
+ +Node Description | +MAC | +IP | +
---|---|---|
CoreOS/etcd/Kubernetes Master | +d0:00:67:13:0d:00 | +10.20.30.40 | +
CoreOS Slave 1 | +d0:00:67:13:0d:01 | +10.20.30.41 | +
CoreOS Slave 2 | +d0:00:67:13:0d:02 | +10.20.30.42 | +
To setup CentOS PXELINUX environment there is a complete guide here. This section is the abbreviated version.
+ +Install packages needed on CentOS
+ + sudo yum install tftp-server dhcp syslinux
+
+ vi /etc/xinetd.d/tftp
to enable tftp service and change disable to ‘no’
+ disable = no
Copy over the syslinux images we will need.
+ + su -
+ mkdir -p /tftpboot
+ cd /tftpboot
+ cp /usr/share/syslinux/pxelinux.0 /tftpboot
+ cp /usr/share/syslinux/menu.c32 /tftpboot
+ cp /usr/share/syslinux/memdisk /tftpboot
+ cp /usr/share/syslinux/mboot.c32 /tftpboot
+ cp /usr/share/syslinux/chain.c32 /tftpboot
+
+ /sbin/service dhcpd start
+ /sbin/service xinetd start
+ /sbin/chkconfig tftp on
+
+ Setup default boot menu
+ + mkdir /tftpboot/pxelinux.cfg
+ touch /tftpboot/pxelinux.cfg/default
+
+ Edit the menu vi /tftpboot/pxelinux.cfg/default
default menu.c32
+ prompt 0
+ timeout 15
+ ONTIMEOUT local
+ display boot.msg
+
+ MENU TITLE Main Menu
+
+ LABEL local
+ MENU LABEL Boot local hard drive
+ LOCALBOOT 0
+
+ Now you should have a working PXELINUX setup to image CoreOS nodes. You can verify the services by using VirtualBox locally or with bare metal servers.
+ +This section describes how to setup the CoreOS images to live alongside a pre-existing PXELINUX environment.
+ +/tftpboot/
is our root directory.Download the CoreOS PXE files provided by the CoreOS team.
+ + MY_TFTPROOT_DIR=/tftpboot
+ mkdir -p $MY_TFTPROOT_DIR/images/coreos/
+ cd $MY_TFTPROOT_DIR/images/coreos/
+ wget http://stable.release.core-os.net/amd64-usr/current/coreos_production_pxe.vmlinuz
+ wget http://stable.release.core-os.net/amd64-usr/current/coreos_production_pxe.vmlinuz.sig
+ wget http://stable.release.core-os.net/amd64-usr/current/coreos_production_pxe_image.cpio.gz
+ wget http://stable.release.core-os.net/amd64-usr/current/coreos_production_pxe_image.cpio.gz.sig
+ gpg --verify coreos_production_pxe.vmlinuz.sig
+ gpg --verify coreos_production_pxe_image.cpio.gz.sig
+
+ Edit the menu vi /tftpboot/pxelinux.cfg/default
again
default menu.c32
+ prompt 0
+ timeout 300
+ ONTIMEOUT local
+ display boot.msg
+
+ MENU TITLE Main Menu
+
+ LABEL local
+ MENU LABEL Boot local hard drive
+ LOCALBOOT 0
+
+ MENU BEGIN CoreOS Menu
+
+ LABEL coreos-master
+ MENU LABEL CoreOS Master
+ KERNEL images/coreos/coreos_production_pxe.vmlinuz
+ APPEND initrd=images/coreos/coreos_production_pxe_image.cpio.gz cloud-config-url=http://<xxx.xxx.xxx.xxx>/pxe-cloud-config-single-master.yml
+
+ LABEL coreos-slave
+ MENU LABEL CoreOS Slave
+ KERNEL images/coreos/coreos_production_pxe.vmlinuz
+ APPEND initrd=images/coreos/coreos_production_pxe_image.cpio.gz cloud-config-url=http://<xxx.xxx.xxx.xxx>/pxe-cloud-config-slave.yml
+ MENU END
+
+ This configuration file will now boot from local drive but have the option to PXE image CoreOS.
+ +This section covers configuring the DHCP server to hand out our new images. In this case we are assuming that there are other servers that will boot alongside other images.
+ +Add the filename
to the host or subnet sections.
filename "/tftpboot/pxelinux.0";
+
+ At this point we want to make pxelinux configuration files that will be the templates for the different CoreOS deployments.
+ + subnet 10.20.30.0 netmask 255.255.255.0 {
+ next-server 10.20.30.242;
+ option broadcast-address 10.20.30.255;
+ filename "<other default image>";
+
+ ...
+ # http://www.syslinux.org/wiki/index.php/PXELINUX
+ host core_os_master {
+ hardware ethernet d0:00:67:13:0d:00;
+ option routers 10.20.30.1;
+ fixed-address 10.20.30.40;
+ option domain-name-servers 10.20.30.242;
+ filename "/pxelinux.0";
+ }
+ host core_os_slave {
+ hardware ethernet d0:00:67:13:0d:01;
+ option routers 10.20.30.1;
+ fixed-address 10.20.30.41;
+ option domain-name-servers 10.20.30.242;
+ filename "/pxelinux.0";
+ }
+ host core_os_slave2 {
+ hardware ethernet d0:00:67:13:0d:02;
+ option routers 10.20.30.1;
+ fixed-address 10.20.30.42;
+ option domain-name-servers 10.20.30.242;
+ filename "/pxelinux.0";
+ }
+ ...
+ }
+
+ We will be specifying the node configuration later in the guide.
+ +To deploy our configuration we need to create an etcd
master. To do so we want to pxe CoreOS with a specific cloud-config.yml. There are two options we have here.
+1. Is to template the cloud config file and programmatically create new static configs for different cluster setups.
+2. Have a service discovery protocol running in our stack to do auto discovery.
This demo we just make a static single etcd
server to host our Kubernetes and etcd
master servers.
Since we are OFFLINE here most of the helping processes in CoreOS and Kubernetes are then limited. To do our setup we will then have to download and serve up our binaries for Kubernetes in our local environment.
+ +An easy solution is to host a small web server on the DHCP/TFTP host for all our binaries to make them available to the local CoreOS PXE machines.
+ +To get this up and running we are going to setup a simple apache
server to serve our binaries needed to bootstrap Kubernetes.
This is on the PXE server from the previous section:
+ +rm /etc/httpd/conf.d/welcome.conf
+cd /var/www/html/
+wget -O kube-register https://github.com/kelseyhightower/kube-register/releases/download/v0.0.2/kube-register-0.0.2-linux-amd64
+wget -O setup-network-environment https://github.com/kelseyhightower/setup-network-environment/releases/download/v1.0.0/setup-network-environment
+wget https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kubernetes --no-check-certificate
+wget https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kube-apiserver --no-check-certificate
+wget https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kube-controller-manager --no-check-certificate
+wget https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kube-scheduler --no-check-certificate
+wget https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kubectl --no-check-certificate
+wget https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kubecfg --no-check-certificate
+wget https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kubelet --no-check-certificate
+wget https://storage.googleapis.com/kubernetes-release/release/v0.15.0/bin/linux/amd64/kube-proxy --no-check-certificate
+wget -O flanneld https://storage.googleapis.com/k8s/flanneld --no-check-certificate
+
+
+This sets up our binaries we need to run Kubernetes. This would need to be enhanced to download from the Internet for updates in the future.
+ +Now for the good stuff!
+ +The following config files are tailored for the OFFLINE version of a Kubernetes deployment.
+ +These are based on the work found here: master.yml, node.yml
+ +To make the setup work, you need to replace a few placeholders:
+ +<PXE_SERVER_IP>
with your PXE server ip address (e.g. 10.20.30.242)<MASTER_SERVER_IP>
with the Kubernetes master ip address (e.g. 10.20.30.40)rdocker.example.com
with your docker registry dns name.rproxy.example.com
with your proxy server (and port)On the PXE server make and fill in the variables vi /var/www/html/coreos/pxe-cloud-config-master.yml
.
#cloud-config
+---
+write_files:
+ - path: /opt/bin/waiter.sh
+ owner: root
+ content: |
+ #! /usr/bin/bash
+ until curl http://127.0.0.1:4001/v2/machines; do sleep 2; done
+ - path: /opt/bin/kubernetes-download.sh
+ owner: root
+ permissions: 0755
+ content: |
+ #! /usr/bin/bash
+ /usr/bin/wget -N -P "/opt/bin" "http://<PXE_SERVER_IP>/kubectl"
+ /usr/bin/wget -N -P "/opt/bin" "http://<PXE_SERVER_IP>/kubernetes"
+ /usr/bin/wget -N -P "/opt/bin" "http://<PXE_SERVER_IP>/kubecfg"
+ chmod +x /opt/bin/*
+ - path: /etc/profile.d/opt-path.sh
+ owner: root
+ permissions: 0755
+ content: |
+ #! /usr/bin/bash
+ PATH=$PATH/opt/bin
+coreos:
+ units:
+ - name: 10-eno1.network
+ runtime: true
+ content: |
+ [Match]
+ Name=eno1
+ [Network]
+ DHCP=yes
+ - name: 20-nodhcp.network
+ runtime: true
+ content: |
+ [Match]
+ Name=en*
+ [Network]
+ DHCP=none
+ - name: get-kube-tools.service
+ runtime: true
+ command: start
+ content: |
+ [Service]
+ ExecStartPre=-/usr/bin/mkdir -p /opt/bin
+ ExecStart=/opt/bin/kubernetes-download.sh
+ RemainAfterExit=yes
+ Type=oneshot
+ - name: setup-network-environment.service
+ command: start
+ content: |
+ [Unit]
+ Description=Setup Network Environment
+ Documentation=https://github.com/kelseyhightower/setup-network-environment
+ Requires=network-online.target
+ After=network-online.target
+ [Service]
+ ExecStartPre=-/usr/bin/mkdir -p /opt/bin
+ ExecStartPre=/usr/bin/wget -N -P /opt/bin http://<PXE_SERVER_IP>/setup-network-environment
+ ExecStartPre=/usr/bin/chmod +x /opt/bin/setup-network-environment
+ ExecStart=/opt/bin/setup-network-environment
+ RemainAfterExit=yes
+ Type=oneshot
+ - name: etcd.service
+ command: start
+ content: |
+ [Unit]
+ Description=etcd
+ Requires=setup-network-environment.service
+ After=setup-network-environment.service
+ [Service]
+ EnvironmentFile=/etc/network-environment
+ User=etcd
+ PermissionsStartOnly=true
+ ExecStart=/usr/bin/etcd \
+ --name ${DEFAULT_IPV4} \
+ --addr ${DEFAULT_IPV4}:4001 \
+ --bind-addr 0.0.0.0 \
+ --cluster-active-size 1 \
+ --data-dir /var/lib/etcd \
+ --http-read-timeout 86400 \
+ --peer-addr ${DEFAULT_IPV4}:7001 \
+ --snapshot true
+ Restart=always
+ RestartSec=10s
+ - name: fleet.socket
+ command: start
+ content: |
+ [Socket]
+ ListenStream=/var/run/fleet.sock
+ - name: fleet.service
+ command: start
+ content: |
+ [Unit]
+ Description=fleet daemon
+ Wants=etcd.service
+ After=etcd.service
+ Wants=fleet.socket
+ After=fleet.socket
+ [Service]
+ Environment="FLEET_ETCD_SERVERS=http://127.0.0.1:4001"
+ Environment="FLEET_METADATA=role=master"
+ ExecStart=/usr/bin/fleetd
+ Restart=always
+ RestartSec=10s
+ - name: etcd-waiter.service
+ command: start
+ content: |
+ [Unit]
+ Description=etcd waiter
+ Wants=network-online.target
+ Wants=etcd.service
+ After=etcd.service
+ After=network-online.target
+ Before=flannel.service
+ Before=setup-network-environment.service
+ [Service]
+ ExecStartPre=/usr/bin/chmod +x /opt/bin/waiter.sh
+ ExecStart=/usr/bin/bash /opt/bin/waiter.sh
+ RemainAfterExit=true
+ Type=oneshot
+ - name: flannel.service
+ command: start
+ content: |
+ [Unit]
+ Wants=etcd-waiter.service
+ After=etcd-waiter.service
+ Requires=etcd.service
+ After=etcd.service
+ After=network-online.target
+ Wants=network-online.target
+ Description=flannel is an etcd backed overlay network for containers
+ [Service]
+ Type=notify
+ ExecStartPre=-/usr/bin/mkdir -p /opt/bin
+ ExecStartPre=/usr/bin/wget -N -P /opt/bin http://<PXE_SERVER_IP>/flanneld
+ ExecStartPre=/usr/bin/chmod +x /opt/bin/flanneld
+ ExecStartPre=-/usr/bin/etcdctl mk /coreos.com/network/config '{"Network":"10.100.0.0/16", "Backend": {"Type": "vxlan"}}'
+ ExecStart=/opt/bin/flanneld
+ - name: kube-apiserver.service
+ command: start
+ content: |
+ [Unit]
+ Description=Kubernetes API Server
+ Documentation=https://github.com/kubernetes/kubernetes
+ Requires=etcd.service
+ After=etcd.service
+ [Service]
+ ExecStartPre=-/usr/bin/mkdir -p /opt/bin
+ ExecStartPre=/usr/bin/wget -N -P /opt/bin http://<PXE_SERVER_IP>/kube-apiserver
+ ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-apiserver
+ ExecStart=/opt/bin/kube-apiserver \
+ --address=0.0.0.0 \
+ --port=8080 \
+ --service-cluster-ip-range=10.100.0.0/16 \
+ --etcd-servers=http://127.0.0.1:4001 \
+ --logtostderr=true
+ Restart=always
+ RestartSec=10
+ - name: kube-controller-manager.service
+ command: start
+ content: |
+ [Unit]
+ Description=Kubernetes Controller Manager
+ Documentation=https://github.com/kubernetes/kubernetes
+ Requires=kube-apiserver.service
+ After=kube-apiserver.service
+ [Service]
+ ExecStartPre=/usr/bin/wget -N -P /opt/bin http://<PXE_SERVER_IP>/kube-controller-manager
+ ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-controller-manager
+ ExecStart=/opt/bin/kube-controller-manager \
+ --master=127.0.0.1:8080 \
+ --logtostderr=true
+ Restart=always
+ RestartSec=10
+ - name: kube-scheduler.service
+ command: start
+ content: |
+ [Unit]
+ Description=Kubernetes Scheduler
+ Documentation=https://github.com/kubernetes/kubernetes
+ Requires=kube-apiserver.service
+ After=kube-apiserver.service
+ [Service]
+ ExecStartPre=/usr/bin/wget -N -P /opt/bin http://<PXE_SERVER_IP>/kube-scheduler
+ ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-scheduler
+ ExecStart=/opt/bin/kube-scheduler --master=127.0.0.1:8080
+ Restart=always
+ RestartSec=10
+ - name: kube-register.service
+ command: start
+ content: |
+ [Unit]
+ Description=Kubernetes Registration Service
+ Documentation=https://github.com/kelseyhightower/kube-register
+ Requires=kube-apiserver.service
+ After=kube-apiserver.service
+ Requires=fleet.service
+ After=fleet.service
+ [Service]
+ ExecStartPre=/usr/bin/wget -N -P /opt/bin http://<PXE_SERVER_IP>/kube-register
+ ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-register
+ ExecStart=/opt/bin/kube-register \
+ --metadata=role=node \
+ --fleet-endpoint=unix:///var/run/fleet.sock \
+ --healthz-port=10248 \
+ --api-endpoint=http://127.0.0.1:8080
+ Restart=always
+ RestartSec=10
+ update:
+ group: stable
+ reboot-strategy: off
+ssh_authorized_keys:
+ - ssh-rsa AAAAB3NzaC1yc2EAAAAD...
+
+
+On the PXE server make and fill in the variables vi /var/www/html/coreos/pxe-cloud-config-slave.yml
.
#cloud-config
+---
+write_files:
+ - path: /etc/default/docker
+ content: |
+ DOCKER_EXTRA_OPTS='--insecure-registry="rdocker.example.com:5000"'
+coreos:
+ units:
+ - name: 10-eno1.network
+ runtime: true
+ content: |
+ [Match]
+ Name=eno1
+ [Network]
+ DHCP=yes
+ - name: 20-nodhcp.network
+ runtime: true
+ content: |
+ [Match]
+ Name=en*
+ [Network]
+ DHCP=none
+ - name: etcd.service
+ mask: true
+ - name: docker.service
+ drop-ins:
+ - name: 50-insecure-registry.conf
+ content: |
+ [Service]
+ Environment="HTTP_PROXY=http://rproxy.example.com:3128/" "NO_PROXY=localhost,127.0.0.0/8,rdocker.example.com"
+ - name: fleet.service
+ command: start
+ content: |
+ [Unit]
+ Description=fleet daemon
+ Wants=fleet.socket
+ After=fleet.socket
+ [Service]
+ Environment="FLEET_ETCD_SERVERS=http://<MASTER_SERVER_IP>:4001"
+ Environment="FLEET_METADATA=role=node"
+ ExecStart=/usr/bin/fleetd
+ Restart=always
+ RestartSec=10s
+ - name: flannel.service
+ command: start
+ content: |
+ [Unit]
+ After=network-online.target
+ Wants=network-online.target
+ Description=flannel is an etcd backed overlay network for containers
+ [Service]
+ Type=notify
+ ExecStartPre=-/usr/bin/mkdir -p /opt/bin
+ ExecStartPre=/usr/bin/wget -N -P /opt/bin http://<PXE_SERVER_IP>/flanneld
+ ExecStartPre=/usr/bin/chmod +x /opt/bin/flanneld
+ ExecStart=/opt/bin/flanneld -etcd-endpoints http://<MASTER_SERVER_IP>:4001
+ - name: docker.service
+ command: start
+ content: |
+ [Unit]
+ After=flannel.service
+ Wants=flannel.service
+ Description=Docker Application Container Engine
+ Documentation=http://docs.docker.io
+ [Service]
+ EnvironmentFile=-/etc/default/docker
+ EnvironmentFile=/run/flannel/subnet.env
+ ExecStartPre=/bin/mount --make-rprivate /
+ ExecStart=/usr/bin/docker -d --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU} -s=overlay -H fd:// ${DOCKER_EXTRA_OPTS}
+ [Install]
+ WantedBy=multi-user.target
+ - name: setup-network-environment.service
+ command: start
+ content: |
+ [Unit]
+ Description=Setup Network Environment
+ Documentation=https://github.com/kelseyhightower/setup-network-environment
+ Requires=network-online.target
+ After=network-online.target
+ [Service]
+ ExecStartPre=-/usr/bin/mkdir -p /opt/bin
+ ExecStartPre=/usr/bin/wget -N -P /opt/bin http://<PXE_SERVER_IP>/setup-network-environment
+ ExecStartPre=/usr/bin/chmod +x /opt/bin/setup-network-environment
+ ExecStart=/opt/bin/setup-network-environment
+ RemainAfterExit=yes
+ Type=oneshot
+ - name: kube-proxy.service
+ command: start
+ content: |
+ [Unit]
+ Description=Kubernetes Proxy
+ Documentation=https://github.com/kubernetes/kubernetes
+ Requires=setup-network-environment.service
+ After=setup-network-environment.service
+ [Service]
+ ExecStartPre=/usr/bin/wget -N -P /opt/bin http://<PXE_SERVER_IP>/kube-proxy
+ ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-proxy
+ ExecStart=/opt/bin/kube-proxy \
+ --etcd-servers=http://<MASTER_SERVER_IP>:4001 \
+ --logtostderr=true
+ Restart=always
+ RestartSec=10
+ - name: kube-kubelet.service
+ command: start
+ content: |
+ [Unit]
+ Description=Kubernetes Kubelet
+ Documentation=https://github.com/kubernetes/kubernetes
+ Requires=setup-network-environment.service
+ After=setup-network-environment.service
+ [Service]
+ EnvironmentFile=/etc/network-environment
+ ExecStartPre=/usr/bin/wget -N -P /opt/bin http://<PXE_SERVER_IP>/kubelet
+ ExecStartPre=/usr/bin/chmod +x /opt/bin/kubelet
+ ExecStart=/opt/bin/kubelet \
+ --address=0.0.0.0 \
+ --port=10250 \
+ --hostname-override=${DEFAULT_IPV4} \
+ --api-servers=<MASTER_SERVER_IP>:8080 \
+ --healthz-bind-address=0.0.0.0 \
+ --healthz-port=10248 \
+ --logtostderr=true
+ Restart=always
+ RestartSec=10
+ update:
+ group: stable
+ reboot-strategy: off
+ssh_authorized_keys:
+ - ssh-rsa AAAAB3NzaC1yc2EAAAAD...
+
+
+Create a pxelinux target file for a slave node: vi /tftpboot/pxelinux.cfg/coreos-node-slave
default coreos
+prompt 1
+timeout 15
+
+display boot.msg
+
+label coreos
+ menu default
+ kernel images/coreos/coreos_production_pxe.vmlinuz
+ append initrd=images/coreos/coreos_production_pxe_image.cpio.gz cloud-config-url=http://<pxe-host-ip>/coreos/pxe-cloud-config-slave.yml console=tty0 console=ttyS0 coreos.autologin=tty1 coreos.autologin=ttyS0
+
+
+And one for the master node: vi /tftpboot/pxelinux.cfg/coreos-node-master
default coreos
+prompt 1
+timeout 15
+
+display boot.msg
+
+label coreos
+ menu default
+ kernel images/coreos/coreos_production_pxe.vmlinuz
+ append initrd=images/coreos/coreos_production_pxe_image.cpio.gz cloud-config-url=http://<pxe-host-ip>/coreos/pxe-cloud-config-master.yml console=tty0 console=ttyS0 coreos.autologin=tty1 coreos.autologin=ttyS0
+
+
+Now that we have our new targets setup for master and slave we want to configure the specific hosts to those targets. We will do this by using the pxelinux mechanism of setting a specific MAC addresses to a specific pxelinux.cfg file.
+ +Refer to the MAC address table in the beginning of this guide. Documentation for more details can be found here.
+ +cd /tftpboot/pxelinux.cfg
+ln -s coreos-node-master 01-d0-00-67-13-0d-00
+ln -s coreos-node-slave 01-d0-00-67-13-0d-01
+ln -s coreos-node-slave 01-d0-00-67-13-0d-02
+
+
+Reboot these servers to get the images PXEd and ready for running containers!
+ +Now that the CoreOS with Kubernetes installed is up and running lets spin up some Kubernetes pods to demonstrate the system.
+ +See a simple nginx example to try out your new cluster.
+ +For more complete applications, please look in the examples directory.
+ +List all keys in etcd:
+ + etcdctl ls --recursive
+
+
+List fleet machines
+ +fleetctl list-machines
+
+
+Check system status of services on master:
+ +systemctl status kube-apiserver
+systemctl status kube-controller-manager
+systemctl status kube-scheduler
+systemctl status kube-register
+
+
+Check system status of services on a node:
+ +systemctl status kube-kubelet
+systemctl status docker.service
+
+
+List Kubernetes
+ +kubectl get pods
+kubectl get nodes
+
+
+Kill all pods:
+ +for i in `kubectl get pods | awk '{print $1}'`; do kubectl stop pod $i; done
+
+
+
+
+
+
+
+
+
+
+ Use the master.yaml and node.yaml cloud-configs to provision a multi-node Kubernetes cluster.
+ +++ +Attention: This requires at least CoreOS version 695.0.0, which includes
+etcd2
.
Attention: Replace <ami_image_id>
below for a suitable version of CoreOS image for AWS.
aws ec2 create-security-group --group-name kubernetes --description "Kubernetes Security Group"
+aws ec2 authorize-security-group-ingress --group-name kubernetes --protocol tcp --port 22 --cidr 0.0.0.0/0
+aws ec2 authorize-security-group-ingress --group-name kubernetes --protocol tcp --port 80 --cidr 0.0.0.0/0
+aws ec2 authorize-security-group-ingress --group-name kubernetes --source-security-group-name kubernetes
+
+aws ec2 run-instances \
+--image-id <ami_image_id> \
+--key-name <keypair> \
+--region us-west-2 \
+--security-groups kubernetes \
+--instance-type m3.medium \
+--user-data file://master.yaml
+
+aws ec2 describe-instances --instance-id <master-instance-id>
+
+Edit node.yaml
and replace all instances of <master-private-ip>
with the private IP address of the master node.
aws ec2 run-instances \
+--count 1 \
+--image-id <ami_image_id> \
+--key-name <keypair> \
+--region us-west-2 \
+--security-groups kubernetes \
+--instance-type m3.medium \
+--user-data file://node.yaml
+
+Attention: Replace <gce_image_id>
below for a suitable version of CoreOS image for Google Compute Engine.
gcloud compute instances create master \
+--image-project coreos-cloud \
+--image <gce_image_id> \
+--boot-disk-size 200GB \
+--machine-type n1-standard-1 \
+--zone us-central1-a \
+--metadata-from-file user-data=master.yaml
+
+gcloud compute instances list
+
+Edit node.yaml
and replace all instances of <master-private-ip>
with the private IP address of the master node.
gcloud compute instances create node1 \
+--image-project coreos-cloud \
+--image <gce_image_id> \
+--boot-disk-size 200GB \
+--machine-type n1-standard-1 \
+--zone us-central1-a \
+--metadata-from-file user-data=node.yaml
+
+Next, setup an ssh tunnel to the master so you can run kubectl from your local host.
+In one terminal, run gcloud compute ssh master --ssh-flag="-L 8080:127.0.0.1:8080"
and in a second
+run gcloud compute ssh master --ssh-flag="-R 8080:127.0.0.1:8080"
.
These instructions are for running on the command line. Most of this you can also do through the Horizon dashboard. +These instructions were tested on the Ice House release on a Metacloud distribution of OpenStack but should be similar if not the same across other versions/distributions of OpenStack.
+ +Make sure the environment variables are set for OpenStack such as:
+ +OS_TENANT_ID
+OS_PASSWORD
+OS_AUTH_URL
+OS_USERNAME
+OS_TENANT_NAME
+
+Test this works with something like:
+ +
+nova list
+
+
+
+You’ll need a suitable version of CoreOS image for OpenStack +Once you download that, upload it to glance. An example is shown below:
+ +glance image-create --name CoreOS723 \
+--container-format bare --disk-format qcow2 \
+--file coreos_production_openstack_image.img \
+--is-public True
+
+nova secgroup-create kubernetes "Kubernetes Security Group"
+nova secgroup-add-rule kubernetes tcp 22 22 0.0.0.0/0
+nova secgroup-add-rule kubernetes tcp 80 80 0.0.0.0/0
+
+nova boot \
+--image <image_name> \
+--key-name <my_key> \
+--flavor <flavor id> \
+--security-group kubernetes \
+--user-data files/master.yaml \
+kube-master
+
+<image_name>
is the CoreOS image name. In our example we can use the image we created in the previous step and put in ‘CoreOS723’
<my_key>
is the keypair name that you already generated to access the instance.
<flavor_id>
is the flavor ID you use to size the instance. Run nova flavor-list
to get the IDs. 3 on the system this was tested with gives the m1.large size.
The important part is to ensure you have the files/master.yml as this is what will do all the post boot configuration. This path is relevant so we are assuming in this example that you are running the nova command in a directory where there is a subdirectory called files that has the master.yml file in it. Absolute paths also work.
+ +Next, assign it a public IP address:
+ +
+nova floating-ip-list
+
+
+
+Get an IP address that’s free and run:
+ +
+nova floating-ip-associate kube-master <ip address>
+
+
+
+where <ip address>
is the IP address that was available from the nova floating-ip-list
command.
Edit node.yaml
and replace all instances of <master-private-ip>
with the private IP address of the master node. You can get this by running nova show kube-master
assuming you named your instance kube master. This is not the floating IP address you just assigned it.
nova boot \
+--image <image_name> \
+--key-name <my_key> \
+--flavor <flavor id> \
+--security-group kubernetes \
+--user-data files/node.yaml \
+minion01
+
+This is basically the same as the master nodes but with the node.yaml post-boot script instead of the master.
+ + + + + + + + + +There are multiple guides on running Kubernetes with CoreOS:
+ +These guides are maintained by CoreOS and deploy Kubernetes the “CoreOS Way” with full TLS, the DNS add-on, and more. These guides pass Kubernetes conformance testing and we encourage you to test this yourself.
+ + + +Guide to setting up a multi-node cluster on Vagrant. The deployer can independently configure the number of etcd nodes, master nodes, and worker nodes to bring up a fully HA control plane.
+ +The quickest way to set up a Kubernetes development environment locally. As easy as git clone
, vagrant up
and configuring kubectl
.
A generic guide to setting up an HA cluster on any cloud or bare metal, with full TLS. Repeat the master or worker steps to configure more machines of that role.
+ +These guides are maintained by community members, cover specific platforms and use cases, and experiment with different ways of configuring Kubernetes on CoreOS.
+ + + +Set up a single master, multi-worker cluster on your choice of platform: AWS, GCE, or VMware Fusion.
+ +Easy Multi-node Cluster on Google Compute Engine
+ +Scripted installation of a single master, multi-worker cluster on GCE. Kubernetes components are managed by fleet.
+ +Multi-node cluster using cloud-config and Weave on Vagrant
+ +Configure a Vagrant-based cluster of 3 machines with networking provided by Weave.
+ +Multi-node cluster using cloud-config and Vagrant
+ +Configure a single master, multi-worker cluster locally, running on your choice of hypervisor: VirtualBox, Parallels, or VMware
+ +Multi-node cluster with Vagrant and fleet units using a small OS X App
+ +Guide to running a single master, multi-worker cluster controlled by an OS X menubar application. Uses Vagrant under the hood.
+ +Resizable multi-node cluster on Azure with Weave
+ +Guide to running an HA etcd cluster with a single master on Azure. Uses the Azure node.js CLI to resize the cluster.
+ +Multi-node cluster using cloud-config, CoreOS and VMware ESXi
+ +Configure a single master, single worker cluster on VMware ESXi.
+ + + + + + + + + +This guide will walk you through installing Kubernetes-Mesos on Datacenter Operating System (DCOS) with the DCOS CLI and operating Kubernetes with the DCOS Kubectl plugin.
+ +Table of Contents +
+ +DCOS is system software that manages computer cluster hardware and software resources and provides common services for distributed applications. Among other services, it provides Apache Mesos as its cluster kernel and Marathon as its init system. With DCOS CLI, Mesos frameworks like Kubernetes-Mesos can be installed with a single command.
+ +Another feature of the DCOS CLI is that it allows plugins like the DCOS Kubectl plugin. This allows for easy access to a version-compatible Kubectl without having to manually download or install.
+ +Further information about the benefits of installing Kubernetes on DCOS can be found in the Kubernetes-Mesos documentation.
+ +For more details about the Kubernetes DCOS packaging, see the Kubernetes-Mesos project.
+ +Since Kubernetes-Mesos is still alpha, it is a good idea to familiarize yourself with the current known issues which may limit or modify the behavior of Kubernetes on DCOS.
+ +If you have problems completing the steps below, please file an issue against the kubernetes-mesos project.
+ +Explore the following resources for more information about Kubernetes, Kubernetes on Mesos/DCOS, and DCOS itself.
+ +Configure and validate the Mesosphere Multiverse as a package source repository
+ +```
+ +$ dcos config prepend package.sources https://github.com/mesosphere/multiverse/archive/version-1.x.zip + $ dcos package update –validate
+ +```
+Install etcd
+ +By default, the Kubernetes DCOS package starts a single-node etcd. In order to avoid state loss in the event of Kubernetes component container failure, install an HA etcd-mesos cluster on DCOS.
+ +```
+ +$ dcos package install etcd
+ +```
+Verify that etcd is installed and healthy
+ +The etcd cluster takes a short while to deploy. Verify that /etcd
is healthy before going on to the next step.
```
+ +$ dcos marathon app list + ID MEM CPUS TASKS HEALTH DEPLOYMENT CONTAINER CMD + /etcd 128 0.2 1/1 1/1 — DOCKER None
+ +```
+Create Kubernetes installation configuration
+ +Configure Kubernetes to use the HA etcd installed on DCOS.
+ +```
+ +$ cat >/tmp/options.json «EOF + { + “kubernetes”: { + “etcd-mesos-framework-name”: “etcd” + } + } + EOF
+ +```
+Install Kubernetes
+ +```
+ +$ dcos package install –options=/tmp/options.json kubernetes
+ +```
+Verify that Kubernetes is installed and healthy
+ +The Kubernetes cluster takes a short while to deploy. Verify that /kubernetes
is healthy before going on to the next step.
```
+ +$ dcos marathon app list + ID MEM CPUS TASKS HEALTH DEPLOYMENT CONTAINER CMD + /etcd 128 0.2 1/1 1/1 — DOCKER None + /kubernetes 768 1 1/1 1/1 — DOCKER None
+ +```
+Verify that Kube-DNS & Kube-UI are deployed, running, and ready
+ +```
+ +$ dcos kubectl get pods –namespace=kube-system + NAME READY STATUS RESTARTS AGE + kube-dns-v8-tjxk9 4/4 Running 0 1m + kube-ui-v2-tjq7b 1/1 Running 0 1m
+ +```
+ +Names and ages may vary.
+Now that Kubernetes is installed on DCOS, you may wish to explore the Kubernetes Examples or the Kubernetes User Guide.
+ +Stop and delete all replication controllers and pods in each namespace:
+ +Before uninstalling Kubernetes, destroy all the pods and replication controllers. The uninstall process will try to do this itself, but by default it times out quickly and may leave your cluster in a dirty state.
+ +```
+ +$ dcos kubectl delete rc,pods –all –namespace=default + $ dcos kubectl delete rc,pods –all –namespace=kube-system
+ +```
+Validate that all pods have been deleted
+ +```
+ +$ dcos kubectl get pods –all-namespaces
+ +```
+Uninstall Kubernetes
+ +```
+ +$ dcos package uninstall kubernetes
+ +```
+Testing version v1.1
+ -Test of heading 2 is above
+First of all, download the template dns rc and svc file from
-Test of heading 3 is above
+ -Test of heading 4 is above
+Then you need to set DNS_REPLICAS
, DNS_DOMAIN
, DNS_SERVER_IP
, KUBE_SERVER
ENV.
Testing inline code
and below is a code example:
+$ export DNS_REPLICAS=1
+
+$ export DNS_DOMAIN=cluster.local # specify in startup parameter `--cluster-domain` for containerized kubelet
+
+$ export DNS_SERVER_IP=10.0.0.10 # specify in startup parameter `--cluster-dns` for containerized kubelet
+
+$ export KUBE_SERVER=10.10.103.250 # your master server ip, you may change it
-console.log("Hello, world!");
-Okeedoke.
+Replace the corresponding value in the template.
+
+
+$ sed -e "s/{{ pillar\['dns_replicas'\] }}/${DNS_REPLICAS}/g;s/{{ pillar\['dns_domain'\] }}/${DNS_DOMAIN}/g;s/{kube_server_url}/${KUBE_SERVER}/g;" skydns-rc.yaml.in > ./skydns-rc.yaml
+
+$ sed -e "s/{{ pillar\['dns_server'\] }}/${DNS_SERVER_IP}/g" skydns-svc.yaml.in > ./skydns-svc.yaml
+
+
+
+Use kubectl
to create skydns rc and service
+
+
+$ kubectl -s "$KUBE_SERVER:8080" --namespace=kube-system create -f ./skydns-rc.yaml
+
+$ kubectl -s "$KUBE_SERVER:8080" --namespace=kube-system create -f ./skydns-svc.yaml
+
+
+
+Test if DNS works
+
+Follow this link to check it out.
+
+
+
+
+
+
+
+
Running Multi-Node Kubernetes Using Docker +——————————————
+ +Note: +These instructions are somewhat significantly more advanced than the single node instructions. If you are +interested in just starting to explore Kubernetes, we recommend that you start there.
+ +Note: +There is a bug in Docker 1.7.0 that prevents this from working correctly. +Please install Docker 1.6.2 or Docker 1.7.1.
+ +Table of Contents
+ +This guide will set up a 2-node Kubernetes cluster, consisting of a master node which hosts the API server and orchestrates work +and a worker node which receives work from the master. You can repeat the process of adding worker nodes an arbitrary number of +times to create larger clusters.
+ +Here’s a diagram of what the final result will look like: +
+ +This guide also uses a pattern of running two instances of the Docker daemon
+ 1) A bootstrap Docker instance which is used to start system daemons like flanneld
and etcd
+ 2) A main Docker instance which is used for the Kubernetes infrastructure and user’s scheduled containers
This pattern is necessary because the flannel
daemon is responsible for setting up and managing the network that interconnects
+all of the Docker containers created by Kubernetes. To achieve this, it must run outside of the main Docker daemon. However,
+it is still useful to use containers for deployment and management, so we create a simpler bootstrap daemon to achieve this.
You can specify k8s version on very node before install:
+ +
+export K8S_VERSION=<your_k8s_version (e.g. 1.0.3)>
+
+
+
+Otherwise, we’ll use latest hyperkube
image as default k8s version.
The first step in the process is to initialize the master node.
+ +Clone the Kubernetes repo, and run master.sh on the master machine with root:
+ +cd kubernetes/docs/getting-started-guides/docker-multinode/
+./master.sh
+
+Master done!
See here for detailed instructions explanation.
+ +Once your master is up and running you can add one or more workers on different machines.
+ +Clone the Kubernetes repo, and run worker.sh on the worker machine with root:
+ +export MASTER_IP=<your_master_ip (e.g. 1.2.3.4)>
+cd kubernetes/docs/getting-started-guides/docker-multinode/
+./worker.sh
+
+Worker done!
See here for detailed instructions explanation.
+ +See here for instructions.
+ +Once your cluster has been created you can test it out
+ +For more complete applications, please look in the examples directory
+ + + + + + + + + +We’ll begin by setting up the master node. For the purposes of illustration, we’ll assume that the IP of this machine is ${MASTER_IP}
There are two main phases to installing the master:
+ * Setting up flanneld
and etcd
+ * Starting the Kubernetes master components
Note: +There is a bug in Docker 1.7.0 that prevents this from working correctly. +Please install Docker 1.6.2 or Docker 1.7.1.
+ +We’re going to use flannel
to set up networking between Docker daemons. Flannel itself (and etcd on which it relies) will run inside of
+Docker containers themselves. To achieve this, we need a separate “bootstrap” instance of the Docker daemon. This daemon will be started with
+--iptables=false
so that it can only run containers with --net=host
. That’s sufficient to bootstrap our system.
Run:
+ +sudo sh -c 'docker -d -H unix:///var/run/docker-bootstrap.sock -p /var/run/docker-bootstrap.pid --iptables=false --ip-masq=false --bridge=none --graph=/var/lib/docker-bootstrap 2> /var/log/docker-bootstrap.log 1> /dev/null &'
+
+Important Note: +If you are running this on a long running system, rather than experimenting, you should run the bootstrap Docker instance under something like SysV init, upstart or systemd so that it is restarted +across reboots and failures.
+ +Run:
+ +sudo docker -H unix:///var/run/docker-bootstrap.sock run --net=host -d gcr.io/google_containers/etcd:2.0.12 /usr/local/bin/etcd --addr=127.0.0.1:4001 --bind-addr=0.0.0.0:4001 --data-dir=/var/etcd/data
+
+Next, you need to set a CIDR range for flannel. This CIDR should be chosen to be non-overlapping with any existing network you are using:
+ +sudo docker -H unix:///var/run/docker-bootstrap.sock run --net=host gcr.io/google_containers/etcd:2.0.12 etcdctl set /coreos.com/network/config '{ "Network": "10.1.0.0/16" }'
+
+Flannel is a network abstraction layer build by CoreOS, we will use it to provide simplified networking between our Pods of containers.
+ +Flannel re-configures the bridge that Docker uses for networking. As a result we need to stop Docker, reconfigure its networking, and then restart Docker.
+ +To re-configure Docker to use flannel, we need to take docker down, run flannel and then restart Docker.
+ +Turning down Docker is system dependent, it may be:
+ +sudo /etc/init.d/docker stop
+
+or
+ +sudo systemctl stop docker
+
+or it may be something else.
+ +Now run flanneld itself:
+ +sudo docker -H unix:///var/run/docker-bootstrap.sock run -d --net=host --privileged -v /dev/net:/dev/net quay.io/coreos/flannel:0.5.0
+
+The previous command should have printed a really long hash, copy this hash.
+ +Now get the subnet settings from flannel:
+ +sudo docker -H unix:///var/run/docker-bootstrap.sock exec <really-long-hash-from-above-here> cat /run/flannel/subnet.env
+
+You now need to edit the docker configuration to activate new flags. Again, this is system specific.
+ +This may be in /etc/default/docker
or /etc/systemd/service/docker.service
or it may be elsewhere.
Regardless, you need to add the following to the docker command line:
+ +--bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}
+
+Docker creates a bridge named docker0
by default. You need to remove this:
sudo /sbin/ifconfig docker0 down
+sudo brctl delbr docker0
+
+You may need to install the bridge-utils
package for the brctl
binary.
Again this is system dependent, it may be:
+ +sudo /etc/init.d/docker start
+
+it may be:
+ +systemctl start docker
+
+Ok, now that your networking is set up, you can startup Kubernetes, this is the same as the single-node case, we will use the “main” instance of the Docker daemon for the Kubernetes components.
+ +sudo docker run \
+ --volume=/:/rootfs:ro \
+ --volume=/sys:/sys:ro \
+ --volume=/dev:/dev \
+ --volume=/var/lib/docker/:/var/lib/docker:rw \
+ --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \
+ --volume=/var/run:/var/run:rw \
+ --net=host \
+ --privileged=true \
+ --pid=host \
+ -d \
+ gcr.io/google_containers/hyperkube:v1.0.1 /hyperkube kubelet --api-servers=http://localhost:8080 --v=2 --address=0.0.0.0 --enable-server --hostname-override=127.0.0.1 --config=/etc/kubernetes/manifests-multi --cluster-dns=10.0.0.10 --cluster-domain=cluster.local
+
+++ +Note that
+--cluster-dns
and--cluster-domain
is used to deploy dns, feel free to discard them if dns is not needed.
sudo docker run -d --net=host --privileged gcr.io/google_containers/hyperkube:v1.0.1 /hyperkube proxy --master=http://127.0.0.1:8080 --v=2
+
+At this point, you should have a functioning 1-node cluster. Let’s test it out!
+ +Download the kubectl binary and make it available by editing your PATH ENV. +(OS X) +(linux)
+ +List the nodes
+ +kubectl get nodes
+
+This should print:
+ +NAME LABELS STATUS
+127.0.0.1 kubernetes.io/hostname=127.0.0.1 Ready
+
+If the status of the node is NotReady
or Unknown
please check that all of the containers you created are successfully running.
+If all else fails, ask questions on Slack.
Move on to adding one or more workers or deploy a dns
+ + + + + + + + + +Testing version v1.1
+ -Test of heading 2 is above
+To validate that your node(s) have been added, run:
-Test of heading 3 is above
- -Test of heading 4 is above
- -Testing inline code
and below is a code example:
console.log("Hello, world!");
+
+ kubectl get nodes
+
+
+That should show something like:
+
+
+ NAME LABELS STATUS
+10.240.99.26 kubernetes.io/hostname=10.240.99.26 Ready
+127.0.0.1 kubernetes.io/hostname=127.0.0.1 Ready
+
+
+
+If the status of any node is Unknown
or NotReady
your cluster is broken, double check that all containers are running properly, and if all else fails, contact us on Slack.
+
+Run an application
+
+
+ kubectl -s http://localhost:8080 run nginx --image=nginx --port=80
+
+
+
+now run docker ps
you should see nginx running. You may need to wait a few minutes for the image to get pulled.
+
+Expose it as a service
+
+
+ kubectl expose rc nginx --port=80
+
+
+
+Run the following command to obtain the IP of this service we just created. There are two IPs, the first one is internal (CLUSTER_IP), and the second one is the external load-balanced IP.
+
+
+ kubectl get svc nginx
+
+
+
+Alternatively, you can obtain only the first IP (CLUSTER_IP) by running:
+
+
+ kubectl get svc nginx --template={{.spec.clusterIP}}
+
+
+
+Hit the webserver with the first IP (CLUSTER_IP):
+
+
+ curl <insert-cluster-ip-here>
+
+
+
+Note that you will need run this curl command on your boot2docker VM if you are running on OS X.
+
+Scaling
+
+Now try to scale up the nginx you created before:
+
+
+ kubectl scale rc nginx --replicas=3
+
+
+
+And list the pods
+
+
+ kubectl get pods
+
+
+
+You should see pods landing on the newly added machine.
+
+
+
+
+
+
+
-Okeedoke.
These instructions are very similar to the master set-up above, but they are duplicated for clarity.
+You need to repeat these instructions for each node you want to join the cluster.
+We will assume that the IP address of this node is ${NODE_IP}
and you have the IP address of the master in ${MASTER_IP}
that you created in the master instructions.
For each worker node, there are three steps:
+ * Set up flanneld
on the worker node
+ * Start Kubernetes on the worker node
+ * Add the worker to the cluster
As before, the Flannel daemon is going to provide network connectivity.
+ +Note: +There is a bug in Docker 1.7.0 that prevents this from working correctly. +Please install Docker 1.6.2 or wait for Docker 1.7.1.
+ +As previously, we need a second instance of the Docker daemon running to bootstrap the flannel networking.
+ +Run:
+ +sudo sh -c 'docker -d -H unix:///var/run/docker-bootstrap.sock -p /var/run/docker-bootstrap.pid --iptables=false --ip-masq=false --bridge=none --graph=/var/lib/docker-bootstrap 2> /var/log/docker-bootstrap.log 1> /dev/null &'
+
+Important Note: +If you are running this on a long running system, rather than experimenting, you should run the bootstrap Docker instance under something like SysV init, upstart or systemd so that it is restarted +across reboots and failures.
+ +To re-configure Docker to use flannel, we need to take docker down, run flannel and then restart Docker.
+ +Turning down Docker is system dependent, it may be:
+ +sudo /etc/init.d/docker stop
+
+or
+ +sudo systemctl stop docker
+
+or it may be something else.
+ +Now run flanneld itself, this call is slightly different from the above, since we point it at the etcd instance on the master.
+ +sudo docker -H unix:///var/run/docker-bootstrap.sock run -d --net=host --privileged -v /dev/net:/dev/net quay.io/coreos/flannel:0.5.0 /opt/bin/flanneld --etcd-endpoints=http://${MASTER_IP}:4001
+
+The previous command should have printed a really long hash, copy this hash.
+ +Now get the subnet settings from flannel:
+ +sudo docker -H unix:///var/run/docker-bootstrap.sock exec <really-long-hash-from-above-here> cat /run/flannel/subnet.env
+
+You now need to edit the docker configuration to activate new flags. Again, this is system specific.
+ +This may be in /etc/default/docker
or /etc/systemd/service/docker.service
or it may be elsewhere.
Regardless, you need to add the following to the docker command line:
+ +--bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}
+
+Docker creates a bridge named docker0
by default. You need to remove this:
sudo /sbin/ifconfig docker0 down
+sudo brctl delbr docker0
+
+You may need to install the bridge-utils
package for the brctl
binary.
Again this is system dependent, it may be:
+ +sudo /etc/init.d/docker start
+
+it may be:
+ +systemctl start docker
+
+Again this is similar to the above, but the --api-servers
now points to the master we set up in the beginning.
sudo docker run \
+ --volume=/:/rootfs:ro \
+ --volume=/sys:/sys:ro \
+ --volume=/dev:/dev \
+ --volume=/var/lib/docker/:/var/lib/docker:rw \
+ --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \
+ --volume=/var/run:/var/run:rw \
+ --net=host \
+ --privileged=true \
+ --pid=host \
+ -d \
+ gcr.io/google_containers/hyperkube:v1.0.1 /hyperkube kubelet --api-servers=http://${MASTER_IP}:8080 --v=2 --address=0.0.0.0 --enable-server --hostname-override=$(hostname -i) --cluster-dns=10.0.0.10 --cluster-domain=cluster.local
+
+The service proxy provides load-balancing between groups of containers defined by Kubernetes Services
sudo docker run -d --net=host --privileged gcr.io/google_containers/hyperkube:v1.0.1 /hyperkube proxy --master=http://${MASTER_IP}:8080 --v=2
+
+Move on to testing your cluster or add another node
+ + + + + + + + + +Running Kubernetes locally via Docker +————————————-
+ +Table of Contents
+ +The following instructions show you how to set up a simple, single node Kubernetes cluster using Docker.
+ +Here’s a diagram of what the final result will look like: +
+ +CONFIG_RESOURCE_COUNTERS=y
+ CONFIG_MEMCG=y
+ CONFIG_MEMCG_SWAP=y
+ CONFIG_MEMCG_SWAP_ENABLED=y
+ CONFIG_MEMCG_KMEM=y
+
+GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1"
+
+NOTE: The above is specifically for GRUB2.
+You can check the command line parameters passed to your kernel by looking at the
+output of /proc/cmdline:
+
+
+$cat /proc/cmdline
+ BOOT_IMAGE=/boot/vmlinuz-3.18.4-aufs root=/dev/sda5 ro cgroup_enable=memory
+ swapaccount=1
+
+docker run --net=host -d gcr.io/google_containers/etcd:2.0.12 /usr/local/bin/etcd --addr=127.0.0.1:4001 --bind-addr=0.0.0.0:4001 --data-dir=/var/etcd/data
+
+docker run \
+ --volume=/:/rootfs:ro \
+ --volume=/sys:/sys:ro \
+ --volume=/dev:/dev \
+ --volume=/var/lib/docker/:/var/lib/docker:ro \
+ --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \
+ --volume=/var/run:/var/run:rw \
+ --net=host \
+ --pid=host \
+ --privileged=true \
+ -d \
+ gcr.io/google_containers/hyperkube:v1.0.1 \
+ /hyperkube kubelet --containerized --hostname-override="127.0.0.1" --address="0.0.0.0" --api-servers=http://localhost:8080 --config=/etc/kubernetes/manifests
+
+This actually runs the kubelet, which in turn runs a pod that contains the other master components.
+ +docker run -d --net=host --privileged gcr.io/google_containers/hyperkube:v1.0.1 /hyperkube proxy --master=http://127.0.0.1:8080 --v=2
+
+At this point you should have a running Kubernetes cluster. You can test this by downloading the kubectl +binary +(OS X) +(linux)
+ +Note: +On OS/X you will need to set up port forwarding via ssh:
+ +boot2docker ssh -L8080:localhost:8080
+
+List the nodes in your cluster by running:
+ +kubectl get nodes
+
+This should print:
+ +NAME LABELS STATUS
+127.0.0.1 <none> Ready
+
+If you are running different Kubernetes clusters, you may need to specify -s http://localhost:8080
to select the local cluster.
kubectl -s http://localhost:8080 run nginx --image=nginx --port=80
+
+Now run docker ps
you should see nginx running. You may need to wait a few minutes for the image to get pulled.
kubectl expose rc nginx --port=80
+
+Run the following command to obtain the IP of this service we just created. There are two IPs, the first one is internal (CLUSTER_IP), and the second one is the external load-balanced IP.
+ +kubectl get svc nginx
+
+Alternatively, you can obtain only the first IP (CLUSTER_IP) by running:
+ +kubectl get svc nginx --template={{.spec.clusterIP}}
+
+Hit the webserver with the first IP (CLUSTER_IP):
+ +curl <insert-cluster-ip-here>
+
+Note that you will need run this curl command on your boot2docker VM if you are running on OS X.
+ +Many of these containers run under the management of the kubelet
binary, which attempts to keep containers running, even if they fail. So, in order to turn down
+the cluster, you need to first kill the kubelet container, and then any other containers.
You may use docker kill $(docker ps -aq)
, note this removes all containers running under Docker, so use with caution.
You need two or more Fedora 22 droplets on Digital Ocean with Private Networking enabled.
+ +This guide will walk you through the process of getting a Kubernetes Fedora cluster running on Digital Ocean with networking powered by Calico networking. It will cover the installation and configuration of the following systemd processes on the following hosts:
+ +Kubernetes Master:
+- kube-apiserver
+- kube-controller-manager
+- kube-scheduler
+- etcd
+- docker
+- calico-node
Kubernetes Node:
+- kubelet
+- kube-proxy
+- docker
+- calico-node
For this demo, we will be setting up one Master and one Node with the following information:
+ +Hostname | +IP | +
---|---|
kube-master | +10.134.251.56 | +
kube-node-1 | +10.134.251.55 | +
This guide is scalable to multiple nodes provided you configure interface-cbr0 with its own subnet on each Node and add an entry to /etc/hosts for each host.
+ +Ensure you substitute the IP Addresses and Hostnames used in this guide with ones in your own setup.
+ +Digital Ocean private networking configures a private network on eth1 for each host. To simplify communication between the hosts, we will add an entry to /etc/hosts so that all hosts in the cluster can hostname-resolve one another to this interface. It is important that the hostname resolves to this interface instead of eth0, as all Kubernetes and Calico services will be running on it.
+ +
+echo "10.134.251.56 kube-master" >> /etc/hosts
+echo "10.134.251.55 kube-node-1" >> /etc/hosts
+
+
+
+++ +Make sure that communication works between kube-master and each kube-node by using a utility such as ping.
+
+yum -y install etcd
+
+
+
+/etc/etcd/etcd.conf
+ETCD_LISTEN_CLIENT_URLS="http://kube-master:4001"
+
+ETCD_ADVERTISE_CLIENT_URLS="http://kube-master:4001"
+
+
+
+
+yum -y install kubernetes
+
+
+
+/etc/kubernetes/config
+# How the controller-manager, scheduler, and proxy find the apiserver
+KUBE_MASTER="--master=http://kube-master:8080"
+
+
+
+/etc/kubernetes/apiserver
+# The address on the local server to listen to.
+KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
+
+KUBE_ETCD_SERVERS="--etcd-servers=http://kube-master:4001"
+
+# Remove ServiceAccount from this line to run without API Tokens
+KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ResourceQuota"
+
+
+
+
+mkdir /var/run/kubernetes
+chown kube:kube /var/run/kubernetes
+chmod 750 /var/run/kubernetes
+
+
+
+
+for SERVICE in etcd kube-apiserver kube-controller-manager kube-scheduler; do
+ systemctl restart $SERVICE
+ systemctl enable $SERVICE
+ systemctl status $SERVICE
+done
+
+
+
+Next, we’ll launch Calico on Master to allow communication between Pods and any services running on the Master. +* Install calicoctl, the calico configuration tool.
+ +
+wget https://github.com/Metaswitch/calico-docker/releases/download/v0.5.5/calicoctl
+chmod +x ./calicoctl
+sudo mv ./calicoctl /usr/bin
+
+
+
+/etc/systemd/system/calico-node.service
+[Unit]
+Description=calicoctl node
+Requires=docker.service
+After=docker.service
+
+[Service]
+User=root
+Environment="ETCD_AUTHORITY=kube-master:4001"
+PermissionsStartOnly=true
+ExecStartPre=/usr/bin/calicoctl checksystem --fix
+ExecStart=/usr/bin/calicoctl node --ip=10.134.251.56 --detach=false
+
+[Install]
+WantedBy=multi-user.target
+
+
+
+++ +Be sure to substitute
+--ip=10.134.251.56
with your Master’s eth1 IP Address.
+systemctl enable calico-node.service
+systemctl start calico-node.service
+
+
+
+++ +Starting calico for the first time may take a few minutes as the calico-node docker image is downloaded.
+
By default, docker will create and run on a virtual interface called docker0
. This interface is automatically assigned the address range 172.17.42.1/16. In order to set our own address range, we will create a new virtual interface called cbr0
and then start docker on it.
/etc/sysconfig/network-scripts/ifcfg-cbr0
:
+DEVICE=cbr0
+TYPE=Bridge
+IPADDR=192.168.1.1
+NETMASK=255.255.255.0
+ONBOOT=yes
+BOOTPROTO=static
+
+
+
+++ +Note for Multi-Node Clusters: Each node should be assigned an IP address on a unique subnet. In this example, node-1 is using 192.168.1.1/24, so node-2 should be assigned another pool on the 192.168.x.0/24 subnet, e.g. 192.168.2.1/24.
+
+systemctl restart network.service
+
+
+
+
+yum -y install docker
+
+
+
+cbr0
by editing /etc/sysconfig/docker-network
:
+DOCKER_NETWORK_OPTIONS="--bridge=cbr0 --iptables=false --ip-masq=false"
+
+
+
+
+systemctl start docker
+
+
+
+
+wget https://github.com/Metaswitch/calico-docker/releases/download/v0.5.5/calicoctl
+chmod +x ./calicoctl
+sudo mv ./calicoctl /usr/bin
+
+
+
+/etc/systemd/system/calico-node.service
+[Unit]
+Description=calicoctl node
+Requires=docker.service
+After=docker.service
+
+[Service]
+User=root
+Environment="ETCD_AUTHORITY=kube-master:4001"
+PermissionsStartOnly=true
+ExecStartPre=/usr/bin/calicoctl checksystem --fix
+ExecStart=/usr/bin/calicoctl node --ip=10.134.251.55 --detach=false --kubernetes
+
+[Install]
+WantedBy=multi-user.target
+
+
+
+++ +Note: You must replace the IP address with your node’s eth1 IP Address!
+
+systemctl enable calico-node.service
+systemctl start calico-node.service
+
+
+
+Most Kubernetes application deployments will require communication between Pods and the kube-apiserver on Master. On a standard Digital Ocean Private Network, requests sent from Pods to the kube-apiserver will not be returned as the networking fabric will drop response packets destined for any 192.168.0.0/16 address. To resolve this, you can have calicoctl add a masquerade rule to all outgoing traffic on the node:
+ +
+ETCD_AUTHORITY=kube-master:4001 calicoctl pool add 192.168.0.0/16 --nat-outgoing
+
+
+
+
+yum -y install kubernetes
+
+
+
+/etc/kubernetes/config
+# How the controller-manager, scheduler, and proxy find the apiserver
+KUBE_MASTER="--master=http://kube-master:8080"
+
+
+
+Edit /etc/kubernetes/kubelet
We’ll pass in an extra parameter - --network-plugin=calico
to tell the Kubelet to use the Calico networking plugin. Additionally, we’ll add two environment variables that will be used by the Calico networking plugin.
+# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
+KUBELET_ADDRESS="--address=0.0.0.0"
+
+# You may leave this blank to use the actual hostname
+# KUBELET_HOSTNAME="--hostname-override=127.0.0.1"
+
+# location of the api-server
+KUBELET_API_SERVER="--api-servers=http://kube-master:8080"
+
+# Add your own!
+KUBELET_ARGS="--network-plugin=calico"
+
+# The following are variables which the kubelet will pass to the calico-networking plugin
+ETCD_AUTHORITY="kube-master:4001"
+KUBE_API_ROOT="http://kube-master:8080/api/v1"
+
+
+
+
+for SERVICE in kube-proxy kubelet; do
+ systemctl restart $SERVICE
+ systemctl enable $SERVICE
+ systemctl status $SERVICE
+done
+
+
+
+The cluster should be running! Check that your nodes are reporting as such:
+ +
+kubectl get nodes
+NAME LABELS STATUS
+kube-node-1 kubernetes.io/hostname=kube-node-1 Ready
+
+
+
+
+
+
+
+
+
+
+
+ Configuring Kubernetes on Fedora via Ansible +——————————————————————————————————-
+ +Configuring Kubernetes on Fedora via Ansible offers a simple way to quickly create a clustered environment with little effort.
+ +Table of Contents
+ +The hosts can be virtual or bare metal. Ansible will take care of the rest of the configuration for you - configuring networking, installing packages, handling the firewall, etc. This example will use one master and two nodes.
+ +A Kubernetes cluster requires etcd, a master, and n nodes, so we will create a cluster with three hosts, for example:
+ +master,etcd = kube-master.example.com
+ node1 = kube-node-01.example.com
+ node2 = kube-node-02.example.com
+
+Make sure your local machine has
+ +If not
+ +yum install -y ansible git python-netaddr
+
+Now clone down the Kubernetes repository
+ +git clone https://github.com/kubernetes/contrib.git
+cd contrib/ansible
+
+Tell ansible about each machine and its role in your cluster
+ +Get the IP addresses from the master and nodes. Add those to the ~/contrib/ansible/inventory
file on the host running Ansible.
[masters]
+kube-master.example.com
+
+[etcd]
+kube-master.example.com
+
+[nodes]
+kube-node-01.example.com
+kube-node-02.example.com
+
+If you already are running on a machine which has passwordless ssh access to the kube-master and kube-node-{01,02} nodes, and ‘sudo’ privileges, simply set the value of ansible_ssh_user
in ~/contrib/ansible/group_vars/all.yaml
to the username which you use to ssh to the nodes (i.e. fedora
), and proceed to the next step…
Otherwise setup ssh on the machines like so (you will need to know the root password to all machines in the cluster).
+ +edit: ~/contrib/ansible/group_vars/all.yml
+ +ansible_ssh_user: root
+
+Configuring ssh access to the cluster
+ +If you already have ssh access to every machine using ssh public keys you may skip to setting up the cluster
+ +Make sure your local machine (root) has an ssh key pair if not
+ +ssh-keygen
+
+Copy the ssh public key to all nodes in the cluster
+ +for node in kube-master.example.com kube-node-01.example.com kube-node-02.example.com; do
+ ssh-copy-id ${node}
+done
+
+Although the default value of variables in ~/contrib/ansible/group_vars/all.yml
should be good enough, if not, change them as needed.
edit: ~/contrib/ansible/group_vars/all.yml
+ +Configure access to kubernetes packages
+ +Modify source_type
as below to access kubernetes packages through the package manager.
source_type: packageManager
+
+Configure the IP addresses used for services
+ +Each Kubernetes service gets its own IP address. These are not real IPs. You need only select a range of IPs which are not in use elsewhere in your environment.
+ +kube_service_addresses: 10.254.0.0/16
+
+Managing flannel
+ +Modify flannel_subnet
, flannel_prefix
and flannel_host_prefix
only if defaults are not appropriate for your cluster.
Managing add on services in your cluster
+ +Set cluster_logging
to false or true (default) to disable or enable logging with elasticsearch.
cluster_logging: true
+
+Turn cluster_monitoring
to true (default) or false to enable or disable cluster monitoring with heapster and influxdb.
cluster_monitoring: true
+
+Turn dns_setup
to true (recommended) or false to enable or disable whole DNS configuration.
dns_setup: true
+
+Tell ansible to get to work!
+ +This will finally setup your whole Kubernetes cluster for you.
+ +cd ~/contrib/ansible/
+
+./setup.sh
+
+That’s all there is to it. It’s really that easy. At this point you should have a functioning Kubernetes cluster.
+ +Show kubernetes nodes
+ +Run the following on the kube-master:
+ +kubectl get nodes
+
+Show services running on masters and nodes
+ +systemctl | grep -i kube
+
+Show firewall rules on the masters and nodes
+ +iptables -nvL
+
+Create /tmp/apache.json on the master with the following contents and deploy pod
+ +{
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "fedoraapache",
+ "labels": {
+ "name": "fedoraapache"
+ }
+ },
+ "spec": {
+ "containers": [
+ {
+ "name": "fedoraapache",
+ "image": "fedora/apache",
+ "ports": [
+ {
+ "hostPort": 80,
+ "containerPort": 80
+ }
+ ]
+ }
+ ]
+ }
+}
+
+kubectl create -f /tmp/apache.json
+
+Check where the pod was created
+ +kubectl get pods
+
+Check Docker status on nodes
+ +docker ps
+docker images
+
+After the pod is ‘Running’ Check web server access on the node
+ +curl http://localhost
+
+That’s it !
+ + + + + + + + + +Getting started on Fedora +—————————————————–
+ +Table of Contents
+ +This is a getting started guide for Fedora. It is a manual configuration so you understand all the underlying packages / services / ports, etc…
+ +This guide will only get ONE node (previously minion) working. Multiple nodes require a functional networking configuration done outside of Kubernetes. Although the additional Kubernetes configuration requirements should be obvious.
+ +The Kubernetes package provides a few services: kube-apiserver, kube-scheduler, kube-controller-manager, kubelet, kube-proxy. These services are managed by systemd and the configuration resides in a central location: /etc/kubernetes. We will break the services up between the hosts. The first host, fed-master, will be the Kubernetes master. This host will run the kube-apiserver, kube-controller-manager, and kube-scheduler. In addition, the master will also run etcd (not needed if etcd runs on a different host but this guide assumes that etcd and Kubernetes master run on the same host). The remaining host, fed-node will be the node and run kubelet, proxy and docker.
+ +System Information:
+ +Hosts:
+ +
+fed-master = 192.168.121.9
+fed-node = 192.168.121.65
+
+
+
+Prepare the hosts:
+ +yum -y install --enablerepo=updates-testing kubernetes
+
+yum -y install etcd iptables
+
+echo "192.168.121.9 fed-master
+192.168.121.65 fed-node" >> /etc/hosts
+
+# Comma separated list of nodes in the etcd cluster
+KUBE_MASTER="--master=http://fed-master:8080"
+
+# logging to stderr means we get it in the systemd journal
+KUBE_LOGTOSTDERR="--logtostderr=true"
+
+# journal message level, 0 is debug
+KUBE_LOG_LEVEL="--v=0"
+
+# Should this cluster be allowed to run privileged docker containers
+KUBE_ALLOW_PRIV="--allow-privileged=false"
+
+systemctl disable iptables-services firewalld
+systemctl stop iptables-services firewalld
+
+Configure the Kubernetes services on the master.
+ +# The address on the local server to listen to.
+KUBE_API_ADDRESS="--address=0.0.0.0"
+
+# Comma separated list of nodes in the etcd cluster
+KUBE_ETCD_SERVERS="--etcd-servers=http://127.0.0.1:4001"
+
+# Address range to use for services
+KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
+
+# Add your own!
+KUBE_API_ARGS=""
+
+ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:4001"
+
+mkdir /var/run/kubernetes
+chown kube:kube /var/run/kubernetes
+chmod 750 /var/run/kubernetes
+
+for SERVICES in etcd kube-apiserver kube-controller-manager kube-scheduler; do
+ systemctl restart $SERVICES
+ systemctl enable $SERVICES
+ systemctl status $SERVICES
+done
+
+Addition of nodes:
+Create following node.json file on Kubernetes master node:
+{
+ "apiVersion": "v1",
+ "kind": "Node",
+ "metadata": {
+ "name": "fed-node",
+ "labels":{ "name": "fed-node-label"}
+ },
+ "spec": {
+ "externalID": "fed-node"
+ }
+}
+
+Now create a node object internally in your Kubernetes cluster by running:
+ +$ kubectl create -f ./node.json
+
+$ kubectl get nodes
+NAME LABELS STATUS
+fed-node name=fed-node-label Unknown
+
+Please note that in the above, it only creates a representation for the node
+fed-node internally. It does not provision the actual fed-node. Also, it
+is assumed that fed-node (as specified in name
) can be resolved and is
+reachable from Kubernetes master node. This guide will discuss how to provision
+a Kubernetes node (fed-node) below.
Configure the Kubernetes services on the node.
+ +We need to configure the kubelet on the node.
+ +###
+# Kubernetes kubelet (node) config
+
+# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
+KUBELET_ADDRESS="--address=0.0.0.0"
+
+# You may leave this blank to use the actual hostname
+KUBELET_HOSTNAME="--hostname-override=fed-node"
+
+# location of the api-server
+KUBELET_API_SERVER="--api-servers=http://fed-master:8080"
+
+# Add your own!
+#KUBELET_ARGS=""
+
+for SERVICES in kube-proxy kubelet docker; do
+ systemctl restart $SERVICES
+ systemctl enable $SERVICES
+ systemctl status $SERVICES
+done
+
+kubectl get nodes
+NAME LABELS STATUS
+fed-node name=fed-node-label Ready
+
+To delete fed-node from your Kubernetes cluster, one should run the following on fed-master (Please do not do it, it is just for information):
+ +kubectl delete -f ./node.json
+
+You should be finished!
+ +The cluster should be running! Launch a test pod.
+ +You should have a functional cluster, check out 101!
+ + + + + + + + + +Kubernetes multiple nodes cluster with flannel on Fedora +——————————————————–
+ +Table of Contents
+ +This document describes how to deploy Kubernetes on multiple hosts to set up a multi-node cluster and networking with flannel. Follow fedora getting started guide to setup 1 master (fed-master) and 2 or more nodes. Make sure that all nodes have different names (fed-node1, fed-node2 and so on) and labels (fed-node1-label, fed-node2-label, and so on) to avoid any conflict. Also make sure that the Kubernetes master host is running etcd, kube-controller-manager, kube-scheduler, and kube-apiserver services, and the nodes are running docker, kube-proxy and kubelet services. Now install flannel on Kubernetes nodes. flannel on each node configures an overlay network that docker uses. flannel runs on each node to setup a unique class-C container network.
+ +Perform following commands on the Kubernetes master
+ +flannel-config.json
in your current directory on fed-master. flannel provides udp and vxlan among other overlay networking backend options. In this guide, we choose kernel based vxlan backend. The contents of the json are:{
+ "Network": "18.16.0.0/16",
+ "SubnetLen": 24,
+ "Backend": {
+ "Type": "vxlan",
+ "VNI": 1
+ }
+}
+
+NOTE: Choose an IP range that is NOT part of the public IP address range.
+ +etcdctl set /coreos.com/network/config < flannel-config.json
+
+etcdctl get /coreos.com/network/config
+
+Perform following commands on all Kubernetes nodes
+ +# Flanneld configuration options
+
+# etcd url location. Point this to the server where etcd runs
+FLANNEL_ETCD="http://fed-master:4001"
+
+# etcd config key. This is the configuration key that flannel queries
+# For address range assignment
+FLANNEL_ETCD_KEY="/coreos.com/network"
+
+# Any additional options that you want to pass
+FLANNEL_OPTIONS=""
+
+Note: By default, flannel uses the interface for the default route. If you have multiple interfaces and would like to use an interface other than the default route one, you could add “-iface=” to FLANNEL_OPTIONS. For additional options, run flanneld --help
on command line.
systemctl enable flanneld
+
+systemctl start flanneld
+
+systemctl reboot
).systemctl stop docker
+ip link delete docker0
+systemctl start flanneld
+systemctl start docker
+
+# ip -4 a|grep inet
+ inet 127.0.0.1/8 scope host lo
+ inet 192.168.122.77/24 brd 192.168.122.255 scope global dynamic eth0
+ inet 18.16.29.0/16 scope global flannel.1
+ inet 18.16.29.1/24 scope global docker0
+
+grep -E "\{|\}|key|value"
). If you set up a 1 master and 3 nodes cluster, you should see one block for each node showing the subnets they have been assigned. You can associate those subnets to each node by the MAC address (VtepMAC) and IP address (Public IP) that is listed in the output.curl -s http://fed-master:4001/v2/keys/coreos.com/network/subnets | python -mjson.tool
+
+{
+ "node": {
+ "key": "/coreos.com/network/subnets",
+ {
+ "key": "/coreos.com/network/subnets/18.16.29.0-24",
+ "value": "{\"PublicIP\":\"192.168.122.77\",\"BackendType\":\"vxlan\",\"BackendData\":{\"VtepMAC\":\"46:f1:d0:18:d0:65\"}}"
+ },
+ {
+ "key": "/coreos.com/network/subnets/18.16.83.0-24",
+ "value": "{\"PublicIP\":\"192.168.122.36\",\"BackendType\":\"vxlan\",\"BackendData\":{\"VtepMAC\":\"ca:38:78:fc:72:29\"}}"
+ },
+ {
+ "key": "/coreos.com/network/subnets/18.16.90.0-24",
+ "value": "{\"PublicIP\":\"192.168.122.127\",\"BackendType\":\"vxlan\",\"BackendData\":{\"VtepMAC\":\"92:e2:80:ba:2d:4d\"}}"
+ }
+ }
+}
+
+/run/flannel/subnet.env
file. This file was generated automatically by flannel.# cat /run/flannel/subnet.env
+FLANNEL_SUBNET=18.16.29.1/24
+FLANNEL_MTU=1450
+FLANNEL_IPMASQ=false
+
+At this point, we have etcd running on the Kubernetes master, and flannel / docker running on Kubernetes nodes. Next steps are for testing cross-host container communication which will confirm that docker and flannel are configured properly.
+Issue the following commands on any 2 nodes:
+# docker run -it fedora:latest bash
+bash-4.3#
+
+bash-4.3# yum -y install iproute iputils
+bash-4.3# setcap cap_net_raw-ep /usr/bin/ping
+
+bash-4.3# ip -4 a l eth0 | grep inet
+ inet 18.16.29.4/24 scope global eth0
+
+bash-4.3# ip a l eth0 | grep inet
+ inet 18.16.90.4/24 scope global eth0
+
+bash-4.3# ping 18.16.90.4
+PING 18.16.90.4 (18.16.90.4) 56(84) bytes of data.
+64 bytes from 18.16.90.4: icmp_seq=1 ttl=62 time=0.275 ms
+64 bytes from 18.16.90.4: icmp_seq=2 ttl=62 time=0.372 ms
+
+Getting started on Google Compute Engine +—————————————-
+ +Table of Contents
+ +The example below creates a Kubernetes cluster with 4 worker node Virtual Machines and a master Virtual Machine (i.e. 5 VMs in your cluster). This cluster is set up and controlled from your workstation (or wherever you find convenient).
+ +If you want a simplified getting started experience and GUI for managing clusters, please consider trying Google Container Engine (GKE) for hosted cluster installation and management.
+ +If you want to use custom binaries or pure open source Kubernetes, please continue with the instructions below.
+ +gcloud
as necessary. gcloud
can be installed as a part of the Google Cloud SDK.gcloud preview
command line component installed. Run gcloud preview
at the command line - if it asks to install any components, go ahead and install them. If it simply shows help text, you’re good to go. This is required as the cluster setup script uses GCE Instance Groups, which are in the gcloud preview namespace. You will also need to enable Compute Engine Instance Group Manager API
in the developers console.gcloud config list project
and change it via gcloud config set project <project-id>
.You can install a client and start a cluster with either one of these commands (we list both in case only one is installed on your machine):
+ +curl -sS https://get.k8s.io | bash
+
+or
+ +wget -q -O - https://get.k8s.io | bash
+
+Once this command completes, you will have a master VM and four worker VMs, running as a Kubernetes cluster.
+ +By default, some containers will already be running on your cluster. Containers like kibana
and elasticsearch
provide logging, while heapster
provides monitoring services.
The script run by the commands above creates a cluster with the name/prefix “kubernetes”. It defines one specific cluster config, so you can’t run it more than once.
+ +Alternately, you can download and install the latest Kubernetes release from this page, then run the <kubernetes>/cluster/kube-up.sh
script to start the cluster:
cd kubernetes
+cluster/kube-up.sh
+
+If you want more than one cluster running in your project, want to use a different name, or want a different number of worker nodes, see the <kubernetes>/cluster/gce/config-default.sh
file for more fine-grained configuration before you start up your cluster.
If you run into trouble, please see the section on troubleshooting, post to the +google-containers group, or come ask questions on Slack.
+ +The next few steps will show you:
+ +The cluster startup script will leave you with a running cluster and a kubernetes
directory on your workstation.
+The next step is to make sure the kubectl
tool is in your path.
The kubectl tool controls the Kubernetes cluster manager. It lets you inspect your cluster resources, create, delete, and update components, and much more. +You will use it to look at your new cluster and bring up example apps.
+ +Add the appropriate binary folder to your PATH
to access kubectl:
# OS X
+export PATH=<path/to/kubernetes-directory>/platforms/darwin/amd64:$PATH
+
+# Linux
+export PATH=<path/to/kubernetes-directory>/platforms/linux/amd64:$PATH
+
+Note: gcloud also ships with kubectl
, which by default is added to your path.
+However the gcloud bundled kubectl version may be older than the one downloaded by the
+get.k8s.io install script. We recommend you use the downloaded binary to avoid
+potential issues with client/server version skew.
You may find it useful to enable kubectl
bash completion:
+$ source ./contrib/completions/bash/kubectl
+
+
+
+Note: This will last for the duration of your bash session. If you want to make this permanent you need to add this line in your bash profile.
+ +Alternatively, on most linux distributions you can also move the completions file to your bash_completions.d like this:
+ +
+$ cp ./contrib/completions/bash/kubectl /etc/bash_completion.d/
+
+
+
+but then you have to update it when you update kubectl.
+ +Once kubectl
is in your path, you can use it to look at your cluster. E.g., running:
$ kubectl get --all-namespaces services
+
+should show a set of services that look something like this:
+ +NAMESPACE NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE
+default kubernetes 10.0.0.1 <none> 443/TCP <none> 1d
+kube-system kube-dns 10.0.0.2 <none> 53/TCP,53/UDP k8s-app=kube-dns 1d
+kube-system kube-ui 10.0.0.3 <none> 80/TCP k8s-app=kube-ui 1d
+...
+
+Similarly, you can take a look at the set of pods that were created during cluster startup. +You can do this via the
+ +$ kubectl get --all-namespaces pods
+
+command.
+ +You’ll see a list of pods that looks something like this (the name specifics will be different):
+ +NAMESPACE NAME READY STATUS RESTARTS AGE
+kube-system fluentd-cloud-logging-kubernetes-minion-63uo 1/1 Running 0 14m
+kube-system fluentd-cloud-logging-kubernetes-minion-c1n9 1/1 Running 0 14m
+kube-system fluentd-cloud-logging-kubernetes-minion-c4og 1/1 Running 0 14m
+kube-system fluentd-cloud-logging-kubernetes-minion-ngua 1/1 Running 0 14m
+kube-system kube-dns-v5-7ztia 3/3 Running 0 15m
+kube-system kube-ui-v1-curt1 1/1 Running 0 15m
+kube-system monitoring-heapster-v5-ex4u3 1/1 Running 1 15m
+kube-system monitoring-influx-grafana-v1-piled 2/2 Running 0 15m
+
+Some of the pods may take a few seconds to start up (during this time they’ll show Pending
), but check that they all show as Running
after a short period.
Then, see a simple nginx example to try out your new cluster.
+ +For more complete applications, please look in the examples directory. The guestbook example is a good “getting started” walkthrough.
+ +To remove/delete/teardown the cluster, use the kube-down.sh
script.
cd kubernetes
+cluster/kube-down.sh
+
+Likewise, the kube-up.sh
in the same directory will bring it back up. You do not need to rerun the curl
or wget
command: everything needed to setup the Kubernetes cluster is now on your workstation.
The script above relies on Google Storage to stage the Kubernetes release. It
+then will start (by default) a single master VM along with 4 worker VMs. You
+can tweak some of these parameters by editing kubernetes/cluster/gce/config-default.sh
+You can view a transcript of a successful cluster creation
+here.
You need to have the Google Cloud Storage API, and the Google Cloud Storage +JSON API enabled. It is activated by default for new projects. Otherwise, it +can be done in the Google Cloud Console. See the Google Cloud Storage JSON +API Overview for more +details.
+ +Also ensure that– as listed in the Prerequsites section– you’ve enabled the Compute Engine Instance Group Manager API
, and can start up a GCE VM from the command line as in the GCE Quickstart instructions.
If the Kubernetes startup script hangs waiting for the API to be reachable, you can troubleshoot by SSHing into the master and node VMs and looking at logs such as /var/log/startupscript.log
.
Once you fix the issue, you should run kube-down.sh
to cleanup after the partial cluster creation, before running kube-up.sh
to try again.
If you’re having trouble SSHing into your instances, ensure the GCE firewall
+isn’t blocking port 22 to your VMs. By default, this should work but if you
+have edited firewall rules or created a new non-default network, you’ll need to
+expose it: gcloud compute firewall-rules create default-ssh --network=<network-name>
+--description "SSH allowed from anywhere" --allow tcp:22
Additionally, your GCE SSH key must either have no passcode or you need to be
+using ssh-agent
.
The instances must be able to connect to each other using their private IP. The
+script uses the “default” network which should have a firewall rule called
+“default-allow-internal” which allows traffic on any port on the private IPs.
+If this rule is missing from the default network or if you change the network
+being used in cluster/config-default.sh
create a new rule with the following
+field values:
10.0.0.0/8
tcp:1-65535;udp:1-65535;icmp
Kubernetes can run on a range of platforms, from your laptop, to VMs on a cloud provider, to rack of +bare metal servers. The effort required to set up a cluster varies from running a single command to +crafting your own customized cluster. We’ll guide you in picking a solution that fits for your needs.
+ +Table of Contents +
+ +If you just want to “kick the tires” on Kubernetes, we recommend the local Docker-based solution.
+ +The local Docker-based solution is one of several Local cluster solutions +that are quick to set up, but are limited to running on one machine.
+ +When you are ready to scale up to more machines and higher availability, a Hosted +solution is the easiest to create and maintain.
+ +Turn-key cloud solutions require only a few commands to create +and cover a wider range of cloud providers.
+ +Custom solutions require more effort to setup but cover and even +they vary from step-by-step instructions to general advice for setting up +a Kubernetes cluster from scratch.
+ +Local-machine solutions create a single cluster with one or more Kubernetes nodes on a single +physical machine. Setup is completely automated and doesn’t require a cloud provider account. +But their size and availability is limited to that of a single machine.
+ +The local-machine solutions are:
+ +Google Container Engine offers managed Kubernetes +clusters.
+ +These solutions allow you to create Kubernetes clusters on a range of Cloud IaaS providers with only a +few commands, and have active community support.
+ + + +Kubernetes can run on a wide range of Cloud providers and bare-metal environments, and with many +base operating systems.
+ +If you can find a guide below that matches your needs, use it. It may be a little out of date, but +it will be easier than starting from scratch. If you do want to start from scratch because you +have special requirements or just because you want to understand what is underneath a Kubernetes +cluster, try the Getting Started from Scratch guide.
+ +If you are interested in supporting Kubernetes on a new platform, check out our advice for +writing a new solution.
+ +These solutions are combinations of cloud provider and OS not covered by the above solutions.
+ + + +These solutions provide integration with 3rd party schedulers, resource managers, and/or lower level platforms.
+ +Here are all the solutions mentioned above in table form.
+ +IaaS Provider | +Config. Mgmt | +OS | +Networking | +Docs | +Conforms | +Support Level | +
---|---|---|---|---|---|---|
GKE | ++ | + | GCE | +docs | +✓ | +Commercial | +
Vagrant | +Saltstack | +Fedora | +flannel | +docs | +✓ | +Project | +
GCE | +Saltstack | +Debian | +GCE | +docs | +✓ | +Project | +
Azure | +CoreOS | +CoreOS | +Weave | +docs | ++ | Community (@errordeveloper, @squillace, @chanezon, @crossorigin) | +
Docker Single Node | +custom | +N/A | +local | +docs | ++ | Project (@brendandburns) | +
Docker Multi Node | +Flannel | +N/A | +local | +docs | ++ | Project (@brendandburns) | +
Bare-metal | +Ansible | +Fedora | +flannel | +docs | ++ | Project | +
Digital Ocean | +custom | +Fedora | +Calico | +docs | ++ | Community (@djosborne) | +
Bare-metal | +custom | +Fedora | +none | +docs | ++ | Project | +
Bare-metal | +custom | +Fedora | +flannel | +docs | ++ | Community (@aveshagarwal) | +
libvirt | +custom | +Fedora | +flannel | +docs | ++ | Community (@aveshagarwal) | +
KVM | +custom | +Fedora | +flannel | +docs | ++ | Community (@aveshagarwal) | +
Mesos/Docker | +custom | +Ubuntu | +Docker | +docs | ++ | Community (Kubernetes-Mesos Authors) | +
Mesos/GCE | ++ | + | + | docs | ++ | Community (Kubernetes-Mesos Authors) | +
DCOS | +Marathon | +CoreOS/Alpine | +custom | +docs | ++ | Community (Kubernetes-Mesos Authors) | +
AWS | +CoreOS | +CoreOS | +flannel | +docs | ++ | Community | +
GCE | +CoreOS | +CoreOS | +flannel | +docs | ++ | Community (@pires) | +
Vagrant | +CoreOS | +CoreOS | +flannel | +docs | ++ | Community (@pires, @AntonioMeireles) | +
Bare-metal (Offline) | +CoreOS | +CoreOS | +flannel | +docs | ++ | Community (@jeffbean) | +
Bare-metal | +CoreOS | +CoreOS | +Calico | +docs | ++ | Community (@caseydavenport) | +
CloudStack | +Ansible | +CoreOS | +flannel | +docs | ++ | Community (@runseb) | +
Vmware | ++ | Debian | +OVS | +docs | ++ | Community (@pietern) | +
Bare-metal | +custom | +CentOS | +none | +docs | ++ | Community (@coolsvap) | +
AWS | +Juju | +Ubuntu | +flannel | +docs | ++ | Community ( @whit, @matt, @chuck ) | +
OpenStack/HPCloud | +Juju | +Ubuntu | +flannel | +docs | ++ | Community ( @whit, @matt, @chuck ) | +
Joyent | +Juju | +Ubuntu | +flannel | +docs | ++ | Community ( @whit, @matt, @chuck ) | +
AWS | +Saltstack | +Ubuntu | +OVS | +docs | ++ | Community (@justinsb) | +
Bare-metal | +custom | +Ubuntu | +Calico | +docs | ++ | Community (@djosborne) | +
Bare-metal | +custom | +Ubuntu | +flannel | +docs | ++ | Community (@resouer, @WIZARD-CXY) | +
Local | ++ | + | none | +docs | ++ | Community (@preillyme) | +
libvirt/KVM | +CoreOS | +CoreOS | +libvirt/KVM | +docs | ++ | Community (@lhuard1A) | +
oVirt | ++ | + | + | docs | ++ | Community (@simon3z) | +
Rackspace | +CoreOS | +CoreOS | +flannel | +docs | ++ | Community (@doublerr) | +
any | +any | +any | +any | +docs | ++ | Community (@erictune) | +
Note: The above table is ordered by version test/used in notes followed by support level.
+ +Definition of columns:
+ +Juju makes it easy to deploy +Kubernetes by provisioning, installing and configuring all the systems in +the cluster. Once deployed the cluster can easily scale up with one command +to increase the cluster size.
+ +Table of Contents
+ +++ +Note: If you’re running kube-up, on Ubuntu - all of the dependencies +will be handled for you. You may safely skip to the section: +Launch Kubernetes Cluster
+
Install the Juju client on your +local Ubuntu system:
+ +sudo add-apt-repository ppa:juju/stable
+sudo apt-get update
+sudo apt-get install juju-core juju-quickstart
+
+
+If you are not using Ubuntu or prefer the isolation of Docker, you may +run the following:
+ +mkdir ~/.juju
+sudo docker run -v ~/.juju:/home/ubuntu/.juju -ti jujusolutions/jujubox:latest
+
+
+At this point from either path you will have access to the juju
+quickstart
command.
To set up the credentials for your chosen cloud run:
+ +juju quickstart --constraints="mem=3.75G" -i
+
+
+++ +The
+constraints
flag is optional, it changes the size of virtual machines +that Juju will generate when it requests a new machine. Larger machines +will run faster but cost more money than smaller machines.
Follow the dialogue and choose save
and use
. Quickstart will now
+bootstrap the juju root node and setup the juju web based user
+interface.
You will need to export the KUBERNETES_PROVIDER
environment variable before
+bringing up the cluster.
export KUBERNETES_PROVIDER=juju
+cluster/kube-up.sh
+
+
+If this is your first time running the kube-up.sh
script, it will install
+the required dependencies to get started with Juju, additionally it will
+launch a curses based configuration utility allowing you to select your cloud
+provider and enter the proper access credentials.
Next it will deploy the kubernetes master, etcd, 2 nodes with flannel based +Software Defined Networking (SDN) so containers on different hosts can +communicate with each other.
+ +The juju status
command provides information about each unit in the cluster:
$ juju status --format=oneline
+- docker/0: 52.4.92.78 (started)
+ - flannel-docker/0: 52.4.92.78 (started)
+ - kubernetes/0: 52.4.92.78 (started)
+- docker/1: 52.6.104.142 (started)
+ - flannel-docker/1: 52.6.104.142 (started)
+ - kubernetes/1: 52.6.104.142 (started)
+- etcd/0: 52.5.216.210 (started) 4001/tcp
+- juju-gui/0: 52.5.205.174 (started) 80/tcp, 443/tcp
+- kubernetes-master/0: 52.6.19.238 (started) 8080/tcp
+
+
+You can use juju ssh
to access any of the units:
juju ssh kubernetes-master/0
+
+
+kubectl
is available on the Kubernetes master node. We’ll ssh in to
+launch some containers, but one could use kubectl
locally by setting
+KUBERNETES_MASTER
to point at the ip address of “kubernetes-master/0”.
No pods will be available before starting a container:
+ +kubectl get pods
+NAME READY STATUS RESTARTS AGE
+
+kubectl get replicationcontrollers
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+
+
+We’ll follow the aws-coreos example. Create a pod manifest: pod.json
{
+ "apiVersion": "v1",
+ "kind": "Pod",
+ "metadata": {
+ "name": "hello",
+ "labels": {
+ "name": "hello",
+ "environment": "testing"
+ }
+ },
+ "spec": {
+ "containers": [{
+ "name": "hello",
+ "image": "quay.io/kelseyhightower/hello",
+ "ports": [{
+ "containerPort": 80,
+ "hostPort": 80
+ }]
+ }]
+ }
+}
+
+Create the pod with kubectl:
+ +kubectl create -f pod.json
+
+
+Get info on the pod:
+ +kubectl get pods
+
+
+To test the hello app, we need to locate which node is hosting
+the container. Better tooling for using Juju to introspect container
+is in the works but we can use juju run
and juju status
to find
+our hello app.
Exit out of our ssh session and run:
+ +juju run --unit kubernetes/0 "docker ps -n=1"
+...
+juju run --unit kubernetes/1 "docker ps -n=1"
+CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
+02beb61339d8 quay.io/kelseyhightower/hello:latest /hello About an hour ago Up About an hour k8s_hello....
+
+
+We see “kubernetes/1” has our container, we can open port 80:
+ +juju run --unit kubernetes/1 "open-port 80"
+juju expose kubernetes
+sudo apt-get install curl
+curl $(juju status --format=oneline kubernetes/1 | cut -d' ' -f3)
+
+
+Finally delete the pod:
+ +juju ssh kubernetes-master/0
+kubectl delete pods hello
+
+
+We can add node units like so:
+ +juju add-unit docker # creates unit docker/2, kubernetes/2, docker-flannel/2
+
+
+The k8petstore example is available as a +juju action.
+ +juju action do kubernetes-master/0
+
+
+++ +Note: this example includes curl statements to exercise the app, which +automatically generates “petstore” transactions written to redis, and allows +you to visualize the throughput in your browser.
+
./kube-down.sh
+
+
+or destroy your current Juju environment (using the juju env
command):
juju destroy-environment --force `juju env`
+
+
+The Kubernetes charms and bundles can be found in the kubernetes
project on
+github.com:
Juju runs natively against a variety of public cloud providers. Juju currently +works with Amazon Web Service, +Windows Azure, +DigitalOcean, +Google Compute Engine, +HP Public Cloud, +Joyent, +LXC, any +OpenStack deployment, +Vagrant, and +Vmware vSphere.
+ +If you do not see your favorite cloud provider listed many clouds can be +configured for manual provisioning.
+ +The Kubernetes bundle has been tested on GCE and AWS and found to work with +version 1.0.0.
+ + + + + + + + + +Getting started with libvirt CoreOS +———————————–
+ +Table of Contents
+ +libvirt-coreos
use casekube-*
scripts.libvirt-coreos
use caseThe primary goal of the libvirt-coreos
cluster provider is to deploy a multi-node Kubernetes cluster on local VMs as fast as possible and to be as light as possible in term of resources used.
In order to achieve that goal, its deployment is very different from the “standard production deployment” method used on other providers. This was done on purpose in order to implement some optimizations made possible by the fact that we know that all VMs will be running on the same physical machine.
+ +The libvirt-coreos
cluster provider doesn’t aim at being production look-alike.
Another difference is that no security is enforced on libvirt-coreos
at all. For example,
So, an k8s application developer should not validate its interaction with Kubernetes on libvirt-coreos
because he might technically succeed in doing things that are prohibited on a production environment like:
On the other hand, libvirt-coreos
might be useful for people investigating low level implementation of Kubernetes because debugging techniques like sniffing the network traffic or introspecting the etcd content are easier on libvirt-coreos
than on a production deployment.
systemctl enable libvirtd
systemctl start libvirtd
You can test it with the following command:
+ +virsh -c qemu:///system pool-list
+
+If you have access error messages, please read https://libvirt.org/acl.html and https://libvirt.org/aclpolkit.html .
+ +In short, if your libvirt has been compiled with Polkit support (ex: Arch, Fedora 21), you can create /etc/polkit-1/rules.d/50-org.libvirt.unix.manage.rules
as follows to grant full access to libvirt to $USER
sudo /bin/sh -c "cat - > /etc/polkit-1/rules.d/50-org.libvirt.unix.manage.rules" << EOF
+polkit.addRule(function(action, subject) {
+ if (action.id == "org.libvirt.unix.manage" &&
+ subject.user == "$USER") {
+ return polkit.Result.YES;
+ polkit.log("action=" + action);
+ polkit.log("subject=" + subject);
+ }
+});
+EOF
+
+If your libvirt has not been compiled with Polkit (ex: Ubuntu 14.04.1 LTS), check the permissions on the libvirt unix socket:
+ +$ ls -l /var/run/libvirt/libvirt-sock
+srwxrwx--- 1 root libvirtd 0 févr. 12 16:03 /var/run/libvirt/libvirt-sock
+
+$ usermod -a -G libvirtd $USER
+# $USER needs to logout/login to have the new group be taken into account
+
+(Replace $USER
with your login name)
All the disk drive resources needed by the VM (CoreOS disk image, Kubernetes binaries, cloud-init files, etc.) are put inside ./cluster/libvirt-coreos/libvirt_storage_pool
.
As we’re using the qemu:///system
instance of libvirt, qemu will run with a specific user:group
distinct from your user. It is configured in /etc/libvirt/qemu.conf
. That qemu user must have access to that libvirt storage pool.
If your $HOME
is world readable, everything is fine. If your $HOME is private, cluster/kube-up.sh
will fail with an error message like:
error: Cannot access storage file '$HOME/.../kubernetes/cluster/libvirt-coreos/libvirt_storage_pool/kubernetes_master.img' (as uid:99, gid:78): Permission denied
+
+In order to fix that issue, you have several possibilities:
+* set POOL_PATH
inside cluster/libvirt-coreos/config-default.sh
to a directory:
+ * backed by a filesystem with a lot of free disk space
+ * writable by your user;
+ * accessible by the qemu user.
+* Grant the qemu user access to the storage pool.
On Arch:
+ +setfacl -m g:kvm:--x ~
+
+By default, the libvirt-coreos setup will create a single Kubernetes master and 3 Kubernetes nodes. Because the VM drives use Copy-on-Write and because of memory ballooning and KSM, there is a lot of resource over-allocation.
+ +To start your local cluster, open a shell and run:
+ +cd kubernetes
+
+export KUBERNETES_PROVIDER=libvirt-coreos
+cluster/kube-up.sh
+
+The KUBERNETES_PROVIDER
environment variable tells all of the various cluster management scripts which variant to use. If you forget to set this, the assumption is you are running on Google Compute Engine.
The NUM_MINIONS
environment variable may be set to specify the number of nodes to start. If it is not set, the number of nodes defaults to 3.
The KUBE_PUSH
environment variable may be set to specify which Kubernetes binaries must be deployed on the cluster. Its possible values are:
release
(default if KUBE_PUSH
is not set) will deploy the binaries of _output/release-tars/kubernetes-server-….tar.gz
. This is built with make release
or make release-skip-tests
.local
will deploy the binaries of _output/local/go/bin
. These are built with make
.You can check that your machines are there and running with:
+ +$ virsh -c qemu:///system list
+ Id Name State
+----------------------------------------------------
+ 15 kubernetes_master running
+ 16 kubernetes_minion-01 running
+ 17 kubernetes_minion-02 running
+ 18 kubernetes_minion-03 running
+
+You can check that the Kubernetes cluster is working with:
+ +$ kubectl get nodes
+NAME LABELS STATUS
+192.168.10.2 <none> Ready
+192.168.10.3 <none> Ready
+192.168.10.4 <none> Ready
+
+The VMs are running CoreOS.
+Your ssh keys have already been pushed to the VM. (It looks for ~/.ssh/id_*.pub)
+The user to use to connect to the VM is core
.
+The IP to connect to the master is 192.168.10.1.
+The IPs to connect to the nodes are 192.168.10.2 and onwards.
Connect to kubernetes_master
:
ssh core@192.168.10.1
+
+Connect to kubernetes_minion-01
:
ssh core@192.168.10.2
+
+kube-*
scripts.All of the following commands assume you have set KUBERNETES_PROVIDER
appropriately:
export KUBERNETES_PROVIDER=libvirt-coreos
+
+Bring up a libvirt-CoreOS cluster of 5 nodes
+ +NUM_MINIONS=5 cluster/kube-up.sh
+
+Destroy the libvirt-CoreOS cluster
+ +cluster/kube-down.sh
+
+Update the libvirt-CoreOS cluster with a new Kubernetes release produced by make release
or make release-skip-tests
:
cluster/kube-push.sh
+
+Update the libvirt-CoreOS cluster with the locally built Kubernetes binaries produced by make
:
KUBE_PUSH=local cluster/kube-push.sh
+
+Interact with the cluster
+ +kubectl ...
+
+Build the release tarballs:
+ +make release
+
+Install libvirt
+ +On Arch:
+ +pacman -S qemu libvirt
+
+On Ubuntu 14.04.1:
+ +aptitude install qemu-system-x86 libvirt-bin
+
+On Fedora 21:
+ +yum install qemu libvirt
+
+Start the libvirt daemon
+ +On Arch:
+ +systemctl start libvirtd
+
+On Ubuntu 14.04.1:
+ +service libvirt-bin start
+
+Fix libvirt access permission (Remember to adapt $USER
)
On Arch and Fedora 21:
+ +cat > /etc/polkit-1/rules.d/50-org.libvirt.unix.manage.rules <<EOF
+polkit.addRule(function(action, subject) {
+ if (action.id == "org.libvirt.unix.manage" &&
+ subject.user == "$USER") {
+ return polkit.Result.YES;
+ polkit.log("action=" + action);
+ polkit.log("subject=" + subject);
+ }
+});
+EOF
+
+On Ubuntu:
+ +usermod -a -G libvirtd $USER
+
+Ensure libvirtd has been restarted since ebtables was installed.
+ + + + + + + + + +Getting started locally +———————–
+ +Table of Contents
+ +get pods
and docker ps
don’t show it.Not running Linux? Consider running Linux in a local virtual machine with Vagrant, or on a cloud provider like Google Compute Engine
+ +At least Docker
+1.3+. Ensure the Docker daemon is running and can be contacted (try docker
+ps
). Some of the Kubernetes components need to run as root, which normally
+works fine with docker.
You need an etcd in your path, please make sure it is installed and in your $PATH
.
You need go at least 1.3+ in your path, please make sure it is installed and in your $PATH
.
In a separate tab of your terminal, run the following (since one needs sudo access to start/stop Kubernetes daemons, it is easier to run the entire script as root):
+ +cd kubernetes
+hack/local-up-cluster.sh
+
+This will build and start a lightweight local cluster, consisting of a master +and a single node. Type Control-C to shut it down.
+ +You can use the cluster/kubectl.sh script to interact with the local cluster. hack/local-up-cluster.sh will +print the commands to run to point kubectl at the local cluster.
+ +Your cluster is running, and you want to start running containers!
+ +You can now use any of the cluster/kubectl.sh commands to interact with your local setup.
+ +cluster/kubectl.sh get pods
+cluster/kubectl.sh get services
+cluster/kubectl.sh get replicationcontrollers
+cluster/kubectl.sh run my-nginx --image=nginx --replicas=2 --port=80
+
+
+## begin wait for provision to complete, you can monitor the docker pull by opening a new terminal
+ sudo docker images
+ ## you should see it pulling the nginx image, once the above command returns it
+ sudo docker ps
+ ## you should see your container running!
+ exit
+## end wait
+
+## introspect Kubernetes!
+cluster/kubectl.sh get pods
+cluster/kubectl.sh get services
+cluster/kubectl.sh get replicationcontrollers
+
+Note the difference between a container
+and a pod. Since you only asked for the former, Kubernetes will create a wrapper pod for you.
+However you cannot view the nginx start page on localhost. To verify that nginx is running you need to run curl
within the docker container (try docker exec
).
You can control the specifications of a pod via a user defined manifest, and reach nginx through your browser on the port specified therein:
+ +cluster/kubectl.sh create -f docs/user-guide/pod.yaml
+
+Congratulations!
+ +Some firewall software that uses iptables may not interact well with
+kubernetes. If you have trouble around networking, try disabling any
+firewall or other iptables-using systems, first. Also, you can check
+if SELinux is blocking anything by running a command such as journalctl --since yesterday | grep avc
.
By default the IP range for service cluster IPs is 10.0.. - depending on your +docker installation, this may conflict with IPs for containers. If you find +containers running with IPs in this range, edit hack/local-cluster-up.sh and +change the service-cluster-ip-range flag to something else.
+ +You are running a single node setup. This has the limitation of only supporting a single replica of a given pod. If you are interested in running with larger replica sizes, we encourage you to try the local vagrant setup or one of the cloud providers.
+ +cd kubernetes
+hack/build-go.sh
+hack/local-up-cluster.sh
+
+get pods
and docker ps
don’t show it.One or more of the KUbernetes daemons might’ve crashed. Tail the logs of each in /tmp.
+ +The local-up-cluster.sh script doesn’t start a DNS service. Similar situation can be found here. You can start a manually. Related documents can be found here
+ + + + + + + + + +On the Google Compute Engine (GCE) platform the default cluster level logging support targets +Google Cloud Logging as described at the Logging getting +started page. Here we describe how to set up a cluster to ingest logs into Elasticsearch and view them using Kibana as an +alternative to Google Cloud Logging.
+ +To use Elasticsearch and Kibana for cluster logging you should set the following environment variable as shown below:
+ +KUBE_LOGGING_DESTINATION=elasticsearch
+
+You should also ensure that KUBE_ENABLE_NODE_LOGGING=true
(which is the default for the GCE platform).
Now when you create a cluster a message will indicate that the Fluentd node-level log collectors +will target Elasticsearch:
+ +$ cluster/kube-up.sh
+...
+Project: kubernetes-satnam
+Zone: us-central1-b
+... calling kube-up
+Project: kubernetes-satnam
+Zone: us-central1-b
++++ Staging server tars to Google Storage: gs://kubernetes-staging-e6d0e81793/devel
++++ kubernetes-server-linux-amd64.tar.gz uploaded (sha1 = 6987c098277871b6d69623141276924ab687f89d)
++++ kubernetes-salt.tar.gz uploaded (sha1 = bdfc83ed6b60fa9e3bff9004b542cfc643464cd0)
+Looking for already existing resources
+Starting master and configuring firewalls
+Created [https://www.googleapis.com/compute/v1/projects/kubernetes-satnam/zones/us-central1-b/disks/kubernetes-master-pd].
+NAME ZONE SIZE_GB TYPE STATUS
+kubernetes-master-pd us-central1-b 20 pd-ssd READY
+Created [https://www.googleapis.com/compute/v1/projects/kubernetes-satnam/regions/us-central1/addresses/kubernetes-master-ip].
++++ Logging using Fluentd to elasticsearch
+
+The node level Fluentd collector pods and the Elasticsearech pods used to ingest cluster logs and the pod for the Kibana +viewer should be running in the kube-system namespace soon after the cluster comes to life.
+ +$ kubectl get pods --namespace=kube-system
+NAME READY REASON RESTARTS AGE
+elasticsearch-logging-v1-78nog 1/1 Running 0 2h
+elasticsearch-logging-v1-nj2nb 1/1 Running 0 2h
+fluentd-elasticsearch-kubernetes-minion-5oq0 1/1 Running 0 2h
+fluentd-elasticsearch-kubernetes-minion-6896 1/1 Running 0 2h
+fluentd-elasticsearch-kubernetes-minion-l1ds 1/1 Running 0 2h
+fluentd-elasticsearch-kubernetes-minion-lz9j 1/1 Running 0 2h
+kibana-logging-v1-bhpo8 1/1 Running 0 2h
+kube-dns-v3-7r1l9 3/3 Running 0 2h
+monitoring-heapster-v4-yl332 1/1 Running 1 2h
+monitoring-influx-grafana-v1-o79xf 2/2 Running 0 2h
+
+Here we see that for a four node cluster there is a fluent-elasticsearch
pod running which gathers
+the Docker container logs and sends them to Elasticsearch. The Fluentd collector communicates to
+a Kubernetes service that maps requests to specific Elasticsearch pods. Similarly, Kibana can also be
+accessed via a Kubernetes service definition.
$ kubectl get services --namespace=kube-system
+NAME LABELS SELECTOR IP(S) PORT(S)
+elasticsearch-logging k8s-app=elasticsearch-logging,kubernetes.io/cluster-service=true,kubernetes.io/name=Elasticsearch k8s-app=elasticsearch-logging 10.0.222.57 9200/TCP
+kibana-logging k8s-app=kibana-logging,kubernetes.io/cluster-service=true,kubernetes.io/name=Kibana k8s-app=kibana-logging 10.0.193.226 5601/TCP
+kube-dns k8s-app=kube-dns,kubernetes.io/cluster-service=true,kubernetes.io/name=KubeDNS k8s-app=kube-dns 10.0.0.10 53/UDP
+ 53/TCP
+kubernetes component=apiserver,provider=kubernetes <none> 10.0.0.1 443/TCP
+monitoring-grafana kubernetes.io/cluster-service=true,kubernetes.io/name=Grafana k8s-app=influxGrafana 10.0.167.139 80/TCP
+monitoring-heapster kubernetes.io/cluster-service=true,kubernetes.io/name=Heapster k8s-app=heapster 10.0.208.221 80/TCP
+monitoring-influxdb kubernetes.io/cluster-service=true,kubernetes.io/name=InfluxDB k8s-app=influxGrafana 10.0.188.57 8083/TCP
+
+By default two Elasticsearch replicas are created and one Kibana replica is created.
+ +$ kubectl get rc --namespace=kube-system
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+elasticsearch-logging-v1 elasticsearch-logging gcr.io/google_containers/elasticsearch:1.4 k8s-app=elasticsearch-logging,version=v1 2
+kibana-logging-v1 kibana-logging gcr.io/google_containers/kibana:1.3 k8s-app=kibana-logging,version=v1 1
+kube-dns-v3 etcd gcr.io/google_containers/etcd:2.0.9 k8s-app=kube-dns,version=v3 1
+ kube2sky gcr.io/google_containers/kube2sky:1.9
+ skydns gcr.io/google_containers/skydns:2015-03-11-001
+monitoring-heapster-v4 heapster gcr.io/google_containers/heapster:v0.14.3 k8s-app=heapster,version=v4 1
+monitoring-influx-grafana-v1 influxdb gcr.io/google_containers/heapster_influxdb:v0.3 k8s-app=influxGrafana,version=v1 1
+ grafana gcr.io/google_containers/heapster_grafana:v0.7
+
+The Elasticsearch and Kibana services are not directly exposed via a publicly reachable IP address. Instead,
+they can be accessed via the service proxy running at the master. The URLs for accessing Elasticsearch
+and Kibana via the service proxy can be found using the kubectl cluster-info
command.
$ kubectl cluster-info
+Kubernetes master is running at https://146.148.94.154
+Elasticsearch is running at https://146.148.94.154/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging
+Kibana is running at https://146.148.94.154/api/v1/proxy/namespaces/kube-system/services/kibana-logging
+KubeDNS is running at https://146.148.94.154/api/v1/proxy/namespaces/kube-system/services/kube-dns
+KubeUI is running at https://146.148.94.154/api/v1/proxy/namespaces/kube-system/services/kube-ui
+Grafana is running at https://146.148.94.154/api/v1/proxy/namespaces/kube-system/services/monitoring-grafana
+Heapster is running at https://146.148.94.154/api/v1/proxy/namespaces/kube-system/services/monitoring-heapster
+InfluxDB is running at https://146.148.94.154/api/v1/proxy/namespaces/kube-system/services/monitoring-influxdb
+
+Before accessing the logs ingested into Elasticsearch using a browser and the service proxy URL we need to find out
+the admin
password for the cluster using kubectl config view
.
$ kubectl config view
+...
+- name: kubernetes-satnam_kubernetes-basic-auth
+ user:
+ password: 7GlspJ9Q43OnGIJO
+ username: admin
+...
+
+The first time you try to access the cluster from a browser a dialog box appears asking for the username and password.
+Use the username admin
and provide the basic auth password reported by kubectl config view
for the
+cluster you are trying to connect to. Connecting to the Elasticsearch URL should then give the
+status page for Elasticsearch.
You can now type Elasticsearch queries directly into the browser. Alternatively you can query Elasticsearch
+from your local machine using curl
but first you need to know what your bearer token is:
$ kubectl config view --minify
+apiVersion: v1
+clusters:
+- cluster:
+ certificate-authority-data: REDACTED
+ server: https://146.148.94.154
+ name: kubernetes-satnam_kubernetes
+contexts:
+- context:
+ cluster: kubernetes-satnam_kubernetes
+ user: kubernetes-satnam_kubernetes
+ name: kubernetes-satnam_kubernetes
+current-context: kubernetes-satnam_kubernetes
+kind: Config
+preferences: {}
+users:
+- name: kubernetes-satnam_kubernetes
+ user:
+ client-certificate-data: REDACTED
+ client-key-data: REDACTED
+ token: JsUe2Z3cXqa17UQqQ8qWGGf4nOSLwSnp
+
+Now you can issue requests to Elasticsearch:
+ +$ curl --header "Authorization: Bearer JsUe2Z3cXqa17UQqQ8qWGGf4nOSLwSnp" --insecure https://146.148.94.154/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging/
+{
+ "status" : 200,
+ "name" : "Vance Astrovik",
+ "cluster_name" : "kubernetes-logging",
+ "version" : {
+ "number" : "1.5.2",
+ "build_hash" : "62ff9868b4c8a0c45860bebb259e21980778ab1c",
+ "build_timestamp" : "2015-04-27T09:21:06Z",
+ "build_snapshot" : false,
+ "lucene_version" : "4.10.4"
+ },
+ "tagline" : "You Know, for Search"
+}
+
+Note that you need the trailing slash at the end of the service proxy URL. Here is an example of a search:
+ +$ curl --header "Authorization: Bearer JsUe2Z3cXqa17UQqQ8qWGGf4nOSLwSnp" --insecure https://146.148.94.154/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging/_search?pretty=true
+{
+ "took" : 7,
+ "timed_out" : false,
+ "_shards" : {
+ "total" : 6,
+ "successful" : 6,
+ "failed" : 0
+ },
+ "hits" : {
+ "total" : 123711,
+ "max_score" : 1.0,
+ "hits" : [ {
+ "_index" : ".kibana",
+ "_type" : "config",
+ "_id" : "4.0.2",
+ "_score" : 1.0,
+ "_source":{"buildNum":6004,"defaultIndex":"logstash-*"}
+ }, {
+...
+ "_index" : "logstash-2015.06.22",
+ "_type" : "fluentd",
+ "_id" : "AU4c_GvFZL5p_gZ8dxtx",
+ "_score" : 1.0,
+ "_source":{"log":"synthetic-logger-10lps-pod: 31: 2015-06-22 20:35:33.597918073+00:00\n","stream":"stdout","tag":"kubernetes.synthetic-logger-10lps-pod_default_synth-lgr","@timestamp":"2015-06-22T20:35:33+00:00"}
+ }, {
+ "_index" : "logstash-2015.06.22",
+ "_type" : "fluentd",
+ "_id" : "AU4c_GvFZL5p_gZ8dxt2",
+ "_score" : 1.0,
+ "_source":{"log":"synthetic-logger-10lps-pod: 36: 2015-06-22 20:35:34.108780133+00:00\n","stream":"stdout","tag":"kubernetes.synthetic-logger-10lps-pod_default_synth-lgr","@timestamp":"2015-06-22T20:35:34+00:00"}
+ } ]
+ }
+}
+
+The Elasticsearch website contains information about URI search queries which can be used to extract the required logs.
+ +Alternatively you can view the ingested logs using Kibana. The first time you visit the Kibana URL you will be
+presented with a page that asks you to configure your view of the ingested logs. Select the option for
+timeseries values and select @timestamp
. On the following page select the Discover
tab and then you
+should be able to see the ingested logs. You can set the refresh interval to 5 seconds to have the logs
+regulary refreshed. Here is a typical view of ingested logs from the Kibana viewer.
Another way to access Elasticsearch and Kibana in the cluster is to use kubectl proxy
which will serve
+a local proxy to the remote master:
$ kubectl proxy
+Starting to serve on localhost:8001
+
+Now you can visit the URL http://localhost:8001/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging to contact Elasticsearch and http://localhost:8001/api/v1/proxy/namespaces/kube-system/services/kibana-logging to access the Kibana viewer.
+ + + + + + + + + +A Kubernetes cluster will typically be humming along running many system and application pods. How does the system administrator collect, manage and query the logs of the system pods? How does a user query the logs of their application which is composed of many pods which may be restarted or automatically generated by the Kubernetes system? These questions are addressed by the Kubernetes cluster level logging services.
+ +Cluster level logging for Kubernetes allows us to collect logs which persist beyond the lifetime of the pod’s container images or the lifetime of the pod or even cluster. In this article we assume that a Kubernetes cluster has been created with cluster level logging support for sending logs to Google Cloud Logging. After a cluster has been created you will have a collection of system pods running in the kube-system
namespace that support monitoring,
+logging and DNS resolution for names of Kubernetes services:
$ kubectl get pods --namespace=kube-system
+NAME READY REASON RESTARTS AGE
+fluentd-cloud-logging-kubernetes-minion-0f64 1/1 Running 0 32m
+fluentd-cloud-logging-kubernetes-minion-27gf 1/1 Running 0 32m
+fluentd-cloud-logging-kubernetes-minion-pk22 1/1 Running 0 31m
+fluentd-cloud-logging-kubernetes-minion-20ej 1/1 Running 0 31m
+kube-dns-v3-pk22 3/3 Running 0 32m
+monitoring-heapster-v1-20ej 0/1 Running 9 32m
+
+Here is the same information in a picture which shows how the pods might be placed on specific nodes.
+ + + +This diagram shows four nodes created on a Google Compute Engine cluster with the name of each VM node on a purple background. The internal and public IPs of each node are shown on gray boxes and the pods running in each node are shown in green boxes. Each pod box shows the name of the pod and the namespace it runs in, the IP address of the pod and the images which are run as part of the pod’s execution. Here we see that every node is running a fluentd-cloud-logging pod which is collecting the log output of the containers running on the same node and sending them to Google Cloud Logging. A pod which provides the +cluster DNS service runs on one of the nodes and a pod which provides monitoring support runs on another node.
+ +To help explain how cluster level logging works let’s start off with a synthetic log generator pod specification counter-pod.yaml:
+ + + +apiVersion: v1
+kind: Pod
+metadata:
+ name: counter
+spec:
+ containers:
+ - name: count
+ image: ubuntu:14.04
+ args: [bash, -c,
+ 'for ((i = 0; ; i++)); do echo "$i: $(date)"; sleep 1; done']
+
+This pod specification has one container which runs a bash script when the container is born. This script simply writes out the value of a counter and the date once per second and runs indefinitely. Let’s create the pod in the default +namespace.
+ +$ kubectl create -f examples/blog-logging/counter-pod.yaml
+ pods/counter
+
+We can observe the running pod:
+ +$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+counter 1/1 Running 0 5m
+
+This step may take a few minutes to download the ubuntu:14.04 image during which the pod status will be shown as Pending
.
One of the nodes is now running the counter pod:
+ + + +When the pod status changes to Running
we can use the kubectl logs command to view the output of this counter pod.
$ kubectl logs counter
+0: Tue Jun 2 21:37:31 UTC 2015
+1: Tue Jun 2 21:37:32 UTC 2015
+2: Tue Jun 2 21:37:33 UTC 2015
+3: Tue Jun 2 21:37:34 UTC 2015
+4: Tue Jun 2 21:37:35 UTC 2015
+5: Tue Jun 2 21:37:36 UTC 2015
+...
+
+This command fetches the log text from the Docker log file for the image that is running in this container. We can connect to the running container and observe the running counter bash script.
+ +$ kubectl exec -i counter bash
+ps aux
+USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
+root 1 0.0 0.0 17976 2888 ? Ss 00:02 0:00 bash -c for ((i = 0; ; i++)); do echo "$i: $(date)"; sleep 1; done
+root 468 0.0 0.0 17968 2904 ? Ss 00:05 0:00 bash
+root 479 0.0 0.0 4348 812 ? S 00:05 0:00 sleep 1
+root 480 0.0 0.0 15572 2212 ? R 00:05 0:00 ps aux
+
+What happens if for any reason the image in this pod is killed off and then restarted by Kubernetes? Will we still see the log lines from the previous invocation of the container followed by the log lines for the started container? Or will we lose the log lines from the original container’s execution and only see the log lines for the new container? Let’s find out. First let’s stop the currently running counter.
+ +$ kubectl stop pod counter
+pods/counter
+
+Now let’s restart the counter.
+ +$ kubectl create -f examples/blog-logging/counter-pod.yaml
+pods/counter
+
+Let’s wait for the container to restart and get the log lines again.
+ +$ kubectl logs counter
+0: Tue Jun 2 21:51:40 UTC 2015
+1: Tue Jun 2 21:51:41 UTC 2015
+2: Tue Jun 2 21:51:42 UTC 2015
+3: Tue Jun 2 21:51:43 UTC 2015
+4: Tue Jun 2 21:51:44 UTC 2015
+5: Tue Jun 2 21:51:45 UTC 2015
+6: Tue Jun 2 21:51:46 UTC 2015
+7: Tue Jun 2 21:51:47 UTC 2015
+8: Tue Jun 2 21:51:48 UTC 2015
+
+We’ve lost the log lines from the first invocation of the container in this pod! Ideally, we want to preserve all the log lines from each invocation of each container in the pod. Furthermore, even if the pod is restarted we would still like to preserve all the log lines that were ever emitted by the containers in the pod. But don’t fear, this is the functionality provided by cluster level logging in Kubernetes. When a cluster is created, the standard output and standard error output of each container can be ingested using a Fluentd agent running on each node into either Google Cloud Logging or into Elasticsearch and viewed with Kibana.
+ +When a Kubernetes cluster is created with logging to Google Cloud Logging enabled, the system creates a pod called fluentd-cloud-logging
on each node of the cluster to collect Docker container logs. These pods were shown at the start of this blog article in the response to the first get pods command.
This log collection pod has a specification which looks something like this:
+ + + +apiVersion: v1
+kind: Pod
+metadata:
+ name: fluentd-cloud-logging
+ namespace: kube-system
+spec:
+ containers:
+ - name: fluentd-cloud-logging
+ image: gcr.io/google_containers/fluentd-gcp:1.14
+ resources:
+ limits:
+ cpu: 100m
+ memory: 200Mi
+ env:
+ - name: FLUENTD_ARGS
+ value: -q
+ volumeMounts:
+ - name: varlog
+ mountPath: /var/log
+ - name: varlibdockercontainers
+ mountPath: /var/lib/docker/containers
+ readOnly: true
+ terminationGracePeriodSeconds: 30
+ volumes:
+ - name: varlog
+ hostPath:
+ path: /var/log
+ - name: varlibdockercontainers
+ hostPath:
+ path: /var/lib/docker/containers
+
+This pod specification maps the directory on the host containing the Docker log files, /var/lib/docker/containers
, to a directory inside the container which has the same path. The pod runs one image, gcr.io/google_containers/fluentd-gcp:1.6
, which is configured to collect the Docker log files from the logs directory and ingest them into Google Cloud Logging. One instance of this pod runs on each node of the cluster. Kubernetes will notice if this pod fails and automatically restart it.
We can click on the Logs item under the Monitoring section of the Google Developer Console and select the logs for the counter container, which will be called kubernetes.counter_default_count. This identifies the name of the pod (counter), the namespace (default) and the name of the container (count) for which the log collection occurred. Using this name we can select just the logs for our counter container from the drop down menu:
+ + + +When we view the logs in the Developer Console we observe the logs for both invocations of the container.
+ + + +Note the first container counted to 108 and then it was terminated. When the next container image restarted the counting process resumed from 0. Similarly if we deleted the pod and restarted it we would capture the logs for all instances of the containers in the pod whenever the pod was running.
+ +Logs ingested into Google Cloud Logging may be exported to various other destinations including Google Cloud Storage buckets and BigQuery. Use the Exports tab in the Cloud Logging console to specify where logs should be streamed to. You can also follow this link to the + settings tab.
+ +We could query the ingested logs from BigQuery using the SQL query which reports the counter log lines showing the newest lines first:
+ +SELECT metadata.timestamp, structPayload.log
+ FROM [mylogs.kubernetes_counter_default_count_20150611]
+ ORDER BY metadata.timestamp DESC
+
+Here is some sample output:
+ + + +We could also fetch the logs from Google Cloud Storage buckets to our desktop or laptop and then search them locally. The following command fetches logs for the counter pod running in a cluster which is itself in a Compute Engine project called myproject
. Only logs for the date 2015-06-11 are fetched.
$ gsutil -m cp -r gs://myproject/kubernetes.counter_default_count/2015/06/11 .
+
+Now we can run queries over the ingested logs. The example below uses the jq program to extract just the log lines.
+ +$ cat 21\:00\:00_21\:59\:59_S0.json | jq '.structPayload.log'
+"0: Thu Jun 11 21:39:38 UTC 2015\n"
+"1: Thu Jun 11 21:39:39 UTC 2015\n"
+"2: Thu Jun 11 21:39:40 UTC 2015\n"
+"3: Thu Jun 11 21:39:41 UTC 2015\n"
+"4: Thu Jun 11 21:39:42 UTC 2015\n"
+"5: Thu Jun 11 21:39:43 UTC 2015\n"
+"6: Thu Jun 11 21:39:44 UTC 2015\n"
+"7: Thu Jun 11 21:39:45 UTC 2015\n"
+...
+
+This page has touched briefly on the underlying mechanisms that support gathering cluster level logs on a Kubernetes deployment. The approach here only works for gathering the standard output and standard error output of the processes running in the pod’s containers. To gather other logs that are stored in files one can use a sidecar container to gather the required files as described at the page Collecting log files within containers with Fluentd and sending them to the Google Cloud Logging service.
+ +Some of the material in this section also appears in the blog article Cluster Level Logging with Kubernetes.
+ + + + + + + + + +The mesos/docker provider uses docker-compose to launch Kubernetes as a Mesos framework, running in docker with its +dependencies (etcd & mesos).
+ +Non-Goals:
+ +The cluster consists of several docker containers linked together by docker-managed hostnames:
+ +Component | +Hostname | +Description | +
---|---|---|
docker-grand-ambassador | ++ | Proxy to allow circular hostname linking in docker | +
etcd | +etcd | +Key/Value store used by Mesos | +
Mesos Master | +mesosmaster1 | +REST endpoint for interacting with Mesos | +
Mesos Slave (x2) | +mesosslave1, mesosslave2 | +Mesos agents that offer resources and run framework executors (e.g. Kubernetes Kublets) | +
Kubernetes API Server | +apiserver | +REST endpoint for interacting with Kubernetes | +
Kubernetes Controller Manager | +controller | ++ |
Kubernetes Scheduler | +scheduler | +Schedules container deployment by accepting Mesos offers | +
Required:
+ +Optional:
+ +It’s possible to install all of the above via Homebrew on a Mac.
+ +Some steps print instructions for configuring or launching. Make sure each is properly set up before continuing to the next step.
+ +brew install git
+brew install caskroom/cask/brew-cask
+brew cask install virtualbox
+brew install docker
+brew install boot2docker
+boot2docker init
+boot2docker up
+brew install docker-compose
+
+
+Most of the above are available via apt and yum, but depending on your distribution, you may have to install via other +means to get the latest versions.
+ +It is recommended to use Ubuntu, simply because it best supports AUFS, used by docker to mount volumes. Alternate file +systems may not fully support docker-in-docker.
+ +In order to build Kubernetes, the current user must be in a docker group with sudo privileges. +See the docker docs for instructions.
+ +If on a mac using boot2docker, the following steps will make the docker IPs (in the virtualbox VM) reachable from the +host machine (mac).
+ +Set the VM’s host-only network to “promiscuous mode”:
+ +boot2docker stop + VBoxManage modifyvm boot2docker-vm –nicpromisc2 allow-all + boot2docker start
+ +This allows the VM to accept packets that were sent to a different IP.
+ +Since the host-only network routes traffic between VMs and the host, other VMs will also be able to access the docker + IPs, if they have the following route.
+Route traffic to docker through the boot2docker IP:
+ +sudo route -n add -net 172.17.0.0 $(boot2docker ip)
+ +Since the boot2docker IP can change when the VM is restarted, this route may need to be updated over time.
+ To delete the route later: sudo route delete 172.17.0.0
Checkout source
+ +git clone https://github.com/kubernetes/kubernetes + cd kubernetes
+ +By default, that will get you the bleeding edge of master branch. + You may want a release branch instead, + if you have trouble with master.
+Build binaries
+ +You’ll need to build kubectl (CLI) for your local architecture and operating system and the rest of the server binaries for linux/amd64.
+ +Building a new release covers both cases:
+ +KUBERNETES_CONTRIB=mesos build/release.sh
+ +For developers, it may be faster to build locally.
+[Optional] Build docker images
+ +The following docker images are built as part of ./cluster/kube-up.sh
, but it may make sense to build them manually the first time because it may take a while.
Test image includes all the dependencies required for running e2e tests.
+ +./cluster/mesos/docker/test/build.sh
+ +In the future, this image may be available to download. It doesn’t contain anything specific to the current release, except its build dependencies.
+Kubernetes-Mesos image includes the compiled linux binaries.
+ +./cluster/mesos/docker/km/build.sh
+ +This image needs to be built every time you recompile the server binaries.
+[Optional] Configure Mesos resources
+ +By default, the mesos-slaves are configured to offer a fixed amount of resources (cpus, memory, disk, ports).
+ If you want to customize these values, update the MESOS_RESOURCES
environment variables in ./cluster/mesos/docker/docker-compose.yml
.
+ If you delete the MESOS_RESOURCES
environment variables, the resource amounts will be auto-detected based on the host resources, which will over-provision by > 2x.
If the configured resources are not available on the host, you may want to increase the resources available to Docker Engine. + You may have to increase you VM disk, memory, or cpu allocation in VirtualBox, + Docker Machine, or + Boot2Docker.
+Configure provider
+ +export KUBERNETES_PROVIDER=mesos/docker
+ +This tells cluster scripts to use the code within cluster/mesos/docker
.
Create cluster
+ +./cluster/kube-up.sh
+ +If you manually built all the above docker images, you can skip that step during kube-up:
+ +MESOS_DOCKER_SKIP_BUILD=true ./cluster/kube-up.sh
+ +After deploying the cluster, ~/.kube/config
will be created or updated to configure kubectl to target the new cluster.
Explore examples
+ +To learn more about Pods, Volumes, Labels, Services, and Replication Controllers, start with the + Kubernetes Walkthrough.
+ +To skip to a more advanced example, see the Guestbook Example
+Destroy cluster
+ +./cluster/kube-down.sh
+The kube-up
for the mesos/docker provider will automatically deploy KubeDNS and KubeUI addons as pods/services.
Check their status with:
+ +./cluster/kubectl.sh get pods --namespace=kube-system
+
+
+The web-based Kubernetes UI is accessible in a browser through the API Server proxy: https://<apiserver>:6443/ui/
.
By default, basic-auth is configured with user admin
and password admin
.
The IP of the API Server can be found using ./cluster/kubectl.sh cluster-info
.
Warning: e2e tests can take a long time to run. You may not want to run them immediately if you’re just getting started.
+ +While your cluster is up, you can run the end-to-end tests:
+ +./cluster/test-e2e.sh
+
+
+Notable parameters:
+- Increase the logging verbosity: -v=2
+- Run only a subset of the tests (regex matching): -ginkgo.focus=<pattern>
To build, deploy, test, and destroy, all in one command (plus unit & integration tests):
+ +make test_e2e
+
+
+When compiling from source, it’s simplest to use the ./cluster/kubectl.sh
script, which detects your platform &
+architecture and proxies commands to the appropriate kubectl
binary.
`./cluster/kubectl.sh get pods`
+
+
+Kill all docker containers
+ +docker ps -q -a | xargs docker rm -f
+
+
+Clean up unused docker volumes
+ +docker run -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/docker:/var/lib/docker --rm martin/docker-cleanup-volumes
+
+
+The steps above tell you how to build in a container, for minimal local dependencies. But if you have Go and Make installed you can build locally much faster:
+ +KUBERNETES_CONTRIB=mesos make
+
+
+However, if you’re not on linux, you’ll still need to compile the linux/amd64 server binaries:
+ +KUBERNETES_CONTRIB=mesos build/run.sh hack/build-go.sh
+
+
+The above two steps should be significantly faster than cross-compiling a whole new release for every supported platform (which is what ./build/release.sh
does).
Breakdown:
+ +KUBERNETES_CONTRIB=mesos
- enables building of the contrib/mesos binarieshack/build-go.sh
- builds the Go binaries for the current architecture (linux/amd64 when in a docker container)make
- delegates to hack/build-go.sh
build/run.sh
- executes a command in the build containerbuild/release.sh
- cross compiles Kubernetes for all supported architectures and operating systems (slow)Table of Contents +
+ +Mesos allows dynamic sharing of cluster resources between Kubernetes and other first-class Mesos frameworks such as Hadoop, Spark, and Chronos. +Mesos also ensures applications from different frameworks running on your cluster are isolated and that resources are allocated fairly among them.
+ +Mesos clusters can be deployed on nearly every IaaS cloud provider infrastructure or in your own physical datacenter. Kubernetes on Mesos runs on-top of that and therefore allows you to easily move Kubernetes workloads from one of these environments to the other.
+ +This tutorial will walk you through setting up Kubernetes on a Mesos cluster. +It provides a step by step walk through of adding Kubernetes to a Mesos cluster and starting your first pod with an nginx webserver.
+ +NOTE: There are known issues with the current implementation and support for centralized logging and monitoring is not yet available. +Please file an issue against the kubernetes-mesos project if you have problems completing the steps below.
+ +Further information is available in the Kubernetes on Mesos contrib directory.
+ +Note: You can, but you don’t have to deploy Kubernetes-Mesos on the same machine the Mesos master is running on.
+ +Log into the future Kubernetes master node over SSH, replacing the placeholder below with the correct IP address.
+ +ssh jclouds@${ip_address_of_master_node}
+
+Build Kubernetes-Mesos.
+ +git clone https://github.com/kubernetes/kubernetes
+cd kubernetes
+export KUBERNETES_CONTRIB=mesos
+make
+
+Set some environment variables.
+The internal IP address of the master may be obtained via hostname -i
.
export KUBERNETES_MASTER_IP=$(hostname -i)
+export KUBERNETES_MASTER=http://${KUBERNETES_MASTER_IP}:8888
+
+Note that KUBERNETES_MASTER is used as the api endpoint. If you have existing ~/.kube/config
and point to another endpoint, you need to add option --server=${KUBERNETES_MASTER}
to kubectl in later steps.
Start etcd and verify that it is running:
+ +sudo docker run -d --hostname $(uname -n) --name etcd \
+ -p 4001:4001 -p 7001:7001 quay.io/coreos/etcd:v2.0.12 \
+ --listen-client-urls http://0.0.0.0:4001 \
+ --advertise-client-urls http://${KUBERNETES_MASTER_IP}:4001
+
+$ sudo docker ps
+CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
+fd7bac9e2301 quay.io/coreos/etcd:v2.0.12 "/etcd" 5s ago Up 3s 2379/tcp, 2380/... etcd
+
+It’s also a good idea to ensure your etcd instance is reachable by testing it
+ +curl -L http://${KUBERNETES_MASTER_IP}:4001/v2/keys/
+
+If connectivity is OK, you will see an output of the available keys in etcd (if any).
+ +Update your PATH to more easily run the Kubernetes-Mesos binaries:
+ +export PATH="$(pwd)/_output/local/go/bin:$PATH"
+
+Identify your Mesos master: depending on your Mesos installation this is either a host:port
like mesos-master:5050
or a ZooKeeper URL like zk://zookeeper:2181/mesos
.
+In order to let Kubernetes survive Mesos master changes, the ZooKeeper URL is recommended for production environments.
export MESOS_MASTER=<host:port or zk:// url>
+
+Create a cloud config file mesos-cloud.conf
in the current directory with the following contents:
$ cat <<EOF >mesos-cloud.conf
+[mesos-cloud]
+ mesos-master = ${MESOS_MASTER}
+EOF
+
+Now start the kubernetes-mesos API server, controller manager, and scheduler on the master node:
+ +$ km apiserver \
+ --address=${KUBERNETES_MASTER_IP} \
+ --etcd-servers=http://${KUBERNETES_MASTER_IP}:4001 \
+ --service-cluster-ip-range=10.10.10.0/24 \
+ --port=8888 \
+ --cloud-provider=mesos \
+ --cloud-config=mesos-cloud.conf \
+ --secure-port=0 \
+ --v=1 >apiserver.log 2>&1 &
+
+$ km controller-manager \
+ --master=${KUBERNETES_MASTER_IP}:8888 \
+ --cloud-provider=mesos \
+ --cloud-config=./mesos-cloud.conf \
+ --v=1 >controller.log 2>&1 &
+
+$ km scheduler \
+ --address=${KUBERNETES_MASTER_IP} \
+ --mesos-master=${MESOS_MASTER} \
+ --etcd-servers=http://${KUBERNETES_MASTER_IP}:4001 \
+ --mesos-user=root \
+ --api-servers=${KUBERNETES_MASTER_IP}:8888 \
+ --cluster-dns=10.10.10.10 \
+ --cluster-domain=cluster.local \
+ --v=2 >scheduler.log 2>&1 &
+
+Disown your background jobs so that they’ll stay running if you log out.
+ +disown -a
+
+Add the appropriate binary folder to your PATH
to access kubectl:
export PATH=<path/to/kubernetes-directory>/platforms/linux/amd64:$PATH
+
+Interact with the kubernetes-mesos framework via kubectl
:
$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+
+# NOTE: your service IPs will likely differ
+$ kubectl get services
+NAME LABELS SELECTOR IP(S) PORT(S)
+k8sm-scheduler component=scheduler,provider=k8sm <none> 10.10.10.113 10251/TCP
+kubernetes component=apiserver,provider=kubernetes <none> 10.10.10.1 443/TCP
+
+Lastly, look for Kubernetes in the Mesos web GUI by pointing your browser to
+http://<mesos-master-ip:port>
. Make sure you have an active VPN connection.
+Go to the Frameworks tab, and look for an active framework named “Kubernetes”.
Write a JSON pod description to a local file:
+ +$ cat <<EOPOD >nginx.yaml
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: nginx
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+EOPOD
+
+Send the pod description to Kubernetes using the kubectl
CLI:
$ kubectl create -f ./nginx.yaml
+pods/nginx
+
+Wait a minute or two while dockerd
downloads the image layers from the internet.
+We can use the kubectl
interface to monitor the status of our pod:
$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+nginx 1/1 Running 0 14s
+
+Verify that the pod task is running in the Mesos web GUI. Click on the +Kubernetes framework. The next screen should show the running Mesos task that +started the Kubernetes pod.
+ +Kube-dns is an addon for Kubernetes which adds DNS-based service discovery to the cluster. For a detailed explanation see DNS in Kubernetes.
+ +The kube-dns addon runs as a pod inside the cluster. The pod consists of three co-located containers:
+ +The skydns container offers DNS service via port 53 to the cluster. The etcd communication works via local 127.0.0.1 communication
+ +We assume that kube-dns will use
+ +10.10.10.10
cluster.local
domain.Note that we have passed these two values already as parameter to the apiserver above.
+ +A template for an replication controller spinning up the pod with the 3 containers can be found at cluster/addons/dns/skydns-rc.yaml.in in the repository. The following steps are necessary in order to get a valid replication controller yaml file:
+ +1
cluster.local.
--kube_master_url=${KUBERNETES_MASTER}
parameter to the kube2sky container command.In addition the service template at cluster/addons/dns/skydns-svc.yaml.in needs the following replacement:
+ +10.10.10.10
.To do this automatically:
+ +sed -e "s/{{ pillar\['dns_replicas'\] }}/1/g;"\
+"s,\(command = \"/kube2sky\"\),\\1\\"$'\n'" - --kube_master_url=${KUBERNETES_MASTER},;"\
+"s/{{ pillar\['dns_domain'\] }}/cluster.local/g" \
+ cluster/addons/dns/skydns-rc.yaml.in > skydns-rc.yaml
+sed -e "s/{{ pillar\['dns_server'\] }}/10.10.10.10/g" \
+ cluster/addons/dns/skydns-svc.yaml.in > skydns-svc.yaml
+
+Now the kube-dns pod and service are ready to be launched:
+ +kubectl create -f ./skydns-rc.yaml
+kubectl create -f ./skydns-svc.yaml
+
+Check with kubectl get pods --namespace=kube-system
that 3/3 containers of the pods are eventually up and running. Note that the kube-dns pods run in the kube-system
namespace, not in default
.
To check that the new DNS service in the cluster works, we start a busybox pod and use that to do a DNS lookup. First create the busybox.yaml
pod spec:
cat <<EOF >busybox.yaml
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: busybox
+ namespace: default
+spec:
+ containers:
+ - image: busybox
+ command:
+ - sleep
+ - "3600"
+ imagePullPolicy: IfNotPresent
+ name: busybox
+ restartPolicy: Always
+EOF
+
+Then start the pod:
+ +kubectl create -f ./busybox.yaml
+
+When the pod is up and running, start a lookup for the Kubernetes master service, made available on 10.10.10.1 by default:
+ +kubectl exec busybox -- nslookup kubernetes
+
+If everything works fine, you will get this output:
+ +Server: 10.10.10.10
+Address 1: 10.10.10.10
+
+Name: kubernetes
+Address 1: 10.10.10.1
+
+Try out some of the standard Kubernetes examples.
+ +Read about Kubernetes on Mesos’ architecture in the contrib directory.
+ +NOTE: Some examples require Kubernetes DNS to be installed on the cluster. +Future work will add instructions to this guide to enable support for Kubernetes DNS.
+ +NOTE: Please be aware that there are known issues with the current Kubernetes-Mesos implementation.
+ + + + + + + + + +Getting started on oVirt +————————
+ +Table of Contents
+ +oVirt is a virtual datacenter manager that delivers powerful management of multiple virtual machines on multiple hosts. Using KVM and libvirt, oVirt can be installed on Fedora, CentOS, or Red Hat Enterprise Linux hosts to set up and manage your virtual data center.
+ +The oVirt cloud provider allows to easily discover and automatically add new VM instances as nodes to your Kubernetes cluster. +At the moment there are no community-supported or pre-loaded VM images including Kubernetes but it is possible to import or install Project Atomic (or Fedora) in a VM to generate a template. Any other distribution that includes Kubernetes may work as well.
+ +It is mandatory to install the ovirt-guest-agent in the guests for the VM ip address and hostname to be reported to ovirt-engine and ultimately to Kubernetes.
+ +Once the Kubernetes template is available it is possible to start instantiating VMs that can be discovered by the cloud provider.
+ +The oVirt Cloud Provider requires access to the oVirt REST-API to gather the proper information, the required credential should be specified in the ovirt-cloud.conf
file:
[connection]
+uri = https://localhost:8443/ovirt-engine/api
+username = admin@internal
+password = admin
+
+
+In the same file it is possible to specify (using the filters
section) what search query to use to identify the VMs to be reported to Kubernetes:
[filters]
+# Search query used to find nodes
+vms = tag=kubernetes
+
+
+In the above example all the VMs tagged with the kubernetes
label will be reported as nodes to Kubernetes.
The ovirt-cloud.conf
file then must be specified in kube-controller-manager:
kube-controller-manager ... --cloud-provider=ovirt --cloud-config=/path/to/ovirt-cloud.conf ...
+
+
+This short screencast demonstrates how the oVirt Cloud Provider can be used to dynamically add VMs to your Kubernetes cluster.
+ + + + + + + + + + + +Getting started on Rackspace +—————————-
+ +Table of Contents
+ +In general, the dev-build-and-up.sh workflow for Rackspace is the similar to Google Compute Engine. The specific implementation is different due to the use of CoreOS, Rackspace Cloud Files and the overall network design.
+ +These scripts should be used to deploy development environments for Kubernetes. If your account leverages RackConnect or non-standard networking, these scripts will most likely not work without modification.
+ +NOTE: The rackspace scripts do NOT rely on saltstack
and instead rely on cloud-init for configuration.
The current cluster design is inspired by: +- corekube +- Angus Lees
+ +nova
and swiftly
installed. It’s recommended to use a python virtualenv to install these packages into.export KUBERNETES_PROVIDER=rackspace
and run the bash hack/dev-build-and-up.sh
export KUBERNETES_PROVIDER=rackspace; wget -q -O - https://get.k8s.io | bash
build/
.KUBERNETES_PROVIDER=rackspace
, the scripts will upload kubernetes-server-linux-amd64.tar.gz
to Cloud Files.swiftly
CLI and a temp URL will be enabled on the object.kubernetes-server-linux-amd64.tar.gz
will be uploaded to this container and the URL will be passed to master/nodes when booted.There is a specific cluster/rackspace
directory with the scripts for the following steps:
nova
CLI. A cloud-config.yaml
is generated and provided as user-data with the entire configuration for the systems.$NUM_MINIONS
.eth2
to be the cloud network that the containers will communicate across.config-default.sh
are overridable via environment variables.v0.9
with git checkout v0.9
v0.9
v0.3
with git checkout v0.3
v0.3
cloud-config
files use the special CoreOS identifier $private_ipv4
to configure the services.This document describes how to run Kubernetes using rkt as a container runtime. +We still have a bunch of work to do to make the experience with rkt wonderful, please stay tuned!
+ +systemd should be installed on the machine and should be enabled. The minimum version required at this moment (2015/09/01) is 219 +(Note that systemd is not required by rkt itself, we are using it here to monitor and manage the pods launched by kubelet.)
+Install the latest rkt release according to the instructions here. +The minimum version required for now is v0.8.0.
+Note that for rkt version later than v0.7.0, metadata service
is not required for running pods in private networks. So now rkt pods will not register the metadata service be default.
To use rkt as the container runtime, we need to supply --container-runtime=rkt
and --rkt-path=$PATH_TO_RKT_BINARY
to kubelet. Additionally we can provide --rkt-stage1-image
flag
+as well to select which stage1 image we want to use.
If you are using the hack/local-up-cluster.sh script to launch the local cluster, then you can edit the environment variable CONTAINER_RUNTIME
, RKT_PATH
and RKT_STAGE1_IMAGE
to
+set these flags:
$ export CONTAINER_RUNTIME=rkt
+$ export RKT_PATH=$PATH_TO_RKT_BINARY
+$ export RKT_STAGE1_IMAGE=PATH=$PATH_TO_STAGE1_IMAGE
+
+Then we can launch the local cluster using the script:
+ +$ hack/local-up-cluster.sh
+
+To use rkt as the container runtime for your CoreOS cluster on GCE, you need to specify the OS distribution, project, image:
+ +$ export KUBE_OS_DISTRIBUTION=coreos
+$ export KUBE_GCE_MINION_IMAGE=<image_id>
+$ export KUBE_GCE_MINION_PROJECT=coreos-cloud
+$ export KUBE_CONTAINER_RUNTIME=rkt
+
+You can optionally choose the version of rkt used by setting KUBE_RKT_VERSION
:
$ export KUBE_RKT_VERSION=0.8.0
+
+Then you can launch the cluster by:
+ +$ kube-up.sh
+
+Note that we are still working on making all containerized the master components run smoothly in rkt. Before that we are not able to run the master node with rkt yet.
+ +To use rkt as the container runtime for your CoreOS cluster on AWS, you need to specify the provider and OS distribution:
+ +$ export KUBERNETES_PROVIDER=aws
+$ export KUBE_OS_DISTRIBUTION=coreos
+$ export KUBE_CONTAINER_RUNTIME=rkt
+
+You can optionally choose the version of rkt used by setting KUBE_RKT_VERSION
:
$ export KUBE_RKT_VERSION=0.8.0
+
+You can optionally choose the CoreOS channel by setting COREOS_CHANNEL
:
$ export COREOS_CHANNEL=stable
+
+Then you can launch the cluster by:
+ +$ kube-up.sh
+
+Note: CoreOS is not supported as the master using the automated launch +scripts. The master node is always Ubuntu.
+ +See a simple nginx example to try out your new cluster.
+ +For more complete applications, please look in the examples directory.
+ +Here are severals tips for you when you run into any issues.
+ +By default, the log verbose level is 2. In order to see more logs related to rkt, we can set the verbose level to 4.
+For local cluster, we can set the environment variable: LOG_LEVEL=4
.
+If the cluster is using salt, we can edit the logging.sls in the saltbase.
To check the pods’ status, we can use rkt command, such as rkt list
, rkt status
, rkt image list
, etc.
+More information about rkt command line can be found here
As we use systemd to launch rkt pods(by creating service files which will run rkt run-prepared
, we can check the pods’ log
+using journalctl
:
$ sudo journalctl -u $SERVICE_FILE
+
+where $SERVICE_FILE
is the name of the service file created for the pod, you can find it in the kubelet logs.
$ sudo journalctl -M rkt-$UUID -u $CONTAINER_NAME
+
+where $UUID
is the rkt pod’s UUID, which you can find via rkt list --full
, and $CONTAINER_NAME
is the container’s name.
Besides above tricks, Kubernetes also provides us handy tools for debugging the pods. More information can be found here
+ + + + + + + + + +This document describes how to run Kubernetes using rkt as a container runtime. +We still have a bunch of work to do to make the experience with rkt wonderful, please stay tuned!
+ +systemd should be installed on the machine and should be enabled. The minimum version required at this moment (2015/09/01) is 219 +(Note that systemd is not required by rkt itself, we are using it here to monitor and manage the pods launched by kubelet.)
+Install the latest rkt release according to the instructions here. +The minimum version required for now is v0.8.0.
+Note that for rkt version later than v0.7.0, metadata service
is not required for running pods in private networks. So now rkt pods will not register the metadata service be default.
To use rkt as the container runtime, we need to supply --container-runtime=rkt
and --rkt-path=$PATH_TO_RKT_BINARY
to kubelet. Additionally we can provide --rkt-stage1-image
flag
+as well to select which stage1 image we want to use.
If you are using the hack/local-up-cluster.sh script to launch the local cluster, then you can edit the environment variable CONTAINER_RUNTIME
, RKT_PATH
and RKT_STAGE1_IMAGE
to
+set these flags:
$ export CONTAINER_RUNTIME=rkt
+$ export RKT_PATH=$PATH_TO_RKT_BINARY
+$ export RKT_STAGE1_IMAGE=PATH=$PATH_TO_STAGE1_IMAGE
+
+Then we can launch the local cluster using the script:
+ +$ hack/local-up-cluster.sh
+
+To use rkt as the container runtime for your CoreOS cluster on GCE, you need to specify the OS distribution, project, image:
+ +$ export KUBE_OS_DISTRIBUTION=coreos
+$ export KUBE_GCE_MINION_IMAGE=<image_id>
+$ export KUBE_GCE_MINION_PROJECT=coreos-cloud
+$ export KUBE_CONTAINER_RUNTIME=rkt
+
+You can optionally choose the version of rkt used by setting KUBE_RKT_VERSION
:
$ export KUBE_RKT_VERSION=0.8.0
+
+Then you can launch the cluster by:
+ +$ kube-up.sh
+
+Note that we are still working on making all containerized the master components run smoothly in rkt. Before that we are not able to run the master node with rkt yet.
+ +To use rkt as the container runtime for your CoreOS cluster on AWS, you need to specify the provider and OS distribution:
+ +$ export KUBERNETES_PROVIDER=aws
+$ export KUBE_OS_DISTRIBUTION=coreos
+$ export KUBE_CONTAINER_RUNTIME=rkt
+
+You can optionally choose the version of rkt used by setting KUBE_RKT_VERSION
:
$ export KUBE_RKT_VERSION=0.8.0
+
+You can optionally choose the CoreOS channel by setting COREOS_CHANNEL
:
$ export COREOS_CHANNEL=stable
+
+Then you can launch the cluster by:
+ +$ kube-up.sh
+
+Note: CoreOS is not supported as the master using the automated launch +scripts. The master node is always Ubuntu.
+ +See a simple nginx example to try out your new cluster.
+ +For more complete applications, please look in the examples directory.
+ +Here are severals tips for you when you run into any issues.
+ +By default, the log verbose level is 2. In order to see more logs related to rkt, we can set the verbose level to 4.
+For local cluster, we can set the environment variable: LOG_LEVEL=4
.
+If the cluster is using salt, we can edit the logging.sls in the saltbase.
To check the pods’ status, we can use rkt command, such as rkt list
, rkt status
, rkt image list
, etc.
+More information about rkt command line can be found here
As we use systemd to launch rkt pods(by creating service files which will run rkt run-prepared
, we can check the pods’ log
+using journalctl
:
$ sudo journalctl -u $SERVICE_FILE
+
+where $SERVICE_FILE
is the name of the service file created for the pod, you can find it in the kubelet logs.
$ sudo journalctl -M rkt-$UUID -u $CONTAINER_NAME
+
+where $UUID
is the rkt pod’s UUID, which you can find via rkt list --full
, and $CONTAINER_NAME
is the container’s name.
Besides above tricks, Kubernetes also provides us handy tools for debugging the pods. More information can be found here
+ + + + + + + + + +Getting started from Scratch +—————————-
+ +This guide is for people who want to craft a custom Kubernetes cluster. If you +can find an existing Getting Started Guide that meets your needs on this +list, then we recommend using it, as you will be able to benefit +from the experience of others. However, if you have specific IaaS, networking, +configuration management, or operating system requirements not met by any of +those guides, then this guide will provide an outline of the steps you need to +take. Note that it requires considerably more effort than using one of the +pre-defined guides.
+ +This guide is also useful for those wanting to understand at a high level some of the +steps that existing cluster setup scripts are making.
+ +Table of Contents
+ + + +kubectl
installed on your desktop. This will happen as a side
+ effect of completing one of the other Getting Started Guides. If not, follow the instructions
+ here.Kubernetes has the concept of a Cloud Provider, which is a module which provides
+an interface for managing TCP Load Balancers, Nodes (Instances) and Networking Routes.
+The interface is defined in pkg/cloudprovider/cloud.go
. It is possible to
+create a custom cluster without implementing a cloud provider (for example if using
+bare-metal), and not all parts of the interface need to be implemented, depending
+on how flags are set on various components.
Kubernetes has a distinctive networking model.
+ +Kubernetes allocates an IP address to each pod. When creating a cluster, you +need to allocate a block of IPs for Kubernetes to use as Pod IPs. The simplest +approach is to allocate a different block of IPs to each node in the cluster as +the node is added. A process in one pod should be able to communicate with +another pod using the IP of the second pod. This connectivity can be +accomplished in two ways: +- Configure network to route Pod IPs + - Harder to setup from scratch. + - Google Compute Engine (GCE) and AWS guides use this approach. + - Need to make the Pod IPs routable by programming routers, switches, etc. + - Can be configured external to Kubernetes, or can implement in the “Routes” interface of a Cloud Provider module. + - Generally highest performance. +- Create an Overlay network + - Easier to setup + - Traffic is encapsulated, so per-pod IPs are routable. + - Examples: + - Flannel + - Weave + - Open vSwitch (OVS) + - Does not require “Routes” portion of Cloud Provider module. + - Reduced performance (exactly how much depends on your solution).
+ +You need to select an address range for the Pod IPs.
+- Various approaches:
+ - GCE: each project has its own 10.0.0.0/8
. Carve off a /16
for each
+ Kubernetes cluster from that space, which leaves room for several clusters.
+ Each node gets a further subdivision of this space.
+ - AWS: use one VPC for whole organization, carve off a chunk for each
+ cluster, or use different VPC for different clusters.
+ - IPv6 is not supported yet.
+- Allocate one CIDR subnet for each node’s PodIPs, or a single large CIDR
+ from which smaller CIDRs are automatically allocated to each node (if nodes
+ are dynamically added).
+ - You need max-pods-per-node * max-number-of-nodes IPs in total. A /24
per
+ node supports 254 pods per machine and is a common choice. If IPs are
+ scarce, a /26
(62 pods per machine) or even a /27
(30 pods) may be sufficient.
+ - e.g. use 10.10.0.0/16
as the range for the cluster, with up to 256 nodes
+ using 10.10.0.0/24
through 10.10.255.0/24
, respectively.
+ - Need to make these routable or connect with overlay.
Kubernetes also allocates an IP to each service. However,
+service IPs do not necessarily need to be routable. The kube-proxy takes care
+of translating Service IPs to Pod IPs before traffic leaves the node. You do
+need to Allocate a block of IPs for services. Call this
+SERVICE_CLUSTER_IP_RANGE
. For example, you could set
+SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16"
, allowing 65534 distinct services to
+be active at once. Note that you can grow the end of this range, but you
+cannot move it without disrupting the services and pods that already use it.
Also, you need to pick a static IP for master node.
+- Call this MASTER_IP
.
+- Open any firewalls to allow access to the apiserver ports 80 and/or 443.
+- Enable ipv4 forwarding sysctl, net.ipv4.ip_forward = 1
You should pick a name for your cluster. Pick a short name for each cluster
+which is unique from future cluster names. This will be used in several ways:
+ - by kubectl to distinguish between various clusters you have access to. You will probably want a
+ second one sometime later, such as for testing new Kubernetes releases, running in a different
+region of the world, etc.
+ - Kubernetes clusters can create cloud provider resources (e.g. AWS ELBs) and different clusters
+ need to distinguish which resources each created. Call this CLUSTERNAME
.
You will need binaries for: + - etcd + - A container runner, one of: + - docker + - rkt + - Kubernetes + - kubelet + - kube-proxy + - kube-apiserver + - kube-controller-manager + - kube-scheduler
+ +A Kubernetes binary release includes all the Kubernetes binaries as well as the supported release of etcd. +You can use a Kubernetes binary release (recommended) or build your Kubernetes binaries following the instructions in the +Developer Documentation. Only using a binary release is covered in this guide.
+ +Download the latest binary release and unzip it.
+Then locate ./kubernetes/server/kubernetes-server-linux-amd64.tar.gz
and unzip that.
+Then, within the second set of unzipped files, locate ./kubernetes/server/bin
, which contains
+all the necessary binaries.
You will run docker, kubelet, and kube-proxy outside of a container, the same way you would run any system daemon, so +you just need the bare binaries. For etcd, kube-apiserver, kube-controller-manager, and kube-scheduler, +we recommend that you run these as containers, so you need an image to be built.
+ +You have several choices for Kubernetes images:
+- Use images hosted on Google Container Registry (GCR):
+ - e.g gcr.io/google_containers/hyperkube:$TAG
, where TAG
is the latest
+ release tag, which can be found on the latest releases page.
+ - Ensure $TAG is the same tag as the release tag you are using for kubelet and kube-proxy.
+ - The hyperkube binary is an all in one binary
+ - hyperkube kubelet ...
runs the kublet, hyperkube apiserver ...
runs an apiserver, etc.
+- Build your own images.
+ - Useful if you are using a private registry.
+ - The release contains files such as ./kubernetes/server/bin/kube-apiserver.tar
which
+ can be converted into docker images using a command like
+ docker load -i kube-apiserver.tar
+ - You can verify if the image is loaded successfully with the right repository and tag using
+ command like docker images
For etcd, you can:
+- Use images hosted on Google Container Registry (GCR), such as gcr.io/google_containers/etcd:2.0.12
+- Use images hosted on Docker Hub or Quay.io, such as quay.io/coreos/etcd:v2.2.0
+- Use etcd binary included in your OS distro.
+- Build your own image
+ - You can do: cd kubernetes/cluster/images/etcd; make
We recommend that you use the etcd version which is provided in the Kubernetes binary distribution. The Kubernetes binaries in the release
+were tested extensively with this version of etcd and not with any other version.
+The recommended version number can also be found as the value of ETCD_VERSION
in kubernetes/cluster/images/etcd/Makefile
.
The remainder of the document assumes that the image identifiers have been chosen and stored in corresponding env vars. Examples (replace with latest tags and appropriate registry):
+ - HYPERKUBE_IMAGE==gcr.io/google_containers/hyperkube:$TAG
+ - ETCD_IMAGE=gcr.io/google_containers/etcd:$ETCD_VERSION
There are two main options for security: +- Access the apiserver using HTTP. + - Use a firewall for security. + - This is easier to setup. +- Access the apiserver using HTTPS + - Use https with certs, and credentials for user. + - This is the recommended approach. + - Configuring certs can be tricky.
+ +If following the HTTPS approach, you will need to prepare certs and credentials.
+ +You need to prepare several certs: +- The master needs a cert to act as an HTTPS server. +- The kubelets optionally need certs to identify themselves as clients of the master, and when + serving its own API over HTTPS.
+ +Unless you plan to have a real CA generate your certs, you will need to generate a root cert and use that to sign the master, kubelet, and kubectl certs.
+- see function create-certs
in cluster/gce/util.sh
+- see also cluster/saltbase/salt/generate-cert/make-ca-cert.sh
and
+ cluster/saltbase/salt/generate-cert/make-cert.sh
You will end up with the following files (we will use these variables later on)
+- CA_CERT
+ - put in on node where apiserver runs, in e.g. /srv/kubernetes/ca.crt
.
+- MASTER_CERT
+ - signed by CA_CERT
+ - put in on node where apiserver runs, in e.g. /srv/kubernetes/server.crt
+- MASTER_KEY
+ - put in on node where apiserver runs, in e.g. /srv/kubernetes/server.key
+- KUBELET_CERT
+ - optional
+- KUBELET_KEY
+ - optional
The admin user (and any users) need:
+ - a token or a password to identify them.
+ - tokens are just long alphanumeric strings, e.g. 32 chars. See
+ - TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
Your tokens and passwords need to be stored in a file for the apiserver
+to read. This guide uses /var/lib/kube-apiserver/known_tokens.csv
.
+The format for this file is described in the authentication documentation.
For distributing credentials to clients, the convention in Kubernetes is to put the credentials +into a kubeconfig file.
+ +The kubeconfig file for the administrator can be created as follows:
+ - If you have already used Kubernetes with a non-custom cluster (for example, used a Getting Started
+ Guide), you will already have a $HOME/.kube/config
file.
+ - You need to add certs, keys, and the master IP to the kubeconfig file:
+ - If using the firewall-only security option, set the apiserver this way:
+ - kubectl config set-cluster $CLUSTER_NAME --server=http://$MASTER_IP --insecure-skip-tls-verify=true
+ - Otherwise, do this to set the apiserver ip, client certs, and user credentials.
+ - kubectl config set-cluster $CLUSTER_NAME --certificate-authority=$CA_CERT --embed-certs=true --server=https://$MASTER_IP
+ - kubectl config set-credentials $USER --client-certificate=$CLI_CERT --client-key=$CLI_KEY --embed-certs=true --token=$TOKEN
+ - Set your cluster as the default cluster to use:
+ - kubectl config set-context $CONTEXT_NAME --cluster=$CLUSTER_NAME --user=$USER
+ - kubectl config use-context $CONTEXT_NAME
Next, make a kubeconfig file for the kubelets and kube-proxy. There are a couple of options for how +many distinct files to make: + 1. Use the same credential as the admin + - This is simplest to setup. + 1. One token and kubeconfig file for all kubelets, one for all kube-proxy, one for admin. + - This mirrors what is done on GCE today + 1. Different credentials for every kubelet, etc. + - We are working on this but all the pieces are not ready yet.
+ +You can make the files by copying the $HOME/.kube/config
, by following the code
+in cluster/gce/configure-vm.sh
or by using the following template:
apiVersion: v1
+kind: Config
+users:
+- name: kubelet
+ user:
+ token: ${KUBELET_TOKEN}
+clusters:
+- name: local
+ cluster:
+ certificate-authority-data: ${CA_CERT_BASE64_ENCODED}
+contexts:
+- context:
+ cluster: local
+ user: kubelet
+ name: service-account-context
+current-context: service-account-context
+
+Put the kubeconfig(s) on every node. The examples later in this
+guide assume that there are kubeconfigs in /var/lib/kube-proxy/kubeconfig
and
+/var/lib/kubelet/kubeconfig
.
This section discusses how to configure machines to be Kubernetes nodes.
+ +You should run three daemons on every node: + - docker or rkt + - kubelet + - kube-proxy
+ +You will also need to do assorted other configuration on top of a +base OS install.
+ +Tip: One possible starting point is to setup a cluster using an existing Getting +Started Guide. After getting a cluster running, you can then copy the init.d scripts or systemd unit files from that +cluster, and then modify them for use on your custom cluster.
+ +The minimum required Docker version will vary as the kubelet version changes. The newest stable release is a good choice. Kubelet will log a warning and refuse to start pods if the version is too old, so pick a version and try it.
+ +If you previously had Docker installed on a node without setting Kubernetes-specific +options, you may have a Docker-created bridge and iptables rules. You may want to remove these +as follows before proceeding to configure Docker for Kubernetes.
+ +iptables -t nat -F
+ifconfig docker0 down
+brctl delbr docker0
+
+The way you configure docker will depend in whether you have chosen the routable-vip or overlay-network approaches for your network.
+Some suggested docker options:
+ - create your own bridge for the per-node CIDR ranges, call it cbr0, and set --bridge=cbr0
option on docker.
+ - set --iptables=false
so docker will not manipulate iptables for host-ports (too coarse on older docker versions, may be fixed in newer versions)
+so that kube-proxy can manage iptables instead of docker.
+ - --ip-masq=false
+ - if you have setup PodIPs to be routable, then you want this false, otherwise, docker will
+ rewrite the PodIP source-address to a NodeIP.
+ - some environments (e.g. GCE) still need you to masquerade out-bound traffic when it leaves the cloud environment. This is very environment specific.
+ - if you are using an overlay network, consult those instructions.
+ - --mtu=
+ - may be required when using Flannel, because of the extra packet size due to udp encapsulation
+ - --insecure-registry $CLUSTER_SUBNET
+ - to connect to a private registry, if you set one up, without using SSL.
You may want to increase the number of open files for docker:
+ - DOCKER_NOFILE=1000000
Where this config goes depends on your node OS. For example, GCE’s Debian-based distro uses /etc/default/docker
.
Ensure docker is working correctly on your system before proceeding with the rest of the +installation, by following examples given in the Docker documentation.
+ +rkt is an alternative to Docker. You only need to install one of Docker or rkt. +The minimum version required is v0.5.6.
+ +systemd is required on your node to run rkt. The +minimum version required to match rkt v0.5.6 is +systemd 215.
+ +rkt metadata service is also required
+for rkt networking support. You can start rkt metadata service by using command like
+sudo systemd-run rkt metadata-service
Then you need to configure your kubelet with flag:
+ - --container-runtime=rkt
All nodes should run kubelet. See Selecting Binaries.
+ +Arguments to consider:
+ - If following the HTTPS security approach:
+ - --api-servers=https://$MASTER_IP
+ - --kubeconfig=/var/lib/kubelet/kubeconfig
+ - Otherwise, if taking the firewall-based security approach
+ - --api-servers=http://$MASTER_IP
+ - --config=/etc/kubernetes/manifests
+ - --cluster-dns=
to the address of the DNS server you will setup (see Starting Addons.)
+ - --cluster-domain=
to the dns domain prefix to use for cluster DNS addresses.
+ - --docker-root=
+ - --root-dir=
+ - --configure-cbr0=
(described above)
+ - --register-node
(described in Node documentation.)
All nodes should run kube-proxy. (Running kube-proxy on a “master” node is not +strictly required, but being consistent is easier.) Obtain a binary as described for +kubelet.
+ +Arguments to consider:
+ - If following the HTTPS security approach:
+ - --api-servers=https://$MASTER_IP
+ - --kubeconfig=/var/lib/kube-proxy/kubeconfig
+ - Otherwise, if taking the firewall-based security approach
+ - --api-servers=http://$MASTER_IP
Each node needs to be allocated its own CIDR range for pod networking.
+Call this NODE_X_POD_CIDR
.
A bridge called cbr0
needs to be created on each node. The bridge is explained
+further in the networking documentation. The bridge itself
+needs an address from $NODE_X_POD_CIDR
- by convention the first IP. Call
+this NODE_X_BRIDGE_ADDR
. For example, if NODE_X_POD_CIDR
is 10.0.0.0/16
,
+then NODE_X_BRIDGE_ADDR
is 10.0.0.1/16
. NOTE: this retains the /16
suffix
+because of how this is used later.
--configure-cbr0=true
option in kubelet init script and restart kubelet service. Kubelet will configure cbr0 automatically.
+It will wait to do this until the node controller has set Node.Spec.PodCIDR. Since you have not setup apiserver and node controller
+yet, the bridge will not be setup immediately.--configure-cbr0=false
on kubelet and restart.brctl addbr cbr0
.ip link set dev cbr0 mtu 1460
(NOTE: the actual value of MTU will depend on your network environment)ip addr add $NODE_X_BRIDGE_ADDR dev eth0
ip link set dev cbr0 up
If you have turned off Docker’s IP masquerading to allow pods to talk to each +other, then you may need to do masquerading just for destination IPs outside +the cluster network. For example:
+ +iptables -w -t nat -A POSTROUTING -o eth0 -j MASQUERADE \! -d ${CLUSTER_SUBNET}
+
+This will rewrite the source address from +the PodIP to the Node IP for traffic bound outside the cluster, and kernel +connection tracking +will ensure that responses destined to the node still reach +the pod.
+ +NOTE: This is environment specific. Some environments will not need +any masquerading at all. Others, such as GCE, will not allow pod IPs to send +traffic to the internet, but have no problem with them inside your GCE Project.
+ +glusterfs-client
for GlusterFS
+volumes.The previous steps all involved “conventional” system administration techniques for setting up +machines. You may want to use a Configuration Management system to automate the node configuration +process. There are examples of Saltstack, Ansible, Juju, and CoreOS Cloud Config in the +various Getting Started Guides.
+ +While the basic node services (kubelet, kube-proxy, docker) are typically started and managed using +traditional system administration/automation approaches, the remaining master components of Kubernetes are +all configured and managed by Kubernetes: + - their options are specified in a Pod spec (yaml or json) rather than an /etc/init.d file or + systemd unit. + - they are kept running by Kubernetes rather than by init.
+ +You will need to run one or more instances of etcd. + - Recommended approach: run one etcd instance, with its log written to a directory backed + by durable storage (RAID, GCE PD) + - Alternative: run 3 or 5 etcd instances. + - Log can be written to non-durable storage because storage is replicated. + - run a single apiserver which connects to one of the etc nodes. + See cluster-troubleshooting for more discussion on factors affecting cluster +availability.
+ +To run an etcd instance:
+ +cluster/saltbase/salt/etcd/etcd.manifest
The apiserver, controller manager, and scheduler will each run as a pod on the master node.
+ +For each of these components, the steps to start them running are similar:
+ +HYPERKUBE_IMAGE
to the values chosen in Selecting Images.{
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "kube-apiserver"
+ },
+ "spec": {
+ "hostNetwork": true,
+ "containers": [
+ {
+ "name": "kube-apiserver",
+ "image": "${HYPERKUBE_IMAGE}",
+ "command": [
+ "/hyperkube",
+ "apiserver",
+ "$ARG1",
+ "$ARG2",
+ ...
+ "$ARGN"
+ ],
+ "ports": [
+ {
+ "name": "https",
+ "hostPort": 443,
+ "containerPort": 443
+ },
+ {
+ "name": "local",
+ "hostPort": 8080,
+ "containerPort": 8080
+ }
+ ],
+ "volumeMounts": [
+ {
+ "name": "srvkube",
+ "mountPath": "/srv/kubernetes",
+ "readOnly": true
+ },
+ {
+ "name": "etcssl",
+ "mountPath": "/etc/ssl",
+ "readOnly": true
+ }
+ ],
+ "livenessProbe": {
+ "httpGet": {
+ "path": "/healthz",
+ "port": 8080
+ },
+ "initialDelaySeconds": 15,
+ "timeoutSeconds": 15
+ }
+ }
+ ],
+ "volumes": [
+ {
+ "name": "srvkube",
+ "hostPath": {
+ "path": "/srv/kubernetes"
+ }
+ },
+ {
+ "name": "etcssl",
+ "hostPath": {
+ "path": "/etc/ssl"
+ }
+ }
+ ]
+ }
+}
+
+Here are some apiserver flags you may need to set:
+ +--cloud-provider=
see cloud providers--cloud-config=
see cloud providers--address=${MASTER_IP}
or --bind-address=127.0.0.1
and --address=127.0.0.1
if you want to run a proxy on the master node.--cluster-name=$CLUSTER_NAME
--service-cluster-ip-range=$SERVICE_CLUSTER_IP_RANGE
--etcd-servers=http://127.0.0.1:4001
--tls-cert-file=/srv/kubernetes/server.cert
--tls-private-key-file=/srv/kubernetes/server.key
--admission-control=$RECOMMENDED_LIST
+ --allow-privileged=true
, only if you trust your cluster user to run pods as root.If you are following the firewall-only security approach, then use these arguments:
+ +--token-auth-file=/dev/null
--insecure-bind-address=$MASTER_IP
--advertise-address=$MASTER_IP
If you are using the HTTPS approach, then set:
+- --client-ca-file=/srv/kubernetes/ca.crt
+- --token-auth-file=/srv/kubernetes/known_tokens.csv
+- --basic-auth-file=/srv/kubernetes/basic_auth.csv
This pod mounts several node file system directories using the hostPath
volumes. Their purposes are:
+- The /etc/ssl
mount allows the apiserver to find the SSL root certs so it can
+ authenticate external services, such as a cloud provider.
+ - This is not required if you do not use a cloud provider (e.g. bare-metal).
+- The /srv/kubernetes
mount allows the apiserver to read certs and credentials stored on the
+ node disk. These could instead be stored on a persistent disk, such as a GCE PD, or baked into the image.
+- Optionally, you may want to mount /var/log
as well and redirect output there (not shown in template).
+ - Do this if you prefer your logs to be accessible from the root filesystem with tools like journalctl.
TODO document proxy-ssh setup.
+ +Apiserver supports several cloud providers.
+ +--cloud-provider
flag are aws
, gce
, mesos
, openshift
, ovirt
, rackspace
, vagrant
, or unset.Some cloud providers require a config file. If so, you need to put config file into apiserver image or mount through hostPath.
+ +--cloud-config=
set if cloud provider requires a config file.aws
, gce
, mesos
, openshift
, ovirt
and rackspace
.gce.conf
in this fileComplete this template for the scheduler pod:
+ +{
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "kube-scheduler"
+ },
+ "spec": {
+ "hostNetwork": true,
+ "containers": [
+ {
+ "name": "kube-scheduler",
+ "image": "$HYBERKUBE_IMAGE",
+ "command": [
+ "/hyperkube",
+ "scheduler",
+ "--master=127.0.0.1:8080",
+ "$SCHEDULER_FLAG1",
+ ...
+ "$SCHEDULER_FLAGN"
+ ],
+ "livenessProbe": {
+ "httpGet": {
+ "host" : "127.0.0.1",
+ "path": "/healthz",
+ "port": 10251
+ },
+ "initialDelaySeconds": 15,
+ "timeoutSeconds": 15
+ }
+ }
+ ]
+ }
+}
+
+Typically, no additional flags are required for the scheduler.
+ +Optionally, you may want to mount /var/log
as well and redirect output there.
Template for controller manager pod:
+ +{
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "kube-controller-manager"
+ },
+ "spec": {
+ "hostNetwork": true,
+ "containers": [
+ {
+ "name": "kube-controller-manager",
+ "image": "$HYPERKUBE_IMAGE",
+ "command": [
+ "/hyperkube",
+ "controller-manager",
+ "$CNTRLMNGR_FLAG1",
+ ...
+ "$CNTRLMNGR_FLAGN"
+ ],
+ "volumeMounts": [
+ {
+ "name": "srvkube",
+ "mountPath": "/srv/kubernetes",
+ "readOnly": true
+ },
+ {
+ "name": "etcssl",
+ "mountPath": "/etc/ssl",
+ "readOnly": true
+ }
+ ],
+ "livenessProbe": {
+ "httpGet": {
+ "host": "127.0.0.1",
+ "path": "/healthz",
+ "port": 10252
+ },
+ "initialDelaySeconds": 15,
+ "timeoutSeconds": 15
+ }
+ }
+ ],
+ "volumes": [
+ {
+ "name": "srvkube",
+ "hostPath": {
+ "path": "/srv/kubernetes"
+ }
+ },
+ {
+ "name": "etcssl",
+ "hostPath": {
+ "path": "/etc/ssl"
+ }
+ }
+ ]
+ }
+}
+
+Flags to consider using with controller manager:
+ - --cluster-name=$CLUSTER_NAME
+ - --cluster-cidr=
+ - TODO: explain this flag.
+ - --allocate-node-cidrs=
+ - TODO: explain when you want controller to do this and when you want to do it another way.
+ - --cloud-provider=
and --cloud-config
as described in apiserver section.
+ - --service-account-private-key-file=/srv/kubernetes/server.key
, used by the service account feature.
+ - --master=127.0.0.1:8080
Place each completed pod template into the kubelet config dir
+(whatever --config=
argument of kubelet is set to, typically
+/etc/kubernetes/manifests
). The order does not matter: scheduler and
+controller manager will retry reaching the apiserver until it is up.
Use ps
or docker ps
to verify that each process has started. For example, verify that kubelet has started a container for the apiserver like this:
$ sudo docker ps | grep apiserver:
+5783290746d5 gcr.io/google_containers/kube-apiserver:e36bf367342b5a80d7467fd7611ad873 "/bin/sh -c '/usr/lo'" 10 seconds ago Up 9 seconds k8s_kube-apiserver.feb145e7_kube-apiserver-kubernetes-master_default_eaebc600cf80dae59902b44225f2fc0a_225a4695
+
+Then try to connect to the apiserver:
+ +$ echo $(curl -s http://localhost:8080/healthz)
+ok
+$ curl -s http://localhost:8080/api
+{
+ "versions": [
+ "v1"
+ ]
+}
+
+If you have selected the --register-node=true
option for kubelets, they will now begin self-registering with the apiserver.
+You should soon be able to see all your nodes by running the kubectl get nodes
command.
+Otherwise, you will need to manually create node objects.
TODO talk about starting Logging.
+ +TODO talk about starting Monitoring.
+ +TODO talk about starting DNS.
+ +TODO explain how to use cluster/validate-cluster.sh
Try to run through the “Inspect your cluster” section in one of the other Getting Started Guides, such as GCE. +You should see some services. You should also see “mirror pods” for the apiserver, scheduler and controller-manager, plus any add-ons you started.
+ +At this point you should be able to run through one of the basic examples, such as the nginx example.
+ +You may want to try to run the Conformance test. Any failures may give a hint as to areas that need more attention.
+ +The nodes must be able to connect to each other using their private IP. Verify this by +pinging or SSH-ing from one node to another.
+ +If you run into trouble, please see the section on troubleshooting, post to the +google-containers group, or come ask questions on Slack.
+ + + + + + + + + +Kubernetes Deployment On Bare-metal Ubuntu Nodes with Calico Networking +————————————————
+ +This document describes how to deploy Kubernetes on Ubuntu bare metal nodes with Calico Networking plugin. See projectcalico.org for more information on what Calico is, and the calicoctl github for more information on the command-line tool, calicoctl
.
This guide will set up a simple Kubernetes cluster with a master and two nodes. We will start the following processes with systemd:
+ +On the Master:
+- etcd
+- kube-apiserver
+- kube-controller-manager
+- kube-scheduler
+- calico-node
On each Node:
+- kube-proxy
+- kube-kubelet
+- calico-node
systemd
and thus uses Ubuntu 15.04 which supports systemd natively.First, get the sample configurations for this tutorial
+ +
+wget https://github.com/Metaswitch/calico-kubernetes-ubuntu-demo/archive/master.tar.gz
+tar -xvf master.tar.gz
+
+
+
+Many of the sample systemd services provided rely on environment variables on a per-node basis. Here we’ll edit those environment variables and move them into place.
+ +1.) Copy the network-environment-template from the master
directory for editing.
+cp calico-kubernetes-ubuntu-demo-master/master/network-environment-template network-environment
+
+
+
+2.) Edit network-environment
to represent your current host’s settings.
3.) Move the network-environment
into /etc
+sudo mv -f network-environment /etc
+
+
+
+1.) Build & Install Kubernetes binaries
+ +
+# Get the Kubernetes Source
+wget https://github.com/kubernetes/kubernetes/releases/download/v1.0.3/kubernetes.tar.gz
+
+# Untar it
+tar -xf kubernetes.tar.gz
+tar -xf kubernetes/server/kubernetes-server-linux-amd64.tar.gz
+kubernetes/cluster/ubuntu/build.sh
+
+# Add binaries to /usr/bin
+sudo cp -f binaries/master/* /usr/bin
+sudo cp -f binaries/kubectl /usr/bin
+
+
+
+2.) Install the sample systemd processes settings for launching kubernetes services
+ +
+sudo cp -f calico-kubernetes-ubuntu-demo-master/master/*.service /etc/systemd
+sudo systemctl enable /etc/systemd/etcd.service
+sudo systemctl enable /etc/systemd/kube-apiserver.service
+sudo systemctl enable /etc/systemd/kube-controller-manager.service
+sudo systemctl enable /etc/systemd/kube-scheduler.service
+
+
+
+3.) Launch the processes.
+ +
+sudo systemctl start etcd.service
+sudo systemctl start kube-apiserver.service
+sudo systemctl start kube-controller-manager.service
+sudo systemctl start kube-scheduler.service
+
+
+
+In order to allow the master to route to pods on our nodes, we will launch the calico-node daemon on our master. This will allow it to learn routes over BGP from the other calico-node daemons in the cluster. The docker daemon should already be running before calico is started.
+ +
+# Install the calicoctl binary, which will be used to launch calico
+wget https://github.com/projectcalico/calico-docker/releases/download/v0.5.5/calicoctl
+chmod +x calicoctl
+sudo cp -f calicoctl /usr/bin
+
+# Install and start the calico service
+sudo cp -f calico-kubernetes-ubuntu-demo-master/master/calico-node.service /etc/systemd
+sudo systemctl enable /etc/systemd/calico-node.service
+sudo systemctl start calico-node.service
+
+
+
+++ +Note: calico-node may take a few minutes on first boot while it downloads the calico-node docker image.
+
Perform these steps once on each node, ensuring you appropriately set the environment variables on each node
+ +1.) Get the sample configurations for this tutorial
+ +
+wget https://github.com/Metaswitch/calico-kubernetes-ubuntu-demo/archive/master.tar.gz
+tar -xvf master.tar.gz
+
+
+
+2.) Copy the network-environment-template from the node
directory
+cp calico-kubernetes-ubuntu-demo-master/node/network-environment-template network-environment
+
+
+
+3.) Edit network-environment
to represent your current host’s settings.
4.) Move network-environment
into /etc
+sudo mv -f network-environment /etc
+
+
+
+Instead of using docker’s default interface (docker0), we will configure a new one to use desired IP ranges
+ +
+sudo apt-get install -y bridge-utils
+sudo brctl addbr cbr0
+sudo ifconfig cbr0 up
+sudo ifconfig cbr0 <IP>/24
+
+
+
+++ +Replace <IP> with the subnet for this host’s containers. Example topology:
+
Node | +cbr0 IP | +
---|---|
node-1 | +192.168.1.1/24 | +
node-2 | +192.168.2.1/24 | +
node-X | +192.168.X.1/24 | +
The Docker daemon must be started and told to use the already configured cbr0 instead of using the usual docker0, as well as disabling ip-masquerading and modification of the ip-tables.
+ +1.) Edit the ubuntu-15.04 docker.service for systemd at: /lib/systemd/system/docker.service
2.) Find the line that reads ExecStart=/usr/bin/docker -d -H fd://
and append the following flags: --bridge=cbr0 --iptables=false --ip-masq=false
3.) Reload systemctl and restart docker.
+ +
+sudo systemctl daemon-reload
+sudo systemctl restart docker
+
+
+
+1.) Install Calico
+ +
+# Get the calicoctl binary
+wget https://github.com/projectcalico/calico-docker/releases/download/v0.5.5/calicoctl
+chmod +x calicoctl
+sudo cp -f calicoctl /usr/bin
+
+# Start calico on this node
+sudo cp calico-kubernetes-ubuntu-demo-master/node/calico-node.service /etc/systemd
+sudo systemctl enable /etc/systemd/calico-node.service
+sudo systemctl start calico-node.service
+
+
+
+++ +The calico-node service will automatically get the kubernetes-calico plugin binary and install it on the host system.
+
2.) Use calicoctl to add an IP pool. We must specify the IP and port that the master’s etcd is listening on. +NOTE: This step only needs to be performed once per Kubernetes deployment, as it covers all the node’s IP ranges.
+ +
+ETCD_AUTHORITY=<MASTER_IP>:4001 calicoctl pool add 192.168.0.0/16
+
+
+
+1.) Build & Install Kubernetes binaries
+ +
+# Get the Kubernetes Source
+wget https://github.com/kubernetes/kubernetes/releases/download/v1.0.3/kubernetes.tar.gz
+
+# Untar it
+tar -xf kubernetes.tar.gz
+tar -xf kubernetes/server/kubernetes-server-linux-amd64.tar.gz
+kubernetes/cluster/ubuntu/build.sh
+
+# Add binaries to /usr/bin
+sudo cp -f binaries/minion/* /usr/bin
+
+# Get the iptables based kube-proxy reccomended for this demo
+wget https://github.com/projectcalico/calico-kubernetes/releases/download/v0.1.1/kube-proxy
+sudo cp kube-proxy /usr/bin/
+sudo chmod +x /usr/bin/kube-proxy
+
+
+
+2.) Install and launch the sample systemd processes settings for launching Kubernetes services
+ +
+sudo cp calico-kubernetes-ubuntu-demo-master/node/kube-proxy.service /etc/systemd/
+sudo cp calico-kubernetes-ubuntu-demo-master/node/kube-kubelet.service /etc/systemd/
+sudo systemctl enable /etc/systemd/kube-proxy.service
+sudo systemctl enable /etc/systemd/kube-kubelet.service
+sudo systemctl start kube-proxy.service
+sudo systemctl start kube-kubelet.service
+
+
+
+++ +You may want to consider checking their status after to ensure everything is running
+
Most Kubernetes deployments will require the DNS addon for service discovery. For more on DNS service discovery, check here.
+ +The config repository for this guide comes with manifest files to start the DNS addon. To install DNS, do the following on your Master node.
+ +Replace <MASTER_IP>
in calico-kubernetes-ubuntu-demo-master/dns/skydns-rc.yaml
with your Master’s IP address. Then, create skydns-rc.yaml
and skydns-svc.yaml
using kubectl create -f <FILE>
.
At this point, you have a fully functioning cluster running on kubernetes with a master and 2 nodes networked with Calico. You can now follow any of the standard documentation to set up other services on your cluster.
+ +With this sample configuration, because the containers have private 192.168.0.0/16
IPs, you will need NAT to allow connectivity between containers and the internet. However, in a full datacenter deployment, NAT is not always necessary, since Calico can peer with the border routers over BGP.
The simplest method for enabling connectivity from containers to the internet is to use an iptables masquerade rule. This is the standard mechanism recommended in the Kubernetes GCE environment.
+ +We need to NAT traffic that has a destination outside of the cluster. Internal traffic includes the master/nodes, and the container IP pools. A suitable masquerade chain would follow the pattern below, replacing the following variables:
+- CONTAINER_SUBNET
: The cluster-wide subnet from which container IPs are chosen. All cbr0 bridge subnets fall within this range. The above example uses 192.168.0.0/16
.
+- KUBERNETES_HOST_SUBNET
: The subnet from which Kubernetes node / master IP addresses have been chosen.
+- HOST_INTERFACE
: The interface on the Kubernetes node which is used for external connectivity. The above example uses eth0
+sudo iptables -t nat -N KUBE-OUTBOUND-NAT
+sudo iptables -t nat -A KUBE-OUTBOUND-NAT -d <CONTAINER_SUBNET> -o <HOST_INTERFACE> -j RETURN
+sudo iptables -t nat -A KUBE-OUTBOUND-NAT -d <KUBERNETES_HOST_SUBNET> -o <HOST_INTERFACE> -j RETURN
+sudo iptables -t nat -A KUBE-OUTBOUND-NAT -j MASQUERADE
+sudo iptables -t nat -A POSTROUTING -j KUBE-OUTBOUND-NAT
+
+
+
+This chain should be applied on the master and all nodes. In production, these rules should be persisted, e.g. with iptables-persistent
.
In a datacenter environment, it is recommended to configure Calico to peer with the border routers over BGP. This means that the container IPs will be routable anywhere in the datacenter, and so NAT is not needed on the nodes (though it may be enabled at the datacenter edge to allow outbound-only internet connectivity).
+ + + + + + + + + +Kubernetes Deployment On Bare-metal Ubuntu Nodes +————————————————
+ +This document describes how to deploy kubernetes on ubuntu nodes, 1 master and 3 nodes involved +in the given examples. You can scale to any number of nodes by changing some settings with ease. +The original idea was heavily inspired by @jainvipin ‘s ubuntu single node +work, which has been merge into this document.
+ +Cloud team from Zhejiang University will maintain this work.
+ +First clone the kubernetes github repo
+ +$ git clone https://github.com/kubernetes/kubernetes.git
+
+Then download all the needed binaries into given directory (cluster/ubuntu/binaries)
+ +$ cd kubernetes/cluster/ubuntu
+$ ./build.sh
+
+You can customize your etcd version, flannel version, k8s version by changing corresponding variables
+ETCD_VERSION
, FLANNEL_VERSION
and KUBE_VERSION
in build.sh, by default etcd version is 2.0.12,
+flannel version is 0.4.0 and k8s version is 1.0.3.
Make sure that the involved binaries are located properly in the binaries/master +or binaries/minion directory before you go ahead to the next step .
+ +Note that we use flannel here to set up overlay network, yet it’s optional. Actually you can build up k8s +cluster natively, or use flannel, Open vSwitch or any other SDN tool you like.
+ +An example cluster is listed below:
+ +IP Address | +Role | +
---|---|
10.10.103.223 | +node | +
10.10.103.162 | +node | +
10.10.103.250 | +both master and node | +
First configure the cluster information in cluster/ubuntu/config-default.sh, below is a simple sample.
+ +export nodes="vcap@10.10.103.250 vcap@10.10.103.162 vcap@10.10.103.223"
+
+export role="ai i i"
+
+export NUM_MINIONS=${NUM_MINIONS:-3}
+
+export SERVICE_CLUSTER_IP_RANGE=192.168.3.0/24
+
+export FLANNEL_NET=172.16.0.0/16
+
+The first variable nodes
defines all your cluster nodes, MASTER node comes first and
+separated with blank space like <user_1@ip_1> <user_2@ip_2> <user_3@ip_3>
Then the role
variable defines the role of above machine in the same order, “ai” stands for machine
+acts as both master and node, “a” stands for master, “i” stands for node.
The NUM_MINIONS
variable defines the total number of nodes.
The SERVICE_CLUSTER_IP_RANGE
variable defines the kubernetes service IP range. Please make sure
+that you do have a valid private ip range defined here, because some IaaS provider may reserve private ips.
+You can use below three private network range according to rfc1918. Besides you’d better not choose the one
+that conflicts with your own private network range.
10.0.0.0 - 10.255.255.255 (10/8 prefix)
+
+ 172.16.0.0 - 172.31.255.255 (172.16/12 prefix)
+
+ 192.168.0.0 - 192.168.255.255 (192.168/16 prefix)
+
+
+The FLANNEL_NET
variable defines the IP range used for flannel overlay network,
+should not conflict with above SERVICE_CLUSTER_IP_RANGE
.
Note: When deploying, master needs to connect the Internet to download the necessary files. If your machines locate in a private network that need proxy setting to connect the Internet, you can set the config PROXY_SETTING
in cluster/ubuntu/config-default.sh such as:
PROXY_SETTING="http_proxy=http://server:port https_proxy=https://server:port"
+
+
+After all the above variables being set correctly, we can use following command in cluster/ directory to bring up the whole cluster.
+ +$ KUBERNETES_PROVIDER=ubuntu ./kube-up.sh
The scripts automatically scp binaries and config files to all the machines and start the k8s service on them. +The only thing you need to do is to type the sudo password when promoted.
+ +Deploying minion on machine 10.10.103.223
+...
+[sudo] password to copy files and start minion:
+
+If all things goes right, you will see the below message from console indicating the k8s is up.
+ +Cluster validation succeeded
+
+You can use kubectl
command to check if the newly created k8s is working correctly.
+The kubectl
binary is under the cluster/ubuntu/binaries
directory.
+You can make it available via PATH, then you can use the below command smoothly.
For example, use $ kubectl get nodes
to see if all of your nodes are ready.
$ kubectl get nodes
+NAME LABELS STATUS
+10.10.103.162 kubernetes.io/hostname=10.10.103.162 Ready
+10.10.103.223 kubernetes.io/hostname=10.10.103.223 Ready
+10.10.103.250 kubernetes.io/hostname=10.10.103.250 Ready
+
+Also you can run Kubernetes guest-example to build a redis backend cluster on the k8s.
+ +Assuming you have a starting cluster now, this section will tell you how to deploy addons like DNS +and UI onto the existing cluster.
+ +The configuration of DNS is configured in cluster/ubuntu/config-default.sh.
+ +ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}"
+
+DNS_SERVER_IP="192.168.3.10"
+
+DNS_DOMAIN="cluster.local"
+
+DNS_REPLICAS=1
+
+The DNS_SERVER_IP
is defining the ip of dns server which must be in the SERVICE_CLUSTER_IP_RANGE
.
+The DNS_REPLICAS
describes how many dns pod running in the cluster.
By default, we also take care of kube-ui addon.
+ +ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}"
+
+After all the above variables have been set, just type the following command.
+ +$ cd cluster/ubuntu
+$ KUBERNETES_PROVIDER=ubuntu ./deployAddons.sh
+
+After some time, you can use $ kubectl get pods --namespace=kube-system
to see the DNS and UI pods are running in the cluster.
We are working on these features which we’d like to let everybody know:
+ +Generally, what this approach does is quite simple:
+ +etcd
using IPs based on input from userSo if you encounter a problem, check etcd configuration first
+ +Please try:
+ +/var/log/upstart/etcd.log
for suspicious etcd logCheck /etc/default/etcd
, as we do not have much input validation, a right config should be like:
sh
+ ETCD_OPTS="-name infra1 -initial-advertise-peer-urls <http://ip_of_this_node:2380> -listen-peer-urls <http://ip_of_this_node:2380> -initial-cluster-token etcd-cluster-1 -initial-cluster infra1=<http://ip_of_this_node:2380>,infra2=<http://ip_of_another_node:2380>,infra3=<http://ip_of_another_node:2380> -initial-cluster-state new"
+
$ KUBERNETES_PROVIDER=ubuntu ./kube-down.sh
+ $ KUBERNETES_PROVIDER=ubuntu ./kube-up.sh
+
+/etc/default/{component_name}
.If you already have a kubernetes cluster, and want to upgrade to a new version, +you can use following command in cluster/ directory to update the whole cluster or a specified node to a new version.
+ +$ KUBERNETES_PROVIDER=ubuntu ./kube-push.sh [-m|-n <node id>] <version>
+
+It can be done for all components (by default), master(-m
) or specified node(-n
).
+If the version is not specified, the script will try to use local binaries.You should ensure all the binaries are well prepared in path cluster/ubuntu/binaries
.
$ tree cluster/ubuntu/binaries
+binaries/
+├── kubectl
+├── master
+│ ├── etcd
+│ ├── etcdctl
+│ ├── flanneld
+│ ├── kube-apiserver
+│ ├── kube-controller-manager
+│ └── kube-scheduler
+└── minion
+ ├── flanneld
+ ├── kubelet
+ └── kube-proxy
+
+Upgrading single node is experimental now. You can use following command to get a help.
+ +$ KUBERNETES_PROVIDER=ubuntu ./kube-push.sh -h
+
+Some examples are as follows:
+ +$ KUBERNETES_PROVIDER=ubuntu ./kube-push.sh -m 1.0.5
$ KUBERNETES_PROVIDER=ubuntu ./kube-push.sh -n 10.10.103.223 1.0.5
$ KUBERNETES_PROVIDER=ubuntu ./kube-push.sh 1.0.5
The script will not delete any resources of your cluster, it just replaces the binaries.
+You can use kubectl
command to check if the newly upgraded k8s is working correctly.
+For example, use $ kubectl get nodes
to see if all of your nodes are ready.Or refer to test-it-out
Running Kubernetes with Vagrant (and VirtualBox) is an easy way to run/test/develop on your local machine (Linux, Mac OS X).
+ +Table of Contents
+ +yum install vagrant-libvirt
Setting up a cluster is as simple as running:
+ +export KUBERNETES_PROVIDER=vagrant
+curl -sS https://get.k8s.io | bash
+
+Alternatively, you can download Kubernetes release and extract the archive. To start your local cluster, open a shell and run:
+ +cd kubernetes
+
+export KUBERNETES_PROVIDER=vagrant
+./cluster/kube-up.sh
+
+The KUBERNETES_PROVIDER
environment variable tells all of the various cluster management scripts which variant to use. If you forget to set this, the assumption is you are running on Google Compute Engine.
By default, the Vagrant setup will create a single master VM (called kubernetes-master) and one node (called kubernetes-minion-1). Each VM will take 1 GB, so make sure you have at least 2GB to 4GB of free memory (plus appropriate free disk space).
+ +Vagrant will provision each machine in the cluster with all the necessary components to run Kubernetes. The initial setup can take a few minutes to complete on each machine.
+ +If you installed more than one Vagrant provider, Kubernetes will usually pick the appropriate one. However, you can override which one Kubernetes will use by setting the VAGRANT_DEFAULT_PROVIDER
environment variable:
export VAGRANT_DEFAULT_PROVIDER=parallels
+export KUBERNETES_PROVIDER=vagrant
+./cluster/kube-up.sh
+
+By default, each VM in the cluster is running Fedora.
+ +To access the master or any node:
+ +vagrant ssh master
+vagrant ssh minion-1
+
+If you are running more than one node, you can access the others by:
+ +vagrant ssh minion-2
+vagrant ssh minion-3
+
+Each node in the cluster installs the docker daemon and the kubelet.
+ +The master node instantiates the Kubernetes master components as pods on the machine.
+ +To view the service status and/or logs on the kubernetes-master:
+ +[vagrant@kubernetes-master ~] $ vagrant ssh master
+[vagrant@kubernetes-master ~] $ sudo su
+
+[root@kubernetes-master ~] $ systemctl status kubelet
+[root@kubernetes-master ~] $ journalctl -ru kubelet
+
+[root@kubernetes-master ~] $ systemctl status docker
+[root@kubernetes-master ~] $ journalctl -ru docker
+
+[root@kubernetes-master ~] $ tail -f /var/log/kube-apiserver.log
+[root@kubernetes-master ~] $ tail -f /var/log/kube-controller-manager.log
+[root@kubernetes-master ~] $ tail -f /var/log/kube-scheduler.log
+
+To view the services on any of the nodes:
+ +[vagrant@kubernetes-master ~] $ vagrant ssh minion-1
+[vagrant@kubernetes-master ~] $ sudo su
+
+[root@kubernetes-master ~] $ systemctl status kubelet
+[root@kubernetes-master ~] $ journalctl -ru kubelet
+
+[root@kubernetes-master ~] $ systemctl status docker
+[root@kubernetes-master ~] $ journalctl -ru docker
+
+With your Kubernetes cluster up, you can manage the nodes in your cluster with the regular Vagrant commands.
+ +To push updates to new Kubernetes code after making source changes:
+ +./cluster/kube-push.sh
+
+To stop and then restart the cluster:
+ +vagrant halt
+./cluster/kube-up.sh
+
+To destroy the cluster:
+ +vagrant destroy
+
+Once your Vagrant machines are up and provisioned, the first thing to do is to check that you can use the kubectl.sh
script.
You may need to build the binaries first, you can do this with make
$ ./cluster/kubectl.sh get nodes
+
+NAME LABELS
+10.245.1.4 <none>
+10.245.1.5 <none>
+10.245.1.3 <none>
+
+When using the vagrant provider in Kubernetes, the cluster/kubectl.sh
script will cache your credentials in a ~/.kubernetes_vagrant_auth
file so you will not be prompted for them in the future.
cat ~/.kubernetes_vagrant_auth
+
+{ "User": "vagrant",
+ "Password": "vagrant",
+ "CAFile": "/home/k8s_user/.kubernetes.vagrant.ca.crt",
+ "CertFile": "/home/k8s_user/.kubecfg.vagrant.crt",
+ "KeyFile": "/home/k8s_user/.kubecfg.vagrant.key"
+}
+
+You should now be set to use the cluster/kubectl.sh
script. For example try to list the nodes that you have started with:
./cluster/kubectl.sh get nodes
+
+Your cluster is running, you can list the nodes in your cluster:
+ +$ ./cluster/kubectl.sh get nodes
+
+NAME LABELS
+10.245.2.4 <none>
+10.245.2.3 <none>
+10.245.2.2 <none>
+
+Now start running some containers!
+ +You can now use any of the cluster/kube-*.sh
commands to interact with your VM machines.
+Before starting a container there will be no pods, services and replication controllers.
$ ./cluster/kubectl.sh get pods
+NAME READY STATUS RESTARTS AGE
+
+$ ./cluster/kubectl.sh get services
+NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE
+
+$ ./cluster/kubectl.sh get replicationcontrollers
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+
+Start a container running nginx with a replication controller and three replicas
+ +$ ./cluster/kubectl.sh run my-nginx --image=nginx --replicas=3 --port=80
+
+When listing the pods, you will see that three containers have been started and are in Waiting state:
+ +$ ./cluster/kubectl.sh get pods
+NAME READY STATUS RESTARTS AGE
+my-nginx-5kq0g 0/1 Pending 0 10s
+my-nginx-gr3hh 0/1 Pending 0 10s
+my-nginx-xql4j 0/1 Pending 0 10s
+
+You need to wait for the provisioning to complete, you can monitor the nodes by doing:
+ +$ vagrant ssh minion-1 -c 'sudo docker images'
+kubernetes-minion-1:
+ REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE
+ <none> <none> 96864a7d2df3 26 hours ago 204.4 MB
+ google/cadvisor latest e0575e677c50 13 days ago 12.64 MB
+ kubernetes/pause latest 6c4579af347b 8 weeks ago 239.8 kB
+
+Once the docker image for nginx has been downloaded, the container will start and you can list it:
+ +$ vagrant ssh minion-1 -c 'sudo docker ps'
+kubernetes-minion-1:
+ CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
+ dbe79bf6e25b nginx:latest "nginx" 21 seconds ago Up 19 seconds k8s--mynginx.8c5b8a3a--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1.etcd--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1--fcfa837f
+ fa0e29c94501 kubernetes/pause:latest "/pause" 8 minutes ago Up 8 minutes 0.0.0.0:8080->80/tcp k8s--net.a90e7ce4--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1.etcd--7813c8bd_-_3ffe_-_11e4_-_9036_-_0800279696e1--baf5b21b
+ aa2ee3ed844a google/cadvisor:latest "/usr/bin/cadvisor" 38 minutes ago Up 38 minutes k8s--cadvisor.9e90d182--cadvisor_-_agent.file--4626b3a2
+ 65a3a926f357 kubernetes/pause:latest "/pause" 39 minutes ago Up 39 minutes 0.0.0.0:4194->8080/tcp k8s--net.c5ba7f0e--cadvisor_-_agent.file--342fd561
+
+Going back to listing the pods, services and replicationcontrollers, you now have:
+ +$ ./cluster/kubectl.sh get pods
+NAME READY STATUS RESTARTS AGE
+my-nginx-5kq0g 1/1 Running 0 1m
+my-nginx-gr3hh 1/1 Running 0 1m
+my-nginx-xql4j 1/1 Running 0 1m
+
+$ ./cluster/kubectl.sh get services
+NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE
+my-nginx 10.0.0.1 <none> 80/TCP run=my-nginx 1h
+
+We did not start any services, hence there are none listed. But we see three replicas displayed properly. +Check the guestbook application to learn how to create a service. +You can already play with scaling the replicas with:
+ +$ ./cluster/kubectl.sh scale rc my-nginx --replicas=2
+$ ./cluster/kubectl.sh get pods
+NAME READY STATUS RESTARTS AGE
+my-nginx-5kq0g 1/1 Running 0 2m
+my-nginx-gr3hh 1/1 Running 0 2m
+
+Congratulations!
+ +By default the Vagrantfile will download the box from S3. You can change this (and cache the box locally) by providing a name and an alternate URL when calling kube-up.sh
export KUBERNETES_BOX_NAME=choose_your_own_name_for_your_kuber_box
+export KUBERNETES_BOX_URL=path_of_your_kuber_box
+export KUBERNETES_PROVIDER=vagrant
+./cluster/kube-up.sh
+
+You probably have an incorrect ~/.kubernetes_vagrant_auth file for the cluster you are attempting to contact.
+ +rm ~/.kubernetes_vagrant_auth
+
+After using kubectl.sh make sure that the correct credentials are set:
+ +cat ~/.kubernetes_vagrant_auth
+
+{
+ "User": "vagrant",
+ "Password": "vagrant"
+}
+
+If this is your first time creating the cluster, the kubelet on each node schedules a number of docker pull requests to fetch prerequisite images. This can take some time and as a result may delay your initial pod getting provisioned.
+ +To set up a vagrant cluster for hacking, follow the vagrant developer guide.
+ +Log on to one of the nodes (vagrant ssh minion-1
) and inspect the salt minion log (sudo cat /var/log/salt/minion
).
You can control the number of nodes that are instantiated via the environment variable NUM_MINIONS
on your host machine. If you plan to work with replicas, we strongly encourage you to work with enough nodes to satisfy your largest intended replica size. If you do not plan to work with replicas, you can save some system resources by running with a single node. You do this, by setting NUM_MINIONS
to 1 like so:
export NUM_MINIONS=1
+
+You can control the memory allotted to virtual machines with the KUBERNETES_MEMORY
environment variable.
+Just set it to the number of megabytes you would like the machines to have. For example:
export KUBERNETES_MEMORY=2048
+
+If you need more granular control, you can set the amount of memory for the master and nodes independently. For example:
+ +export KUBERNETES_MASTER_MEMORY=1536
+export KUBERNETES_MINION_MEMORY=2048
+
+vagrant suspend
seems to mess up the network. This is not supported at this time.
You can ensure that vagrant uses nfs to sync folders with virtual machines by setting the KUBERNETES_VAGRANT_USE_NFS environment variable to ‘true’. nfs is faster than virtualbox or vmware’s ‘shared folders’ and does not require guest additions. See the vagrant docs for details on configuring nfs on the host. This setting will have no effect on the libvirt provider, which uses nfs by default. For example:
+ +export KUBERNETES_VAGRANT_USE_NFS=true
+
+Getting started with vSphere +——————————-
+ +The example below creates a Kubernetes cluster with 4 worker node Virtual +Machines and a master Virtual Machine (i.e. 5 VMs in your cluster). This +cluster is set up and controlled from your workstation (or wherever you find +convenient).
+ +Table of Contents
+ + + +GOPATH
set up and include $GOPATH/bin
in your PATH
.export GOPATH=$HOME/src/go
+ mkdir -p $GOPATH
+ export PATH=$PATH:$GOPATH/bin
+
+go get github.com/vmware/govmomi/govc
+
+Download a prebuilt Debian 7.7 VMDK that we’ll use as a base image:
+ +curl --remote-name-all https://storage.googleapis.com/govmomi/vmdk/2014-11-11/kube.vmdk.gz{,.md5}
+md5sum -c kube.vmdk.gz.md5
+gzip -d kube.vmdk.gz
+
+Import this VMDK into your vSphere datastore:
+ +export GOVC_URL='user:pass@hostname'
+export GOVC_INSECURE=1 # If the host above uses a self-signed cert
+export GOVC_DATASTORE='target datastore'
+export GOVC_RESOURCE_POOL='resource pool or cluster with access to datastore'
+
+govc import.vmdk kube.vmdk ./kube/
+
+Verify that the VMDK was correctly uploaded and expanded to ~3GiB:
+ +govc datastore.ls ./kube/
+
+Take a look at the file cluster/vsphere/config-common.sh
fill in the required
+parameters. The guest login for the image that you imported is kube:kube
.
Now, let’s continue with deploying Kubernetes. +This process takes about ~10 minutes.
+ +cd kubernetes # Extracted binary release OR repository root
+export KUBERNETES_PROVIDER=vsphere
+cluster/kube-up.sh
+
+Refer to the top level README and the getting started guide for Google Compute +Engine. Once you have successfully reached this point, your vSphere Kubernetes +deployment works just as any other one!
+ +Enjoy!
+ +The output of kube-up.sh
displays the IP addresses of the VMs it deploys. You
+can log into any VM as the kube
user to poke around and figure out what is
+going on (find yourself authorized with your SSH key, or use the password
+kube
otherwise).
The User’s guide is for anyone who wants to run programs and +services on an existing Kubernetes cluster.
+The Cluster Admin’s guide is for anyone setting up +a Kubernetes cluster or administering it.
+The Developer guide is for anyone wanting to write +programs that access the Kubernetes API, write plugins or extensions, or +modify the core code of Kubernetes.
+The Kubectl Command Line Interface is a detailed reference on
+the kubectl
CLI.
The API object documentation +is a detailed description of all fields found in core API objects.
+An overview of the Design of Kubernetes
+There are example files and walkthroughs in the examples +folder.
+If something went wrong, see the troubleshooting document for how to debug. +You should also check the known issues for the release you’re using.
+To report a security issue, see Reporting a Security Issue.
+If you believe you have discovered a vulnerability or a have a security incident to report, please follow the steps below. This applies to Kubernetes releases v1.0 or later.
+ +To watch for security and major API announcements, please join our kubernetes-announce group.
+ +To report an issue, please:
+- Submit a bug report here.
+ - Select “I want to report a technical security bug in a Google product (SQLi, XSS, etc.).”
+ - Select “Other” as the Application Type.
+- Under reproduction steps, please additionally include
+ - the words “Kubernetes Security issue”
+ - Description of the issue
+ - Kubernetes release (e.g. output of kubectl version
command, which includes server version.)
+ - Environment setup (e.g. which “Getting Started Guide” you followed, if any; what node operating system used; what service or software creates your virtual machines, if any)
An online submission will have the fastest response; however, if you prefer email, please send mail to security@google.com. If you feel the need, please use the PGP public key to encrypt communications.
+ + + + + + + + + +We are targetting late October for our 1.1 release of Kubernetes. We plan on cutting a first release candidate +in early October. We will enter feature freeze for the 1.1 release on September 21st. Note this does not mean +that the master branch is fully frozen, but all 1.1 features must be in by September 21st and large-scale +refactors of the codebase will be blocked until the 1.1 release is finalized to ensure easy cherry-picks.
+ +The 1.1 release of Kubernetes will be a purely additive releases, the v1
API will be maintained, with a set
+of newly added features.
The following features are considered blocking for the 1.1 release: + * Docker 1.8.x + * Graceful pod termination + * IPtables based kube-proxy (tbd if this is the default for all platforms) + * Improvements to kubectl usability and features + * Support for 250 node clusters + * Horizontal Pod autoscaling + * Support for experimental APIs and API groups. + * Job objects
+ +The following features will be part of 1.1 if complete, but will not block the release: + * Deployment API + * ScheduledJob API + * Daemon Controller + * ConfigData API + * HTTP(S) load balancer support + * Rolling update improvements + * Third party CRUD resources
+ +We’re in the process of prioritizing changes to be made after 1.1.
+ +Please watch the [Github milestones] (https://github.com/kubernetes/kubernetes/milestones) for our future plans.
+ + + + + + + + + +Sometimes things go wrong. This guide is aimed at making them right. It has two sections: + * Troubleshooting your application - Useful for users who are deploying code into Kubernetes and wondering why it is not working. + * Troubleshooting your cluster - Useful for cluster administrators and people whose Kubernetes cluster is unhappy.
+ +You should also check the known issues for the release you’re using.
+ +If your problem isn’t answered by any of the guides above, there are variety of ways for you to get help from the Kubernetes team.
+ +If you aren’t familiar with it, many of your questions may be answered by the user guide.
+ +We also have a number of FAQ pages: + * User FAQ + * Debugging FAQ + * Services FAQ
+ +You may also find the Stack Overflow topics relevant: + * Kubernetes + * Google Container Engine - GKE
+ +Someone else from the community may have already asked a similar question or may be able to help with your problem. The Kubernetes team will also monitor posts tagged kubernetes. If there aren’t any existing questions that help, please ask a new one!
+ +The Kubernetes team hangs out on Slack in the #kubernetes-users
channel. You can participate in the Kubernetes team here. Slack requires registration, but the Kubernetes team is open invitation to anyone to register here. Feel free to come and ask any and all questions.
The Google Container Engine mailing list is google-containers@googlegroups.com
+ +If you have what looks like a bug, or you would like to make a feature request, please use the Github issue tracking system.
+ +Before you file an issue, please search existing issues to see if your issue is already covered.
+ +If filing a bug, please include detailed information about how to reproduce the problem, such as:
+* Kubernetes version: kubectl version
+* Cloud provider, OS distro, network configuration, and Docker version
+* Steps to reproduce the problem
Table of Contents +
+ + + + + +The user guide is intended for anyone who wants to run programs and services on an existing Kubernetes cluster. Setup and administration of a Kubernetes cluster is described in the Cluster Admin Guide. The Developer Guide is for anyone wanting to either write code which directly accesses the Kubernetes API, or to contribute directly to the Kubernetes project.
+ +Please ensure you have completed the prerequisites for running examples from the user guide.
+ +If you don’t have much familiarity with Kubernetes, we recommend you read the following sections in order:
+ +Table of Contents +
+ +When accessing the Kubernetes API for the first time, we suggest using the
+Kubernetes CLI, kubectl
.
To access a cluster, you need to know the location of the cluster and have credentials +to access it. Typically, this is automatically set-up when you work through +though a Getting started guide, +or someone else setup the cluster and provided you with credentials and a location.
+ +Check the location and credentials that kubectl knows about with this command:
+ +$ kubectl config view
+
+Many of the examples provide an introduction to using +kubectl and complete documentation is found in the kubectl manual.
+ +Kubectl handles locating and authenticating to the apiserver. +If you want to directly access the REST API with an http client like +curl or wget, or a browser, there are several ways to locate and authenticate: + - Run kubectl in proxy mode. + - Recommended approach. + - Uses stored apiserver location. + - Verifies identity of apiserver using self-signed cert. No MITM possible. + - Authenticates to apiserver. + - In future, may do intelligent client-side load-balancing and failover. + - Provide the location and credentials directly to the http client. + - Alternate approach. + - Works with some types of client code that are confused by using a proxy. + - Need to import a root cert into your browser to protect against MITM.
+ +The following command runs kubectl in a mode where it acts as a reverse proxy. It handles +locating the apiserver and authenticating. +Run it like this:
+ +$ kubectl proxy --port=8080 &
+
+See kubectl proxy for more details.
+ +Then you can explore the API with curl, wget, or a browser, like so:
+ +$ curl http://localhost:8080/api/
+{
+ "versions": [
+ "v1"
+ ]
+}
+
+It is also possible to avoid using kubectl proxy by passing an authentication token +directly to the apiserver, like this:
+ +$ APISERVER=$(kubectl config view | grep server | cut -f 2- -d ":" | tr -d " ")
+$ TOKEN=$(kubectl config view | grep token | cut -f 2 -d ":" | tr -d " ")
+$ curl $APISERVER/api --header "Authorization: Bearer $TOKEN" --insecure
+{
+ "versions": [
+ "v1"
+ ]
+}
+
+The above example uses the --insecure
flag. This leaves it subject to MITM
+attacks. When kubectl accesses the cluster it uses a stored root certificate
+and client certificates to access the server. (These are installed in the
+~/.kube
directory). Since cluster certificates are typically self-signed, it
+make take special configuration to get your http client to use root
+certificate.
On some clusters, the apiserver does not require authentication; it may serve +on localhost, or be protected by a firewall. There is not a standard +for this. Configuring Access to the API +describes how a cluster admin can configure this. Such approaches may conflict +with future high-availability support.
+ +There are client libraries for accessing the API +from several languages. The Kubernetes project-supported +Go +client library can use the same kubeconfig file +as the kubectl CLI does to locate and authenticate to the apiserver.
+ +See documentation for other libraries for how they authenticate.
+ +When accessing the API from a pod, locating and authenticating +to the api server are somewhat different.
+ +The recommended way to locate the apiserver within the pod is with
+the kubernetes
DNS name, which resolves to a Service IP which in turn
+will be routed to an apiserver.
The recommended way to authenticate to the apiserver is with a
+service account credential. By kube-system, a pod
+is associated with a service account, and a credential (token) for that
+service account is placed into the filesystem tree of each container in that pod,
+at /var/run/secrets/kubernetes.io/serviceaccount/token
.
From within a pod the recommended ways to connect to API are:
+ - run a kubectl proxy as one of the containers in the pod, or as a background
+ process within a container. This proxies the
+ Kubernetes API to the localhost interface of the pod, so that other processes
+ in any container of the pod can access it. See this example of using kubectl proxy
+ in a pod.
+ - use the Go client library, and create a client using the client.NewInCluster()
factory.
+ This handles locating and authenticating to the apiserver.
+In each case, the credentials of the pod are used to communicate securely with the apiserver.
The previous section was about connecting the Kubernetes API server. This section is about +connecting to other services running on Kubernetes cluster. In Kubernetes, the +nodes, pods and services all have +their own IPs. In many cases, the node IPs, pod IPs, and some service IPs on a cluster will not be +routable, so they will not be reachable from a machine outside the cluster, +such as your desktop machine.
+ +You have several options for connecting to nodes, pods and services from outside the cluster:
+ - Access services through public IPs.
+ - Use a service with type NodePort
or LoadBalancer
to make the service reachable outside
+ the cluster. See the services and
+ kubectl expose documentation.
+ - Depending on your cluster environment, this may just expose the service to your corporate network,
+ or it may expose it to the internet. Think about whether the service being exposed is secure.
+ Does it do its own authentication?
+ - Place pods behind services. To access one specific pod from a set of replicas, such as for debugging,
+ place a unique label on the pod it and create a new service which selects this label.
+ - In most cases, it should not be necessary for application developer to directly access
+ nodes via their nodeIPs.
+ - Access services, nodes, or pods using the Proxy Verb.
+ - Does apiserver authentication and authorization prior to accessing the remote service.
+ Use this if the services are not secure enough to expose to the internet, or to gain
+ access to ports on the node IP, or for debugging.
+ - Proxies may cause problems for some web applications.
+ - Only works for HTTP/HTTPS.
+ - Described here.
+ - Access from a node or pod in the cluster.
+ - Run a pod, and then connect to a shell in it using kubectl exec.
+ Connect to other nodes, pods, and services from that shell.
+ - Some clusters may allow you to ssh to a node in the cluster. From there you may be able to
+ access cluster services. This is a non-standard method, and will work on some clusters but
+ not others. Browsers and other tools may or may not be installed. Cluster DNS may not work.
Typically, there are several services which are started on a cluster by kube-system. Get a list of these
+with the kubectl cluster-info
command:
$ kubectl cluster-info
+
+ Kubernetes master is running at https://104.197.5.247
+ elasticsearch-logging is running at https://104.197.5.247/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging
+ kibana-logging is running at https://104.197.5.247/api/v1/proxy/namespaces/kube-system/services/kibana-logging
+ kube-dns is running at https://104.197.5.247/api/v1/proxy/namespaces/kube-system/services/kube-dns
+ grafana is running at https://104.197.5.247/api/v1/proxy/namespaces/kube-system/services/monitoring-grafana
+ heapster is running at https://104.197.5.247/api/v1/proxy/namespaces/kube-system/services/monitoring-heapster
+
+This shows the proxy-verb URL for accessing each service.
+For example, this cluster has cluster-level logging enabled (using Elasticsearch), which can be reached
+at https://104.197.5.247/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging/
if suitable credentials are passed, or through a kubectl proxy at, for example:
+http://localhost:8080/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging/
.
+(See above for how to pass credentials or use kubectl proxy.)
As mentioned above, you use the kubectl cluster-info
command to retrieve the service’s proxy URL. To create proxy URLs that include service endpoints, suffixes, and parameters, you simply append to the service’s proxy URL:
+http://
kubernetes_master_address
/
service_path
/
service_name
/
service_endpoint-suffix-parameter
+
_search?q=user:kimchy
, you would use: http://104.197.5.247/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging/_search?q=user:kimchy
_cluster/health?pretty=true
, you would use: https://104.197.5.247/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging/_cluster/health?pretty=true
{
+ "cluster_name" : "kubernetes_logging",
+ "status" : "yellow",
+ "timed_out" : false,
+ "number_of_nodes" : 1,
+ "number_of_data_nodes" : 1,
+ "active_primary_shards" : 5,
+ "active_shards" : 5,
+ "relocating_shards" : 0,
+ "initializing_shards" : 0,
+ "unassigned_shards" : 5
+ }
+
+You may be able to put an apiserver proxy url into the address bar of a browser. However: + - Web browsers cannot usually pass tokens, so you may need to use basic (password) auth. Apiserver can be configured to accept basic auth, + but your cluster may not be configured to accept basic auth. + - Some web apps may not work, particularly those with client side javascript that construct urls in a + way that is unaware of the proxy path prefix.
+ +The redirect capabilities have been deprecated and removed. Please use a proxy (see below) instead.
+ +There are several different proxies you may encounter when using Kubernetes:
+ 1. The kubectl proxy:
+ - runs on a user’s desktop or in a pod
+ - proxies from a localhost address to the Kubernetes apiserver
+ - client to proxy uses HTTP
+ - proxy to apiserver uses HTTPS
+ - locates apiserver
+ - adds authentication headers
+ 1. The apiserver proxy:
+ - is a bastion built into the apiserver
+ - connects a user outside of the cluster to cluster IPs which otherwise might not be reachable
+ - runs in the apiserver processes
+ - client to proxy uses HTTPS (or http if apiserver so configured)
+ - proxy to target may use HTTP or HTTPS as chosen by proxy using available information
+ - can be used to reach a Node, Pod, or Service
+ - does load balancing when used to reach a Service
+ 1. The kube proxy:
+ - runs on each node
+ - proxies UDP and TCP
+ - does not understand HTTP
+ - provides load balancing
+ - is just used to reach services
+ 1. A Proxy/Load-balancer in front of apiserver(s):
+ - existence and implementation varies from cluster to cluster (e.g. nginx)
+ - sits between all clients and one or more apiservers
+ - acts as load balancer if there are several apiservers.
+ 1. Cloud Load Balancers on external services:
+ - are provided by some cloud providers (e.g. AWS ELB, Google Cloud Load Balancer)
+ - are created automatically when the Kubernetes service has type LoadBalancer
+ - use UDP/TCP only
+ - implementation varies by cloud provider.
Kubernetes users will typically not need to worry about anything other than the first two types. The cluster admin +will typically ensure that the latter types are setup correctly.
+ + + + + + + + + +We have labels for identifying metadata.
+ +It is also useful to be able to attach arbitrary non-identifying metadata, for retrieval by API clients such as tools, libraries, etc. This information may be large, may be structured or unstructured, may include characters not permitted by labels, etc. Such information would not be used for object selection and therefore doesn’t belong in labels.
+ +Like labels, annotations are key-value maps.
+ +"annotations": {
+ "key1" : "value1",
+ "key2" : "value2"
+}
+
+Possible information that could be recorded in annotations:
+ +Yes, this information could be stored in an external database or directory, but that would make it much harder to produce shared client libraries and tools for deployment, management, introspection, etc.
+ + + + + + + + + +This guide is to help users debug applications that are deployed into Kubernetes and not behaving correctly. +This is not a guide for people who want to debug their cluster. For that you should check out +this guide
+ +Table of Contents +
+ +Users are highly encouraged to check out our FAQ
+ +The first step in troubleshooting is triage. What is the problem? Is it your Pods, your Replication Controller or +your Service? + * Debugging Pods + * Debugging Replication Controllers + * Debugging Services
+ +The first step in debugging a Pod is taking a look at it. Check the current state of the Pod and recent events with the following command:
+ +$ kubectl describe pods ${POD_NAME}
+
+Look at the state of the containers in the pod. Are they all Running
? Have there been recent restarts?
Continue debugging depending on the state of the pods.
+ +If a Pod is stuck in Pending
it means that it can not be scheduled onto a node. Generally this is because
+there are insufficient resources of one type or another that prevent scheduling. Look at the output of the
+kubectl describe ...
command above. There should be messages from the scheduler about why it can not schedule
+your pod. Reasons include:
You don’t have enough resources: You may have exhausted the supply of CPU or Memory in your cluster, in this case +you need to delete Pods, adjust resource requests, or add new nodes to your cluster. See Compute Resources document for more information.
+You are using hostPort
: When you bind a Pod to a hostPort
there are a limited number of places that pod can be
+scheduled. In most cases, hostPort
is unnecessary, try using a Service object to expose your Pod. If you do require
+hostPort
then you can only schedule as many Pods as there are nodes in your Kubernetes cluster.
If a Pod is stuck in the Waiting
state, then it has been scheduled to a worker node, but it can’t run on that machine.
+Again, the information from kubectl describe ...
should be informative. The most common cause of Waiting
pods is a failure to pull the image. There are three things to check:
+* Make sure that you have the name of the image correct
+* Have you pushed the image to the repository?
+* Run a manual docker pull <image>
on your machine to see if the image can be pulled.
First, take a look at the logs of +the current container:
+ +$ kubectl logs ${POD_NAME} ${CONTAINER_NAME}
+
+If your container has previously crashed, you can access the previous container’s crash log with:
+ +$ kubectl logs --previous ${POD_NAME} ${CONTAINER_NAME}
+
+Alternately, you can run commands inside that container with exec
:
$ kubectl exec ${POD_NAME} -c ${CONTAINER_NAME} -- ${CMD} ${ARG1} ${ARG2} ... ${ARGN}
+
+Note that -c ${CONTAINER_NAME}
is optional and can be omitted for Pods that only contain a single container.
As an example, to look at the logs from a running Cassandra pod, you might run
+ +$ kubectl exec cassandra -- cat /var/log/cassandra/system.log
+
+If none of these approaches work, you can find the host machine that the pod is running on and SSH into that host, +but this should generally not be necessary given tools in the Kubernetes API. Therefore, if you find yourself needing to ssh into a machine, please file a +feature request on GitHub describing your use case and why these tools are insufficient.
+ +If your pod is not behaving as you expected, it may be that there was an error in your
+pod description (e.g. mypod.yaml
file on your local machine), and that the error
+was silently ignored when you created the pod. Often a section of the pod description
+is nested incorrectly, or a key name is typed incorrectly, and so the key is ignored.
+For example, if you misspelled command
as commnd
then the pod will be created but
+will not use the command line you intended it to use.
The first thing to do is to delete your pod and try creating it again with the --validate
option.
+For example, run kubectl create --validate -f mypod.yaml
.
+If you misspelled command
as commnd
then will give an error like this:
+I0805 10:43:25.129850 46757 schema.go:126] unknown field: commnd
+I0805 10:43:25.129973 46757 schema.go:129] this may be a false alarm, see https://github.com/kubernetes/kubernetes/issues/6842
+pods/mypod
+
+
+
+
+
+The next thing to check is whether the pod on the apiserver
+matches the pod you meant to create (e.g. in a yaml file on your local machine).
+For example, run kubectl get pods/mypod -o yaml > mypod-on-apiserver.yaml
and then
+manually compare the original pod description, mypod.yaml
with the one you got
+back from apiserver, mypod-on-apiserver.yaml
. There will typically be some
+lines on the “apiserver” version that are not on the original version. This is
+expected. However, if there are lines on the original that are not on the apiserver
+version, then this may indicate a problem with your pod spec.
Replication controllers are fairly straightforward. They can either create Pods or they can’t. If they can’t +create pods, then please refer to the instructions above to debug your pods.
+ +You can also use kubectl describe rc ${CONTROLLER_NAME}
to introspect events related to the replication
+controller.
Services provide load balancing across a set of pods. There are several common problems that can make Services +not work properly. The following instructions should help debug Service problems.
+ +First, verify that there are endpoints for the service. For every Service object, the apiserver makes an endpoints
resource available.
You can view this resource with:
+ +$ kubectl get endpoints ${SERVICE_NAME}
+
+Make sure that the endpoints match up with the number of containers that you expect to be a member of your service. +For example, if your Service is for an nginx container with 3 replicas, you would expect to see three different +IP addresses in the Service’s endpoints.
+ +If you are missing endpoints, try listing pods using the labels that Service uses. Imagine that you have +a Service where the labels are:
+ +...
+spec:
+ - selector:
+ name: nginx
+ type: frontend
+
+You can use:
+ +$ kubectl get pods --selector=name=nginx,type=frontend
+
+to list pods that match this selector. Verify that the list matches the Pods that you expect to provide your Service.
+ +If the list of pods matches expectations, but your endpoints are still empty, it’s possible that you don’t
+have the right ports exposed. If your service has a containerPort
specified, but the Pods that are
+selected don’t have that port listed, then they won’t be added to the endpoints list.
Verify that the pod’s containerPort
matches up with the Service’s containerPort
If you can connect to the service, but the connection is immediately dropped, and there are endpoints +in the endpoints list, it’s likely that the proxy can’t contact your pods.
+ +There are three things to
+check:
+ * Are your pods working correctly? Look for restart count, and debug pods
+ * Can you connect to your pods directly? Get the IP address for the Pod, and try to connect directly to that IP
+ * Is your application serving on the port that you configured? Kubernetes doesn’t do port remapping, so if your application serves on 8080, the containerPort
field needs to be 8080.
If none of the above solves your problem, follow the instructions in Debugging Service document to make sure that your Service
is running, has Endpoints
, and your Pods
are actually serving; you have DNS working, iptables rules installed, and kube-proxy does not seem to be misbehaving.
You may also visit troubleshooting document for more information.
+ + + + + + + + + +Table of Contents +
+ +When specifying a pod, you can optionally specify how much CPU and memory (RAM) each +container needs. When containers have their resource requests specified, the scheduler is +able to make better decisions about which nodes to place pods on; and when containers have their +limits specified, contention for resources on a node can be handled in a specified manner. For +more details about the difference between requests and limits, please refer to +Resource QoS.
+ +CPU and memory are each a resource type. A resource type has a base unit. CPU is specified +in units of cores. Memory is specified in units of bytes.
+ +CPU and RAM are collectively referred to as compute resources, or just resources. Compute +resources are measureable quantities which can be requested, allocated, and consumed. They are +distinct from API resources. API resources, such as pods and +services are objects that can be written to and retrieved from the Kubernetes API +server.
+ +Each container of a Pod can optionally specify spec.container[].resources.limits.cpu
and/or
+spec.container[].resources.limits.memory
and/or spec.container[].resources.requests.cpu
+and/or spec.container[].resources.requests.memory
.
Specifying resource requests and/or limits is optional. In some clusters, unset limits or requests +may be replaced with default values when a pod is created or updated. The default value depends on +how the cluster is configured. If value of requests is not specified, they are set to be equal +to limits by default. Please note that resource limits must be greater than or equal to resource +requests.
+ +Although requests/limits can only be specified on individual containers, it is convenient to talk +about pod resource requests/limits. A pod resource request/limit for a particular resource +type is the sum of the resource requests/limits of that type for each container in the pod, with +unset values treated as zero (or equal to default values in some cluster configurations).
+ +The following pod has two containers. Each has a request of 0.25 core of cpu and 64MiB +(220 bytes) of memory and a limit of 0.5 core of cpu and 128MiB of memory. The pod can +be said to have a request of 0.5 core and 128 MiB of memory and a limit of 1 core and 256MiB of +memory.
+ +apiVersion: v1
+kind: Pod
+metadata:
+ name: frontend
+spec:
+ containers:
+ - name: db
+ image: mysql
+ resources:
+ requests:
+ memory: "64Mi"
+ cpu: "250m"
+ limits:
+ memory: "128Mi"
+ cpu: "500m"
+ - name: wp
+ image: wordpress
+ resources:
+ requests:
+ memory: "64Mi"
+ cpu: "250m"
+ limits:
+ memory: "128Mi"
+ cpu: "500m"
+
+When a pod is created, the Kubernetes scheduler selects a node for the pod to +run on. Each node has a maximum capacity for each of the resource types: the +amount of CPU and memory it can provide for pods. The scheduler ensures that, +for each resource type (CPU and memory), the sum of the resource requests of the +containers scheduled to the node is less than the capacity of the node. Note +that although actual memory or CPU resource usage on nodes is very low, the +scheduler will still refuse to place pods onto nodes if the capacity check +fails. This protects against a resource shortage on a node when resource usage +later increases, such as due to a daily peak in request rate.
+ +When kubelet starts a container of a pod, it passes the CPU and memory limits to the container +runner (Docker or rkt).
+ +When using Docker:
+- The spec.container[].resources.limits.cpu
is multiplied by 1024, converted to an integer, and
+ used as the value of the --cpu-shares
flag to the docker run
+ command.
+- The spec.container[].resources.limits.memory
is converted to an integer, and used as the value
+ of the --memory
flag
+ to the docker run
command.
TODO: document behavior for rkt
+ +If a container exceeds its memory limit, it may be terminated. If it is restartable, it will be +restarted by kubelet, as will any other type of runtime failure.
+ +A container may or may not be allowed to exceed its CPU limit for extended periods of time. +However, it will not be killed for excessive CPU usage.
+ +To determine if a container cannot be scheduled or is being killed due to resource limits, see the +“Troubleshooting” section below.
+ +The resource usage of a pod is reported as part of the Pod status.
+ +If optional monitoring is configured for your cluster, +then pod resource usage can be retrieved from the monitoring system.
+ +If the scheduler cannot find any node where a pod can fit, then the pod will remain unscheduled +until a place can be found. An event will be produced each time the scheduler fails to find a +place for the pod, like this:
+ +$ kubectl describe pod frontend | grep -A 3 Events
+Events:
+ FirstSeen LastSeen Count From Subobject PathReason Message
+ 36s 5s 6 {scheduler } FailedScheduling Failed for reason PodExceedsFreeCPU and possibly others
+
+In the case shown above, the pod “frontend” fails to be scheduled due to insufficient
+CPU resource on the node. Similar error messages can also suggest failure due to insufficient
+memory (PodExceedsFreeMemory). In general, if a pod or pods are pending with this message and
+alike, then there are several things to try:
+- Add more nodes to the cluster.
+- Terminate unneeded pods to make room for pending pods.
+- Check that the pod is not larger than all the nodes. For example, if all the nodes
+have a capacity of cpu: 1
, then a pod with a limit of cpu: 1.1
will never be scheduled.
You can check node capacities and amounts allocated with the kubectl describe nodes
command.
+For example:
$ kubectl describe nodes gke-cluster-4-386701dd-node-ww4p
+Name: gke-cluster-4-386701dd-node-ww4p
+[ ... lines removed for clarity ...]
+Capacity:
+ cpu: 1
+ memory: 464Mi
+ pods: 40
+Allocated resources (total requests):
+ cpu: 910m
+ memory: 2370Mi
+ pods: 4
+[ ... lines removed for clarity ...]
+Pods: (4 in total)
+ Namespace Name CPU(milliCPU) Memory(bytes)
+ frontend webserver-ffj8j 500 (50% of total) 2097152000 (50% of total)
+ kube-system fluentd-cloud-logging-gke-cluster-4-386701dd-node-ww4p 100 (10% of total) 209715200 (5% of total)
+ kube-system kube-dns-v8-qopgw 310 (31% of total) 178257920 (4% of total)
+TotalResourceLimits:
+ CPU(milliCPU): 910 (91% of total)
+ Memory(bytes): 2485125120 (59% of total)
+[ ... lines removed for clarity ...]
+
+Here you can see from the Allocated resources
section that that a pod which ask for more than
+90 millicpus or more than 1341MiB of memory will not be able to fit on this node.
Looking at the Pods
section, you can see which pods are taking up space on the node.
The resource quota feature can be configured +to limit the total amount of resources that can be consumed. If used in conjunction +with namespaces, it can prevent one team from hogging all the resources.
+ +Your container may be terminated because it’s resource-starved. To check if a container is being killed because it is hitting a resource limit, call kubectl describe pod
+on the pod you are interested in:
[12:54:41] $ ./cluster/kubectl.sh describe pod simmemleak-hra99
+Name: simmemleak-hra99
+Namespace: default
+Image(s): saadali/simmemleak
+Node: kubernetes-minion-tf0f/10.240.216.66
+Labels: name=simmemleak
+Status: Running
+Reason:
+Message:
+IP: 10.244.2.75
+Replication Controllers: simmemleak (1/1 replicas created)
+Containers:
+ simmemleak:
+ Image: saadali/simmemleak
+ Limits:
+ cpu: 100m
+ memory: 50Mi
+ State: Running
+ Started: Tue, 07 Jul 2015 12:54:41 -0700
+ Last Termination State: Terminated
+ Exit Code: 1
+ Started: Fri, 07 Jul 2015 12:54:30 -0700
+ Finished: Fri, 07 Jul 2015 12:54:33 -0700
+ Ready: False
+ Restart Count: 5
+Conditions:
+ Type Status
+ Ready False
+Events:
+ FirstSeen LastSeen Count From SubobjectPath Reason Message
+ Tue, 07 Jul 2015 12:53:51 -0700 Tue, 07 Jul 2015 12:53:51 -0700 1 {scheduler } scheduled Successfully assigned simmemleak-hra99 to kubernetes-minion-tf0f
+ Tue, 07 Jul 2015 12:53:51 -0700 Tue, 07 Jul 2015 12:53:51 -0700 1 {kubelet kubernetes-minion-tf0f} implicitly required container POD pulled Pod container image "gcr.io/google_containers/pause:0.8.0" already present on machine
+ Tue, 07 Jul 2015 12:53:51 -0700 Tue, 07 Jul 2015 12:53:51 -0700 1 {kubelet kubernetes-minion-tf0f} implicitly required container POD created Created with docker id 6a41280f516d
+ Tue, 07 Jul 2015 12:53:51 -0700 Tue, 07 Jul 2015 12:53:51 -0700 1 {kubelet kubernetes-minion-tf0f} implicitly required container POD started Started with docker id 6a41280f516d
+ Tue, 07 Jul 2015 12:53:51 -0700 Tue, 07 Jul 2015 12:53:51 -0700 1 {kubelet kubernetes-minion-tf0f} spec.containers{simmemleak} created Created with docker id 87348f12526a
+
+The Restart Count: 5
indicates that the simmemleak
container in this pod was terminated and restarted 5 times.
You can call get pod
with the -o go-template=...
option to fetch the status of previously terminated containers:
[13:59:01] $ ./cluster/kubectl.sh get pod -o go-template='{{range.status.containerStatuses}}{{"Container Name: "}}{{.name}}{{"\r\nLastState: "}}{{.lastState}}{{end}}' simmemleak-60xbc
+Container Name: simmemleak
+LastState: map[terminated:map[exitCode:137 reason:OOM Killed startedAt:2015-07-07T20:58:43Z finishedAt:2015-07-07T20:58:43Z containerID:docker://0e4095bba1feccdfe7ef9fb6ebffe972b4b14285d5acdec6f0d3ae8a22fad8b2]][13:59:03] clusterScaleDoc ~/go/src/github.com/kubernetes/kubernetes $
+
+We can see that this container was terminated because reason:OOM Killed
, where OOM stands for Out Of Memory.
The current system only allows resource quantities to be specified on a container. +It is planned to improve accounting for resources which are shared by all containers in a pod, +such as EmptyDir volumes.
+ +The current system only supports container requests and limits for CPU and Memory. +It is planned to add new resource types, including a node disk space +resource, and a framework for adding custom resource types.
+ +Kubernetes supports overcommitment of resources by supporting multiple levels of Quality of Service.
+ +Currently, one unit of CPU means different things on different cloud providers, and on different +machine types within the same cloud providers. For example, on AWS, the capacity of a node +is reported in ECUs, while in GCE it is reported in logical +cores. We plan to revise the definition of the cpu resource to allow for more consistency +across providers and platforms.
+ + + + + + + + + +This document is meant to highlight and consolidate in one place configuration best practices that are introduced throughout the user-guide and getting-started documentation and examples. This is a living document so if you think of something that is not on this list but might be useful to others, please don’t hesitate to file an issue or submit a PR.
+ +kubectl create -f <directory>
where possible. This looks for config objects in all .yaml
, .yml
, and .json
files in <directory>
and passes them to create.hostPort
unless absolutely necessary (e.g., for a node daemon) as it will prevent certain scheduling configurations due to port conflicts. Use the apiserver proxying or port forwarding for debug/admin access, or a service for external service access. If you need to expose a pod’s port on the host machine, consider using a NodePort service before resorting to hostPort
. If you only need access to the port for debugging purposes, you can also use the kubectl proxy and apiserver proxy or kubectl port-forward.hostNetwork
for the same reasons as hostPort
.service: myservice
) and another to represent the replication controller managing the pods (e.g., controller: mycontroller
), attach labels that identify semantic attributes of your application or deployment and select the appropriate subsets in your service and replication controller, such as { app: myapp, tier: frontend, deployment: v3 }
. A service can be made to span multiple deployments, such as across rolling updates, by simply omitting release-specific labels from its selector, rather than updating a service’s selector to match the replication controller’s selector fully.restartPolicy: Never
scenarios). A minimal Job is coming. See #1624. Naked pods will not be rescheduled in the event of node failure.Table of Contents +
+ + + + + +In addition to the imperative-style commands, such as kubectl run
and kubectl expose
, described elsewhere, Kubernetes supports declarative configuration. Often times, configuration files are preferable to imperative commands, since they can be checked into version control and changes to the files can be code reviewed, which is especially important for more complex configurations, producing a more robust, reliable and archival system.
In the declarative style, all configuration is stored in YAML or JSON configuration files using Kubernetes’s API resource schemas as the configuration schemas. kubectl
can create, update, delete, and get API resources. The apiVersion
(currently “v1”), resource kind
, and resource name
are used by kubectl
to construct the appropriate API path to invoke for the specified operation.
Kubernetes executes containers in Pods. A pod containing a simple Hello World container can be specified in YAML as follows:
+ +apiVersion: v1
+kind: Pod
+metadata:
+ name: hello-world
+spec: # specification of the pod’s contents
+ restartPolicy: Never
+ containers:
+ - name: hello
+ image: "ubuntu:14.04"
+ command: ["/bin/echo","hello”,”world"]
+
+The value of metadata.name
, hello-world
, will be the name of the pod resource created, and must be unique within the cluster, whereas containers[0].name
is just a nickname for the container within that pod. image
is the name of the Docker image, which Kubernetes expects to be able to pull from a registry, the Docker Hub by default.
restartPolicy: Never
indicates that we just want to run the container once and then terminate the pod.
The command
overrides the Docker container’s Entrypoint
. Command arguments (corresponding to Docker’s Cmd
) may be specified using args
, as follows:
command: ["/bin/echo"]
+ args: ["hello","world"]
+
+This pod can be created using the create
command:
$ kubectl create -f ./hello-world.yaml
+pods/hello-world
+
+kubectl
prints the resource type and name of the resource created when successful.
If you’re not sure you specified the resource correctly, you can ask kubectl
to validate it for you:
$ kubectl create -f ./hello-world.yaml --validate
+
+Let’s say you specified entrypoint
instead of command
. You’d see output as follows:
I0709 06:33:05.600829 14160 schema.go:126] unknown field: entrypoint
+I0709 06:33:05.600988 14160 schema.go:129] this may be a false alarm, see http://issue.k8s.io/6842
+pods/hello-world
+
+kubectl create --validate
currently warns about problems it detects, but creates the resource anyway, unless a required field is absent or a field value is invalid. Unknown API fields are ignored, so be careful. This pod was created, but with no command
, which is an optional field, since the image may specify an Entrypoint
.
+View the Pod API
+object
+to see the list of valid fields.
Kubernetes does not automatically run commands in a shell (not all images contain shells). If you would like to run your command in a shell, such as to expand environment variables (specified using env
), you could do the following:
apiVersion: v1
+kind: Pod
+metadata:
+ name: hello-world
+spec: # specification of the pod’s contents
+ restartPolicy: Never
+ containers:
+ - name: hello
+ image: "ubuntu:14.04"
+ env:
+ - name: MESSAGE
+ value: "hello world"
+ command: ["/bin/sh","-c"]
+ args: ["/bin/echo \"${MESSAGE}\""]
+
+However, a shell isn’t necessary just to expand environment variables. Kubernetes will do it for you if you use $(ENVVAR)
syntax:
command: ["/bin/echo"]
+ args: ["$(MESSAGE)"]
+
+You can see the pod you created (actually all of your cluster’s pods) using the get
command.
If you’re quick, it will look as follows:
+ +$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+hello-world 0/1 Pending 0 0s
+
+Initially, a newly created pod is unscheduled – no node has been selected to run it. Scheduling happens after creation, but is fast, so you normally shouldn’t see pods in an unscheduled state unless there’s a problem.
+ +After the pod has been scheduled, the image may need to be pulled to the node on which it was scheduled, if it hadn’t been pulled already. After a few seconds, you should see the container running:
+ +$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+hello-world 1/1 Running 0 5s
+
+The READY
column shows how many containers in the pod are running.
Almost immediately after it starts running, this command will terminate. kubectl
shows that the container is no longer running and displays the exit status:
$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+hello-world 0/1 ExitCode:0 0 15s
+
+You probably want to see the output of the command you ran. As with docker logs
, kubectl logs
will show you the output:
$ kubectl logs hello-world
+hello world
+
+When you’re done looking at the output, you should delete the pod:
+ +$ kubectl delete pod hello-world
+pods/hello-world
+
+As with create
, kubectl
prints the resource type and name of the resource deleted when successful.
You can also use the resource/name format to specify the pod:
+ +$ kubectl delete pods/hello-world
+pods/hello-world
+
+Terminated pods aren’t currently automatically deleted, so that you can observe their final status, so be sure to clean up your dead pods.
+ +On the other hand, containers and their logs are eventually deleted automatically in order to free up disk space on the nodes.
+ +Table of Contents +
+ +Now that you have a continuously running, replicated application you can expose it on a network. Before discussing the Kubernetes approach to networking, it is worthwhile to contrast it with the “normal” way networking works with Docker.
+ +By default, Docker uses host-private networking, so containers can talk to other containers only if they are on the same machine. In order for Docker containers to communicate across nodes, they must be allocated ports on the machine’s own IP address, which are then forwarded or proxied to the containers. This obviously means that containers must either coordinate which ports they use very carefully or else be allocated ports dynamically.
+ +Coordinating ports across multiple developers is very difficult to do at scale and exposes users to cluster-level issues outside of their control. Kubernetes assumes that pods can communicate with other pods, regardless of which host they land on. We give every pod its own cluster-private-IP address so you do not need to explicitly create links between pods or mapping container ports to host ports. This means that containers within a Pod can all reach each other’s ports on localhost, and all pods in a cluster can see each other without NAT. The rest of this document will elaborate on how you can run reliable services on such a networking model.
+ +This guide uses a simple nginx server to demonstrate proof of concept. The same principles are embodied in a more complete Jenkins CI application.
+ +We did this in a previous example, but lets do it once again and focus on the networking perspective. Create an nginx pod, and note that it has a container port specification:
+ +$ cat nginxrc.yaml
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: my-nginx
+spec:
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+
+This makes it accessible from any node in your cluster. Check the nodes the pod is running on:
+ +$ kubectl create -f ./nginxrc.yaml
+$ kubectl get pods -l app=nginx -o wide
+my-nginx-6isf4 1/1 Running 0 2h e2e-test-beeps-minion-93ly
+my-nginx-t26zt 1/1 Running 0 2h e2e-test-beeps-minion-93ly
+
+Check your pods’ IPs:
+ +$ kubectl get pods -l app=nginx -o json | grep podIP
+ "podIP": "10.245.0.15",
+ "podIP": "10.245.0.14",
+
+You should be able to ssh into any node in your cluster and curl both IPs. Note that the containers are not using port 80 on the node, nor are there any special NAT rules to route traffic to the pod. This means you can run multiple nginx pods on the same node all using the same containerPort and access them from any other pod or node in your cluster using IP. Like Docker, ports can still be published to the host node’s interface(s), but the need for this is radically diminished because of the networking model.
+ +You can read more about how we achieve this if you’re curious.
+ +So we have pods running nginx in a flat, cluster wide, address space. In theory, you could talk to these pods directly, but what happens when a node dies? The pods die with it, and the replication controller will create new ones, with different IPs. This is the problem a Service solves.
+ +A Kubernetes Service is an abstraction which defines a logical set of Pods running somewhere in your cluster, that all provide the same functionality. When created, each Service is assigned a unique IP address (also called clusterIP). This address is tied to the lifespan of the Service, and will not change while the Service is alive. Pods can be configured to talk to the Service, and know that communication to the Service will be automatically load-balanced out to some pod that is a member of the Service.
+ +You can create a Service for your 2 nginx replicas with the following yaml:
+ +$ cat nginxsvc.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginxsvc
+ labels:
+ app: nginx
+spec:
+ ports:
+ - port: 80
+ protocol: TCP
+ selector:
+ app: nginx
+
+This specification will create a Service which targets TCP port 80 on any Pod with the app=nginx
label, and expose it on an abstracted Service port (targetPort
: is the port the container accepts traffic on, port
: is the abstracted Service port, which can be any port other pods use to access the Service). View service API object to see the list of supported fields in service definition.
+Check your Service:
$ kubectl get svc
+NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE
+kubernetes 10.179.240.1 <none> 443/TCP <none> 8d
+nginxsvc 10.179.252.126 122.222.183.144 80/TCP,81/TCP,82/TCP run=nginx2 11m
+
+As mentioned previously, a Service is backed by a group of pods. These pods are exposed through endpoints
. The Service’s selector will be evaluated continuously and the results will be POSTed to an Endpoints object also named nginxsvc
. When a pod dies, it is automatically removed from the endpoints, and new pods matching the Service’s selector will automatically get added to the endpoints. Check the endpoints, and note that the IPs are the same as the pods created in the first step:
$ kubectl describe svc nginxsvc
+Name: nginxsvc
+Namespace: default
+Labels: app=nginx
+Selector: app=nginx
+Type: ClusterIP
+IP: 10.0.116.146
+Port: <unnamed> 80/TCP
+Endpoints: 10.245.0.14:80,10.245.0.15:80
+Session Affinity: None
+No events.
+
+$ kubectl get ep
+NAME ENDPOINTS
+nginxsvc 10.245.0.14:80,10.245.0.15:80
+
+You should now be able to curl the nginx Service on 10.0.116.146:80
from any node in your cluster. Note that the Service IP is completely virtual, it never hits the wire, if you’re curious about how this works you can read more about the service proxy.
Kubernetes supports 2 primary modes of finding a Service - environment variables and DNS. The former works out of the box while the latter requires the kube-dns cluster addon.
+ +When a Pod is run on a Node, the kubelet adds a set of environment variables for each active Service. This introduces an ordering problem. To see why, inspect the environment of your running nginx pods:
+ +$ kubectl exec my-nginx-6isf4 -- printenv | grep SERVICE
+KUBERNETES_SERVICE_HOST=10.0.0.1
+KUBERNETES_SERVICE_PORT=443
+
+Note there’s no mention of your Service. This is because you created the replicas before the Service. Another disadvantage of doing this is that the scheduler might put both pods on the same machine, which will take your entire Service down if it dies. We can do this the right way by killing the 2 pods and waiting for the replication controller to recreate them. This time around the Service exists before the replicas. This will given you scheduler level Service spreading of your pods (provided all your nodes have equal capacity), as well as the right environment variables:
+ +$ kubectl scale rc my-nginx --replicas=0; kubectl scale rc my-nginx --replicas=2;
+$ kubectl get pods -l app=nginx -o wide
+NAME READY STATUS RESTARTS AGE NODE
+my-nginx-5j8ok 1/1 Running 0 2m node1
+my-nginx-90vaf 1/1 Running 0 2m node2
+
+$ kubectl exec my-nginx-5j8ok -- printenv | grep SERVICE
+KUBERNETES_SERVICE_PORT=443
+NGINXSVC_SERVICE_HOST=10.0.116.146
+KUBERNETES_SERVICE_HOST=10.0.0.1
+NGINXSVC_SERVICE_PORT=80
+
+Kubernetes offers a DNS cluster addon Service that uses skydns to automatically assign dns names to other Services. You can check if it’s running on your cluster:
+ +$ kubectl get services kube-dns --namespace=kube-system
+NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE
+kube-dns 10.179.240.10 <none> 53/UDP,53/TCP k8s-app=kube-dns 8d
+
+If it isn’t running, you can enable it. The rest of this section will assume you have a Service with a long lived IP (nginxsvc), and a dns server that has assigned a name to that IP (the kube-dns cluster addon), so you can talk to the Service from any pod in your cluster using standard methods (e.g. gethostbyname). Let’s create another pod to test this:
+ +$ cat curlpod.yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: curlpod
+spec:
+ containers:
+ - image: radial/busyboxplus:curl
+ command:
+ - sleep
+ - "3600"
+ imagePullPolicy: IfNotPresent
+ name: curlcontainer
+ restartPolicy: Always
+
+And perform a lookup of the nginx Service
+ +$ kubectl create -f ./curlpod.yaml
+default/curlpod
+$ kubectl get pods curlpod
+NAME READY STATUS RESTARTS AGE
+curlpod 1/1 Running 0 18s
+
+$ kubectl exec curlpod -- nslookup nginxsvc
+Server: 10.0.0.10
+Address 1: 10.0.0.10
+Name: nginxsvc
+Address 1: 10.0.116.146
+
+Till now we have only accessed the nginx server from within the cluster. Before exposing the Service to the internet, you want to make sure the communication channel is secure. For this, you will need: +* Self signed certificates for https (unless you already have an identity certificate) +* An nginx server configured to use the certificates +* A secret that makes the certificates accessible to pods
+ +You can acquire all these from the nginx https example, in short:
+ +$ make keys secret KEY=/tmp/nginx.key CERT=/tmp/nginx.crt SECRET=/tmp/secret.json
+$ kubectl create -f /tmp/secret.json
+secrets/nginxsecret
+$ kubectl get secrets
+NAME TYPE DATA
+default-token-il9rc kubernetes.io/service-account-token 1
+nginxsecret Opaque 2
+
+Now modify your nginx replicas to start a https server using the certificate in the secret, and the Service, to expose both ports (80 and 443):
+ +$ cat nginx-app.yaml
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginxsvc
+ labels:
+ app: nginx
+spec:
+ type: NodePort
+ ports:
+ - port: 8080
+ targetPort: 80
+ protocol: TCP
+ name: http
+ - port: 443
+ protocol: TCP
+ name: https
+ selector:
+ app: nginx
+---
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: my-nginx
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ volumes:
+ - name: secret-volume
+ secret:
+ secretName: nginxsecret
+ containers:
+ - name: nginxhttps
+ image: bprashanth/nginxhttps:1.0
+ ports:
+ - containerPort: 443
+ - containerPort: 80
+ volumeMounts:
+ - mountPath: /etc/nginx/ssl
+ name: secret-volume
+
+Noteworthy points about the nginx-app manifest: +- It contains both rc and service specification in the same file +- The nginx server serves http traffic on port 80 and https traffic on 443, and nginx Service exposes both ports. +- Each container has access to the keys through a volume mounted at /etc/nginx/ssl. This is setup before the nginx server is started.
+ +$ kubectl delete rc,svc -l app=nginx; kubectl create -f ./nginx-app.yaml
+replicationcontrollers/my-nginx
+services/nginxsvc
+services/nginxsvc
+replicationcontrollers/my-nginx
+
+At this point you can reach the nginx server from any node.
+ +$ kubectl get pods -o json | grep -i podip
+ "podIP": "10.1.0.80",
+node $ curl -k https://10.1.0.80
+...
+<h1>Welcome to nginx!</h1>
+
+Note how we supplied the -k
parameter to curl in the last step, this is because we don’t know anything about the pods running nginx at certificate generation time,
+so we have to tell curl to ignore the CName mismatch. By creating a Service we linked the CName used in the certificate with the actual DNS name used by pods during Service lookup.
+Lets test this from a pod (the same secret is being reused for simplicity, the pod only needs nginx.crt to access the Service):
$ cat curlpod.yaml
+vapiVersion: v1
+kind: ReplicationController
+metadata:
+ name: curlrc
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ app: curlpod
+ spec:
+ volumes:
+ - name: secret-volume
+ secret:
+ secretName: nginxsecret
+ containers:
+ - name: curlpod
+ command:
+ - sh
+ - -c
+ - while true; do sleep 1; done
+ image: radial/busyboxplus:curl
+ volumeMounts:
+ - mountPath: /etc/nginx/ssl
+ name: secret-volume
+
+$ kubectl create -f ./curlpod.yaml
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+curlpod 1/1 Running 0 2m
+my-nginx-7006w 1/1 Running 0 24m
+
+$ kubectl exec curlpod -- curl https://nginxsvc --cacert /etc/nginx/ssl/nginx.crt
+...
+<title>Welcome to nginx!</title>
+...
+
+For some parts of your applications you may want to expose a Service onto an external IP address. Kubernetes supports two ways of doing this: NodePorts and LoadBalancers. The Service created in the last section already used NodePort
, so your nginx https replica is ready to serve traffic on the internet if your node has a public IP.
$ kubectl get svc nginxsvc -o json | grep -i nodeport -C 5
+ {
+ "name": "http",
+ "protocol": "TCP",
+ "port": 80,
+ "targetPort": 80,
+ "nodePort": 32188
+ },
+ {
+ "name": "https",
+ "protocol": "TCP",
+ "port": 443,
+ "targetPort": 443,
+ "nodePort": 30645
+ }
+
+$ kubectl get nodes -o json | grep ExternalIP -C 2
+ {
+ "type": "ExternalIP",
+ "address": "104.197.63.17"
+ }
+--
+ },
+ {
+ "type": "ExternalIP",
+ "address": "104.154.89.170"
+ }
+$ curl https://104.197.63.17:30645 -k
+...
+<h1>Welcome to nginx!</h1>
+
+Lets now recreate the Service to use a cloud load balancer, just change the Type
of Service in the nginx-app.yaml from NodePort
to LoadBalancer
:
$ kubectl delete rc, svc -l app=nginx
+$ kubectl create -f ./nginx-app.yaml
+$ kubectl get svc nginxsvc
+NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE
+nginxsvc 10.179.252.126 162.222.184.144 80/TCP,81/TCP,82/TCP run=nginx2 13m
+
+$ curl https://162.22.184.144 -k
+...
+<title>Welcome to nginx!</title>
+
+The IP address in the EXTERNAL_IP
column is the one that is available on the public internet. The CLUSTER_IP
is only available inside your
+cluster/private cloud network.
Learn about more Kubernetes features that will help you run containers reliably in production.
+ + + + + + + + + +kubectl port-forward forwards connections to a local port to a port on a pod. Its man page is available here. Compared to kubectl proxy, kubectl port-forward
is more generic as it can forward TCP traffic while kubectl proxy
can only forward HTTP traffic. This guide demonstrates how to use kubectl port-forward
to connect to a Redis database, which may be useful for database debugging.
$ kubectl create examples/redis/redis-master.yaml
+pods/redis-master
+
+wait until the Redis master pod is Running and Ready,
+ +$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+redis-master 2/2 Running 0 41s
+
+The Redis master is listening on port 6397, to verify this,
+ +$ kubectl get pods redis-master -t='{{(index (index .spec.containers 0).ports 0).containerPort}}{{"\n"}}'
+6379
+
+then we forward the port 6379 on the local workstation to the port 6379 of pod redis-master,
+ +$ kubectl port-forward redis-master 6379:6379
+I0710 14:43:38.274550 3655 portforward.go:225] Forwarding from 127.0.0.1:6379 -> 6379
+I0710 14:43:38.274797 3655 portforward.go:225] Forwarding from [::1]:6379 -> 6379
+
+To verify the connection is successful, we run a redis-cli on the local workstation,
+ +$ redis-cli
+127.0.0.1:6379> ping
+PONG
+
+Now one can debug the database from the local workstation.
+ + + + + + + + + +You have seen the basics about kubectl proxy
and apiserver proxy
. This guide shows how to use them together to access a service(kube-ui) running on the Kubernetes cluster from your workstation.
kube-ui is deployed as a cluster add-on. To find its apiserver proxy URL,
+ +$ kubectl cluster-info | grep "KubeUI"
+KubeUI is running at https://173.255.119.104/api/v1/proxy/namespaces/kube-system/services/kube-ui
+
+if this command does not find the URL, try the steps here.
+ +The above proxy URL is an access to the kube-ui service provided by the apiserver. To access it, you still need to authenticate to the apiserver. kubectl proxy
can handle the authentication.
$ kubectl proxy --port=8001
+Starting to serve on localhost:8001
+
+Now you can access the kube-ui service on your local workstation at http://localhost:8001/api/v1/proxy/namespaces/kube-system/services/kube-ui
+ + + + + + + + + +Table of Contents +
+ +This document describes the environment for Kubelet managed containers on a Kubernetes node (kNode). In contrast to the Kubernetes cluster API, which provides an API for creating and managing containers, the Kubernetes container environment provides the container access to information about what else is going on in the cluster.
+ +This cluster information makes it possible to build applications that are cluster aware. +Additionally, the Kubernetes container environment defines a series of hooks that are surfaced to optional hook handlers defined as part of individual containers. Container hooks are somewhat analogous to operating system signals in a traditional process model. However these hooks are designed to make it easier to build reliable, scalable cloud applications in the Kubernetes cluster. Containers that participate in this cluster lifecycle become cluster native.
+ +Another important part of the container environment is the file system that is available to the container. In Kubernetes, the filesystem is a combination of an image and one or more volumes.
+ +The following sections describe both the cluster information provided to containers, as well as the hooks and life-cycle that allows containers to interact with the management system.
+ +There are two types of information that are available within the container environment. There is information about the container itself, and there is information about other objects in the system.
+ +Currently, the Pod name for the pod in which the container is running is set as the hostname of the container, and is accessible through all calls to access the hostname within the container (e.g. the hostname command, or the gethostname function call in libc), but this is planned to change in the future and should not be used.
+ +The Pod name and namespace are also available as environment variables via the downward API. Additionally, user-defined environment variables from the pod definition, are also available to the container, as are any environment variables specified statically in the Docker image.
+ +In the future, we anticipate expanding this information with richer information about the container. Examples include available memory, number of restarts, and in general any state that you could get from the call to GET /pods on the API server.
+ +Currently the list of all services that are running at the time when the container was created via the Kubernetes Cluster API are available to the container as environment variables. The set of environment variables matches the syntax of Docker links.
+ +For a service named foo that maps to a container port named bar, the following variables are defined:
+ +FOO_SERVICE_HOST=<the host the service is running on>
+FOO_SERVICE_PORT=<the port the service is running on>
+
+Services have dedicated IP address, and are also surfaced to the container via DNS (If DNS addon is enabled). Of course DNS is still not an enumerable protocol, so we will continue to provide environment variables so that containers can do discovery.
+ +Container hooks provide information to the container about events in its management lifecycle. For example, immediately after a container is started, it receives a PostStart hook. These hooks are broadcast into the container with information about the life-cycle of the container. They are different from the events provided by Docker and other systems which are output from the container. Output events provide a log of what has already happened. Input hooks provide real-time notification about things that are happening, but no historical log.
+ +There are currently two container hooks that are surfaced to containers:
+ +PostStart
+ +This hook is sent immediately after a container is created. It notifies the container that it has been created. No parameters are passed to the handler.
+ +PreStop
+ +This hook is called immediately before a container is terminated. No parameters are passed to the handler. This event handler is blocking, and must complete before the call to delete the container is sent to the Docker daemon. The SIGTERM notification sent by Docker is also still sent. A more complete description of termination behavior can be found in Termination of Pods.
+ +When a management hook occurs, the management system calls into any registered hook handlers in the container for that hook. These hook handler calls are synchronous in the context of the pod containing the container. Typically we expect that users will make their hook handlers as lightweight as possible, but there are cases where long running commands make sense (e.g. saving state prior to container stop).
+ +Hook delivery is intended to be “at least once”, which means that a hook may be called multiple times for any given event (e.g. “start” or “stop”) and it is up to the hook implementer to be able to handle this +correctly.
+ +We expect double delivery to be rare, but in some cases if the Kubelet restarts in the middle of sending a hook, the hook may be resent after the Kubelet comes back up.
+ +Likewise, we only make a single delivery attempt. If (for example) an http hook receiver is down, and unable to take traffic, we do not make any attempts to resend.
+ +Currently, there are (hopefully rare) scenarios where PostStart hooks may not be delivered.
+ +Hook handlers are the way that hooks are surfaced to containers. Containers can select the type of hook handler they would like to implement. Kubernetes currently supports two different hook handler types:
+ +Exec - Executes a specific command (e.g. pre-stop.sh) inside the cgroups and namespaces of the container. Resources consumed by the command are counted against the container.
+HTTP - Executes an HTTP request against a specific endpoint on the container.
+So far the Pods we’ve seen have all used the image
field to indicate what process Kubernetes
+should run in a container. In this case, Kubernetes runs the image’s default command. If we want
+to run a particular command or override the image’s defaults, there are two additional fields that
+we can use:
Command
: Controls the actual command run by the imageArgs
: Controls the arguments passed to the commandDocker images have metadata associated with them that is used to store information about the image.
+The image author may use this to define defaults for the command and arguments to run a container
+when the user does not supply values. Docker calls the fields for commands and arguments
+Entrypoint
and Cmd
respectively. The full details for this feature are too complicated to
+describe here, mostly due to the fact that the docker API allows users to specify both of these
+fields as either a string array or a string and there are subtle differences in how those cases are
+handled. We encourage the curious to check out docker’s documentation for this feature.
Kubernetes allows you to override both the image’s default command (docker Entrypoint
) and args
+(docker Cmd
) with the Command
and Args
fields of Container
. The rules are:
Command
or Args
for a container, the defaults defined by the image
+will be usedCommand
but no Args
for a container, only the supplied Command
will be
+used; the image’s default arguments are ignoredArgs
, the image’s default command will be used with the arguments you
+supplyCommand
and Args
, the image’s defaults will be ignored and the values
+you supply will be usedHere are examples for these rules in table format
+ +Image Entrypoint |
+ Image Cmd |
+ Container Command |
+ Container Args |
+ Command Run | +
---|---|---|---|---|
[/ep-1] |
+ [foo bar] |
+ <not set> | +<not set> | +[ep-1 foo bar] |
+
[/ep-1] |
+ [foo bar] |
+ [/ep-2] |
+ <not set> | +[ep-2] |
+
[/ep-1] |
+ [foo bar] |
+ <not set> | +[zoo boo] |
+ [ep-1 zoo boo] |
+
[/ep-1] |
+ [foo bar] |
+ [/ep-2] |
+ [zoo boo] |
+ [ep-2 zoo boo] |
+
By default, Docker containers are “unprivileged” and cannot, for example, run a Docker daemon inside a Docker container. We can have fine grain control over the capabilities using cap-add and cap-drop.More details here.
+ +The relationship between Docker’s capabilities and Linux capabilities
+ +Docker’s capabilities | +Linux capabilities | +
---|---|
SETPCAP | +CAP_SETPCAP | +
SYS_MODULE | +CAP_SYS_MODULE | +
SYS_RAWIO | +CAP_SYS_RAWIO | +
SYS_PACCT | +CAP_SYS_PACCT | +
SYS_ADMIN | +CAP_SYS_ADMIN | +
SYS_NICE | +CAP_SYS_NICE | +
SYS_RESOURCE | +CAP_SYS_RESOURCE | +
SYS_TIME | +CAP_SYS_TIME | +
SYS_TTY_CONFIG | +CAP_SYS_TTY_CONFIG | +
MKNOD | +CAP_MKNOD | +
AUDIT_WRITE | +CAP_AUDIT_WRITE | +
AUDIT_CONTROL | +CAP_AUDIT_CONTROL | +
MAC_OVERRIDE | +CAP_MAC_OVERRIDE | +
MAC_ADMIN | +CAP_MAC_ADMIN | +
NET_ADMIN | +CAP_NET_ADMIN | +
SYSLOG | +CAP_SYSLOG | +
CHOWN | +CAP_CHOWN | +
NET_RAW | +CAP_NET_RAW | +
DAC_OVERRIDE | +CAP_DAC_OVERRIDE | +
FOWNER | +CAP_FOWNER | +
DAC_READ_SEARCH | +CAP_DAC_READ_SEARCH | +
FSETID | +CAP_FSETID | +
KILL | +CAP_KILL | +
SETGID | +CAP_SETGID | +
SETUID | +CAP_SETUID | +
LINUX_IMMUTABLE | +CAP_LINUX_IMMUTABLE | +
NET_BIND_SERVICE | +CAP_NET_BIND_SERVICE | +
NET_BROADCAST | +CAP_NET_BROADCAST | +
IPC_LOCK | +CAP_IPC_LOCK | +
IPC_OWNER | +CAP_IPC_OWNER | +
SYS_CHROOT | +CAP_SYS_CHROOT | +
SYS_PTRACE | +CAP_SYS_PTRACE | +
SYS_BOOT | +CAP_SYS_BOOT | +
LEASE | +CAP_LEASE | +
SETFCAP | +CAP_SETFCAP | +
WAKE_ALARM | +CAP_WAKE_ALARM | +
BLOCK_SUSPEND | +CAP_BLOCK_SUSPEND | +
An issue that comes up rather frequently for new installations of Kubernetes is
+that Services
are not working properly. You’ve run all your Pod
s and
+ReplicationController
s, but you get no response when you try to access them.
+This document will hopefully help you to figure out what’s going wrong.
Table of Contents +
+ +Throughout this doc you will see various commands that you can run. Some
+commands need to be run within Pod
, others on a Kubernetes Node
, and others
+can run anywhere you have kubectl
and credentials for the cluster. To make it
+clear what is expected, this document will use the following conventions.
If the command “COMMAND” is expected to run in a Pod
and produce “OUTPUT”:
u@pod$ COMMAND
+OUTPUT
+
+If the command “COMMAND” is expected to run on a Node
and produce “OUTPUT”:
u@node$ COMMAND
+OUTPUT
+
+If the command is “kubectl ARGS”:
+ +$ kubectl ARGS
+OUTPUT
+
+For many steps here you will want to see what a Pod
running in the cluster
+sees. Kubernetes does not directly support interactive Pod
s (yet), but you can
+approximate it:
$ cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+ name: busybox-sleep
+spec:
+ containers:
+ - name: busybox
+ image: busybox
+ args:
+ - sleep
+ - "1000000"
+EOF
+pods/busybox-sleep
+
+Now, when you need to run a command (even an interactive shell) in a Pod
-like
+context, use:
$ kubectl exec busybox-sleep -- <COMMAND>
+
+or
+ +$ kubectl exec -ti busybox-sleep sh
+/ #
+
+For the purposes of this walk-through, let’s run some Pod
s. Since you’re
+probably debugging your own Service
you can substitute your own details, or you
+can follow along and get a second data point.
$ kubectl run hostnames --image=gcr.io/google_containers/serve_hostname \
+ --labels=app=hostnames \
+ --port=9376 \
+ --replicas=3
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+hostnames hostnames gcr.io/google_containers/serve_hostname app=hostnames 3
+
+Note that this is the same as if you had started the ReplicationController
with
+the following YAML:
apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: hostnames
+spec:
+ selector:
+ app: hostnames
+ replicas: 3
+ template:
+ metadata:
+ labels:
+ app: hostnames
+ spec:
+ containers:
+ - name: hostnames
+ image: gcr.io/google_containers/serve_hostname
+ ports:
+ - containerPort: 9376
+ protocol: TCP
+
+Confirm your Pod
s are running:
$ kubectl get pods -l app=hostnames
+NAME READY STATUS RESTARTS AGE
+hostnames-0uton 1/1 Running 0 12s
+hostnames-bvc05 1/1 Running 0 12s
+hostnames-yp2kp 1/1 Running 0 12s
+
+The astute reader will have noticed that we did not actually create a Service
+yet - that is intentional. This is a step that sometimes gets forgotten, and
+is the first thing to check.
So what would happen if I tried to access a non-existent Service
? Assuming you
+have another Pod
that consumes this Service
by name you would get something
+like:
u@pod$ wget -qO- hostnames
+wget: bad address 'hostname'
+
+or:
+ +u@pod$ echo $HOSTNAMES_SERVICE_HOST
+
+So the first thing to check is whether that Service
actually exists:
$ kubectl get svc hostnames
+Error from server: service "hostnames" not found
+
+So we have a culprit, let’s create the Service
. As before, this is for the
+walk-through - you can use your own Service
’s details here.
$ kubectl expose rc hostnames --port=80 --target-port=9376
+service "hostnames" exposed
+
+And read it back, just to be sure:
+ +$ kubectl get svc hostnames
+NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE
+hostnames 10.0.0.1 <none> 80/TCP run=hostnames 1h
+
+As before, this is the same as if you had started the Service
with YAML:
apiVersion: v1
+kind: Service
+metadata:
+ name: hostnames
+spec:
+ selector:
+ app: hostnames
+ ports:
+ - name: default
+ protocol: TCP
+ port: 80
+ targetPort: 9376
+
+Now you can confirm that the Service
exists.
From a Pod
in the same Namespace
:
u@pod$ nslookup hostnames
+Server: 10.0.0.10
+Address: 10.0.0.10#53
+
+Name: hostnames
+Address: 10.0.1.175
+
+If this fails, perhaps your Pod
and Service
are in different
+Namespace
s, try a namespace-qualified name:
u@pod$ nslookup hostnames.default
+Server: 10.0.0.10
+Address: 10.0.0.10#53
+
+Name: hostnames.default
+Address: 10.0.1.175
+
+If this works, you’ll need to ensure that Pod
s and Service
s run in the same
+Namespace
. If this still fails, try a fully-qualified name:
u@pod$ nslookup hostnames.default.svc.cluster.local
+Server: 10.0.0.10
+Address: 10.0.0.10#53
+
+Name: hostnames.default.svc.cluster.local
+Address: 10.0.1.175
+
+Note the suffix here: “default.svc.cluster.local”. The “default” is the
+Namespace
we’re operating in. The “svc” denotes that this is a Service
.
+The “cluster.local” is your cluster domain.
You can also try this from a Node
in the cluster (note: 10.0.0.10 is my DNS
+Service
):
u@node$ nslookup hostnames.default.svc.cluster.local 10.0.0.10
+Server: 10.0.0.10
+Address: 10.0.0.10#53
+
+Name: hostnames.default.svc.cluster.local
+Address: 10.0.1.175
+
+If you are able to do a fully-qualified name lookup but not a relative one, you
+need to check that your kubelet
is running with the right flags.
+The --cluster-dns
flag needs to point to your DNS Service
’s IP and the
+--cluster-domain
flag needs to be your cluster’s domain - we assumed
+“cluster.local” in this document, but yours might be different, in which case
+you should change that in all of the commands above.
If the above still fails - DNS lookups are not working for your Service
- we
+can take a step back and see what else is not working. The Kubernetes master
+Service
should always work:
u@pod$ nslookup kubernetes.default
+Server: 10.0.0.10
+Address 1: 10.0.0.10
+
+Name: kubernetes
+Address 1: 10.0.0.1
+
+If this fails, you might need to go to the kube-proxy section of this doc, or
+even go back to the top of this document and start over, but instead of
+debugging your own Service
, debug DNS.
The next thing to test is whether your Service
works at all. From a
+Node
in your cluster, access the Service
’s IP (from kubectl get
above).
u@node$ curl 10.0.1.175:80
+hostnames-0uton
+
+u@node$ curl 10.0.1.175:80
+hostnames-yp2kp
+
+u@node$ curl 10.0.1.175:80
+hostnames-bvc05
+
+If your Service
is working, you should get correct responses. If not, there
+are a number of things that could be going wrong. Read on.
It might sound silly, but you should really double and triple check that your
+Service
is correct and matches your Pods
. Read back your Service
and
+verify it:
$ kubectl get service hostnames -o json
+{
+ "kind": "Service",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "hostnames",
+ "namespace": "default",
+ "selfLink": "/api/v1/namespaces/default/services/hostnames",
+ "uid": "428c8b6c-24bc-11e5-936d-42010af0a9bc",
+ "resourceVersion": "347189",
+ "creationTimestamp": "2015-07-07T15:24:29Z",
+ "labels": {
+ "app": "hostnames"
+ }
+ },
+ "spec": {
+ "ports": [
+ {
+ "name": "default",
+ "protocol": "TCP",
+ "port": 80,
+ "targetPort": 9376,
+ "nodePort": 0
+ }
+ ],
+ "selector": {
+ "app": "hostnames"
+ },
+ "clusterIP": "10.0.1.175",
+ "type": "ClusterIP",
+ "sessionAffinity": "None"
+ },
+ "status": {
+ "loadBalancer": {}
+ }
+}
+
+Is the port you are trying to access in spec.ports[]
? Is the targetPort
+correct for your Pod
s? If you meant it to be a numeric port, is it a number
+(9376) or a string “9376”? If you meant it to be a named port, do your Pod
s
+expose a port with the same name? Is the port’s protocol
the same as the
+Pod
’s?
If you got this far, we assume that you have confirmed that your Service
+exists and resolves by DNS. Now let’s check that the Pod
s you ran are
+actually being selected by the Service
.
Earlier we saw that the Pod
s were running. We can re-check that:
$ kubectl get pods -l app=hostnames
+NAME READY STATUS RESTARTS AGE
+hostnames-0uton 1/1 Running 0 1h
+hostnames-bvc05 1/1 Running 0 1h
+hostnames-yp2kp 1/1 Running 0 1h
+
+The “AGE” column says that these Pod
s are about an hour old, which implies that
+they are running fine and not crashing.
The -l app=hostnames
argument is a label selector - just like our Service
+has. Inside the Kubernetes system is a control loop which evaluates the
+selector of every Service
and save the results into an Endpoints
object.
$ kubectl get endpoints hostnames
+NAME ENDPOINTS
+hostnames 10.244.0.5:9376,10.244.0.6:9376,10.244.0.7:9376
+
+This confirms that the control loop has found the correct Pod
s for your
+Service
. If the hostnames
row is blank, you should check that the
+spec.selector
field of your Service
actually selects for metadata.labels
+values on your Pod
s.
At this point, we know that your Service
exists and has selected your Pod
s.
+Let’s check that the Pod
s are actually working - we can bypass the Service
+mechanism and go straight to the Pod
s.
u@pod$ wget -qO- 10.244.0.5:9376
+hostnames-0uton
+
+pod $ wget -qO- 10.244.0.6:9376
+hostnames-bvc05
+
+u@pod$ wget -qO- 10.244.0.7:9376
+hostnames-yp2kp
+
+We expect each Pod
in the Endpoints
list to return its own hostname. If
+this is not what happens (or whatever the correct behavior is for your own
+Pod
s), you should investigate what’s happening there. You might find
+kubectl logs
to be useful or kubectl exec
directly to your Pod
s and check
+service from there.
If you get here, your Service
is running, has Endpoints
, and your Pod
s
+are actually serving. At this point, the whole Service
proxy mechanism is
+suspect. Let’s confirm it, piece by piece.
Confirm that kube-proxy
is running on your Node
s. You should get something
+like the below:
u@node$ ps auxw | grep kube-proxy
+root 4194 0.4 0.1 101864 17696 ? Sl Jul04 25:43 /usr/local/bin/kube-proxy --master=https://kubernetes-master --kubeconfig=/var/lib/kube-proxy/kubeconfig --v=2
+
+Next, confirm that it is not failing something obvious, like contacting the
+master. To do this, you’ll have to look at the logs. Accessing the logs
+depends on your Node
OS. On some OSes it is a file, such as
+/var/log/kube-proxy.log, while other OSes use journalctl
to access logs. You
+should see something like:
I0707 17:34:53.945651 30031 server.go:88] Running in resource-only container "/kube-proxy"
+I0707 17:34:53.945921 30031 proxier.go:121] Setting proxy IP to 10.240.115.247 and initializing iptables
+I0707 17:34:54.053023 30031 roundrobin.go:262] LoadBalancerRR: Setting endpoints for default/kubernetes: to [10.240.169.188:443]
+I0707 17:34:54.053175 30031 roundrobin.go:262] LoadBalancerRR: Setting endpoints for default/hostnames:default to [10.244.0.5:9376 10.244.0.6:9376 10.244.0.7:9376]
+I0707 17:34:54.053284 30031 roundrobin.go:262] LoadBalancerRR: Setting endpoints for default/kube-dns:dns to [10.244.3.3:53]
+I0707 17:34:54.053310 30031 roundrobin.go:262] LoadBalancerRR: Setting endpoints for default/kube-dns:dns-tcp to [10.244.3.3:53]
+I0707 17:34:54.054780 30031 proxier.go:306] Adding new service "default/kubernetes:" at 10.0.0.1:443/TCP
+I0707 17:34:54.054903 30031 proxier.go:247] Proxying for service "default/kubernetes:" on TCP port 40074
+I0707 17:34:54.079181 30031 proxier.go:306] Adding new service "default/hostnames:default" at 10.0.1.175:80/TCP
+I0707 17:34:54.079273 30031 proxier.go:247] Proxying for service "default/hostnames:default" on TCP port 48577
+I0707 17:34:54.113665 30031 proxier.go:306] Adding new service "default/kube-dns:dns" at 10.0.0.10:53/UDP
+I0707 17:34:54.113776 30031 proxier.go:247] Proxying for service "default/kube-dns:dns" on UDP port 34149
+I0707 17:34:54.120224 30031 proxier.go:306] Adding new service "default/kube-dns:dns-tcp" at 10.0.0.10:53/TCP
+I0707 17:34:54.120297 30031 proxier.go:247] Proxying for service "default/kube-dns:dns-tcp" on TCP port 53476
+I0707 17:34:54.902313 30031 proxysocket.go:130] Accepted TCP connection from 10.244.3.3:42670 to 10.244.3.1:40074
+I0707 17:34:54.903107 30031 proxysocket.go:130] Accepted TCP connection from 10.244.3.3:42671 to 10.244.3.1:40074
+I0707 17:35:46.015868 30031 proxysocket.go:246] New UDP connection from 10.244.3.2:57493
+I0707 17:35:46.017061 30031 proxysocket.go:246] New UDP connection from 10.244.3.2:55471
+
+If you see error messages about not being able to contact the master, you
+should double-check your Node
configuration and installation steps.
One of the main responsibilities of kube-proxy
is to write the iptables
+rules which implement Service
s. Let’s check that those rules are getting
+written.
u@node$ iptables-save | grep hostnames
+-A KUBE-PORTALS-CONTAINER -d 10.0.1.175/32 -p tcp -m comment --comment "default/hostnames:default" -m tcp --dport 80 -j REDIRECT --to-ports 48577
+-A KUBE-PORTALS-HOST -d 10.0.1.175/32 -p tcp -m comment --comment "default/hostnames:default" -m tcp --dport 80 -j DNAT --to-destination 10.240.115.247:48577
+
+There should be 2 rules for each port on your Service
(just one in this
+example) - a “KUBE-PORTALS-CONTAINER” and a “KUBE-PORTALS-HOST”. If you do
+not see these, try restarting kube-proxy
with the -V
flag set to 4, and
+then look at the logs again.
Assuming you do see the above rules, try again to access your Service
by IP:
u@node$ curl 10.0.1.175:80
+hostnames-0uton
+
+If this fails, we can try accessing the proxy directly. Look back at the
+iptables-save
output above, and extract the port number that kube-proxy
is
+using for your Service
. In the above examples it is “48577”. Now connect to
+that:
u@node$ curl localhost:48577
+hostnames-yp2kp
+
+If this still fails, look at the kube-proxy
logs for specific lines like:
Setting endpoints for default/hostnames:default to [10.244.0.5:9376 10.244.0.6:9376 10.244.0.7:9376]
+
+If you don’t see those, try restarting kube-proxy
with the -V
flag set to 4, and
+then look at the logs again.
If you get this far, something very strange is happening. Your Service
is
+running, has Endpoints
, and your Pod
s are actually serving. You have DNS
+working, iptables
rules installed, and kube-proxy
does not seem to be
+misbehaving. And yet your Service
is not working. You should probably let
+us know, so we can help investigate!
Contact us on +Slack or +email or +GitHub.
+ +Visit troubleshooting document for more information.
+ + + + + + + + + +Table of Contents +
+ + + + + +You previously read about how to quickly deploy a simple replicated application using kubectl run
and how to configure and launch single-run containers using pods (Configuring containers). Here you’ll use the configuration-based approach to deploy a continuously running, replicated application.
Kubernetes creates and manages sets of replicated containers (actually, replicated Pods) using Replication Controllers.
+ +A replication controller simply ensures that a specified number of pod “replicas” are running at any one time. If there are too many, it will kill some. If there are too few, it will start more. It’s analogous to Google Compute Engine’s Instance Group Manager or AWS’s Auto-scaling Group (with no scaling policies).
+ +The replication controller created to run nginx by kubectl run
in the Quick start could be specified using YAML as follows:
apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: my-nginx
+spec:
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+
+Some differences compared to specifying just a pod are that the kind
is ReplicationController
, the number of replicas
desired is specified, and the pod specification is under the template
field. The names of the pods don’t need to be specified explicitly because they are generated from the name of the replication controller.
+View the replication controller API
+object
+to view the list of supported fields.
This replication controller can be created using create
, just as with pods:
$ kubectl create -f ./nginx-rc.yaml
+replicationcontrollers/my-nginx
+
+Unlike in the case where you directly create pods, a replication controller replaces pods that are deleted or terminated for any reason, such as in the case of node failure. For this reason, we recommend that you use a replication controller for a continuously running application even if your application requires only a single pod, in which case you can omit replicas
and it will default to a single replica.
You can view the replication controller you created using get
:
$ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+my-nginx nginx nginx app=nginx 2
+
+This tells you that your controller will ensure that you have two nginx replicas.
+ +You can see those replicas using get
, just as with pods you created directly:
$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+my-nginx-065jq 1/1 Running 0 51s
+my-nginx-buaiq 1/1 Running 0 51s
+
+When you want to kill your application, delete your replication controller, as in the Quick start:
+ +$ kubectl delete rc my-nginx
+replicationcontrollers/my-nginx
+
+By default, this will also cause the pods managed by the replication controller to be deleted. If there were a large number of pods, this may take a while to complete. If you want to leave the pods running, specify --cascade=false
.
If you try to delete the pods before deleting the replication controller, it will just replace them, as it is supposed to do.
+ +Kubernetes uses user-defined key-value attributes called labels to categorize and identify sets of resources, such as pods and replication controllers. The example above specified a single label in the pod template, with key app
and value nginx
. All pods created carry that label, which can be viewed using -L
:
$ kubectl get pods -L app
+NAME READY STATUS RESTARTS AGE APP
+my-nginx-afv12 0/1 Running 0 3s nginx
+my-nginx-lg99z 0/1 Running 0 3s nginx
+
+The labels from the pod template are copied to the replication controller’s labels by default, as well – all resources in Kubernetes support labels:
+ +$ kubectl get rc my-nginx -L app
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS APP
+my-nginx nginx nginx app=nginx 2 nginx
+
+More importantly, the pod template’s labels are used to create a selector
that will match pods carrying those labels. You can see this field by requesting it using the Go template output format of kubectl get
:
$ kubectl get rc my-nginx -o template --template="{{.spec.selector}}"
+map[app:nginx]
+
+You could also specify the selector
explicitly, such as if you wanted to specify labels in the pod template that you didn’t want to select on, but you should ensure that the selector will match the labels of the pods created from the pod template, and that it won’t match pods created by other replication controllers. The most straightforward way to ensure the latter is to create a unique label value for the replication controller, and to specify it in both the pod template’s labels and in the selector.
Table of Contents +
+ +A Deployment provides declarative update for Pods and ReplicationControllers. +Users describe the desired state in deployment object and deployment +controller changes the actual state to that at a controlled rate. +Users can define deployments to create new resources, or replace existing ones +by new ones.
+ +A typical use case is: +* Create a deployment to bring up a replication controller and pods. +* Later, update that deployment to recreate the pods (for ex: to use a new image).
+ +Deployments is part of the extensions
API Group and is not enabled by default.
+Set --runtime-config=extensions/v1beta1/deployments=true
on API server to
+enable it.
+This can be achieved by exporting ENABLE_DEPLOYMENTS=true
before running
+kube-up.sh
script on GCE.
Note that Deployment objects effectively have API version
+v1alpha1
.
+Alpha objects may change or even be discontinued in future software releases.
+However, due to to a known issue, they will appear as API version v1beta1
if
+enabled.
Here is an example Deployment. It creates a replication controller to +bring up 3 nginx pods.
+ + + +apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: nginx-deployment
+spec:
+ replicas: 3
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx:1.7.9
+ ports:
+ - containerPort: 80
+
+Run the example by downloading the example file and then running this command:
+ +$ kubectl create -f docs/user-guide/nginx-deployment.yaml
+deployment "nginx-deployment" created
+
+Running a get immediately will give:
+ +$ kubectl get deployments
+NAME UPDATEDREPLICAS AGE
+nginx-deployment 0/3 8s
+
+This indicates that deployment is trying to update 3 replicas. It has not +updated any one of those yet.
+ +Running a get again after a minute, will give:
+ +$ kubectl get deployments
+NAME UPDATEDREPLICAS AGE
+nginx-deployment 3/3 1m
+
+This indicates that deployent has created all the 3 replicas.
+Running kubectl get rc
and kubectl get pods
will show the replication controller (RC) and pods created.
$ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS AGE
+REPLICAS AGE
+deploymentrc-1975012602 nginx nginx:1.7.9 deployment.kubernetes.io/podTemplateHash=1975012602,app=nginx 3 2m
+
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+deploymentrc-1975012602-4f2tb 1/1 Running 0 1m
+deploymentrc-1975012602-j975u 1/1 Running 0 1m
+deploymentrc-1975012602-uashb 1/1 Running 0 1m
+
+The created RC will ensure that there are 3 nginx pods at all time.
+ +Lets say, now we want to update the nginx pods to start using nginx:1.9.1 image +instead of nginx:1.7.9. +For this, we update our deployment to be as follows:
+ + + +apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: nginx-deployment
+spec:
+ replicas: 3
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx:1.9.1
+ ports:
+ - containerPort: 80
+
+$ kubectl apply -f docs/user-guide/new-nginx-deployment.yaml
+deployment "nginx-deployment" configured
+
+Running a get immediately will still give:
+ +$ kubectl get deployments
+NAME UPDATEDREPLICAS AGE
+nginx-deployment 3/3 8s
+
+This indicates that deployment status has not been updated yet (it is still +showing old status). +Running a get again after a minute, will give:
+ +$ kubectl get deployments
+NAME UPDATEDREPLICAS AGE
+nginx-deployment 1/3 1m
+
+This indicates that deployment has updated one of the three pods that it needs +to update. +Eventually, it will get around to updating all the pods.
+ +$ kubectl get deployments
+NAME UPDATEDREPLICAS AGE
+nginx-deployment 3/3 3m
+
+We can run kubectl get rc
to see that deployment updated the pods by creating a new RC
+which it scaled up to 3 and scaled down the old RC to 0.
kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS AGE
+deploymentrc-1562004724 nginx nginx:1.9.1 deployment.kubernetes.io/podTemplateHash=1562004724,app=nginx 3 5m
+deploymentrc-1975012602 nginx nginx:1.7.9 deployment.kubernetes.io/podTemplateHash=1975012602,app=nginx 0 7m
+
+Running get pods, will only show the new pods.
+ +kubectl get pods
+NAME READY STATUS RESTARTS AGE
+deploymentrc-1562004724-0tgk5 1/1 Running 0 9m
+deploymentrc-1562004724-1rkfl 1/1 Running 0 8m
+deploymentrc-1562004724-6v702 1/1 Running 0 8m
+
+Next time we want to update pods, we can just update the deployment again.
+ +Deployment ensures that not all pods are down while they are being updated. By +default, it ensures that minimum of 1 less than the desired number of pods are +up. For example, if you look at the above deployment closely, you will see that +it first created a new pod, then deleted some old pods and created new ones. It +does not kill old pods until a sufficient number of new pods have come up.
+ +$ kubectl describe deployments
+Name: nginx-deployment
+Namespace: default
+CreationTimestamp: Thu, 22 Oct 2015 17:58:49 -0700
+Labels: app=nginx-deployment
+Selector: app=nginx
+Replicas: 3 updated / 3 total
+StrategyType: RollingUpdate
+RollingUpdateStrategy: 1 max unavailable, 1 max surge, 0 min ready seconds
+OldReplicationControllers: deploymentrc-1562004724 (3/3 replicas created)
+NewReplicationController: <none>
+Events:
+ FirstSeen LastSeen Count From SubobjectPath Reason Message
+ ───────── ──────── ───── ──── ───────────── ────── ───────
+ 10m 10m 1 {deployment-controller } ScalingRC Scaled up rc deploymentrc-1975012602 to 3
+ 2m 2m 1 {deployment-controller } ScalingRC Scaled up rc deploymentrc-1562004724 to 1
+ 2m 2m 1 {deployment-controller } ScalingRC Scaled down rc deploymentrc-1975012602 to 1
+ 1m 1m 1 {deployment-controller } ScalingRC Scaled up rc deploymentrc-1562004724 to 3
+ 1m 1m 1 {deployment-controller } ScalingRC Scaled down rc deploymentrc-1975012602 to 0
+
+Here we see that when we first created the deployment, it created an RC and scaled it up to 3 replicas directly. +When we updated the deployment, it created a new RC and scaled it up to 1 and then scaled down the old RC by 1, so that at least 2 pods were available at all times. +It then scaled up the new RC to 3 and when those pods were ready, it scaled down the old RC to 0.
+ +Each time a new deployment object is observed, a replication controller is
+created to bring up the desired pods if there is no existing RC doing so.
+Existing RCs controlling pods whose labels match .spec.selector
but the
+template does not match .spec.template
are scaled down.
+Eventually, the new RC will be scaled to .spec.replicas
and all old RCs will
+be scaled to 0.
+If the user updates the deployment while an existing deployment was in progress,
+deployment will create a new RC as per the update and start scaling that up and
+will roll the RC that it was scaling up before in its list of old RCs and will
+start scaling it down.
+For example: If user creates a deployment to create 5 replicas of nginx:1.7.9.
+But then updates the deployment to create 5 replicas of nging:1.9.1, when only 3
+replicas of nginx:1.7.9 had been created, then deployment will immediately start
+killing the 3 nginx:1.7.9 pods that it had created and will start creating
+nginx:1.9.1 pods. It will not wait for 5 replicas of nginx:1.7.9 to be created
+before changing course.
As with all other Kubernetes configs, a Deployment needs apiVersion
, kind
, and
+metadata
fields. For general information about working with config files,
+see here, here, and here.
A Deployment also needs a .spec
section.
The .spec.template
is the only required field of the .spec
.
The .spec.template
is a pod template. It has exactly
+the same schema as a pod, except it is nested and does not have an
+apiVersion
or kind
.
.spec.replicas
is an optional field that specifies the number of desired pods. Defaults
+to 1.
.spec.selector
is an optional field that specifies label selectors for pods
+targeted by this deployment. Deployment kills some of these pods, if their
+template is different than .spec.template
or if the total number of such pods
+exceeds .spec.replicas
. It will bring up new pods with .spec.template
if
+number of pods are less than the desired number.
.spec.uniqueLabelKey
is an optional field specifying key of the selector that
+is added to existing RCs (and label key that is added to its pods) to prevent
+the existing RCs to select new pods (and old pods being selected by new RC).
+Users can set this to an empty string to indicate that the system should
+not add any selector and label. If unspecified, system uses
+“deployment.kubernetes.io/podTemplateHash”.
+Value of this key is hash of .spec.template
.
+No label is added if this is set to empty string.
.spec.strategy
specifies the strategy to replace old pods by new ones.
+.spec.strategy.type
can be “Recreate” or “RollingUpdate”. “RollingUpdate” is
+the default value.
All existing pods are killed before new ones are created when
+.spec.strategy.type==Recreate
.
+Note: This is not implemented yet.
Deployment updates pods in a [rolling update][update-demo/] fashion
+when .spec.strategy.type==RollingUpdate
.
+Users can specify maxUnavailable
, maxSurge
and minReadySeconds
to control
+the rolling update process.
.spec.strategy.rollingUpdate.maxUnavailable
is an optional field that specifies the
+maximum number of pods that can be unavailable during the update process.
+Value can be an absolute number (ex: 5) or a percentage of desired pods (ex:
+10%).
+Absolute number is calculated from percentage by rounding up.
+This can not be 0 if .spec.strategy.rollingUpdate.maxSurge
is 0.
+By default, a fixed value of 1 is used.
+Example: when this is set to 30%, the old RC can be scaled down to
+70% of desired pods immediately when the rolling update starts. Once new pods are
+ready, old RC can be scaled down further, followed by scaling up the new RC,
+ensuring that the total number of pods available at all times during the
+update is at least 70% of desired pods.
.spec.strategy.rollingUpdate.maxSurge
is an optional field that specifies the
+maximum number of pods that can be created above the desired number of pods.
+Value can be an absolute number (ex: 5) or a percentage of desired pods (ex:
+10%).
+This can not be 0 if MaxUnavailable is 0.
+Absolute number is calculated from percentage by rounding up.
+By default, a value of 1 is used.
+Example: when this is set to 30%, the new RC can be scaled up immediately when
+the rolling update starts, such that the total number of old and new pods do not exceed
+130% of desired pods. Once old pods have been killed,
+new RC can be scaled up further, ensuring that total number of pods running
+at any time during the update is atmost 130% of desired pods.
.spec.strategy.rollingUpdate.minReadySeconds
is an optional field that specifies the
+minimum number of seconds for which a newly created pod should be ready
+without any of its container crashing, for it to be considered available.
+Defaults to 0 (pod will be considered available as soon as it is ready).
+Note: This is not implemented yet.
Kubectl rolling update also updates pods and replication controllers in a similar fashion. +But deployments is declarative and is server side.
+ + + + + + + + + +In this doc, we introduce the Kubernetes command line for interacting with the api to docker-cli users. The tool, kubectl, is designed to be familiar to docker-cli users but there are a few necessary differences. Each section of this doc highlights a docker subcommand explains the kubectl equivalent.
+ -Table of Contents
+ - +In this doc, we introduce the Kubernetes command line for interacting with the api to docker-cli users. The tool, kubectl, is designed to be familiar to docker-cli users but there are a few necessary differences. Each section of this doc highlights a docker subcommand explains the kubectl equivalent.
+ +Table of Contents +
+ +Following this example, you will create a pod with a container that consumes the pod’s name and +namespace using the downward API.
+ +This example assumes you have a Kubernetes cluster installed and running, and that you have
+installed the kubectl
command line tool somewhere in your path. Please see the getting
+started for installation instructions for your platform.
Containers consume the downward API using environment variables. The downward API allows +containers to be injected with the name and namespace of the pod the container is in.
+ +Use the examples/downward-api/dapi-pod.yaml
file to create a Pod with a container that consumes the
+downward API.
$ kubectl create -f docs/user-guide/downward-api/dapi-pod.yaml
+
+This pod runs the env
command in a container that consumes the downward API. You can grep
+through the pod logs to see that the pod was injected with the correct values:
$ kubectl logs dapi-test-pod | grep POD_
+2015-04-30T20:22:18.568024817Z MY_POD_NAME=dapi-test-pod
+2015-04-30T20:22:18.568087688Z MY_POD_NAMESPACE=default
+2015-04-30T20:22:18.568092435Z MY_POD_IP=10.0.1.6
+
+Following this example, you will create a pod with a container that consumes the pod’s name and +namespace using the downward API.
+ +This example assumes you have a Kubernetes cluster installed and running, and that you have
+installed the kubectl
command line tool somewhere in your path. Please see the getting
+started for installation instructions for your platform.
Containers consume the downward API using environment variables. The downward API allows +containers to be injected with the name and namespace of the pod the container is in.
+ +Use the examples/downward-api/dapi-pod.yaml
file to create a Pod with a container that consumes the
+downward API.
$ kubectl create -f docs/user-guide/downward-api/dapi-pod.yaml
+
+This pod runs the env
command in a container that consumes the downward API. You can grep
+through the pod logs to see that the pod was injected with the correct values:
$ kubectl logs dapi-test-pod | grep POD_
+2015-04-30T20:22:18.568024817Z MY_POD_NAME=dapi-test-pod
+2015-04-30T20:22:18.568087688Z MY_POD_NAMESPACE=default
+2015-04-30T20:22:18.568092435Z MY_POD_IP=10.0.1.6
+
+Following this example, you will create a pod with a downward API volume. +A downward API volume is a k8s volume plugin with the ability to save some pod information in a plain text file. The pod information can be for example some metadata.
+ +Supported metadata fields:
+ +metadata.annotations
metadata.namespace
metadata.name
metadata.labels
This example assumes you have a Kubernetes cluster installed and running, and the kubectl
command line tool somewhere in your path. Please see the gettingstarted for installation instructions for your platform.
Use the docs/user-guide/downward-api/dapi-volume.yaml
file to create a Pod with a downward API volume which stores pod labels and pod annotations to /etc/labels
and /etc/annotations
respectively.
$ kubectl create -f docs/user-guide/downward-api/volume/dapi-volume.yaml
+
+The pod displays (every 5 seconds) the content of the dump files which can be executed via the usual kubectl log
command
$ kubectl logs kubernetes-downwardapi-volume-example
+cluster="test-cluster1"
+rack="rack-22"
+zone="us-est-coast"
+build="two"
+builder="john-doe"
+kubernetes.io/config.seen="2015-08-24T13:47:23.432459138Z"
+kubernetes.io/config.source="api"
+
+In pod’s /etc
directory one may find the file created by the plugin (system files elided):
$ kubectl exec kubernetes-downwardapi-volume-example -i -t -- sh
+/ # ls -laR /etc
+/etc:
+total 32
+drwxrwxrwt 3 0 0 180 Aug 24 13:03 .
+drwxr-xr-x 1 0 0 4096 Aug 24 13:05 ..
+drwx------ 2 0 0 80 Aug 24 13:03 ..2015_08_24_13_03_44259413923
+lrwxrwxrwx 1 0 0 30 Aug 24 13:03 ..downwardapi -> ..2015_08_24_13_03_44259413923
+lrwxrwxrwx 1 0 0 25 Aug 24 13:03 annotations -> ..downwardapi/annotations
+lrwxrwxrwx 1 0 0 20 Aug 24 13:03 labels -> ..downwardapi/labels
+
+/etc/..2015_08_24_13_03_44259413923:
+total 8
+drwx------ 2 0 0 80 Aug 24 13:03 .
+drwxrwxrwt 3 0 0 180 Aug 24 13:03 ..
+-rw-r--r-- 1 0 0 115 Aug 24 13:03 annotations
+-rw-r--r-- 1 0 0 53 Aug 24 13:03 labels
+/ #
+
+The file labels
is stored in a temporary directory (..2015_08_24_13_03_44259413923
in the example above) which is symlinked to by ..downwardapi
. Symlinks for annotations and labels in /etc
point to files containing the actual metadata through the ..downwardapi
indirection. This structure allows for dynamic atomic refresh of the metadata: updates are written to a new temporary directory, and the ..downwardapi
symlink is updated atomically using rename(2)
.
Following this example, you will create a pod with a downward API volume. +A downward API volume is a k8s volume plugin with the ability to save some pod information in a plain text file. The pod information can be for example some metadata.
+ +Supported metadata fields:
+ +metadata.annotations
metadata.namespace
metadata.name
metadata.labels
This example assumes you have a Kubernetes cluster installed and running, and the kubectl
command line tool somewhere in your path. Please see the gettingstarted for installation instructions for your platform.
Use the docs/user-guide/downward-api/dapi-volume.yaml
file to create a Pod with a downward API volume which stores pod labels and pod annotations to /etc/labels
and /etc/annotations
respectively.
$ kubectl create -f docs/user-guide/downward-api/volume/dapi-volume.yaml
+
+The pod displays (every 5 seconds) the content of the dump files which can be executed via the usual kubectl log
command
$ kubectl logs kubernetes-downwardapi-volume-example
+cluster="test-cluster1"
+rack="rack-22"
+zone="us-est-coast"
+build="two"
+builder="john-doe"
+kubernetes.io/config.seen="2015-08-24T13:47:23.432459138Z"
+kubernetes.io/config.source="api"
+
+In pod’s /etc
directory one may find the file created by the plugin (system files elided):
$ kubectl exec kubernetes-downwardapi-volume-example -i -t -- sh
+/ # ls -laR /etc
+/etc:
+total 32
+drwxrwxrwt 3 0 0 180 Aug 24 13:03 .
+drwxr-xr-x 1 0 0 4096 Aug 24 13:05 ..
+drwx------ 2 0 0 80 Aug 24 13:03 ..2015_08_24_13_03_44259413923
+lrwxrwxrwx 1 0 0 30 Aug 24 13:03 ..downwardapi -> ..2015_08_24_13_03_44259413923
+lrwxrwxrwx 1 0 0 25 Aug 24 13:03 annotations -> ..downwardapi/annotations
+lrwxrwxrwx 1 0 0 20 Aug 24 13:03 labels -> ..downwardapi/labels
+
+/etc/..2015_08_24_13_03_44259413923:
+total 8
+drwx------ 2 0 0 80 Aug 24 13:03 .
+drwxrwxrwt 3 0 0 180 Aug 24 13:03 ..
+-rw-r--r-- 1 0 0 115 Aug 24 13:03 annotations
+-rw-r--r-- 1 0 0 53 Aug 24 13:03 labels
+/ #
+
+The file labels
is stored in a temporary directory (..2015_08_24_13_03_44259413923
in the example above) which is symlinked to by ..downwardapi
. Symlinks for annotations and labels in /etc
point to files containing the actual metadata through the ..downwardapi
indirection. This structure allows for dynamic atomic refresh of the metadata: updates are written to a new temporary directory, and the ..downwardapi
symlink is updated atomically using rename(2)
.
Environment Guide Example +========================= +This example demonstrates running pods, replication controllers, and +services. It shows two types of pods: frontend and backend, with +services on top of both. Accessing the frontend pod will return +environment information about itself, and a backend pod that it has +accessed through the service. The goal is to illuminate the +environment metadata available to running containers inside the +Kubernetes cluster. The documentation for the Kubernetes environment +is here.
+ + + +This example assumes that you have a Kubernetes cluster installed and
+running, and that you have installed the kubectl
command line tool
+somewhere in your path. Please see the getting
+started for installation instructions
+for your platform.
The code for the containers is under +containers/
+ +kubectl create -f ./backend-rc.yaml
+kubectl create -f ./backend-srv.yaml
+kubectl create -f ./show-rc.yaml
+kubectl create -f ./show-srv.yaml
+
+
+Use kubectl describe service show-srv
to determine the public IP of
+your service.
++ +Note: If your platform does not support external load balancers, + you’ll need to open the proper port and direct traffic to the + internal IP shown for the frontend service with the above command
+
Run curl <public ip>:80
to query the service. You should get
+something like this back:
+Pod Name: show-rc-xxu6i
+Pod Namespace: default
+USER_VAR: important information
+
+Kubernetes environment variables
+BACKEND_SRV_SERVICE_HOST = 10.147.252.185
+BACKEND_SRV_SERVICE_PORT = 5000
+KUBERNETES_RO_SERVICE_HOST = 10.147.240.1
+KUBERNETES_RO_SERVICE_PORT = 80
+KUBERNETES_SERVICE_HOST = 10.147.240.2
+KUBERNETES_SERVICE_PORT = 443
+KUBE_DNS_SERVICE_HOST = 10.147.240.10
+KUBE_DNS_SERVICE_PORT = 53
+
+Found backend ip: 10.147.252.185 port: 5000
+Response from backend
+Backend Container
+Backend Pod Name: backend-rc-6qiya
+Backend Namespace: default
+
+
+
+First the frontend pod’s information is printed. The pod name and
+namespace are retrieved from the
+Downward API. Next, USER_VAR
is the name of
+an environment variable set in the pod
+definition. Then, the dynamic Kubernetes environment
+variables are scanned and printed. These are used to find the backend
+service, named backend-srv
. Finally, the frontend pod queries the
+backend service and prints the information returned. Again the backend
+pod returns its own pod name and namespace.
Try running the curl
command a few times, and notice what
+changes. Ex: watch -n 1 curl -s <ip>
Firstly, the frontend service
+is directing your request to different frontend pods each time. The
+frontend pods are always contacting the backend through the backend
+service. This results in a different backend pod servicing each
+request as well.
kubectl delete rc,service -l type=show-type
+kubectl delete rc,service -l type=backend-type
+
+
+
+
+
+
+
+
+
+
+ Building
+——–
+For each container, the build steps are the same. The examples below
+are for the show
container. Replace show
with backend
for the
+backend container.
docker build -t gcr.io/<project-name>/show .
+gcloud docker push gcr.io/<project-name>/show
+
+
+docker build -t <username>/show .
+docker push <username>/show
+
+
+Edit both show-rc.yaml
and backend-rc.yaml
and replace the
+specified image:
with the one that you built.
Building
+——–
+For each container, the build steps are the same. The examples below
+are for the show
container. Replace show
with backend
for the
+backend container.
docker build -t gcr.io/<project-name>/show .
+gcloud docker push gcr.io/<project-name>/show
+
+
+docker build -t <username>/show .
+docker push <username>/show
+
+
+Edit both show-rc.yaml
and backend-rc.yaml
and replace the
+specified image:
with the one that you built.
Environment Guide Example +========================= +This example demonstrates running pods, replication controllers, and +services. It shows two types of pods: frontend and backend, with +services on top of both. Accessing the frontend pod will return +environment information about itself, and a backend pod that it has +accessed through the service. The goal is to illuminate the +environment metadata available to running containers inside the +Kubernetes cluster. The documentation for the Kubernetes environment +is here.
+ + + +This example assumes that you have a Kubernetes cluster installed and
+running, and that you have installed the kubectl
command line tool
+somewhere in your path. Please see the getting
+started for installation instructions
+for your platform.
The code for the containers is under +containers/
+ +kubectl create -f ./backend-rc.yaml
+kubectl create -f ./backend-srv.yaml
+kubectl create -f ./show-rc.yaml
+kubectl create -f ./show-srv.yaml
+
+
+Use kubectl describe service show-srv
to determine the public IP of
+your service.
++ +Note: If your platform does not support external load balancers, + you’ll need to open the proper port and direct traffic to the + internal IP shown for the frontend service with the above command
+
Run curl <public ip>:80
to query the service. You should get
+something like this back:
+Pod Name: show-rc-xxu6i
+Pod Namespace: default
+USER_VAR: important information
+
+Kubernetes environment variables
+BACKEND_SRV_SERVICE_HOST = 10.147.252.185
+BACKEND_SRV_SERVICE_PORT = 5000
+KUBERNETES_RO_SERVICE_HOST = 10.147.240.1
+KUBERNETES_RO_SERVICE_PORT = 80
+KUBERNETES_SERVICE_HOST = 10.147.240.2
+KUBERNETES_SERVICE_PORT = 443
+KUBE_DNS_SERVICE_HOST = 10.147.240.10
+KUBE_DNS_SERVICE_PORT = 53
+
+Found backend ip: 10.147.252.185 port: 5000
+Response from backend
+Backend Container
+Backend Pod Name: backend-rc-6qiya
+Backend Namespace: default
+
+
+
+First the frontend pod’s information is printed. The pod name and
+namespace are retrieved from the
+Downward API. Next, USER_VAR
is the name of
+an environment variable set in the pod
+definition. Then, the dynamic Kubernetes environment
+variables are scanned and printed. These are used to find the backend
+service, named backend-srv
. Finally, the frontend pod queries the
+backend service and prints the information returned. Again the backend
+pod returns its own pod name and namespace.
Try running the curl
command a few times, and notice what
+changes. Ex: watch -n 1 curl -s <ip>
Firstly, the frontend service
+is directing your request to different frontend pods each time. The
+frontend pods are always contacting the backend through the backend
+service. This results in a different backend pod servicing each
+request as well.
kubectl delete rc,service -l type=show-type
+kubectl delete rc,service -l type=backend-type
+
+
+
+
+
+
+
+
+
+
+ Developers can use kubectl exec
to run commands in a container. This guide demonstrates two use cases.
Kubernetes exposes services through environment variables. It is convenient to check these environment variables using kubectl exec
.
We first create a pod and a service,
+ +$ kubectl create -f examples/guestbook/redis-master-controller.yaml
+$ kubectl create -f examples/guestbook/redis-master-service.yaml
+
+wait until the pod is Running and Ready,
+ +$ kubectl get pod
+NAME READY REASON RESTARTS AGE
+redis-master-ft9ex 1/1 Running 0 12s
+
+then we can check the environment variables of the pod,
+ +$ kubectl exec redis-master-ft9ex env
+...
+REDIS_MASTER_SERVICE_PORT=6379
+REDIS_MASTER_SERVICE_HOST=10.0.0.219
+...
+
+We can use these environment variables in applications to find the service.
+ +It is convenient to use kubectl exec
to check if the volumes are mounted as expected.
+We first create a Pod with a volume mounted at /data/redis,
kubectl create -f docs/user-guide/walkthrough/pod-redis.yaml
+
+wait until the pod is Running and Ready,
+ +$ kubectl get pods
+NAME READY REASON RESTARTS AGE
+storage 1/1 Running 0 1m
+
+we then use kubectl exec
to verify that the volume is mounted at /data/redis,
$ kubectl exec storage ls /data
+redis
+
+After all, open a terminal in a pod is the most direct way to introspect the pod. Assuming the pod/storage is still running, run
+ +$ kubectl exec -ti storage -- bash
+root@storage:/data#
+
+This gets you a terminal.
+ + + + + + + + + +This document describes the current state of Horizontal Pod Autoscaler in Kubernetes.
+ + + +Horizontal pod autoscaling allows the number of pods in a replication controller or deployment +to scale automatically based on observed CPU utilization. +It is a beta feature in Kubernetes 1.1.
+ +The autoscaler is implemented as a Kubernetes API resource and a controller. +The resource describes behavior of the controller. +The controller periodically adjusts the number of replicas in a replication controller or deployment +to match the observed average CPU utilization to the target specified by user.
+ +The autoscaler is implemented as a control loop.
+It periodically queries CPU utilization for the pods it targets.
+(The period of the autoscaler is controlled by --horizontal-pod-autoscaler-sync-period
flag of controller manager.
+The default value is 30 seconds).
+Then, it compares the arithmetic mean of the pods’ CPU utilization with the target and adjust the number of replicas if needed.
CPU utilization is the recent CPU usage of a pod divided by the sum of CPU requested by the pod’s containers. +Please note that if some of the pod’s containers do not have CPU request set, +CPU utilization for the pod will not be defined and the autoscaler will not take any action. +Further details of the autoscaling algorithm are given here.
+ +Autoscaler uses heapster to collect CPU utilization. +Therefore, it is required to deploy heapster monitoring in your cluster for autoscaling to work.
+ +Autoscaler accesses corresponding replication controller or deployment by scale sub-resource. +Scale is an interface which allows to dynamically set the number of replicas and to learn the current state of them. +More details on scale sub-resource can be found here.
+ +Horizontal pod autoscaler is a top-level resource in the Kubernetes REST API (currently in beta). +More details about the API object can be found at +HorizontalPodAutoscaler Object.
+ +Horizontal pod autoscaler, like every API resource, is supported in a standard way by kubectl
.
+We can create a new autoscaler using kubectl create
command.
+We can list autoscalers by kubectl get hpa
and get detailed description by kubectl describe hpa
.
+Finally, we can delete an autoscaler using kubectl delete hpa
.
In addition, there is a special kubectl autoscale
command that allows for easy creation of horizontal pod autoscaler.
+For instance, executing kubectl autoscale rc foo --min=2 --max=5 --cpu-percent=80
+will create an autoscaler for replication controller foo, with target CPU utilization set to 80%
+and the number of replicas between 2 and 5.
+The detailed documentation of kubectl autoscale
can be found here.
Currently in Kubernetes, it is possible to perform a rolling update by managing replication controllers directly, +or by using the deployment object, which manages the underlying replication controllers for you. +Horizontal pod autoscaler only supports the latter approach: the horizontal pod autoscaler is bound to the deployment object, +it sets the size for the deployment object, and the deployment is responsible for setting sizes of underlying replication controllers.
+ +Horizontal pod autoscaler does not work with rolling update using direct manipulation of replication controllers,
+i.e. you cannot bind a horizontal pod autoscaler to a replication controller and do rolling update (e.g. using kubectl rolling-update
).
+The reason this doesn’t work is that when rolling update creates a new replication controller,
+the horizontal pod autoscaler will not be bound to the new replication controller.
Horizontal pod autoscaling is a beta feature in Kubernetes 1.1. +It allows the number of pods in a replication controller or deployment to scale automatically based on observed CPU usage. +In the future also other metrics will be supported.
+ +In this document we explain how this feature works by walking you through an example of enabling horizontal pod autoscaling with the php-apache server.
+ +This example requires a running Kubernetes cluster and kubectl in the version at least 1.1. +Heapster monitoring needs to be deployed in the cluster +as horizontal pod autoscaler uses it to collect metrics +(if you followed getting started on GCE guide, +heapster monitoring will be turned-on by default).
+ +To demonstrate horizontal pod autoscaler we will use a custom docker image based on php-apache server. +The image can be found here. +It defines index.php page which performs some CPU intensive computations.
+ +First, we will start a replication controller running the image and expose it as an external service:
+ + + +$ kubectl run php-apache --image=gcr.io/google_containers/hpa-example --requests=cpu=200m
+replicationcontroller "php-apache" created
+
+$ kubectl expose rc php-apache --port=80 --type=LoadBalancer
+service "php-apache" exposed
+
+Now, we will wait some time and verify that both the replication controller and the service were correctly created and are running. We will also determine the IP address of the service:
+ +$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+php-apache-wa3t1 1/1 Running 0 12m
+
+$ kubectl describe services php-apache | grep "LoadBalancer Ingress"
+LoadBalancer Ingress: 146.148.24.244
+
+We may now check that php-apache server works correctly by calling curl
with the service’s IP:
$ curl http://146.148.24.244
+OK!
+
+Please notice that when exposing the service we assumed that our cluster runs on a provider which supports load balancers (e.g.: on GCE).
+If load balancers are not supported (e.g.: on Vagrant), we can expose php-apache service as ClusterIP
and connect to it using the proxy on the master:
$ kubectl expose rc php-apache --port=80 --type=ClusterIP
+service "php-apache" exposed
+
+$ kubectl cluster-info | grep master
+Kubernetes master is running at https://146.148.6.215
+
+$ curl -k -u <admin>:<password> https://146.148.6.215/api/v1/proxy/namespaces/default/services/php-apache/
+OK!
+
+Now that the server is running, we will create a horizontal pod autoscaler for it. +To create it, we will use the hpa-php-apache.yaml file, which looks like this:
+ +apiVersion: extensions/v1beta1
+kind: HorizontalPodAutoscaler
+metadata:
+ name: php-apache
+ namespace: default
+spec:
+ scaleRef:
+ kind: ReplicationController
+ name: php-apache
+ namespace: default
+ minReplicas: 1
+ maxReplicas: 10
+ cpuUtilization:
+ targetPercentage: 50
+
+This defines a horizontal pod autoscaler that maintains between 1 and 10 replicas of the Pods +controlled by the php-apache replication controller we created in the first step of these instructions. +Roughly speaking, the horizontal autoscaler will increase and decrease the number of replicas +(via the replication controller) so as to maintain an average CPU utilization across all Pods of 50% +(since each pod requests 200 milli-cores by kubectl run, this means average CPU utilization of 100 milli-cores). +See here for more details on the algorithm.
+ +We will create the autoscaler by executing the following command:
+ +$ kubectl create -f docs/user-guide/horizontal-pod-autoscaling/hpa-php-apache.yaml
+horizontalpodautoscaler "php-apache" created
+
+Alternatively, we can create the autoscaler using kubectl autoscale. +The following command will create the equivalent autoscaler as defined in the hpa-php-apache.yaml file:
+ +
+$ kubectl autoscale rc php-apache --cpu-percent=50 --min=1 --max=10
+replicationcontroller "php-apache" autoscaled
+
+
+
+We may check the current status of autoscaler by running:
+ +$ kubectl get hpa
+NAME REFERENCE TARGET CURRENT MINPODS MAXPODS AGE
+php-apache ReplicationController/default/php-apache/ 50% 0% 1 10 27s
+
+Please note that the current CPU consumption is 0% as we are not sending any requests to the server
+(the CURRENT
column shows the average across all the pods controlled by the corresponding replication controller).
Now, we will see how the autoscaler reacts on the increased load of the server. +We will start an infinite loop of queries to our server (please run it in a different terminal):
+ +$ while true; do curl http://146.148.6.244; done
+
+We may examine, how CPU load was increased (the results should be visible after about 3-4 minutes) by executing:
+ +$ kubectl get hpa
+NAME REFERENCE TARGET CURRENT MINPODS MAXPODS AGE
+php-apache ReplicationController/default/php-apache/ 50% 305% 1 10 4m
+
+In the case presented here, it bumped CPU consumption to 305% of the request. +As a result, the replication controller was resized to 7 replicas:
+ +$ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS AGE
+php-apache php-apache gcr.io/google_containers/hpa-example run=php-apache 7 18m
+
+Now, we may increase the load even more by running yet another infinite loop of queries (in yet another terminal):
+ +$ while true; do curl http://146.148.6.244; done
+
+In the case presented here, it increased the number of serving pods to 10:
+ +$ kubectl get hpa
+NAME REFERENCE TARGET CURRENT MINPODS MAXPODS AGE
+php-apache ReplicationController/default/php-apache/ 50% 65% 1 10 14m
+
+$ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS AGE
+php-apache php-apache gcr.io/google_containers/hpa-example run=php-apache 10 24m
+
+We will finish our example by stopping the user load.
+We will terminate both infinite while
loops sending requests to the server and verify the result state:
$ kubectl get hpa
+NAME REFERENCE TARGET CURRENT MINPODS MAXPODS AGE
+php-apache ReplicationController/default/php-apache/ 50% 0% 1 10 21m
+
+$ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS AGE
+php-apache php-apache gcr.io/google_containers/hpa-example run=php-apache 1 31m
+
+As we see, in the presented case CPU utilization dropped to 0, and the number of replicas dropped to 1.
+ + + + + + + + +Horizontal pod autoscaling is a beta feature in Kubernetes 1.1. +It allows the number of pods in a replication controller or deployment to scale automatically based on observed CPU usage. +In the future also other metrics will be supported.
+ +In this document we explain how this feature works by walking you through an example of enabling horizontal pod autoscaling with the php-apache server.
+ +This example requires a running Kubernetes cluster and kubectl in the version at least 1.1. +Heapster monitoring needs to be deployed in the cluster +as horizontal pod autoscaler uses it to collect metrics +(if you followed getting started on GCE guide, +heapster monitoring will be turned-on by default).
+ +To demonstrate horizontal pod autoscaler we will use a custom docker image based on php-apache server. +The image can be found here. +It defines index.php page which performs some CPU intensive computations.
+ +First, we will start a replication controller running the image and expose it as an external service:
+ + + +$ kubectl run php-apache --image=gcr.io/google_containers/hpa-example --requests=cpu=200m
+replicationcontroller "php-apache" created
+
+$ kubectl expose rc php-apache --port=80 --type=LoadBalancer
+service "php-apache" exposed
+
+Now, we will wait some time and verify that both the replication controller and the service were correctly created and are running. We will also determine the IP address of the service:
+ +$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+php-apache-wa3t1 1/1 Running 0 12m
+
+$ kubectl describe services php-apache | grep "LoadBalancer Ingress"
+LoadBalancer Ingress: 146.148.24.244
+
+We may now check that php-apache server works correctly by calling curl
with the service’s IP:
$ curl http://146.148.24.244
+OK!
+
+Please notice that when exposing the service we assumed that our cluster runs on a provider which supports load balancers (e.g.: on GCE).
+If load balancers are not supported (e.g.: on Vagrant), we can expose php-apache service as ClusterIP
and connect to it using the proxy on the master:
$ kubectl expose rc php-apache --port=80 --type=ClusterIP
+service "php-apache" exposed
+
+$ kubectl cluster-info | grep master
+Kubernetes master is running at https://146.148.6.215
+
+$ curl -k -u <admin>:<password> https://146.148.6.215/api/v1/proxy/namespaces/default/services/php-apache/
+OK!
+
+Now that the server is running, we will create a horizontal pod autoscaler for it. +To create it, we will use the hpa-php-apache.yaml file, which looks like this:
+ +apiVersion: extensions/v1beta1
+kind: HorizontalPodAutoscaler
+metadata:
+ name: php-apache
+ namespace: default
+spec:
+ scaleRef:
+ kind: ReplicationController
+ name: php-apache
+ namespace: default
+ minReplicas: 1
+ maxReplicas: 10
+ cpuUtilization:
+ targetPercentage: 50
+
+This defines a horizontal pod autoscaler that maintains between 1 and 10 replicas of the Pods +controlled by the php-apache replication controller we created in the first step of these instructions. +Roughly speaking, the horizontal autoscaler will increase and decrease the number of replicas +(via the replication controller) so as to maintain an average CPU utilization across all Pods of 50% +(since each pod requests 200 milli-cores by kubectl run, this means average CPU utilization of 100 milli-cores). +See here for more details on the algorithm.
+ +We will create the autoscaler by executing the following command:
+ +$ kubectl create -f docs/user-guide/horizontal-pod-autoscaling/hpa-php-apache.yaml
+horizontalpodautoscaler "php-apache" created
+
+Alternatively, we can create the autoscaler using kubectl autoscale. +The following command will create the equivalent autoscaler as defined in the hpa-php-apache.yaml file:
+ +
+$ kubectl autoscale rc php-apache --cpu-percent=50 --min=1 --max=10
+replicationcontroller "php-apache" autoscaled
+
+
+
+We may check the current status of autoscaler by running:
+ +$ kubectl get hpa
+NAME REFERENCE TARGET CURRENT MINPODS MAXPODS AGE
+php-apache ReplicationController/default/php-apache/ 50% 0% 1 10 27s
+
+Please note that the current CPU consumption is 0% as we are not sending any requests to the server
+(the CURRENT
column shows the average across all the pods controlled by the corresponding replication controller).
Now, we will see how the autoscaler reacts on the increased load of the server. +We will start an infinite loop of queries to our server (please run it in a different terminal):
+ +$ while true; do curl http://146.148.6.244; done
+
+We may examine, how CPU load was increased (the results should be visible after about 3-4 minutes) by executing:
+ +$ kubectl get hpa
+NAME REFERENCE TARGET CURRENT MINPODS MAXPODS AGE
+php-apache ReplicationController/default/php-apache/ 50% 305% 1 10 4m
+
+In the case presented here, it bumped CPU consumption to 305% of the request. +As a result, the replication controller was resized to 7 replicas:
+ +$ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS AGE
+php-apache php-apache gcr.io/google_containers/hpa-example run=php-apache 7 18m
+
+Now, we may increase the load even more by running yet another infinite loop of queries (in yet another terminal):
+ +$ while true; do curl http://146.148.6.244; done
+
+In the case presented here, it increased the number of serving pods to 10:
+ +$ kubectl get hpa
+NAME REFERENCE TARGET CURRENT MINPODS MAXPODS AGE
+php-apache ReplicationController/default/php-apache/ 50% 65% 1 10 14m
+
+$ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS AGE
+php-apache php-apache gcr.io/google_containers/hpa-example run=php-apache 10 24m
+
+We will finish our example by stopping the user load.
+We will terminate both infinite while
loops sending requests to the server and verify the result state:
$ kubectl get hpa
+NAME REFERENCE TARGET CURRENT MINPODS MAXPODS AGE
+php-apache ReplicationController/default/php-apache/ 50% 0% 1 10 21m
+
+$ kubectl get rc
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS AGE
+php-apache php-apache gcr.io/google_containers/hpa-example run=php-apache 1 31m
+
+As we see, in the presented case CPU utilization dropped to 0, and the number of replicas dropped to 1.
+ + + + + + + + +All objects in the Kubernetes REST API are unambiguously identified by a Name and a UID.
+ +For non-unique user-provided attributes, Kubernetes provides labels and annotations.
+ +Names are generally client-provided. Only one object of a given kind can have a given name at a time (i.e., they are spatially unique). But if you delete an object, you can make a new object with the same name. Names are the used to refer to an object in a resource URL, such as /api/v1/pods/some-name
. By convention, the names of Kubernetes resources should be up to maximum length of 253 characters and consist of lower case alphanumeric characters, -
, and .
, but certain resources have more specific restrictions. See the identifiers design doc for the precise syntax rules for names.
UID are generated by Kubernetes. Every object created over the whole lifetime of a Kubernetes cluster has a distinct UID (i.e., they are spatially and temporally unique).
+ + + + + + + + + +Each container in a pod has its own image. Currently, the only type of image supported is a Docker Image.
+ +You create your Docker image and push it to a registry before referring to it in a Kubernetes pod.
+ +The image
property of a container supports the same syntax as the docker
command does, including private registries and tags.
Table of Contents +
+ +The default pull policy is IfNotPresent
which causes the Kubelet to not
+pull an image if it already exists. If you would like to always force a pull
+you must set a pull image policy of Always
or specify a :latest
tag on
+your image.
Private registries may require keys to read images from them. +Credentials can be provided in several ways: + - Using Google Container Registry + - Per-cluster + - automatically configured on Google Compute Engine or Google Container Engine + - all pods can read the project’s private registry + - Configuring Nodes to Authenticate to a Private Registry + - all pods can read any configured private registries + - requires node configuration by cluster administrator + - Pre-pulling Images + - all pods can use any images cached on a node + - requires root access to all nodes to setup + - Specifying ImagePullSecrets on a Pod + - only pods which provide own keys can access the private registry +Each option is described in more detail below.
+ +Kubernetes has native support for the Google Container +Registry (GCR), when running on Google Compute +Engine (GCE). If you are running your cluster on GCE or Google Container Engine (GKE), simply +use the full image name (e.g. gcr.io/my_project/image:tag).
+ +All pods in a cluster will have read access to images in this registry.
+ +The kubelet will authenticate to GCR using the instance’s
+Google service account. The service account on the instance
+will have a https://www.googleapis.com/auth/devstorage.read_only
,
+so it can pull from the project’s GCR, but not push.
Note: if you are running on Google Container Engine (GKE), there will already be a .dockercfg
on each node
+with credentials for Google Container Registry. You cannot use this approach.
Note: this approach is suitable if you can control node configuration. It +will not work reliably on GCE, and any other cloud provider that does automatic +node replacement.
+ +Docker stores keys for private registries in the $HOME/.dockercfg
file. If you put this
+in the $HOME
of root
on a kubelet, then docker will use it.
Here are the recommended steps to configuring your nodes to use a private registry. In this
+example, run these on your desktop/laptop:
+ 1. run docker login [server]
for each set of credentials you want to use.
+ 1. view $HOME/.dockercfg
in an editor to ensure it contains just the credentials you want to use.
+ 1. get a list of your nodes
+ - for example: nodes=$(kubectl get nodes -o template --template=' ')
+ 1. copy your local .dockercfg
to the home directory of root on each node.
+ - for example: for n in $nodes; do scp ~/.dockercfg root@$n:/root/.dockercfg; done
Verify by creating a pod that uses a private image, e.g.:
+ +$ cat <<EOF > /tmp/private-image-test-1.yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: private-image-test-1
+spec:
+ containers:
+ - name: uses-private-image
+ image: $PRIVATE_IMAGE_NAME
+ imagePullPolicy: Always
+ command: [ "echo", "SUCCESS" ]
+EOF
+$ kubectl create -f /tmp/private-image-test-1.yaml
+pods/private-image-test-1
+$
+
+If everything is working, then, after a few moments, you should see:
+ +$ kubectl logs private-image-test-1
+SUCCESS
+
+If it failed, then you will see:
+ +$ kubectl describe pods/private-image-test-1 | grep "Failed"
+ Fri, 26 Jun 2015 15:36:13 -0700 Fri, 26 Jun 2015 15:39:13 -0700 19 {kubelet node-i2hq} spec.containers{uses-private-image} failed Failed to pull image "user/privaterepo:v1": Error: image user/privaterepo:v1 not found
+
+You must ensure all nodes in the cluster have the same .dockercfg
. Otherwise, pods will run on
+some nodes and fail to run on others. For example, if you use node autoscaling, then each instance
+template needs to include the .dockercfg
or mount a drive that contains it.
All pods will have read access to images in any private registry once private
+registry keys are added to the .dockercfg
.
This was tested with a private docker repository as of 26 June with Kubernetes version v0.19.3. +It should also work for a private registry such as quay.io, but that has not been tested.
+ +Note: if you are running on Google Container Engine (GKE), there will already be a .dockercfg
on each node
+with credentials for Google Container Registry. You cannot use this approach.
Note: this approach is suitable if you can control node configuration. It +will not work reliably on GCE, and any other cloud provider that does automatic +node replacement.
+ +Be default, the kubelet will try to pull each image from the specified registry.
+However, if the imagePullPolicy
property of the container is set to IfNotPresent
or Never
,
+then a local image is used (preferentially or exclusively, respectively).
If you want to rely on pre-pulled images as a substitute for registry authentication, +you must ensure all nodes in the cluster have the same pre-pulled images.
+ +This can be used to preload certain images for speed or as an alternative to authenticating to a private registry.
+ +All pods will have read access to any pre-pulled images.
+ +Note: This approach is currently the recommended approach for GKE, GCE, and any cloud-providers +where node creation is automated.
+ +Kubernetes supports specifying registry keys on a pod.
+ +First, create a .dockercfg
, such as running docker login <registry.domain>
.
+Then put the resulting .dockercfg
file into a secret resource. For example:
$ docker login
+Username: janedoe
+Password: ●●●●●●●●●●●
+Email: jdoe@example.com
+WARNING: login credentials saved in /Users/jdoe/.dockercfg.
+Login Succeeded
+
+$ echo $(cat ~/.dockercfg)
+{ "https://index.docker.io/v1/": { "auth": "ZmFrZXBhc3N3b3JkMTIK", "email": "jdoe@example.com" } }
+
+$ cat ~/.dockercfg | base64
+eyAiaHR0cHM6Ly9pbmRleC5kb2NrZXIuaW8vdjEvIjogeyAiYXV0aCI6ICJabUZyWlhCaGMzTjNiM0prTVRJSyIsICJlbWFpbCI6ICJqZG9lQGV4YW1wbGUuY29tIiB9IH0K
+
+$ cat > /tmp/image-pull-secret.yaml <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+ name: myregistrykey
+data:
+ .dockercfg: eyAiaHR0cHM6Ly9pbmRleC5kb2NrZXIuaW8vdjEvIjogeyAiYXV0aCI6ICJabUZyWlhCaGMzTjNiM0prTVRJSyIsICJlbWFpbCI6ICJqZG9lQGV4YW1wbGUuY29tIiB9IH0K
+type: kubernetes.io/dockercfg
+EOF
+
+$ kubectl create -f /tmp/image-pull-secret.yaml
+secrets/myregistrykey
+$
+
+If you get the error message error: no objects passed to create
, it may mean the base64 encoded string is invalid.
+If you get an error message like Secret "myregistrykey" is invalid: data[.dockercfg]: invalid value ...
it means
+the data was successfully un-base64 encoded, but could not be parsed as a dockercfg file.
This process only needs to be done one time (per namespace).
+ +Now, you can create pods which reference that secret by adding an imagePullSecrets
+section to a pod definition.
apiVersion: v1
+kind: Pod
+metadata:
+ name: foo
+spec:
+ containers:
+ - name: foo
+ image: janedoe/awesomeapp:v1
+ imagePullSecrets:
+ - name: myregistrykey
+
+This needs to be done for each pod that is using a private registry. +However, setting of this field can be automated by setting the imagePullSecrets +in a serviceAccount resource.
+ +Currently, all pods will potentially have read access to any images which were +pulled using imagePullSecrets. That is, imagePullSecrets does NOT protect your +images from being seen by other users in the cluster. Our intent +is to fix that.
+ +You can use this in conjunction with a per-node .dockerfile
. The credentials
+will be merged. This approach will work on Google Container Engine (GKE).
There are a number of solutions for configuring private registries. Here are some +common use cases and suggested solutions.
+ +imagePullSecrets
.Table of Contents +
+ + + + + +The user guide is intended for anyone who wants to run programs and services on an existing Kubernetes cluster. Setup and administration of a Kubernetes cluster is described in the Cluster Admin Guide. The Developer Guide is for anyone wanting to either write code which directly accesses the Kubernetes API, or to contribute directly to the Kubernetes project.
+ +Please ensure you have completed the prerequisites for running examples from the user guide.
+ +If you don’t have much familiarity with Kubernetes, we recommend you read the following sections in order:
+ +Table of Contents +
+ +Terminology
+ +Throughout this doc you will see a few terms that are sometimes used interchangably elsewhere, that might cause confusion. This section attempts to clarify them.
+ +Typically, services and pods have IPs only routable by the cluster network. All traffic that ends up at an edge router is either dropped or forwarded elsewhere. Conceptually, this might look like:
+ +
+ internet
+ |
+ ------------
+ [ Services ]
+
+
+
+An Ingress is a collection of rules that allow inbound connections to reach the cluster services.
+ +
+ internet
+ |
+ [ Ingress ]
+ --|-----|--
+ [ Services ]
+
+
+
+It can be configured to give services externally-reachable urls, load balance traffic, terminate SSL, offer name based virtual hosting etc. Users request ingress by POSTing the Ingress resource to the API server. An Ingress controller is responsible for fulfilling the Ingress, usually with a loadbalancer, though it may also configure your edge router or additional frontends to help handle the traffic in an HA manner.
+ +Before you start using the Ingress resource, there are a few things you should understand: +* The Ingress resource is not available in any Kubernetes release prior to 1.1 +* You need an Ingress controller to satisfy an Ingress. Simply creating the resource will have no effect. +* On GCE/GKE there should be a L7 cluster addon, on other platforms you either need to write your own or deploy an existing controller as a pod. +* The resource currently does not support HTTPS, but will do so before it leaves beta.
+ +A minimal Ingress might look like:
+ +01. apiVersion: extensions/v1beta1
+02. kind: Ingress
+03. metadata:
+04. name: test-ingress
+05. spec:
+06. rules:
+07. - http:
+08. paths:
+09. - path: /testpath
+10. backend:
+11. serviceName: test
+12. servicePort: 80
+
+POSTing this to the API server will have no effect if you have not configured an Ingress controller.
+ +Lines 1-4: As with all other Kubernetes config, an Ingress needs apiVersion
, kind
, and metadata
fields. For general information about working with config files, see here, here, and here.
Lines 5-7: Ingress spec has all the information needed to configure a loadbalancer or proxy server. Most importantly, it contains a list of rules matched against all incoming requests. Currently the Ingress resource only supports http rules.
+ +Lines 8-9: Each http rule contains the following information: A host (eg: foo.bar.com, defaults to * in this example), a list of paths (eg: /testpath) each of which has an associated backend (test:80). Both the host and path must match the content of an incoming request before the loadbalancer directs traffic to the backend.
+ +Lines 10-12: A backend is a service:port combination as described in the services doc. Ingress traffic is typically sent directly to the endpoints matching a backend.
+ +Global Parameters: For the sake of simplicity the example Ingress has no global parameters, see the api-reference for a full definition of the resource. One can specify a global default backend in the absence of which requests that don’t match a path in the spec are sent to the default backend of the Ingress controller. Though the Ingress resource doesn’t support HTTPS yet, security configs would also be global.
+ +In order for the Ingress resource to work, the cluster must have an Ingress controller running. This is unlike other types of controllers, which typically run as part of the kube-controller-manager
binary, and which are typically started automatically as part of cluster creation. You need to choose the ingress controller implementation that is the best fit for your cluster, or implement one. Examples and instructions can be found here.
There are existing Kubernetes concepts that allow you to expose a single service (see alternatives), however you can do so through an Ingress as well, by specifying a default backend with no rules.
+ + + +apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+ name: test-ingress
+spec:
+ backend:
+ serviceName: testsvc
+ servicePort: 80
+
+If you create it using kubectl -f
you should see:
$ kubectl get ing
+NAME RULE BACKEND ADDRESS
+test-ingress - testsvc:80 107.178.254.228
+
+Where 107.178.254.228
is the IP allocated by the Ingress controller to satisfy this Ingress. The RULE
column shows that all traffic send to the IP is directed to the Kubernetes Service listed under BACKEND
.
As described previously, pods within kubernetes have ips only visible on the cluster network, so we need something at the edge accepting ingress traffic and proxying it to the right endpoints. This component is usually a highly available loadbalancer/s. An Ingress allows you to keep the number of loadbalancers down to a minimum, for example, a setup like:
+ +
+foo.bar.com -> 178.91.123.132 -> / foo s1:80
+ / bar s2:80
+
+
+
+would require an Ingress such as:
+ +apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+ name: test
+spec:
+ rules:
+ - host: foo.bar.com
+ http:
+ paths:
+ - path: /foo
+ backend:
+ serviceName: s1
+ servicePort: 80
+ - path: /bar
+ backend:
+ serviceName: s2
+ servicePort: 80
+
+When you create the Ingress with kubectl create -f
:
+$ kubectl get ing
+NAME RULE BACKEND ADDRESS
+test -
+ foo.bar.com
+ /foo s1:80
+ /bar s2:80
+
+
+
+The Ingress controller will provision an implementation specific loadbalancer that satisfies the Ingress, as long as the services (s1, s2) exist. When it has done so, you will see the address of the loadbalancer under the last column of the Ingress.
+ +Name-based virtual hosts use multiple host names for the same IP address.
+ +
+
+foo.bar.com --| |-> foo.bar.com s1:80
+ | 178.91.123.132 |
+bar.foo.com --| |-> bar.foo.com s2:80
+
+
+
+The following Ingress tells the backing loadbalancer to route requests based on the Host header.
+ +apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+ name: test
+spec:
+ rules:
+ - host: foo.bar.com
+ http:
+ paths:
+ - backend:
+ serviceName: s1
+ servicePort: 80
+ - host: bar.foo.com
+ http:
+ paths:
+ - backend:
+ serviceName: s2
+ servicePort: 80
+
+Default Backends: An Ingress with no rules, like the one shown in the previous section, sends all traffic to a single default backend. You can use the same technique to tell a loadbalancer where to find your website’s 404 page, by specifying a set of rules and a default backend. Traffic is routed to your default backend if none of the Hosts in your Ingress match the Host in the request header, and/or none of the paths match the url of the request.
+ +An Ingress controller is bootstrapped with some loadbalancing policy settings that it applies to all Ingress, such as the loadbalancing algorithm, backend weight scheme etc. More advanced loadbalancing concepts (eg: persistent sessions, dynamic weights) are not yet exposed through the Ingress. You can still get these features through the service loadbalancer. With time, we plan to distil loadbalancing patterns that are applicable cross platform into the Ingress resource.
+ +It’s also worth noting that even though health checks are not exposed directly through the Ingress, there exist parallel concepts in Kubernetes such as readiness probes which allow you to achieve the same end result.
+ +Say you’d like to add a new Host to an existing Ingress, you can update it by editing the resource:
+ +$ kubectl get ing
+NAME RULE BACKEND ADDRESS
+test - 178.91.123.132
+ foo.bar.com
+ /foo s1:80
+$ kubectl edit ing test
+
+This should pop up an editor with the existing yaml, modify it to include the new Host.
+ +spec:
+ rules:
+ - host: foo.bar.com
+ http:
+ paths:
+ - backend:
+ serviceName: s1
+ servicePort: 80
+ path: /foo
+ - host: bar.baz.com
+ http:
+ paths:
+ - backend:
+ serviceName: s2
+ servicePort: 80
+ path: /foo
+..
+
+saving it will update the resource in the API server, which should tell the Ingress controller to reconfigure the loadbalancer.
+ +$ kubectl get ing
+NAME RULE BACKEND ADDRESS
+test - 178.91.123.132
+ foo.bar.com
+ /foo s1:80
+ bar.baz.com
+ /foo s2:80
+
+You can achieve the same by invoking kubectl replace -f
on a modified Ingress yaml file.
Please track the L7 and Ingress proposal for more details on the evolution of the resource, and the Ingress sub-repository for more details on the evolution of various Ingress controllers.
+ +You can expose a Service in multiple ways that don’t directly involve the Ingress resource: +* Use Service.Type=LoadBalancer +* Use Service.Type=NodePort +* Use a [Port Proxy] (https://github.com/kubernetes/contrib/tree/master/for-demos/proxy-to-service) +* Deploy the Service loadbalancer. This allows you to share a single IP among multiple Services and achieve more advanced loadbalancing through Service Annotations.
+ + + + + + + + + +Once your application is running, you’ll inevitably need to debug problems with it.
+Earlier we described how you can use kubectl get pods
to retrieve simple status information about
+your pods. But there are a number of ways to get even more information about your application.
Table of Contents +
+ + + + + +kubectl describe pod
to fetch details about podsFor this example we’ll use a ReplicationController to create two pods, similar to the earlier example.
+ +apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: my-nginx
+spec:
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx
+ resources:
+ limits:
+ memory: "128Mi"
+ cpu: "500m"
+ ports:
+ - containerPort: 80
+
+$ kubectl create -f ./my-nginx-rc.yaml
+replicationcontrollers/my-nginx
+
+$ kubectl get pods
+NAME READY REASON RESTARTS AGE
+my-nginx-gy1ij 1/1 Running 0 1m
+my-nginx-yv5cn 1/1 Running 0 1m
+
+We can retrieve a lot more information about each of these pods using kubectl describe pod
. For example:
$ kubectl describe pod my-nginx-gy1ij
+Name: my-nginx-gy1ij
+Image(s): nginx
+Node: kubernetes-minion-y3vk/10.240.154.168
+Labels: app=nginx
+Status: Running
+Reason:
+Message:
+IP: 10.244.1.4
+Replication Controllers: my-nginx (2/2 replicas created)
+Containers:
+ nginx:
+ Image: nginx
+ Limits:
+ cpu: 500m
+ memory: 128Mi
+ State: Running
+ Started: Thu, 09 Jul 2015 15:33:07 -0700
+ Ready: True
+ Restart Count: 0
+Conditions:
+ Type Status
+ Ready True
+Events:
+ FirstSeen LastSeen Count From SubobjectPath Reason Message
+ Thu, 09 Jul 2015 15:32:58 -0700 Thu, 09 Jul 2015 15:32:58 -0700 1 {scheduler } scheduled Successfully assigned my-nginx-gy1ij to kubernetes-minion-y3vk
+ Thu, 09 Jul 2015 15:32:58 -0700 Thu, 09 Jul 2015 15:32:58 -0700 1 {kubelet kubernetes-minion-y3vk} implicitly required container POD pulled Pod container image "gcr.io/google_containers/pause:0.8.0" already present on machine
+ Thu, 09 Jul 2015 15:32:58 -0700 Thu, 09 Jul 2015 15:32:58 -0700 1 {kubelet kubernetes-minion-y3vk} implicitly required container POD created Created with docker id cd1644065066
+ Thu, 09 Jul 2015 15:32:58 -0700 Thu, 09 Jul 2015 15:32:58 -0700 1 {kubelet kubernetes-minion-y3vk} implicitly required container POD started Started with docker id cd1644065066
+ Thu, 09 Jul 2015 15:33:06 -0700 Thu, 09 Jul 2015 15:33:06 -0700 1 {kubelet kubernetes-minion-y3vk} spec.containers{nginx} pulled Successfully pulled image "nginx"
+ Thu, 09 Jul 2015 15:33:06 -0700 Thu, 09 Jul 2015 15:33:06 -0700 1 {kubelet kubernetes-minion-y3vk} spec.containers{nginx} created Created with docker id 56d7a7b14dac
+ Thu, 09 Jul 2015 15:33:07 -0700 Thu, 09 Jul 2015 15:33:07 -0700 1 {kubelet kubernetes-minion-y3vk} spec.containers{nginx} started Started with docker id 56d7a7b14dac
+
+Here you can see configuration information about the container(s) and Pod (labels, resource requirements, etc.), as well as status information about the container(s) and Pod (state, readiness, restart count, events, etc.)
+ +The container state is one of Waiting, Running, or Terminated. Depending on the state, additional information will be provided – here you can see that for a container in Running state, the system tells you when the container started.
+ +Ready tells you whether the container passed its last readiness probe. (In this case, the container does not have a readiness probe configured; the container is assumed to be ready if no readiness probe is configured.)
+ +Restart Count tells you how many times the container has restarted; this information can be useful for detecting crash loops in containers that are configured with a restart policy of “always.”
+ +Currently the only Condition associated with a Pod is the binary Ready condition, which indicates that the pod is able to service requests and should be added to the load balancing pools of all matching services.
+ +Lastly, you see a log of recent events related to your Pod. The system compresses multiple identical events by indicating the first and last time it was seen and the number of times it was seen. “From” indicates the component that is logging the event, “SubobjectPath” tells you which object (e.g. container within the pod) is being referred to, and “Reason” and “Message” tell you what happened.
+ +A common scenario that you can detect using events is when you’ve created a Pod that won’t fit on any node. For example, the Pod might request more resources than are free on any node, or it might specify a label selector that doesn’t match any nodes. Let’s say we created the previous Replication Controller with 5 replicas (instead of 2) and requesting 600 millicores instead of 500, on a four-node cluster where each (virtual) machine has 1 CPU. In that case one of the Pods will not be able to schedule. (Note that because of the cluster addon pods such as fluentd, skydns, etc., that run on each node, if we requested 1000 millicores then none of the Pods would be able to schedule.)
+ +$ kubectl get pods
+NAME READY REASON RESTARTS AGE
+my-nginx-9unp9 0/1 Pending 0 8s
+my-nginx-b7zs9 0/1 Running 0 8s
+my-nginx-i595c 0/1 Running 0 8s
+my-nginx-iichp 0/1 Running 0 8s
+my-nginx-tc2j9 0/1 Running 0 8s
+
+To find out why the my-nginx-9unp9 pod is not running, we can use kubectl describe pod
on the pending Pod and look at its events:
$ kubectl describe pod my-nginx-9unp9
+Name: my-nginx-9unp9
+Image(s): nginx
+Node: /
+Labels: app=nginx
+Status: Pending
+Reason:
+Message:
+IP:
+Replication Controllers: my-nginx (5/5 replicas created)
+Containers:
+ nginx:
+ Image: nginx
+ Limits:
+ cpu: 600m
+ memory: 128Mi
+ State: Waiting
+ Ready: False
+ Restart Count: 0
+Events:
+ FirstSeen LastSeen Count From SubobjectPath Reason Message
+ Thu, 09 Jul 2015 23:56:21 -0700 Fri, 10 Jul 2015 00:01:30 -0700 21 {scheduler } failedScheduling Failed for reason PodFitsResources and possibly others
+
+Here you can see the event generated by the scheduler saying that the Pod failed to schedule for reason PodFitsResources
(and possibly others). PodFitsResources
means there were not enough resources for the Pod on any of the nodes. Due to the way the event is generated, there may be other reasons as well, hence “and possibly others.”
To correct this situation, you can use kubectl scale
to update your Replication Controller to specify four or fewer replicas. (Or you could just leave the one Pod pending, which is harmless.)
Events such as the ones you saw at the end of kubectl describe pod
are persisted in etcd and provide high-level information on what is happening in the cluster. To list all events you can use
+kubectl get events
+
+
+
+but you have to remember that events are namespaced. This means that if you’re interested in events for some namespaced object (e.g. what happened with Pods in namespace my-namespace
) you need to explicitly provide a namespace to the command:
+kubectl get events --namespace=my-namespace
+
+
+
+To see events from all namespaces, you can use the --all-namespaces
argument.
In addition to kubectl describe pod
, another way to get extra information about a pod (beyond what is provided by kubectl get pod
) is to pass the -o yaml
output format flag to kubectl get pod
. This will give you, in YAML format, even more information than kubectl describe pod
–essentially all of the information the system has about the Pod. Here you will see things like annotations (which are key-value metadata without the label restrictions, that is used internally by Kubernetes system components), restart policy, ports, and volumes.
$ kubectl get pod my-nginx-i595c -o yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ annotations:
+ kubernetes.io/created-by: '{"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"ReplicationController","namespace":"default","name":"my-nginx","uid":"c555c14f-26d0-11e5-99cb-42010af00e4b","apiVersion":"v1","resourceVersion":"26174"}}'
+ creationTimestamp: 2015-07-10T06:56:21Z
+ generateName: my-nginx-
+ labels:
+ app: nginx
+ name: my-nginx-i595c
+ namespace: default
+ resourceVersion: "26243"
+ selfLink: /api/v1/namespaces/default/pods/my-nginx-i595c
+ uid: c558e44b-26d0-11e5-99cb-42010af00e4b
+spec:
+ containers:
+ - image: nginx
+ imagePullPolicy: IfNotPresent
+ name: nginx
+ ports:
+ - containerPort: 80
+ protocol: TCP
+ resources:
+ limits:
+ cpu: 600m
+ memory: 128Mi
+ terminationMessagePath: /dev/termination-log
+ volumeMounts:
+ - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+ name: default-token-zkhkk
+ readOnly: true
+ dnsPolicy: ClusterFirst
+ nodeName: kubernetes-minion-u619
+ restartPolicy: Always
+ serviceAccountName: default
+ volumes:
+ - name: default-token-zkhkk
+ secret:
+ secretName: default-token-zkhkk
+status:
+ conditions:
+ - status: "True"
+ type: Ready
+ containerStatuses:
+ - containerID: docker://9506ace0eb91fbc31aef1d249e0d1d6d6ef5ebafc60424319aad5b12e3a4e6a9
+ image: nginx
+ imageID: docker://319d2015d149943ff4d2a20ddea7d7e5ce06a64bbab1792334c0d3273bbbff1e
+ lastState: {}
+ name: nginx
+ ready: true
+ restartCount: 0
+ state:
+ running:
+ startedAt: 2015-07-10T06:56:28Z
+ hostIP: 10.240.112.234
+ phase: Running
+ podIP: 10.244.3.4
+ startTime: 2015-07-10T06:56:21Z
+
+Sometimes when debugging it can be useful to look at the status of a node – for example, because you’ve noticed strange behavior of a Pod that’s running on the node, or to find out why a Pod won’t schedule onto the node. As with Pods, you can use kubectl describe node
and kubectl get node -o yaml
to retrieve detailed information about nodes. For example, here’s what you’ll see if a node is down (disconnected from the network, or kubelet dies and won’t restart, etc.). Notice the events that show the node is NotReady, and also notice that the pods are no longer running (they are evicted after five minutes of NotReady status).
$ kubectl get nodes
+NAME LABELS STATUS
+kubernetes-minion-861h kubernetes.io/hostname=kubernetes-minion-861h NotReady
+kubernetes-minion-bols kubernetes.io/hostname=kubernetes-minion-bols Ready
+kubernetes-minion-st6x kubernetes.io/hostname=kubernetes-minion-st6x Ready
+kubernetes-minion-unaj kubernetes.io/hostname=kubernetes-minion-unaj Ready
+
+$ kubectl describe node kubernetes-minion-861h
+Name: kubernetes-minion-861h
+Labels: kubernetes.io/hostname=kubernetes-minion-861h
+CreationTimestamp: Fri, 10 Jul 2015 14:32:29 -0700
+Conditions:
+ Type Status LastHeartbeatTime LastTransitionTime Reason Message
+ Ready Unknown Fri, 10 Jul 2015 14:34:32 -0700 Fri, 10 Jul 2015 14:35:15 -0700 Kubelet stopped posting node status.
+Addresses: 10.240.115.55,104.197.0.26
+Capacity:
+ cpu: 1
+ memory: 3800808Ki
+ pods: 100
+Version:
+ Kernel Version: 3.16.0-0.bpo.4-amd64
+ OS Image: Debian GNU/Linux 7 (wheezy)
+ Container Runtime Version: docker://Unknown
+ Kubelet Version: v0.21.1-185-gffc5a86098dc01
+ Kube-Proxy Version: v0.21.1-185-gffc5a86098dc01
+PodCIDR: 10.244.0.0/24
+ExternalID: 15233045891481496305
+Pods: (0 in total)
+ Namespace Name
+Events:
+ FirstSeen LastSeen Count From SubobjectPath Reason Message
+ Fri, 10 Jul 2015 14:32:28 -0700 Fri, 10 Jul 2015 14:32:28 -0700 1 {kubelet kubernetes-minion-861h} NodeNotReady Node kubernetes-minion-861h status is now: NodeNotReady
+ Fri, 10 Jul 2015 14:32:30 -0700 Fri, 10 Jul 2015 14:32:30 -0700 1 {kubelet kubernetes-minion-861h} NodeNotReady Node kubernetes-minion-861h status is now: NodeNotReady
+ Fri, 10 Jul 2015 14:33:00 -0700 Fri, 10 Jul 2015 14:33:00 -0700 1 {kubelet kubernetes-minion-861h} starting Starting kubelet.
+ Fri, 10 Jul 2015 14:33:02 -0700 Fri, 10 Jul 2015 14:33:02 -0700 1 {kubelet kubernetes-minion-861h} NodeReady Node kubernetes-minion-861h status is now: NodeReady
+ Fri, 10 Jul 2015 14:35:15 -0700 Fri, 10 Jul 2015 14:35:15 -0700 1 {controllermanager } NodeNotReady Node kubernetes-minion-861h status is now: NodeNotReady
+
+
+$ kubectl get node kubernetes-minion-861h -o yaml
+apiVersion: v1
+kind: Node
+metadata:
+ creationTimestamp: 2015-07-10T21:32:29Z
+ labels:
+ kubernetes.io/hostname: kubernetes-minion-861h
+ name: kubernetes-minion-861h
+ resourceVersion: "757"
+ selfLink: /api/v1/nodes/kubernetes-minion-861h
+ uid: 2a69374e-274b-11e5-a234-42010af0d969
+spec:
+ externalID: "15233045891481496305"
+ podCIDR: 10.244.0.0/24
+ providerID: gce://striped-torus-760/us-central1-b/kubernetes-minion-861h
+status:
+ addresses:
+ - address: 10.240.115.55
+ type: InternalIP
+ - address: 104.197.0.26
+ type: ExternalIP
+ capacity:
+ cpu: "1"
+ memory: 3800808Ki
+ pods: "100"
+ conditions:
+ - lastHeartbeatTime: 2015-07-10T21:34:32Z
+ lastTransitionTime: 2015-07-10T21:35:15Z
+ reason: Kubelet stopped posting node status.
+ status: Unknown
+ type: Ready
+ nodeInfo:
+ bootID: 4e316776-b40d-4f78-a4ea-ab0d73390897
+ containerRuntimeVersion: docker://Unknown
+ kernelVersion: 3.16.0-0.bpo.4-amd64
+ kubeProxyVersion: v0.21.1-185-gffc5a86098dc01
+ kubeletVersion: v0.21.1-185-gffc5a86098dc01
+ machineID: ""
+ osImage: Debian GNU/Linux 7 (wheezy)
+ systemUUID: ABE5F6B4-D44B-108B-C46A-24CCE16C8B6E
+
+Learn about additional debugging tools, including:
+* Logging
+* Monitoring
+* Getting into containers via exec
+* Connecting to containers via proxies
+* Connecting to containers via port forwarding
Table of Contents +
+ +A job creates one or more pods and ensures that a specified number of them successfully terminate. +As pods successfully complete, the job tracks the successful completions. When a specified number +of successful completions is reached, the job itself is complete. Deleting a Job will cleanup the +pods it created.
+ +A simple case is to create 1 Job object in order to reliably run one Pod to completion. +A Job can also be used to run multiple pods in parallel.
+ +Here is an example Job config. It computes π to 2000 places and prints it out. +It takes around 10s to complete. +
+ +apiVersion: extensions/v1beta1
+kind: Job
+metadata:
+ name: pi
+spec:
+ selector:
+ matchLabels:
+ app: pi
+ template:
+ metadata:
+ name: pi
+ labels:
+ app: pi
+ spec:
+ containers:
+ - name: pi
+ image: perl
+ command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"]
+ restartPolicy: Never
+
+Run the example job by downloading the example file and then running this command:
+ +$ kubectl create -f ./job.yaml
+jobs/pi
+
+Check on the status of the job using this command:
+ +$ kubectl describe jobs/pi
+Name: pi
+Namespace: default
+Image(s): perl
+Selector: app=pi
+Parallelism: 2
+Completions: 1
+Labels: <none>
+Pods Statuses: 1 Running / 0 Succeeded / 0 Failed
+Events:
+ FirstSeen LastSeen Count From SubobjectPath Reason Message
+ ───────── ──────── ───── ──── ───────────── ────── ───────
+ 1m 1m 1 {job } SuccessfulCreate Created pod: pi-z548a
+
+To view completed pods of a job, use kubectl get pods --show-all
. The --show-all
will show completed pods too.
To list all the pods that belong to job in a machine readable form, you can use a command like this:
+ +$ pods=$(kubectl get pods --selector=app=pi --output=jsonpath={.items..metadata.name})
+echo $pods
+pi-aiw0a
+
+Here, the selector is the same as the selector for the job. The --output=jsonpath
option specifies an expression
+that just gets the name from each pod in the returned list.
View the standard output of one of the pods:
+ +$ kubectl logs pi-aiw0a
+3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679821480865132823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461284756482337867831652712019091456485669234603486104543266482133936072602491412737245870066063155881748815209209628292540917153643678925903600113305305488204665213841469519415116094330572703657595919530921861173819326117931051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798609437027705392171762931767523846748184676694051320005681271452635608277857713427577896091736371787214684409012249534301465495853710507922796892589235420199561121290219608640344181598136297747713099605187072113499999983729780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083814206171776691473035982534904287554687311595628638823537875937519577818577805321712268066130019278766111959092164201989380952572010654858632788659361533818279682303019520353018529689957736225994138912497217752834791315155748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035637076601047101819429555961989467678374494482553797747268471040475346462080466842590694912933136770289891521047521620569660240580381501935112533824300355876402474964732639141992726042699227967823547816360093417216412199245863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818347977535663698074265425278625518184175746728909777727938000816470600161452491921732172147723501414419735685481613611573525521334757418494684385233239073941433345477624168625189835694855620992192221842725502542568876717904946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886269456042419652850222106611863067442786220391949450471237137869609563643719172874677646575739624138908658326459958133904780275901
+
+As with all other Kubernetes config, a Job needs apiVersion
, kind
, and metadata
fields. For
+general information about working with config files, see here,
+here, and here.
A Job also needs a .spec
section.
The .spec.template
is the only required field of the .spec
.
The .spec.template
is a pod template. It has exactly
+the same schema as a pod, except it is nested and does not have an apiVersion
or
+kind
.
In addition to required fields for a Pod, a pod template in a job must specify appropriate +lables (see pod selector and an appropriate restart policy.
+ +Only a RestartPolicy
equal to Never
or OnFailure
are allowed.
The .spec.selector
field is a label query over a set of pods.
The spec.selector
is an object consisting of two fields:
+* matchLabels
- works the same as the .spec.selector
of a ReplicationController
+* matchExpressions
- allows to build more sophisticated selectors by specyfing key,
+ list of values and an operator that relates the key and values.
When the two are specified the result is ANDed.
+ +If .spec.selector
is unspecified, .spec.selector.matchLabels
will be defaulted to
+.spec.template.metadata.labels
.
Also you should not normally create any pods whose labels match this selector, either directly, +via another Job, or via another controller such as ReplicationController. Otherwise, the Job will +think that those pods were created by it. Kubernetes will not stop you from doing this.
+ +By default, a Job is complete when one Pod runs to successful completion. You can also specify that
+this needs to happen multiple times by specifying .spec.completions
with a value greater than 1.
+When multiple completions are requested, each Pod created by the Job controller has an identical
+spec
. In particular, all pods will have
+the same command line and the same image, the same volumes, and mostly the same environment
+variables. It is up to the user to arrange for the pods to do work on different things. For
+example, the pods might all access a shared work queue service to acquire work units.
To create multiple pods which are similar, but have slightly different arguments, environment +variables or images, use multiple Jobs.
+ +You can suggest how many pods should run concurrently by setting .spec.parallelism
to the number
+of pods you would like to have running concurrently. This number is a suggestion. The number
+running concurrently may be lower or higher for a variety of reasons. For example, it may be lower
+if the number of remaining completions is less, or as the controller is ramping up, or if it is
+throttling the job due to excessive failures. It may be higher for example if a pod is gracefully
+shutdown, and the replacement starts early.
If you do not specify .spec.parallelism
, then it defaults to .spec.completions
.
A Container in a Pod may fail for a number of reasons, such as because the process in it exited with
+a non-zero exit code, or the Container was killed for exceeding a memory limit, etc. If this
+happens, and the .spec.template.containers[].restartPolicy = "OnFailure"
, then the Pod stays
+on the node, but the Container is re-run. Therefore, your program needs to handle the the case when it is
+restarted locally, or else specify .spec.template.containers[].restartPolicy = "Never"
.
+See pods-states for more information on restartPolicy
.
An entire Pod can also fail, for a number of reasons, such as when the pod is kicked off the node
+(node is upgraded, rebooted, delelted, etc.), or if a container of the Pod fails and the
+.spec.template.containers[].restartPolicy = "Never"
. When a Pod fails, then the Job controller
+starts a new Pod. Therefore, your program needs to handle the case when it is restarted in a new
+pod. In particular, it needs to handle temporary files, locks, incomplete output and the like
+caused by previous runs.
Note that even if you specify .spec.parallelism = 1
and .spec.completions = 1
and
+.spec.template.containers[].restartPolicy = "Never"
, the same program may
+sometimes be started twice.
If you do specify .spec.parallelism
and .spec.completions
both greater than 1, then there may be
+multiple pods running at once. Therefore, your pods must also be tolerant of concurrency.
When the node that a pod is running on reboots or fails, the pod is terminated +and will not be restarted. However, a Job will create new pods to replace terminated ones. +For this reason, we recommend that you use a job rather than a bare pod, even if your application +requires only a single pod.
+ +Jobs are complementary to Replication Controllers. +A Replication Controller manages pods which are not expected to terminate (e.g. web servers), and a Job +manages pods that are expected to terminate (e.g. batch jobs).
+ +As discussed in life of a pod, Job
is only appropriate for pods with
+RestartPolicy
equal to OnFailure
or Never
. (Note: If RestartPolicy
is not set, the default
+value is Always
.)
Job objects are in the extensions
API Group.
Job objects have API version v1beta1
. Beta objects may
+undergo changes to their schema and/or semantics in future software releases, but
+similar functionality will be supported.
Support for creating Jobs at specified times/dates (i.e. cron) is expected in the next minor +release.
+ + + + + + + + + +JSONPath template is composed of JSONPath expressions enclosed by {}. +And we add three functions in addition to the original JSONPath syntax:
+ +$
operator is optional since the expression always start from the root object by default.""
to quote text inside JSONPath expression.range
operator to iterate list.The result object is printed as its String() function.
+ +Given the input:
+ +{
+ "kind": "List",
+ "items":[
+ {
+ "kind":"None",
+ "metadata":{"name":"127.0.0.1"},
+ "status":{
+ "capacity":{"cpu":"4"},
+ "addresses":[{"type": "LegacyHostIP", "address":"127.0.0.1"}]
+ }
+ },
+ {
+ "kind":"None",
+ "metadata":{"name":"127.0.0.2"},
+ "status":{
+ "capacity":{"cpu":"8"},
+ "addresses":[
+ {"type": "LegacyHostIP", "address":"127.0.0.2"},
+ {"type": "another", "address":"127.0.0.3"}
+ ]
+ }
+ }
+ ],
+ "users":[
+ {
+ "name": "myself",
+ "user": {}
+ },
+ {
+ "name": "e2e",
+ "user": {"username": "admin", "password": "secret"}
+ }
+ ]
+}
+
+Function | +Description | +Example | +Result | +
---|---|---|---|
text | +the plain text | +kind is {.kind} | +kind is List | +
”” | +quote | +{“{“} | +{ | +
@ | +the current object | +{@} | +the same as input | +
. or [] | +child operator | +{.kind} or {[‘kind’]} | +List | +
.. | +recursive descent | +{..name} | +127.0.0.1 127.0.0.2 myself e2e | +
* | +wildcard. Get all objects | +{.items[*].metadata.name} | +[127.0.0.1 127.0.0.2] | +
[start:end :step] | +subscript operator | +{.users[0].name} | +myself | +
[,] | +union operator | +{.items[*][‘metadata.name’, ‘status.capacity’]} | +127.0.0.1 127.0.0.2 map[cpu:4] map[cpu:8] | +
?() | +filter | +{.users[?(@.name==”e2e”)].user.password} | +secret | +
range, end | +iterate list | +{range .items[*]}[{.metadata.name}, {.status.capacity}] {end} | +[127.0.0.1, map[cpu:4]] [127.0.0.2, map[cpu:8]] | +
This document summarizes known issues with existing Kubernetes releases.
+ +Please consult this document before filing new bugs.
+ +exec
liveness/readiness probes leak resources due to Docker exec leaking resources (#10659)docker load
sometimes hangs which causes the kube-apiserver
not to start. Restarting the Docker daemon should fix the issue (#10868)kube-apiserver
so statistics aren’t collected for master daemons (#10891)type=LoadBalancer
can not use port 10250
because of Google Compute Engine firewall limitationskubectl
or the Kubernetes API (#11435)rados -p poolname listwatchers image_name.rbd
can list RBD clients that are mapping the image.Authentication in kubernetes can differ for different individuals.
+ +So in order to easily switch between multiple clusters, for multiple users, a kubeconfig file was defined.
+ +This file contains a series of authentication mechanisms and cluster connection information associated with nicknames. It also introduces the concept of a tuple of authentication information (user) and cluster connection information called a context that is also associated with a nickname.
+ +Multiple kubeconfig files are allowed. At runtime they are loaded and merged together along with override options specified from the command line (see rules below).
+ +http://issue.k8s.io/1755
+ +The below file contains a current-context
which will be used by default by clients which are using the file to connect to a cluster. Thus, this kubeconfig file has more information in it then we will necessarily have to use in a given session. You can see it defines many clusters, and users associated with those clusters. The context itself is associated with both a cluster AND a user.
current-context: federal-context
+apiVersion: v1
+clusters:
+- cluster:
+ api-version: v1
+ server: http://cow.org:8080
+ name: cow-cluster
+- cluster:
+ certificate-authority: path/to/my/cafile
+ server: https://horse.org:4443
+ name: horse-cluster
+- cluster:
+ insecure-skip-tls-verify: true
+ server: https://pig.org:443
+ name: pig-cluster
+contexts:
+- context:
+ cluster: horse-cluster
+ namespace: chisel-ns
+ user: green-user
+ name: federal-context
+- context:
+ cluster: pig-cluster
+ namespace: saw-ns
+ user: black-user
+ name: queen-anne-context
+kind: Config
+preferences:
+ colors: true
+users:
+- name: blue-user
+ user:
+ token: blue-token
+- name: green-user
+ user:
+ client-certificate: path/to/my/client/cert
+ client-key: path/to/my/client/key
+
+NOTE, that if you are deploying k8s via kube-up.sh, you do not need to create your own kubeconfig files, the script will do it for you.
+ +In any case, you can easily use this file as a template to create your own kubeconfig files.
+ +So, lets do a quick walk through the basics of the above file so you can easily modify it as needed…
+ +The above file would likely correspond to an api-server which was launched using the --token-auth-file=tokens.csv
option, where the tokens.csv file looked something like this:
+blue-user,blue-user,1
+mister-red,mister-red,2
+
+
+
+Also, since we have other users who validate using other mechanisms, the api-server would have probably been launched with other authentication options (there are many such options, make sure you understand which ones YOU care about before crafting a kubeconfig file, as nobody needs to implement all the different permutations of possible authentication schemes).
+ +In the above scenario, green-user would have to log in by providing certificates, whereas blue-user would just provide the token. All this information would be handled for us by the
+ +The rules for loading and merging the kubeconfig files are straightforward, but there are a lot of them. The final config is built in this order: + 1. Get the kubeconfig from disk. This is done with the following hierarchy and merge rules:
+ + If the CommandLineLocation (the value of the `kubeconfig` command line option) is set, use this file only. No merging. Only one instance of this flag is allowed.
+
+
+ Else, if EnvVarLocation (the value of $KUBECONFIG) is available, use it as a list of files that should be merged.
+ Merge files together based on the following rules.
+ Empty filenames are ignored. Files with non-deserializable content produced errors.
+ The first file to set a particular value or map key wins and the value or map key is never changed.
+ This means that the first file to set CurrentContext will have its context preserved. It also means that if two files specify a "red-user", only values from the first file's red-user are used. Even non-conflicting entries from the second file's "red-user" are discarded.
+
+
+ Otherwise, use HomeDirectoryLocation (~/.kube/config) with no merging. 1. Determine the context to use based on the first hit in this chain
+ 1. command line argument - the value of the `context` command line option
+ 1. current-context from the merged kubeconfig file
+ 1. Empty is allowed at this stage 1. Determine the cluster info and user to use. At this point, we may or may not have a context. They are built based on the first hit in this chain. (run it twice, once for user, once for cluster)
+ 1. command line argument - `user` for user name and `cluster` for cluster name
+ 1. If context is present, then use the context's value
+ 1. Empty is allowed 1. Determine the actual cluster info to use. At this point, we may or may not have a cluster info. Build each piece of the cluster info based on the chain (first hit wins):
+ 1. command line arguments - `server`, `api-version`, `certificate-authority`, and `insecure-skip-tls-verify`
+ 1. If cluster info is present and a value for the attribute is present, use it.
+ 1. If you don't have a server location, error. 1. Determine the actual user info to use. User is built using the same rules as cluster info, EXCEPT that you can only have one authentication technique per user.
+ 1. Load precedence is 1) command line flag, 2) user fields from kubeconfig
+ 1. The command line flags are: `client-certificate`, `client-key`, `username`, `password`, and `token`.
+ 1. If there are two conflicting techniques, fail. 1. For any information still missing, use default values and potentially prompt for authentication information
+
+
+kubectl config <subcommand>
In order to more easily manipulate kubeconfig files, there are a series of subcommands to kubectl config
to help.
+See kubectl/kubectl_config.md for help.
$ kubectl config set-credentials myself --username=admin --password=secret
+$ kubectl config set-cluster local-server --server=http://localhost:8080
+$ kubectl config set-context default-context --cluster=local-server --user=myself
+$ kubectl config use-context default-context
+$ kubectl config set contexts.default-context.namespace the-right-prefix
+$ kubectl config view
+
+produces this output
+ +apiVersion: v1
+clusters:
+- cluster:
+ server: http://localhost:8080
+ name: local-server
+contexts:
+- context:
+ cluster: local-server
+ namespace: the-right-prefix
+ user: myself
+ name: default-context
+current-context: default-context
+kind: Config
+preferences: {}
+users:
+- name: myself
+ user:
+ password: secret
+ username: admin
+
+and a kubeconfig file that looks like this
+ +apiVersion: v1
+clusters:
+- cluster:
+ server: http://localhost:8080
+ name: local-server
+contexts:
+- context:
+ cluster: local-server
+ namespace: the-right-prefix
+ user: myself
+ name: default-context
+current-context: default-context
+kind: Config
+preferences: {}
+users:
+- name: myself
+ user:
+ password: secret
+ username: admin
+
+$ kubectl config set preferences.colors true
+$ kubectl config set-cluster cow-cluster --server=http://cow.org:8080 --api-version=v1
+$ kubectl config set-cluster horse-cluster --server=https://horse.org:4443 --certificate-authority=path/to/my/cafile
+$ kubectl config set-cluster pig-cluster --server=https://pig.org:443 --insecure-skip-tls-verify=true
+$ kubectl config set-credentials blue-user --token=blue-token
+$ kubectl config set-credentials green-user --client-certificate=path/to/my/client/cert --client-key=path/to/my/client/key
+$ kubectl config set-context queen-anne-context --cluster=pig-cluster --user=black-user --namespace=saw-ns
+$ kubectl config set-context federal-context --cluster=horse-cluster --user=green-user --namespace=chisel-ns
+$ kubectl config use-context federal-context
+
+So, tying this all together, a quick start to creating your own kubeconfig file:
+ +Take a good look and understand how you’re api-server is being launched: You need to know YOUR security requirements and policies before you can design a kubeconfig file for convenient authentication.
+Replace the snippet above with information for your cluster’s api-server endpoint.
+Make sure your api-server is launched in such a way that at least one user (i.e. green-user) credentials are provided to it. You will of course have to look at api-server documentation in order to determine the current state-of-the-art in terms of providing authentication details.
+Use this overview of the kubectl
command line interface to help you start running commands against Kubernetes clusters. This overview quickly covers kubectl
syntax, describes the command operations, and provides common examples. For details about each command, including all the supported flags and subcommands, see the kubectl reference documentation.
Table of contents: +
+ +TODO: Auto-generate this file to ensure it’s always in sync with any kubectl
changes, see #14177.
Use the following syntax to run kubectl
commands from your terminal window:
+kubectl [command] [TYPE] [NAME] [flags]
+
+
+
+where command
, TYPE
, NAME
, and flags
are:
command
: Specifies the operation that you want to perform on one or more resources, for example create
, get
, describe
, delete
.TYPE
: Specifies the resource type. Resource types are case-sensitive and you can specify the singular, plural, or abbreviated forms. For example, the following commands produce the same output:
```
+ +$ kubectl get pod pod1 + $ kubectl get pods pod1 + $ kubectl get po pod1
+ +```
+NAME
: Specifies the name of the resource. Names are case-sensitive. If the name is omitted, details for all resources are displayed, for example $ kubectl get pods
.
When performing an operation on multiple resources, you can specify each resource by type and name or specify one or more files:
+ * To specify resources by type and name:
+ * To group resources if they are all the same type: TYPE1 name1 name2 name<#>
+ Example: $ kubectl get pod example-pod1 example-pod2
+ * To specify multiple resource types individually: TYPE1/name1 TYPE1/name2 TYPE2/name3 TYPE<#>/name<#>
+ Example: $ kubectl get pod/example-pod1 replicationcontroller/example-rc1
+ * To specify resources with one or more files: -f file1 -f file2 -f file<#>
+ Use YAML rather than JSON since YAML tends to be more user-friendly, especially for configuration files.
+ Example: $ kubectl get pod -f ./pod.yaml
flags
: Specifies optional flags. For example, you can use the -s
or --server
flags to specify the address and port of the Kubernetes API server.If you need help, just run kubectl help
from the terminal window.
The following table includes short descriptions and the general syntax for all of the kubectl
operations:
Operation | +Syntax | +Description | +
---|---|---|
annotate |
+ kubectl annotate (-f FILENAME | TYPE NAME | TYPE/NAME) KEY_1=VAL_1 ... KEY_N=VAL_N [--overwrite] [--all] [--resource-version=version] [flags] |
+ Add or update the annotations of one or more resources. | +
api-versions |
+ kubectl api-versions [flags] |
+ List the API versions that are available. | +
apply |
+ kubectl apply -f FILENAME [flags] |
+ Apply a configuration change to a resource from a file or stdin. | +
attach |
+ kubectl attach POD -c CONTAINER [-i] [-t] [flags] |
+ Attach to a running container either to view the output stream or interact with the container (stdin). | +
autoscale |
+ autoscale (-f FILENAME | TYPE NAME | TYPE/NAME) [--min=MINPODS] --max=MAXPODS [--cpu-percent=CPU] [flags] |
+ Automatically scale the set of pods that are managed by a replication controller. | +
cluster-info |
+ kubectl cluster-info [flags] |
+ Display endpoint information about the master and services in the cluster. | +
config |
+ kubectl config SUBCOMMAND [flags] |
+ Modifies kubeconfig files. See the individual subcommands for details. | +
create |
+ kubectl create -f FILENAME [flags] |
+ Create one or more resources from a file or stdin. | +
delete |
+ kubectl delete (-f FILENAME | TYPE [NAME | /NAME | -l label | --all]) [flags] |
+ Delete resources either from a file, stdin, or specifying label selectors, names, resource selectors, or resources. | +
describe |
+ kubectl describe (-f FILENAME | TYPE [NAME_PREFIX | /NAME | -l label]) [flags] |
+ Display the detailed state of one or more resources. | +
edit |
+ kubectl edit (-f FILENAME | TYPE NAME | TYPE/NAME) [flags] |
+ Edit and update the definition of one or more resources on the server by using the default editor. | +
exec |
+ kubectl exec POD [-c CONTAINER] [-i] [-t] [flags] [-- COMMAND [args...]] |
+ Execute a command against a container in a pod. | +
expose |
+ kubectl expose (-f FILENAME | TYPE NAME | TYPE/NAME) [--port=port] [--protocol=TCP|UDP] [--target-port=number-or-name] [--name=name] [----external-ip=external-ip-of-service] [--type=type] [flags] |
+ Expose a replication controller, service, or pod as a new Kubernetes service. | +
get |
+ kubectl get (-f FILENAME | TYPE [NAME | /NAME | -l label]) [--watch] [--sort-by=FIELD] [[-o | --output]=OUTPUT_FORMAT] [flags] |
+ List one or more resources. | +
label |
+ kubectl label (-f FILENAME | TYPE NAME | TYPE/NAME) KEY_1=VAL_1 ... KEY_N=VAL_N [--overwrite] [--all] [--resource-version=version] [flags] |
+ Add or update the labels of one or more resources. | +
logs |
+ kubectl logs POD [-c CONTAINER] [--follow] [flags] |
+ Print the logs for a container in a pod. | +
patch |
+ kubectl patch (-f FILENAME | TYPE NAME | TYPE/NAME) --patch PATCH [flags] |
+ Update one or more fields of a resource by using the strategic merge patch process. | +
port-forward |
+ kubectl port-forward POD [LOCAL_PORT:]REMOTE_PORT [...[LOCAL_PORT_N:]REMOTE_PORT_N] [flags] |
+ Forward one or more local ports to a pod. | +
proxy |
+ kubectl proxy [--port=PORT] [--www=static-dir] [--www-prefix=prefix] [--api-prefix=prefix] [flags] |
+ Run a proxy to the Kubernetes API server. | +
replace |
+ kubectl replace -f FILENAME |
+ Replace a resource from a file or stdin. | +
rolling-update |
+ kubectl rolling-update OLD_CONTROLLER_NAME ([NEW_CONTROLLER_NAME] --image=NEW_CONTAINER_IMAGE | -f NEW_CONTROLLER_SPEC) [flags] |
+ Perform a rolling update by gradually replacing the specified replication controller and its pods. | +
run |
+ kubectl run NAME --image=image [--env="key=value"] [--port=port] [--replicas=replicas] [--dry-run=bool] [--overrides=inline-json] [flags] |
+ Run a specified image on the cluster. | +
scale |
+ kubectl scale (-f FILENAME | TYPE NAME | TYPE/NAME) --replicas=COUNT [--resource-version=version] [--current-replicas=count] [flags] |
+ Update the size of the specified replication controller. | +
stop |
+ kubectl stop |
+ Deprecated: Instead, see kubectl delete . |
+
version |
+ kubectl version [--client] [flags] |
+ Display the Kubernetes version running on the client and server. | +
Remember: For more about command operations, see the kubectl reference documentation.
+ +The following table includes a list of all the supported resource types and their abbreviated aliases:
+ +Resource type | +Abbreviated alias | +
---|---|
componentstatuses |
+ cs |
+
events |
+ ev |
+
endpoints |
+ ep |
+
horizontalpodautoscalers |
+ hpa |
+
limitranges |
+ limits |
+
nodes |
+ no |
+
namespaces |
+ ns |
+
pods |
+ po |
+
persistentvolumes |
+ pv |
+
persistentvolumeclaims |
+ pvc |
+
resourcequotas |
+ quota |
+
replicationcontrollers |
+ rc |
+
secrets |
+ + |
serviceaccounts |
+ + |
services |
+ svc |
+
ingress |
+ ing |
+
Use the following sections for information about how you can format or sort the output of certain commands. For details about which commands support the various output options, see the kubectl reference documentation.
+ +The default output format for all kubectl
commands is the human readable plain-text format. To output details to your terminal window in a specific format, you can add either the -o
or -output
flags to a supported kubectl
command.
+kubectl [command] [TYPE] [NAME] -o=<output_format>
+
+
+
+Depending on the kubectl
operation, the following output formats are supported:
Output format | +Description | +
---|---|
-o=custom-columns=<spec> |
+ Print a table using a comma separated list of custom columns. | +
-o=custom-columns-file=<filename> |
+ Print a table using the custom columns template in the <filename> file. |
+
-o=json |
+ Output a JSON formatted API object. | +
-o=jsonpath=<template> |
+ Print the fields defined in a jsonpath expression. | +
-o=jsonpath-file=<filename> |
+ Print the fields defined by the jsonpath expression in the <filename> file. |
+
-o=name |
+ Print only the resource name and nothing else. | +
-o=wide |
+ Output in the plain-text format with any additional information. For pods, the node name is included. | +
-o=yaml |
+ Output a YAML formatted API object. | +
In this example, the following command outputs the details for a single pod as a YAML formatted object:
+ +$ kubectl get pod web-pod-13je7 -o=yaml
Remember: See the kubectl reference documentation for details about which output format is supported by each command.
+ +To define custom columns and output only the details that you want into a table, you can use the custom-columns
option. You can choose to define the custom columns inline or use a template file: -o=custom-columns=<spec>
or -o=custom-columns-file=<filename>
.
$ kubectl get pods <pod-name> -o=custom-columns=NAME:.metadata.name,RSRC:.metadata.resourceVersion
+
+$ kubectl get pods <pod-name> -o=custom-columns-file=template.txt
+
+ where the `template.txt` file contains:
+
+ ```
+
+ NAME RSRC
+ metadata.name metadata.resourceVersion
+
+ ```
+
+
+The result of running either command is:
+ +NAME RSRC
+submit-queue 610995
+
+To output objects to a sorted list in your terminal window, you can add the --sort-by
flag to a supported kubectl
command. Sort your objects by specifying any numeric or string field with the --sort-by
flag. To specify a field, use a jsonpath expression.
+kubectl [command] [TYPE] [NAME] --sort-by=<jsonpath_exp>
+
+
+
+To print a list of pods sorted by name, you run:
+ +$ kubectl get pods --sort-by=.metadata.name
Use the following set of examples to help you familiarize yourself with running the commonly used kubectl
operations:
kubectl create
- Create a resource from a file or stdin.
// Create a service using the definition in example-service.yaml.
+ $ kubectl create -f example-service.yaml
+
+ // Create a replication controller using the definition in example-controller.yaml.
+ $ kubectl create -f example-controller.yaml
+
+ // Create the objects that are defined in any .yaml, .yml, or .json file within the <directory> directory.
+ $ kubectl create -f <directory>
+
+ kubectl get
- List one or more resources.
// List all pods in plain-text output format.
+ $ kubectl get pods
+
+ // List all pods in plain-text output format and includes additional information (such as node name).
+ $ kubectl get pods -o wide
+
+ // List the replication controller with the specified name in plain-text output format. Tip: You can shorten and replace the 'replicationcontroller' resource type with the alias 'rc'.
+ $ kubectl get replicationcontroller <rc-name>
+
+ // List all replication controllers and services together in plain-text output format.
+ $ kubectl get rc,services
+
+ kubectl describe
- Display detailed state of one or more resources.
// Display the details of the node with name <node-name>.
+ $ kubectl describe nodes <node-name>
+
+ // Display the details of the pod with name <pod-name>.
+ $ kubectl describe pods/<pod-name>
+
+ // Display the details of all the pods that are managed by the replication controller named <rc-name>.
+ // Remember: Any pods that are created by the replication controller get prefixed with the name of the replication controller.
+ $ kubectl describe pods <rc-name>
+
+ kubectl delete
- Delete resources either from a file, stdin, or specifying label selectors, names, resource selectors, or resources.
// Delete a pod using the type and name specified in the pod.yaml file.
+ $ kubectl delete -f pod.yaml
+
+ // Delete all the pods and services that have the label name=<label-name>.
+ $ kubectl delete pods,services -l name=<label-name>
+
+ // Delete all pods.
+ $ kubectl delete pods --all
+
+ kubectl exec
- Execute a command against a container in a pod.
// Get output from running 'date' from pod <pod-name>. By default, output is from the first container.
+ $ kubectl exec <pod-name> date
+
+ // Get output from running 'date' in container <container-name> of pod <pod-name>.
+ $ kubectl exec <pod-name> -c <container-name> date
+
+ // Get an interactive TTY and run /bin/bash from pod <pod-name>. By default, output is from the first container.
+ $ kubectl exec -ti <pod-name> /bin/bash
+
+ kubectl logs
- Print the logs for a container in a pod.
// Return a snapshot of the logs from pod <pod-name>.
+ $ kubectl logs <pod-name>
+
+ // Start streaming the logs from pod <pod-name>. This is similiar to the 'tail -f' Linux command.
+ $ kubectl logs -f <pod-name>
+
+ Start using the kubectl commands.
+ + + + + + + + + +kubectl controls the Kubernetes cluster manager
+ +kubectl controls the Kubernetes cluster manager.
+ +Find more information at https://github.com/kubernetes/kubernetes.
+ +
+kubectl
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Update the annotations on a resource
+ +Update the annotations on one or more resources.
+ +An annotation is a key/value pair that can hold larger (compared to a label), and possibly not human-readable, data. +It is intended to store non-identifying auxiliary data, especially data manipulated by tools and system extensions. +If –overwrite is true, then existing annotations can be overwritten, otherwise attempting to overwrite an annotation will result in an error. +If –resource-version is specified, then updates will use this resource version, otherwise the existing resource-version will be used.
+ +Possible resources include (case insensitive): pods (po), services (svc), +replicationcontrollers (rc), nodes (no), events (ev), componentstatuses (cs), +limitranges (limits), persistentvolumes (pv), persistentvolumeclaims (pvc), +horizontalpodautoscalers (hpa), resourcequotas (quota) or secrets.
+ +
+kubectl annotate [--overwrite] (-f FILENAME | TYPE NAME) KEY_1=VAL_1 ... KEY_N=VAL_N [--resource-version=version]
+
+
+
+
+# Update pod 'foo' with the annotation 'description' and the value 'my frontend'.
+# If the same annotation is set multiple times, only the last value will be applied
+$ kubectl annotate pods foo description='my frontend'
+
+# Update a pod identified by type and name in "pod.json"
+$ kubectl annotate -f pod.json description='my frontend'
+
+# Update pod 'foo' with the annotation 'description' and the value 'my frontend running nginx', overwriting any existing value.
+$ kubectl annotate --overwrite pods foo description='my frontend running nginx'
+
+# Update all pods in the namespace
+$ kubectl annotate pods --all description='my frontend running nginx'
+
+# Update pod 'foo' only if the resource is unchanged from version 1.
+$ kubectl annotate pods foo description='my frontend running nginx' --resource-version=1
+
+# Update pod 'foo' by removing an annotation named 'description' if it exists.
+# Does not require the --overwrite flag.
+$ kubectl annotate pods foo description-
+
+
+
+
+ --all[=false]: select all resources in the namespace of the specified resource types
+ -f, --filename=[]: Filename, directory, or URL to a file identifying the resource to update the annotation
+ --overwrite[=false]: If true, allow annotations to be overwritten, otherwise reject annotation updates that overwrite existing annotations.
+ --resource-version="": If non-empty, the annotation update will only succeed if this is the current resource-version for the object. Only valid when specifying a single resource.
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Print the supported API versions on the server, in the form of “group/version”.
+ +Print the supported API versions on the server, in the form of “group/version”.
+ +
+kubectl api-versions
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Apply a configuration to a resource by filename or stdin
+ +Apply a configuration to a resource by filename or stdin.
+ +JSON and YAML formats are accepted.
+ +
+kubectl apply -f FILENAME
+
+
+
+
+# Apply the configuration in pod.json to a pod.
+$ kubectl apply -f ./pod.json
+
+# Apply the JSON passed into stdin to a pod.
+$ cat pod.json | kubectl apply -f -
+
+
+
+
+ -f, --filename=[]: Filename, directory, or URL to file that contains the configuration to apply
+ -o, --output="": Output mode. Use "-o name" for shorter output (resource/name).
+ --schema-cache-dir="/tmp/kubectl.schema": If non-empty, load/store cached API schemas in this directory, default is '/tmp/kubectl.schema'
+ --validate[=true]: If true, use a schema to validate the input before sending it
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Attach to a running container.
+ +Attach to a a process that is already running inside an existing container.
+ +
+kubectl attach POD -c CONTAINER
+
+
+
+
+# Get output from running pod 123456-7890, using the first container by default
+$ kubectl attach 123456-7890
+
+# Get output from ruby-container from pod 123456-7890
+$ kubectl attach 123456-7890 -c ruby-container date
+
+# Switch to raw terminal mode, sends stdin to 'bash' in ruby-container from pod 123456-780
+# and sends stdout/stderr from 'bash' back to the client
+$ kubectl attach 123456-7890 -c ruby-container -i -t
+
+
+
+
+ -c, --container="": Container name
+ -i, --stdin[=false]: Pass stdin to the container
+ -t, --tty[=false]: Stdin is a TTY
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Auto-scale a replication controller
+ +Creates an autoscaler that automatically chooses and sets the number of pods that run in a kubernetes cluster.
+ +Looks up a replication controller by name and creates an autoscaler that uses this replication controller as a reference. +An autoscaler can automatically increase or decrease number of pods deployed within the system as needed.
+ +
+kubectl autoscale (-f FILENAME | TYPE NAME | TYPE/NAME) [--min=MINPODS] --max=MAXPODS [--cpu-percent=CPU] [flags]
+
+
+
+
+# Auto scale a replication controller "foo", with the number of pods between 2 to 10, target CPU utilization at a default value that server applies:
+$ kubectl autoscale rc foo --min=2 --max=10
+
+# Auto scale a replication controller "foo", with the number of pods between 1 to 5, target CPU utilization at 80%:
+$ kubectl autoscale rc foo --max=5 --cpu-percent=80
+
+
+
+
+ --cpu-percent=-1: The target average CPU utilization (represented as a percent of requested CPU) over all the pods. If it's not specified or negative, the server will apply a default value.
+ --dry-run[=false]: If true, only print the object that would be sent, without creating it.
+ -f, --filename=[]: Filename, directory, or URL to a file identifying the resource to get from a server.
+ --generator="horizontalpodautoscaler/v1beta1": The name of the API generator to use. Currently there is only 1 generator.
+ --max=-1: The upper limit for the number of pods that can be set by the autoscaler. Required.
+ --min=-1: The lower limit for the number of pods that can be set by the autoscaler. If it's not specified or negative, the server will apply a default value.
+ --name="": The name for the newly created object. If not specified, the name of the input resource will be used.
+ --no-headers[=false]: When using the default output, don't print headers.
+ -o, --output="": Output format. One of: json|yaml|wide|name|go-template=...|go-template-file=...|jsonpath=...|jsonpath-file=... See golang template [http://golang.org/pkg/text/template/#pkg-overview] and jsonpath template [http://releases.k8s.io/release-1.1/docs/user-guide/jsonpath.md].
+ --output-version="": Output the formatted object with the given version (default api-version).
+ -a, --show-all[=false]: When printing, show all resources (default hide terminated pods.)
+ --sort-by="": If non-empty, sort list types using this field specification. The field specification is expressed as a JSONPath expression (e.g. 'ObjectMeta.Name'). The field in the API resource specified by this JSONPath expression must be an integer or a string.
+ --template="": Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Display cluster info
+ +Display addresses of the master and services with label kubernetes.io/cluster-service=true
+ +
+kubectl cluster-info
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+config modifies kubeconfig files
+ +config modifies kubeconfig files using subcommands like “kubectl config set current-context my-context”
+ +The loading order follows these rules: + 1. If the –kubeconfig flag is set, then only that file is loaded. The flag may only be set once and no merging takes place. + 2. If $KUBECONFIG environment variable is set, then it is used a list of paths (normal path delimitting rules for your system). These paths are merged together. When a value is modified, it is modified in the file that defines the stanza. When a value is created, it is created in the first file that exists. If no files in the chain exist, then it creates the last file in the list. + 3. Otherwise, ${HOME}/.kube/config is used and no merging takes place.
+ +
+kubectl config SUBCOMMAND
+
+
+
+
+ --kubeconfig="": use a particular kubeconfig file
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Sets a cluster entry in kubeconfig
+ +Sets a cluster entry in kubeconfig. +Specifying a name that already exists will merge new fields on top of existing values for those fields.
+ +
+kubectl config set-cluster NAME [--server=server] [--certificate-authority=path/to/certficate/authority] [--api-version=apiversion] [--insecure-skip-tls-verify=true]
+
+
+
+
+# Set only the server field on the e2e cluster entry without touching other values.
+$ kubectl config set-cluster e2e --server=https://1.2.3.4
+
+# Embed certificate authority data for the e2e cluster entry
+$ kubectl config set-cluster e2e --certificate-authority=~/.kube/e2e/kubernetes.ca.crt
+
+# Disable cert checking for the dev cluster entry
+$ kubectl config set-cluster e2e --insecure-skip-tls-verify=true
+
+
+
+
+ --api-version="": api-version for the cluster entry in kubeconfig
+ --certificate-authority="": path to certificate-authority for the cluster entry in kubeconfig
+ --embed-certs=false: embed-certs for the cluster entry in kubeconfig
+ --insecure-skip-tls-verify=false: insecure-skip-tls-verify for the cluster entry in kubeconfig
+ --server="": server for the cluster entry in kubeconfig
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --kubeconfig="": use a particular kubeconfig file
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Sets a context entry in kubeconfig
+ +Sets a context entry in kubeconfig +Specifying a name that already exists will merge new fields on top of existing values for those fields.
+ +
+kubectl config set-context NAME [--cluster=cluster_nickname] [--user=user_nickname] [--namespace=namespace]
+
+
+
+
+# Set the user field on the gce context entry without touching other values
+$ kubectl config set-context gce --user=cluster-admin
+
+
+
+
+ --cluster="": cluster for the context entry in kubeconfig
+ --namespace="": namespace for the context entry in kubeconfig
+ --user="": user for the context entry in kubeconfig
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": use a particular kubeconfig file
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Sets a user entry in kubeconfig
+ +Sets a user entry in kubeconfig +Specifying a name that already exists will merge new fields on top of existing values.
+ +Client-certificate flags: + –client-certificate=certfile –client-key=keyfile
+ +Bearer token flags: + –token=bearer_token
+ +Basic auth flags: + –username=basic_user –password=basic_password
+ +Bearer token and basic auth are mutually exclusive.
+ +
+kubectl config set-credentials NAME [--client-certificate=path/to/certfile] [--client-key=path/to/keyfile] [--token=bearer_token] [--username=basic_user] [--password=basic_password]
+
+
+
+
+# Set only the "client-key" field on the "cluster-admin"
+# entry, without touching other values:
+$ kubectl config set-credentials cluster-admin --client-key=~/.kube/admin.key
+
+# Set basic auth for the "cluster-admin" entry
+$ kubectl config set-credentials cluster-admin --username=admin --password=uXFGweU9l35qcif
+
+# Embed client certificate data in the "cluster-admin" entry
+$ kubectl config set-credentials cluster-admin --client-certificate=~/.kube/admin.crt --embed-certs=true
+
+
+
+
+ --client-certificate="": path to client-certificate for the user entry in kubeconfig
+ --client-key="": path to client-key for the user entry in kubeconfig
+ --embed-certs=false: embed client cert/key for the user entry in kubeconfig
+ --password="": password for the user entry in kubeconfig
+ --token="": token for the user entry in kubeconfig
+ --username="": username for the user entry in kubeconfig
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": use a particular kubeconfig file
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --user="": The name of the kubeconfig user to use
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Sets an individual value in a kubeconfig file
+ +Sets an individual value in a kubeconfig file +PROPERTY_NAME is a dot delimited name where each token represents either a attribute name or a map key. Map keys may not contain dots. +PROPERTY_VALUE is the new value you wish to set.
+ +
+kubectl config set PROPERTY_NAME PROPERTY_VALUE
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": use a particular kubeconfig file
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Unsets an individual value in a kubeconfig file
+ +Unsets an individual value in a kubeconfig file +PROPERTY_NAME is a dot delimited name where each token represents either a attribute name or a map key. Map keys may not contain dots.
+ +
+kubectl config unset PROPERTY_NAME
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": use a particular kubeconfig file
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Sets the current-context in a kubeconfig file
+ +Sets the current-context in a kubeconfig file
+ +
+kubectl config use-context CONTEXT_NAME
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": use a particular kubeconfig file
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Displays merged kubeconfig settings or a specified kubeconfig file.
+ +Displays merged kubeconfig settings or a specified kubeconfig file.
+ +You can use –output=template –template=TEMPLATE to extract specific values.
+ +
+kubectl config view
+
+
+
+
+# Show Merged kubeconfig settings.
+$ kubectl config view
+
+# Get the password for the e2e user
+$ kubectl config view -o template --template='{{range .users}}{{ if eq .name "e2e" }}{{ index .user.password }}{{end}}{{end}}'
+
+
+
+
+ --flatten[=false]: flatten the resulting kubeconfig file into self contained output (useful for creating portable kubeconfig files)
+ --merge=true: merge together the full hierarchy of kubeconfig files
+ --minify[=false]: remove all information not used by current-context from the output
+ --no-headers[=false]: When using the default output, don't print headers.
+ -o, --output="": Output format. One of: json|yaml|wide|name|go-template=...|go-template-file=...|jsonpath=...|jsonpath-file=... See golang template [http://golang.org/pkg/text/template/#pkg-overview] and jsonpath template [http://releases.k8s.io/release-1.1/docs/user-guide/jsonpath.md].
+ --output-version="": Output the formatted object with the given version (default api-version).
+ --raw[=false]: display raw byte data
+ -a, --show-all[=false]: When printing, show all resources (default hide terminated pods.)
+ --sort-by="": If non-empty, sort list types using this field specification. The field specification is expressed as a JSONPath expression (e.g. 'ObjectMeta.Name'). The field in the API resource specified by this JSONPath expression must be an integer or a string.
+ --template="": Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": use a particular kubeconfig file
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Create a resource by filename or stdin
+ +Create a resource by filename or stdin.
+ +JSON and YAML formats are accepted.
+ +
+kubectl create -f FILENAME
+
+
+
+
+# Create a pod using the data in pod.json.
+$ kubectl create -f ./pod.json
+
+# Create a pod based on the JSON passed into stdin.
+$ cat pod.json | kubectl create -f -
+
+
+
+
+ -f, --filename=[]: Filename, directory, or URL to file to use to create the resource
+ -o, --output="": Output mode. Use "-o name" for shorter output (resource/name).
+ --schema-cache-dir="/tmp/kubectl.schema": If non-empty, load/store cached API schemas in this directory, default is '/tmp/kubectl.schema'
+ --validate[=true]: If true, use a schema to validate the input before sending it
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Delete resources by filenames, stdin, resources and names, or by resources and label selector.
+ +Delete resources by filenames, stdin, resources and names, or by resources and label selector.
+ +JSON and YAML formats are accepted.
+ +Only one type of the arguments may be specified: filenames, resources and names, or resources and label selector
+ +Note that the delete command does NOT do resource version checks, so if someone +submits an update to a resource right when you submit a delete, their update +will be lost along with the rest of the resource.
+ +
+kubectl delete ([-f FILENAME] | TYPE [(NAME | -l label | --all)])
+
+
+
+
+# Delete a pod using the type and name specified in pod.json.
+$ kubectl delete -f ./pod.json
+
+# Delete a pod based on the type and name in the JSON passed into stdin.
+$ cat pod.json | kubectl delete -f -
+
+# Delete pods and services with same names "baz" and "foo"
+$ kubectl delete pod,service baz foo
+
+# Delete pods and services with label name=myLabel.
+$ kubectl delete pods,services -l name=myLabel
+
+# Delete a pod with UID 1234-56-7890-234234-456456.
+$ kubectl delete pod 1234-56-7890-234234-456456
+
+# Delete all pods
+$ kubectl delete pods --all
+
+
+
+
+ --all[=false]: [-all] to select all the specified resources.
+ --cascade[=true]: If true, cascade the deletion of the resources managed by this resource (e.g. Pods created by a ReplicationController). Default true.
+ -f, --filename=[]: Filename, directory, or URL to a file containing the resource to delete.
+ --grace-period=-1: Period of time in seconds given to the resource to terminate gracefully. Ignored if negative.
+ --ignore-not-found[=false]: Treat "resource not found" as a successful delete. Defaults to "true" when --all is specified.
+ -o, --output="": Output mode. Use "-o name" for shorter output (resource/name).
+ -l, --selector="": Selector (label query) to filter on.
+ --timeout=0: The length of time to wait before giving up on a delete, zero means determine a timeout from the size of the object
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Show details of a specific resource or group of resources
+ +Show details of a specific resource or group of resources.
+ +This command joins many API calls together to form a detailed description of a +given resource or group of resources.
+ +$ kubectl describe TYPE NAME_PREFIX
+ +will first check for an exact match on TYPE and NAME_PREFIX. If no such resource +exists, it will output details for every resource that has a name prefixed with NAME_PREFIX
+ +Possible resource types include (case insensitive): pods (po), services (svc), +replicationcontrollers (rc), nodes (no), events (ev), limitranges (limits), +persistentvolumes (pv), persistentvolumeclaims (pvc), resourcequotas (quota), +namespaces (ns), serviceaccounts, horizontalpodautoscalers (hpa), or secrets.
+ +
+kubectl describe (-f FILENAME | TYPE [NAME_PREFIX | -l label] | TYPE/NAME)
+
+
+
+
+# Describe a node
+$ kubectl describe nodes kubernetes-minion-emt8.c.myproject.internal
+
+# Describe a pod
+$ kubectl describe pods/nginx
+
+# Describe a pod identified by type and name in "pod.json"
+$ kubectl describe -f pod.json
+
+# Describe all pods
+$ kubectl describe pods
+
+# Describe pods by label name=myLabel
+$ kubectl describe po -l name=myLabel
+
+# Describe all pods managed by the 'frontend' replication controller (rc-created pods
+# get the name of the rc as a prefix in the pod the name).
+$ kubectl describe pods frontend
+
+
+
+
+ -f, --filename=[]: Filename, directory, or URL to a file containing the resource to describe
+ -l, --selector="": Selector (label query) to filter on
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Edit a resource on the server
+ +Edit a resource from the default editor.
+ +The edit command allows you to directly edit any API resource you can retrieve via the +command line tools. It will open the editor defined by your KUBE_EDITOR, GIT_EDITOR, +or EDITOR environment variables, or fall back to ‘vi’. You can edit multiple objects, +although changes are applied one at a time. The command accepts filenames as well as +command line arguments, although the files you point to must be previously saved +versions of resources.
+ +The files to edit will be output in the default API version, or a version specified +by –output-version. The default format is YAML - if you would like to edit in JSON +pass -o json.
+ +In the event an error occurs while updating, a temporary file will be created on disk +that contains your unapplied changes. The most common error when updating a resource +is another editor changing the resource on the server. When this occurs, you will have +to apply your changes to the newer version of the resource, or update your temporary +saved copy to include the latest resource version.
+ +
+kubectl edit (RESOURCE/NAME | -f FILENAME)
+
+
+
+
+ # Edit the service named 'docker-registry':
+ $ kubectl edit svc/docker-registry
+
+ # Use an alternative editor
+ $ KUBE_EDITOR="nano" kubectl edit svc/docker-registry
+
+ # Edit the service 'docker-registry' in JSON using the v1 API format:
+ $ kubectl edit svc/docker-registry --output-version=v1 -o json
+
+
+
+
+ -f, --filename=[]: Filename, directory, or URL to file to use to edit the resource
+ -o, --output="yaml": Output format. One of: yaml|json.
+ --output-version="": Output the formatted object with the given version (default api-version).
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Execute a command in a container.
+ +Execute a command in a container.
+ +
+kubectl exec POD [-c CONTAINER] -- COMMAND [args...]
+
+
+
+
+# Get output from running 'date' from pod 123456-7890, using the first container by default
+$ kubectl exec 123456-7890 date
+
+# Get output from running 'date' in ruby-container from pod 123456-7890
+$ kubectl exec 123456-7890 -c ruby-container date
+
+# Switch to raw terminal mode, sends stdin to 'bash' in ruby-container from pod 123456-780
+# and sends stdout/stderr from 'bash' back to the client
+$ kubectl exec 123456-7890 -c ruby-container -i -t -- bash -il
+
+
+
+
+ -c, --container="": Container name. If omitted, the first container in the pod will be chosen
+ -p, --pod="": Pod name
+ -i, --stdin[=false]: Pass stdin to the container
+ -t, --tty[=false]: Stdin is a TTY
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Take a replication controller, service or pod and expose it as a new Kubernetes Service
+ +Take a replication controller, service or pod and expose it as a new Kubernetes Service.
+ +Looks up a replication controller, service or pod by name and uses the selector for that resource as the +selector for a new Service on the specified port. If no labels are specified, the new service will +re-use the labels from the resource it exposes.
+ +
+kubectl expose (-f FILENAME | TYPE NAME) [--port=port] [--protocol=TCP|UDP] [--target-port=number-or-name] [--name=name] [----external-ip=external-ip-of-service] [--type=type]
+
+
+
+
+# Create a service for a replicated nginx, which serves on port 80 and connects to the containers on port 8000.
+$ kubectl expose rc nginx --port=80 --target-port=8000
+
+# Create a service for a replication controller identified by type and name specified in "nginx-controller.yaml", which serves on port 80 and connects to the containers on port 8000.
+$ kubectl expose -f nginx-controller.yaml --port=80 --target-port=8000
+
+# Create a service for a pod valid-pod, which serves on port 444 with the name "frontend"
+$ kubectl expose pod valid-pod --port=444 --name=frontend
+
+# Create a second service based on the above service, exposing the container port 8443 as port 443 with the name "nginx-https"
+$ kubectl expose service nginx --port=443 --target-port=8443 --name=nginx-https
+
+# Create a service for a replicated streaming application on port 4100 balancing UDP traffic and named 'video-stream'.
+$ kubectl expose rc streamer --port=4100 --protocol=udp --name=video-stream
+
+
+
+
+ --container-port="": Synonym for --target-port
+ --dry-run[=false]: If true, only print the object that would be sent, without creating it.
+ --external-ip="": External IP address to set for the service. The service can be accessed by this IP in addition to its generated service IP.
+ -f, --filename=[]: Filename, directory, or URL to a file identifying the resource to expose a service
+ --generator="service/v2": The name of the API generator to use. There are 2 generators: 'service/v1' and 'service/v2'. The only difference between them is that service port in v1 is named 'default', while it is left unnamed in v2. Default is 'service/v2'.
+ -l, --labels="": Labels to apply to the service created by this call.
+ --load-balancer-ip="": IP to assign to to the Load Balancer. If empty, an ephemeral IP will be created and used(cloud-provider specific).
+ --name="": The name for the newly created object.
+ --no-headers[=false]: When using the default output, don't print headers.
+ -o, --output="": Output format. One of: json|yaml|wide|name|go-template=...|go-template-file=...|jsonpath=...|jsonpath-file=... See golang template [http://golang.org/pkg/text/template/#pkg-overview] and jsonpath template [http://releases.k8s.io/release-1.1/docs/user-guide/jsonpath.md].
+ --output-version="": Output the formatted object with the given version (default api-version).
+ --overrides="": An inline JSON override for the generated object. If this is non-empty, it is used to override the generated object. Requires that the object supply a valid apiVersion field.
+ --port=-1: The port that the service should serve on. Copied from the resource being exposed, if unspecified
+ --protocol="TCP": The network protocol for the service to be created. Default is 'tcp'.
+ --selector="": A label selector to use for this service. If empty (the default) infer the selector from the replication controller.
+ --session-affinity="": If non-empty, set the session affinity for the service to this; legal values: 'None', 'ClientIP'
+ -a, --show-all[=false]: When printing, show all resources (default hide terminated pods.)
+ --sort-by="": If non-empty, sort list types using this field specification. The field specification is expressed as a JSONPath expression (e.g. 'ObjectMeta.Name'). The field in the API resource specified by this JSONPath expression must be an integer or a string.
+ --target-port="": Name or number for the port on the container that the service should direct traffic to. Optional.
+ --template="": Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].
+ --type="": Type for this service: ClusterIP, NodePort, or LoadBalancer. Default is 'ClusterIP'.
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Display one or many resources
+ +Display one or many resources.
+ +Possible resource types include (case insensitive): pods (po), services (svc), +replicationcontrollers (rc), nodes (no), events (ev), componentstatuses (cs), +limitranges (limits), persistentvolumes (pv), persistentvolumeclaims (pvc), +resourcequotas (quota), namespaces (ns), endpoints (ep), +horizontalpodautoscalers (hpa), serviceaccounts or secrets.
+ +By specifying the output as ‘template’ and providing a Go template as the value +of the –template flag, you can filter the attributes of the fetched resource(s).
+ +
+kubectl get [(-o|--output=)json|yaml|wide|go-template=...|go-template-file=...|jsonpath=...|jsonpath-file=...] (TYPE [NAME | -l label] | TYPE/NAME ...) [flags]
+
+
+
+
+# List all pods in ps output format.
+$ kubectl get pods
+
+# List all pods in ps output format with more information (such as node name).
+$ kubectl get pods -o wide
+
+# List all pods in resource/name format (such as pod/nginx).
+$ kubectl get pods -o name
+
+# List a single replication controller with specified NAME in ps output format.
+$ kubectl get replicationcontroller web
+
+# List a single pod in JSON output format.
+$ kubectl get -o json pod web-pod-13je7
+
+# List a pod identified by type and name specified in "pod.yaml" in JSON output format.
+$ kubectl get -f pod.yaml -o json
+
+# Return only the phase value of the specified pod.
+$ kubectl get -o template pod/web-pod-13je7 --template={{.status.phase}} --api-version=v1
+
+# List all replication controllers and services together in ps output format.
+$ kubectl get rc,services
+
+# List one or more resources by their type and names.
+$ kubectl get rc/web service/frontend pods/web-pod-13je7
+
+
+
+
+ --all-namespaces[=false]: If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.
+ -f, --filename=[]: Filename, directory, or URL to a file identifying the resource to get from a server.
+ -L, --label-columns=[]: Accepts a comma separated list of labels that are going to be presented as columns. Names are case-sensitive. You can also use multiple flag statements like -L label1 -L label2...
+ --no-headers[=false]: When using the default output, don't print headers.
+ -o, --output="": Output format. One of: json|yaml|wide|name|go-template=...|go-template-file=...|jsonpath=...|jsonpath-file=... See golang template [http://golang.org/pkg/text/template/#pkg-overview] and jsonpath template [http://releases.k8s.io/release-1.1/docs/user-guide/jsonpath.md].
+ --output-version="": Output the formatted object with the given version (default api-version).
+ -l, --selector="": Selector (label query) to filter on
+ -a, --show-all[=false]: When printing, show all resources (default hide terminated pods.)
+ --sort-by="": If non-empty, sort list types using this field specification. The field specification is expressed as a JSONPath expression (e.g. 'ObjectMeta.Name'). The field in the API resource specified by this JSONPath expression must be an integer or a string.
+ --template="": Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].
+ -w, --watch[=false]: After listing/getting the requested object, watch for changes.
+ --watch-only[=false]: Watch for changes to the requested object(s), without listing/getting first.
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Update the labels on a resource
+ +Update the labels on a resource.
+ +A label must begin with a letter or number, and may contain letters, numbers, hyphens, dots, and underscores, up to 63 characters. +If –overwrite is true, then existing labels can be overwritten, otherwise attempting to overwrite a label will result in an error. +If –resource-version is specified, then updates will use this resource version, otherwise the existing resource-version will be used.
+ +
+kubectl label [--overwrite] (-f FILENAME | TYPE NAME) KEY_1=VAL_1 ... KEY_N=VAL_N [--resource-version=version]
+
+
+
+
+# Update pod 'foo' with the label 'unhealthy' and the value 'true'.
+$ kubectl label pods foo unhealthy=true
+
+# Update pod 'foo' with the label 'status' and the value 'unhealthy', overwriting any existing value.
+$ kubectl label --overwrite pods foo status=unhealthy
+
+# Update all pods in the namespace
+$ kubectl label pods --all status=unhealthy
+
+# Update a pod identified by the type and name in "pod.json"
+$ kubectl label -f pod.json status=unhealthy
+
+# Update pod 'foo' only if the resource is unchanged from version 1.
+$ kubectl label pods foo status=unhealthy --resource-version=1
+
+# Update pod 'foo' by removing a label named 'bar' if it exists.
+# Does not require the --overwrite flag.
+$ kubectl label pods foo bar-
+
+
+
+
+ --all[=false]: select all resources in the namespace of the specified resource types
+ --dry-run[=false]: If true, only print the object that would be sent, without sending it.
+ -f, --filename=[]: Filename, directory, or URL to a file identifying the resource to update the labels
+ --no-headers[=false]: When using the default output, don't print headers.
+ -o, --output="": Output format. One of: json|yaml|wide|name|go-template=...|go-template-file=...|jsonpath=...|jsonpath-file=... See golang template [http://golang.org/pkg/text/template/#pkg-overview] and jsonpath template [http://releases.k8s.io/release-1.1/docs/user-guide/jsonpath.md].
+ --output-version="": Output the formatted object with the given version (default api-version).
+ --overwrite[=false]: If true, allow labels to be overwritten, otherwise reject label updates that overwrite existing labels.
+ --resource-version="": If non-empty, the labels update will only succeed if this is the current resource-version for the object. Only valid when specifying a single resource.
+ -l, --selector="": Selector (label query) to filter on
+ -a, --show-all[=false]: When printing, show all resources (default hide terminated pods.)
+ --sort-by="": If non-empty, sort list types using this field specification. The field specification is expressed as a JSONPath expression (e.g. 'ObjectMeta.Name'). The field in the API resource specified by this JSONPath expression must be an integer or a string.
+ --template="": Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Print the logs for a container in a pod.
+ +Print the logs for a container in a pod. If the pod has only one container, the container name is optional.
+ +
+kubectl logs [-f] [-p] POD [-c CONTAINER]
+
+
+
+
+# Return snapshot logs from pod nginx with only one container
+$ kubectl logs nginx
+
+# Return snapshot of previous terminated ruby container logs from pod web-1
+$ kubectl logs -p -c ruby web-1
+
+# Begin streaming the logs of the ruby container in pod web-1
+$ kubectl logs -f -c ruby web-1
+
+# Display only the most recent 20 lines of output in pod nginx
+$ kubectl logs --tail=20 nginx
+
+# Show all logs from pod nginx written in the last hour
+$ kubectl logs --since=1h nginx
+
+
+
+
+ -c, --container="": Container name
+ -f, --follow[=false]: Specify if the logs should be streamed.
+ --interactive[=true]: If true, prompt the user for input when required. Default true.
+ --limit-bytes=0: Maximum bytes of logs to return. Defaults to no limit.
+ -p, --previous[=false]: If true, print the logs for the previous instance of the container in a pod if it exists.
+ --since=0: Only return logs newer than a relative duration like 5s, 2m, or 3h. Defaults to all logs. Only one of since-time / since may be used.
+ --since-time="": Only return logs after a specific date (RFC3339). Defaults to all logs. Only one of since-time / since may be used.
+ --tail=-1: Lines of recent log file to display. Defaults to -1, showing all log lines.
+ --timestamps[=false]: Include timestamps on each line in the log output
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+SUPERSEDED: Set and view the current Kubernetes namespace
+ +SUPERSEDED: Set and view the current Kubernetes namespace scope for command line requests.
+ +namespace has been superseded by the context.namespace field of .kubeconfig files. See ‘kubectl config set-context –help’ for more details.
+ +
+kubectl namespace [namespace]
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Update field(s) of a resource by stdin.
+ +Update field(s) of a resource using strategic merge patch
+ +JSON and YAML formats are accepted.
+ +Please refer to the models in http://kubernetes.io/v1.1/docs/api-reference/v1/definitions.html to find if a field is mutable.
+ +
+kubectl patch (-f FILENAME | TYPE NAME) -p PATCH
+
+
+
+
+
+# Partially update a node using strategic merge patch
+kubectl patch node k8s-node-1 -p '{"spec":{"unschedulable":true}}'
+
+# Partially update a node identified by the type and name specified in "node.json" using strategic merge patch
+kubectl patch -f node.json -p '{"spec":{"unschedulable":true}}'
+
+# Update a container's image; spec.containers[*].name is required because it's a merge key
+kubectl patch pod valid-pod -p '{"spec":{"containers":[{"name":"kubernetes-serve-hostname","image":"new image"}]}}'
+
+
+
+
+ -f, --filename=[]: Filename, directory, or URL to a file identifying the resource to update
+ -o, --output="": Output mode. Use "-o name" for shorter output (resource/name).
+ -p, --patch="": The patch to be applied to the resource JSON file.
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Forward one or more local ports to a pod.
+ +Forward one or more local ports to a pod.
+ +
+kubectl port-forward POD [LOCAL_PORT:]REMOTE_PORT [...[LOCAL_PORT_N:]REMOTE_PORT_N]
+
+
+
+
+
+# Listen on ports 5000 and 6000 locally, forwarding data to/from ports 5000 and 6000 in the pod
+$ kubectl port-forward mypod 5000 6000
+
+# Listen on port 8888 locally, forwarding to 5000 in the pod
+$ kubectl port-forward mypod 8888:5000
+
+# Listen on a random port locally, forwarding to 5000 in the pod
+$ kubectl port-forward mypod :5000
+
+# Listen on a random port locally, forwarding to 5000 in the pod
+$ kubectl port-forward mypod 0:5000
+
+
+
+
+ -p, --pod="": Pod name
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Run a proxy to the Kubernetes API server
+ +To proxy all of the kubernetes api and nothing else, use:
+ +kubectl proxy –api-prefix=/
+ +To proxy only part of the kubernetes api and also some static files:
+ +kubectl proxy –www=/my/files –www-prefix=/static/ –api-prefix=/api/
+ +The above lets you ‘curl localhost:8001/api/v1/pods’.
+ +To proxy the entire kubernetes api at a different root, use:
+ +kubectl proxy –api-prefix=/custom/
+ +The above lets you ‘curl localhost:8001/custom/api/v1/pods’
+ +
+kubectl proxy [--port=PORT] [--www=static-dir] [--www-prefix=prefix] [--api-prefix=prefix]
+
+
+
+
+# Run a proxy to kubernetes apiserver on port 8011, serving static content from ./local/www/
+$ kubectl proxy --port=8011 --www=./local/www/
+
+# Run a proxy to kubernetes apiserver on an arbitrary local port.
+# The chosen port for the server will be output to stdout.
+$ kubectl proxy --port=0
+
+# Run a proxy to kubernetes apiserver, changing the api prefix to k8s-api
+# This makes e.g. the pods api available at localhost:8011/k8s-api/v1/pods/
+$ kubectl proxy --api-prefix=/k8s-api
+
+
+
+
+ --accept-hosts="^localhost$,^127\\.0\\.0\\.1$,^\\[::1\\]$": Regular expression for hosts that the proxy should accept.
+ --accept-paths="^/.*": Regular expression for paths that the proxy should accept.
+ --api-prefix="/": Prefix to serve the proxied API under.
+ --disable-filter[=false]: If true, disable request filtering in the proxy. This is dangerous, and can leave you vulnerable to XSRF attacks, when used with an accessible port.
+ -p, --port=8001: The port on which to run the proxy. Set to 0 to pick a random port.
+ --reject-methods="POST,PUT,PATCH": Regular expression for HTTP methods that the proxy should reject.
+ --reject-paths="^/api/.*/exec,^/api/.*/run": Regular expression for paths that the proxy should reject.
+ -u, --unix-socket="": Unix socket on which to run the proxy.
+ -w, --www="": Also serve static files from the given directory under the specified prefix.
+ -P, --www-prefix="/static/": Prefix to serve static files under, if static file directory is specified.
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Replace a resource by filename or stdin.
+ +Replace a resource by filename or stdin.
+ +JSON and YAML formats are accepted. If replacing an existing resource, the +complete resource spec must be provided. This can be obtained by +$ kubectl get TYPE NAME -o yaml
+ +Please refer to the models in http://kubernetes.io/v1.1/docs/api-reference/v1/definitions.html to find if a field is mutable.
+ +
+kubectl replace -f FILENAME
+
+
+
+
+# Replace a pod using the data in pod.json.
+$ kubectl replace -f ./pod.json
+
+# Replace a pod based on the JSON passed into stdin.
+$ cat pod.json | kubectl replace -f -
+
+# Update a single-container pod's image version (tag) to v4
+kubectl get pod mypod -o yaml | sed 's/\(image: myimage\):.*$/\1:v4/' | kubectl replace -f -
+
+# Force replace, delete and then re-create the resource
+kubectl replace --force -f ./pod.json
+
+
+
+
+ --cascade[=false]: Only relevant during a force replace. If true, cascade the deletion of the resources managed by this resource (e.g. Pods created by a ReplicationController).
+ -f, --filename=[]: Filename, directory, or URL to file to use to replace the resource.
+ --force[=false]: Delete and re-create the specified resource
+ --grace-period=-1: Only relevant during a force replace. Period of time in seconds given to the old resource to terminate gracefully. Ignored if negative.
+ -o, --output="": Output mode. Use "-o name" for shorter output (resource/name).
+ --schema-cache-dir="/tmp/kubectl.schema": If non-empty, load/store cached API schemas in this directory, default is '/tmp/kubectl.schema'
+ --timeout=0: Only relevant during a force replace. The length of time to wait before giving up on a delete of the old resource, zero means determine a timeout from the size of the object
+ --validate[=true]: If true, use a schema to validate the input before sending it
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Perform a rolling update of the given ReplicationController.
+ +Perform a rolling update of the given ReplicationController.
+ +Replaces the specified replication controller with a new replication controller by updating one pod at a time to use the +new PodTemplate. The new-controller.json must specify the same namespace as the +existing replication controller and overwrite at least one (common) label in its replicaSelector.
+ +
+kubectl rolling-update OLD_CONTROLLER_NAME ([NEW_CONTROLLER_NAME] --image=NEW_CONTAINER_IMAGE | -f NEW_CONTROLLER_SPEC)
+
+
+
+
+# Update pods of frontend-v1 using new replication controller data in frontend-v2.json.
+$ kubectl rolling-update frontend-v1 -f frontend-v2.json
+
+# Update pods of frontend-v1 using JSON data passed into stdin.
+$ cat frontend-v2.json | kubectl rolling-update frontend-v1 -f -
+
+# Update the pods of frontend-v1 to frontend-v2 by just changing the image, and switching the
+# name of the replication controller.
+$ kubectl rolling-update frontend-v1 frontend-v2 --image=image:v2
+
+# Update the pods of frontend by just changing the image, and keeping the old name
+$ kubectl rolling-update frontend --image=image:v2
+
+
+
+
+
+ --deployment-label-key="deployment": The key to use to differentiate between two different controllers, default 'deployment'. Only relevant when --image is specified, ignored otherwise
+ --dry-run[=false]: If true, print out the changes that would be made, but don't actually make them.
+ -f, --filename=[]: Filename or URL to file to use to create the new replication controller.
+ --image="": Image to use for upgrading the replication controller. Can not be used with --filename/-f
+ --no-headers[=false]: When using the default output, don't print headers.
+ -o, --output="": Output format. One of: json|yaml|wide|name|go-template=...|go-template-file=...|jsonpath=...|jsonpath-file=... See golang template [http://golang.org/pkg/text/template/#pkg-overview] and jsonpath template [http://releases.k8s.io/release-1.1/docs/user-guide/jsonpath.md].
+ --output-version="": Output the formatted object with the given version (default api-version).
+ --poll-interval=3s: Time delay between polling for replication controller status after the update. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+ --rollback[=false]: If true, this is a request to abort an existing rollout that is partially rolled out. It effectively reverses current and next and runs a rollout
+ --schema-cache-dir="/tmp/kubectl.schema": If non-empty, load/store cached API schemas in this directory, default is '/tmp/kubectl.schema'
+ -a, --show-all[=false]: When printing, show all resources (default hide terminated pods.)
+ --sort-by="": If non-empty, sort list types using this field specification. The field specification is expressed as a JSONPath expression (e.g. 'ObjectMeta.Name'). The field in the API resource specified by this JSONPath expression must be an integer or a string.
+ --template="": Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].
+ --timeout=5m0s: Max time to wait for a replication controller to update before giving up. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+ --update-period=1m0s: Time to wait between updating pods. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+ --validate[=true]: If true, use a schema to validate the input before sending it
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Run a particular image on the cluster.
+ +Create and run a particular image, possibly replicated. +Creates a replication controller to manage the created container(s).
+ +
+kubectl run NAME --image=image [--env="key=value"] [--port=port] [--replicas=replicas] [--dry-run=bool] [--overrides=inline-json]
+
+
+
+
+# Start a single instance of nginx.
+$ kubectl run nginx --image=nginx
+
+# Start a single instance of hazelcast and let the container expose port 5701 .
+$ kubectl run hazelcast --image=hazelcast --port=5701
+
+# Start a single instance of hazelcast and set environment variables "DNS_DOMAIN=cluster" and "POD_NAMESPACE=default" in the container.
+$ kubectl run hazelcast --image=hazelcast --env="DNS_DOMAIN=local" --env="POD_NAMESPACE=default"
+
+# Start a replicated instance of nginx.
+$ kubectl run nginx --image=nginx --replicas=5
+
+# Dry run. Print the corresponding API objects without creating them.
+$ kubectl run nginx --image=nginx --dry-run
+
+# Start a single instance of nginx, but overload the spec of the replication controller with a partial set of values parsed from JSON.
+$ kubectl run nginx --image=nginx --overrides='{ "apiVersion": "v1", "spec": { ... } }'
+
+# Start a single instance of nginx and keep it in the foreground, don't restart it if it exits.
+$ kubectl run -i -tty nginx --image=nginx --restart=Never
+
+# Start the nginx container using the default command, but use custom arguments (arg1 .. argN) for that command.
+$ kubectl run nginx --image=nginx -- <arg1> <arg2> ... <argN>
+
+# Start the nginx container using a different command and custom arguments
+$ kubectl run nginx --image=nginx --command -- <cmd> <arg1> ... <argN>
+
+
+
+
+ --attach[=false]: If true, wait for the Pod to start running, and then attach to the Pod as if 'kubectl attach ...' were called. Default false, unless '-i/--interactive' is set, in which case the default is true.
+ --command[=false]: If true and extra arguments are present, use them as the 'command' field in the container, rather than the 'args' field which is the default.
+ --dry-run[=false]: If true, only print the object that would be sent, without sending it.
+ --env=[]: Environment variables to set in the container
+ --generator="": The name of the API generator to use. Default is 'run/v1' if --restart=Always, otherwise the default is 'run-pod/v1'.
+ --hostport=-1: The host port mapping for the container port. To demonstrate a single-machine container.
+ --image="": The image for the container to run.
+ -l, --labels="": Labels to apply to the pod(s).
+ --leave-stdin-open[=false]: If the pod is started in interactive mode or with stdin, leave stdin open after the first attach completes. By default, stdin will be closed after the first attach completes.
+ --limits="": The resource requirement limits for this container. For example, 'cpu=200m,memory=512Mi'
+ --no-headers[=false]: When using the default output, don't print headers.
+ -o, --output="": Output format. One of: json|yaml|wide|name|go-template=...|go-template-file=...|jsonpath=...|jsonpath-file=... See golang template [http://golang.org/pkg/text/template/#pkg-overview] and jsonpath template [http://releases.k8s.io/release-1.1/docs/user-guide/jsonpath.md].
+ --output-version="": Output the formatted object with the given version (default api-version).
+ --overrides="": An inline JSON override for the generated object. If this is non-empty, it is used to override the generated object. Requires that the object supply a valid apiVersion field.
+ --port=-1: The port that this container exposes.
+ -r, --replicas=1: Number of replicas to create for this container. Default is 1.
+ --requests="": The resource requirement requests for this container. For example, 'cpu=100m,memory=256Mi'
+ --restart="Always": The restart policy for this Pod. Legal values [Always, OnFailure, Never]. If set to 'Always' a replication controller is created for this pod, if set to OnFailure or Never, only the Pod is created and --replicas must be 1. Default 'Always'
+ -a, --show-all[=false]: When printing, show all resources (default hide terminated pods.)
+ --sort-by="": If non-empty, sort list types using this field specification. The field specification is expressed as a JSONPath expression (e.g. 'ObjectMeta.Name'). The field in the API resource specified by this JSONPath expression must be an integer or a string.
+ -i, --stdin[=false]: Keep stdin open on the container(s) in the pod, even if nothing is attached.
+ --template="": Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].
+ --tty[=false]: Allocated a TTY for each container in the pod. Because -t is currently shorthand for --template, -t is not supported for --tty. This shorthand is deprecated and we expect to adopt -t for --tty soon.
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Set a new size for a Replication Controller.
+ +Set a new size for a Replication Controller.
+ +Scale also allows users to specify one or more preconditions for the scale action. +If –current-replicas or –resource-version is specified, it is validated before the +scale is attempted, and it is guaranteed that the precondition holds true when the +scale is sent to the server.
+ +
+kubectl scale [--resource-version=version] [--current-replicas=count] --replicas=COUNT (-f FILENAME | TYPE NAME)
+
+
+
+
+# Scale replication controller named 'foo' to 3.
+$ kubectl scale --replicas=3 replicationcontrollers foo
+
+# Scale a replication controller identified by type and name specified in "foo-controller.yaml" to 3.
+$ kubectl scale --replicas=3 -f foo-controller.yaml
+
+# If the replication controller named foo's current size is 2, scale foo to 3.
+$ kubectl scale --current-replicas=2 --replicas=3 replicationcontrollers foo
+
+# Scale multiple replication controllers.
+$ kubectl scale --replicas=5 rc/foo rc/bar
+
+
+
+
+ --current-replicas=-1: Precondition for current size. Requires that the current size of the replication controller match this value in order to scale.
+ -f, --filename=[]: Filename, directory, or URL to a file identifying the replication controller to set a new size
+ -o, --output="": Output mode. Use "-o name" for shorter output (resource/name).
+ --replicas=-1: The new desired number of replicas. Required.
+ --resource-version="": Precondition for resource version. Requires that the current resource version match this value in order to scale.
+ --timeout=0: The length of time to wait before giving up on a scale operation, zero means don't wait.
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Deprecated: Gracefully shut down a resource by name or filename.
+ +Deprecated: Gracefully shut down a resource by name or filename.
+ +The stop command is deprecated, all its functionalities are covered by delete command. +See ‘kubectl delete –help’ for more details.
+ +Attempts to shut down and delete a resource that supports graceful termination. +If the resource is scalable it will be scaled to 0 before deletion.
+ +
+kubectl stop (-f FILENAME | TYPE (NAME | -l label | --all))
+
+
+
+
+# Shut down foo.
+$ kubectl stop replicationcontroller foo
+
+# Stop pods and services with label name=myLabel.
+$ kubectl stop pods,services -l name=myLabel
+
+# Shut down the service defined in service.json
+$ kubectl stop -f service.json
+
+# Shut down all resources in the path/to/resources directory
+$ kubectl stop -f path/to/resources
+
+
+
+
+ --all[=false]: [-all] to select all the specified resources.
+ -f, --filename=[]: Filename, directory, or URL to file of resource(s) to be stopped.
+ --grace-period=-1: Period of time in seconds given to the resource to terminate gracefully. Ignored if negative.
+ --ignore-not-found[=false]: Treat "resource not found" as a successful stop.
+ -o, --output="": Output mode. Use "-o name" for shorter output (resource/name).
+ -l, --selector="": Selector (label query) to filter on.
+ --timeout=0: The length of time to wait before giving up on a delete, zero means determine a timeout from the size of the object
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Print the client and server version information.
+ +Print the client and server version information.
+ +
+kubectl version
+
+
+
+
+ -c, --client[=false]: Client version only (no server required).
+
+
+
+
+ --alsologtostderr[=false]: log to standard error as well as files
+ --api-version="": The API version to use when talking to the server
+ --certificate-authority="": Path to a cert. file for the certificate authority.
+ --client-certificate="": Path to a client key file for TLS.
+ --client-key="": Path to a client key file for TLS.
+ --cluster="": The name of the kubeconfig cluster to use
+ --context="": The name of the kubeconfig context to use
+ --insecure-skip-tls-verify[=false]: If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure.
+ --kubeconfig="": Path to the kubeconfig file to use for CLI requests.
+ --log-backtrace-at=:0: when logging hits line file:N, emit a stack trace
+ --log-dir="": If non-empty, write log files in this directory
+ --log-flush-frequency=5s: Maximum number of seconds between log flushes
+ --logtostderr[=true]: log to standard error instead of files
+ --match-server-version[=false]: Require server version to match client version
+ --namespace="": If present, the namespace scope for this CLI request.
+ --password="": Password for basic authentication to the API server.
+ -s, --server="": The address and port of the Kubernetes API server
+ --stderrthreshold=2: logs at or above this threshold go to stderr
+ --token="": Bearer token for authentication to the API server.
+ --user="": The name of the kubeconfig user to use
+ --username="": Username for basic authentication to the API server.
+ --v=0: log level for V logs
+ --vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+
+
+
+Table of Contents +
+ +Labels are key/value pairs that are attached to objects, such as pods. +Labels are intended to be used to specify identifying attributes of objects that are meaningful and relevant to users, but which do not directly imply semantics to the core system. +Labels can be used to organize and to select subsets of objects. Labels can be attached to objects at creation time and subsequently added and modified at any time. +Each object can have a set of key/value labels defined. Each Key must be unique for a given object.
+ +"labels": {
+ "key1" : "value1",
+ "key2" : "value2"
+}
+
+We’ll eventually index and reverse-index labels for efficient queries and watches, use them to sort and group in UIs and CLIs, etc. We don’t want to pollute labels with non-identifying, especially large and/or structured, data. Non-identifying information should be recorded using annotations.
+ +Labels enable users to map their own organizational structures onto system objects in a loosely coupled fashion, without requiring clients to store these mappings.
+ +Service deployments and batch processing pipelines are often multi-dimensional entities (e.g., multiple partitions or deployments, multiple release tracks, multiple tiers, multiple micro-services per tier). Management often requires cross-cutting operations, which breaks encapsulation of strictly hierarchical representations, especially rigid hierarchies determined by the infrastructure rather than by users.
+ +Example labels:
+ +"release" : "stable"
, "release" : "canary"
"environment" : "dev"
, "environment" : "qa"
, "environment" : "production"
"tier" : "frontend"
, "tier" : "backend"
, "tier" : "cache"
"partition" : "customerA"
, "partition" : "customerB"
"track" : "daily"
, "track" : "weekly"
These are just examples; you are free to develop your own conventions.
+ +Labels are key value pairs. Valid label keys have two segments: an optional prefix and name, separated by a slash (/
). The name segment is required and must be 63 characters or less, beginning and ending with an alphanumeric character ([a-z0-9A-Z]
) with dashes (-
), underscores (_
), dots (.
), and alphanumerics between. The prefix is optional. If specified, the prefix must be a DNS subdomain: a series of DNS labels separated by dots (.
), not longer than 253 characters in total, followed by a slash (/
).
+If the prefix is omitted, the label key is presumed to be private to the user. Automated system components (e.g. kube-scheduler
, kube-controller-manager
, kube-apiserver
, kubectl
, or other third-party automation) which add labels to end-user objects must specify a prefix. The kubernetes.io/
prefix is reserved for Kubernetes core components.
Valid label values must be 63 characters or less and must be empty or begin and end with an alphanumeric character ([a-z0-9A-Z]
) with dashes (-
), underscores (_
), dots (.
), and alphanumerics between.
Unlike names and UIDs, labels do not provide uniqueness. In general, we expect many objects to carry the same label(s).
+ +Via a label selector, the client/user can identify a set of objects. The label selector is the core grouping primitive in Kubernetes.
+ +The API currently supports two types of selectors: equality-based and set-based. +A label selector can be made of multiple requirements which are comma-separated. In the case of multiple requirements, all must be satisfied so the comma separator acts as an AND logical operator.
+ +An empty label selector (that is, one with zero requirements) selects every object in the collection.
+ +A null label selector (which is only possible for optional selector fields) selects no objects.
+ +Equality- or inequality-based requirements allow filtering by label keys and values. Matching objects must satisfy all of the specified label constraints, though they may have additional labels as well.
+Three kinds of operators are admitted =
,==
,!=
. The first two represent equality (and are simply synonyms), while the latter represents inequality. For example:
+environment = production
+tier != frontend
+
+
+
+The former selects all resources with key equal to environment
and value equal to production
.
+The latter selects all resources with key equal to tier
and value distinct from frontend
, and all resources with no labels with the tier
key.
+One could filter for resources in production
excluding frontend
using the comma operator: environment=production,tier!=frontend
Set-based label requirements allow filtering keys according to a set of values. Three kinds of operators are supported: in
,notin
and exists (only the key identifier). For example:
+environment in (production, qa)
+tier notin (frontend, backend)
+partition
+!partition
+
+
+
+The first example selects all resources with key equal to environment
and value equal to production
or qa
.
+The second example selects all resources with key equal to tier
and values other than frontend
and backend
, and all resources with no labels with the tier
key.
+The third example selects all resources including a label with key partition
; no values are checked.
+The fourth example selects all resources without a label with key partition
; no values are checked.
+Similarly the comma separator acts as an AND operator. So filtering resources with a partition
key (no matter the value) and with environment
different than qa
can be achieved using partition,environment notin (qa)
.
+The set-based label selector is a general form of equality since environment=production
is equivalent to environment in (production)
; similarly for !=
and notin
.
Set-based requirements can be mixed with equality-based requirements. For example: partition in (customerA, customerB),environment!=qa
.
LIST and WATCH operations may specify label selectors to filter the sets of objects returned using a query parameter. Both requirements are permitted:
+ +* _equality-based_ requirements: `?labelSelector=environment%3Dproduction,tier%3Dfrontend`
+* _set-based_ requirements: `?labelSelector=environment+in+%28production%2Cqa%29%2Ctier+in+%28frontend%29`
+
+
+Both label selector styles can be used to list or watch resources via a REST client. For example targetting apiserver
with kubectl
and using equality-based one may write:
$ kubectl get pods -l environment=production,tier=frontend
+
+or using set-based requirements:
+ +$ kubectl get pods -l 'environment in (production),tier in (frontend)'
+
+As already mentioned set-based requirements are more expressive. For instance, they can implement the OR operator on values:
+ +$ kubectl get pods -l 'environment in (production, qa)'
+
+or restricting negative matching via exists operator:
+ +$ kubectl get pods -l 'environment,environment notin (frontend)'
+
+Some Kubernetes objects, such as service
s and replicationcontroller
s, also use label selectors to specify sets of other resources, such as pods.
The set of pods that a service
targets is defined with a label selector. Similarly, the population of pods that a replicationcontroller
should manage is also defined with a label selector.
Labels selectors for both objects are defined in json
or yaml
files using maps, and only equality-based requirement selectors are supported:
"selector": {
+ "component" : "redis",
+}
+
+or
+ +selector:
+ component: redis
+
+this selector (respectively in json
or yaml
format) is equivalent to component=redis
or component in (redis)
.
Newer resources, such as job, support set-based requirements as well.
+ +selector:
+ matchLabels:
+ component: redis
+ matchExpressions:
+ - {key: tier, operator: In, values: [cache]}
+ - {key: environment, operator: NotIn, values: [dev]}
+
+matchLabels
is a map of {key,value}
pairs. A single {key,value}
in the matchLabels
map is equivalent to an element of matchExpressions
, whose key
field is “key”, the operator
is “In”, and the values
array contains only “value”. matchExpressions
is a list of pod selector requirements. Valid operators include In, NotIn, Exists, and DoesNotExist. The values set must be non-empty in the case of In and NotIn. All of the requirements, from both matchLabels
and matchExpressions
are ANDed together – they must all be satisfied in order to match.
This example shows two types of pod health checks: HTTP checks and container execution checks.
+ +The exec-liveness.yaml demonstrates the container execution check.
+ +livenessProbe:
+ exec:
+ command:
+ - cat
+ - /tmp/health
+ initialDelaySeconds: 15
+ timeoutSeconds: 1
+
+Kubelet executes the command cat /tmp/health
in the container and reports failure if the command returns a non-zero exit code.
Note that the container removes the /tmp/health
file after 10 seconds,
echo ok > /tmp/health; sleep 10; rm -rf /tmp/health; sleep 600
+
+so when Kubelet executes the health check 15 seconds (defined by initialDelaySeconds) after the container started, the check would fail.
+ +The http-liveness.yaml demonstrates the HTTP check.
+ +livenessProbe:
+ httpGet:
+ path: /healthz
+ port: 8080
+ initialDelaySeconds: 15
+ timeoutSeconds: 1
+
+The Kubelet sends an HTTP request to the specified path and port to perform the health check. If you take a look at image/server.go, you will see the server starts to respond with an error code 500 after 10 seconds, so the check fails. The Kubelet sends the probe to the container’s ip address by default which could be specified with host
as part of httpGet probe. If the container listens on 127.0.0.1
, host
should be specified as 127.0.0.1
. In general, if the container listens on its ip address or on all interfaces (0.0.0.0), there is no need to specify the host
as part of the httpGet probe.
This guide has more information on health checks.
+ +To show the health check is actually working, first create the pods:
+ +$ kubectl create -f docs/user-guide/liveness/exec-liveness.yaml
+$ kubectl create -f docs/user-guide/liveness/http-liveness.yaml
+
+Check the status of the pods once they are created:
+ +$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+[...]
+liveness-exec 1/1 Running 0 13s
+liveness-http 1/1 Running 0 13s
+
+Check the status half a minute later, you will see the container restart count being incremented:
+ +$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+[...]
+liveness-exec 1/1 Running 1 36s
+liveness-http 1/1 Running 1 36s
+
+At the bottom of the kubectl describe output there are messages indicating that the liveness probes have failed, and the containers have been killed and recreated.
+ +$ kubectl describe pods liveness-exec
+[...]
+Sat, 27 Jun 2015 13:43:03 +0200 Sat, 27 Jun 2015 13:44:34 +0200 4 {kubelet kubernetes-minion-6fbi} spec.containers{liveness} unhealthy Liveness probe failed: cat: can't open '/tmp/health': No such file or directory
+Sat, 27 Jun 2015 13:44:44 +0200 Sat, 27 Jun 2015 13:44:44 +0200 1 {kubelet kubernetes-minion-6fbi} spec.containers{liveness} killing Killing with docker id 65b52d62c635
+Sat, 27 Jun 2015 13:44:44 +0200 Sat, 27 Jun 2015 13:44:44 +0200 1 {kubelet kubernetes-minion-6fbi} spec.containers{liveness} created Created with docker id ed6bb004ee10
+Sat, 27 Jun 2015 13:44:44 +0200 Sat, 27 Jun 2015 13:44:44 +0200 1 {kubelet kubernetes-minion-6fbi} spec.containers{liveness} started Started with docker id ed6bb004ee10
+
+This example shows two types of pod health checks: HTTP checks and container execution checks.
+ +The exec-liveness.yaml demonstrates the container execution check.
+ +livenessProbe:
+ exec:
+ command:
+ - cat
+ - /tmp/health
+ initialDelaySeconds: 15
+ timeoutSeconds: 1
+
+Kubelet executes the command cat /tmp/health
in the container and reports failure if the command returns a non-zero exit code.
Note that the container removes the /tmp/health
file after 10 seconds,
echo ok > /tmp/health; sleep 10; rm -rf /tmp/health; sleep 600
+
+so when Kubelet executes the health check 15 seconds (defined by initialDelaySeconds) after the container started, the check would fail.
+ +The http-liveness.yaml demonstrates the HTTP check.
+ +livenessProbe:
+ httpGet:
+ path: /healthz
+ port: 8080
+ initialDelaySeconds: 15
+ timeoutSeconds: 1
+
+The Kubelet sends an HTTP request to the specified path and port to perform the health check. If you take a look at image/server.go, you will see the server starts to respond with an error code 500 after 10 seconds, so the check fails. The Kubelet sends the probe to the container’s ip address by default which could be specified with host
as part of httpGet probe. If the container listens on 127.0.0.1
, host
should be specified as 127.0.0.1
. In general, if the container listens on its ip address or on all interfaces (0.0.0.0), there is no need to specify the host
as part of the httpGet probe.
This guide has more information on health checks.
+ +To show the health check is actually working, first create the pods:
+ +$ kubectl create -f docs/user-guide/liveness/exec-liveness.yaml
+$ kubectl create -f docs/user-guide/liveness/http-liveness.yaml
+
+Check the status of the pods once they are created:
+ +$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+[...]
+liveness-exec 1/1 Running 0 13s
+liveness-http 1/1 Running 0 13s
+
+Check the status half a minute later, you will see the container restart count being incremented:
+ +$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+[...]
+liveness-exec 1/1 Running 1 36s
+liveness-http 1/1 Running 1 36s
+
+At the bottom of the kubectl describe output there are messages indicating that the liveness probes have failed, and the containers have been killed and recreated.
+ +$ kubectl describe pods liveness-exec
+[...]
+Sat, 27 Jun 2015 13:43:03 +0200 Sat, 27 Jun 2015 13:44:34 +0200 4 {kubelet kubernetes-minion-6fbi} spec.containers{liveness} unhealthy Liveness probe failed: cat: can't open '/tmp/health': No such file or directory
+Sat, 27 Jun 2015 13:44:44 +0200 Sat, 27 Jun 2015 13:44:44 +0200 1 {kubelet kubernetes-minion-6fbi} spec.containers{liveness} killing Killing with docker id 65b52d62c635
+Sat, 27 Jun 2015 13:44:44 +0200 Sat, 27 Jun 2015 13:44:44 +0200 1 {kubelet kubernetes-minion-6fbi} spec.containers{liveness} created Created with docker id ed6bb004ee10
+Sat, 27 Jun 2015 13:44:44 +0200 Sat, 27 Jun 2015 13:44:44 +0200 1 {kubelet kubernetes-minion-6fbi} spec.containers{liveness} started Started with docker id ed6bb004ee10
+
+This directory contains two pod specifications which can be used as synthetic +logging sources. The pod specification in synthetic_0_25lps.yaml +describes a pod that just emits a log message once every 4 seconds. The pod specification in +synthetic_10lps.yaml +describes a pod that just emits 10 log lines per second.
+ +See logging document for more details about logging. To observe the ingested log lines when using Google Cloud Logging please see the getting +started instructions +at Cluster Level Logging to Google Cloud Logging. +To observe the ingested log lines when using Elasticsearch and Kibana please see the getting +started instructions +at Cluster Level Logging with Elasticsearch and Kibana.
+ + + + + + + + + +This directory contains two pod specifications which can be used as synthetic +logging sources. The pod specification in synthetic_0_25lps.yaml +describes a pod that just emits a log message once every 4 seconds. The pod specification in +synthetic_10lps.yaml +describes a pod that just emits 10 log lines per second.
+ +See logging document for more details about logging. To observe the ingested log lines when using Google Cloud Logging please see the getting +started instructions +at Cluster Level Logging to Google Cloud Logging. +To observe the ingested log lines when using Elasticsearch and Kibana please see the getting +started instructions +at Cluster Level Logging with Elasticsearch and Kibana.
+ + + + + + + + + +Kubernetes components, such as kubelet and apiserver, use the glog logging library. Developer conventions for logging severity are described in docs/devel/logging.md.
+ +The logs of a running container may be fetched using the command kubectl logs
. For example, given
+this pod specification counter-pod.yaml, which has a container which writes out some text to standard
+output every second. (You can find different pod specifications here.)
apiVersion: v1
+kind: Pod
+metadata:
+ name: counter
+spec:
+ containers:
+ - name: count
+ image: ubuntu:14.04
+ args: [bash, -c,
+ 'for ((i = 0; ; i++)); do echo "$i: $(date)"; sleep 1; done']
+
+we can run the pod:
+ +$ kubectl create -f ./counter-pod.yaml
+pods/counter
+
+and then fetch the logs:
+ +$ kubectl logs counter
+0: Tue Jun 2 21:37:31 UTC 2015
+1: Tue Jun 2 21:37:32 UTC 2015
+2: Tue Jun 2 21:37:33 UTC 2015
+3: Tue Jun 2 21:37:34 UTC 2015
+4: Tue Jun 2 21:37:35 UTC 2015
+5: Tue Jun 2 21:37:36 UTC 2015
+...
+
+If a pod has more than one container then you need to specify which container’s log files should +be fetched e.g.
+ +$ kubectl logs kube-dns-v3-7r1l9 etcd
+2015/06/23 00:43:10 etcdserver: start to snapshot (applied: 30003, lastsnap: 20002)
+2015/06/23 00:43:10 etcdserver: compacted log at index 30003
+2015/06/23 00:43:10 etcdserver: saved snapshot at index 30003
+2015/06/23 02:05:42 etcdserver: start to snapshot (applied: 40004, lastsnap: 30003)
+2015/06/23 02:05:42 etcdserver: compacted log at index 40004
+2015/06/23 02:05:42 etcdserver: saved snapshot at index 40004
+2015/06/23 03:28:31 etcdserver: start to snapshot (applied: 50005, lastsnap: 40004)
+2015/06/23 03:28:31 etcdserver: compacted log at index 50005
+2015/06/23 03:28:31 etcdserver: saved snapshot at index 50005
+2015/06/23 03:28:56 filePurge: successfully removed file default.etcd/member/wal/0000000000000000-0000000000000000.wal
+2015/06/23 04:51:03 etcdserver: start to snapshot (applied: 60006, lastsnap: 50005)
+2015/06/23 04:51:03 etcdserver: compacted log at index 60006
+2015/06/23 04:51:03 etcdserver: saved snapshot at index 60006
+...
+
+The getting started guide Cluster Level Logging to Google Cloud Logging +explains how container logs are ingested into Google Cloud Logging +and shows how to query the ingested logs.
+ +The getting started guide Cluster Level Logging with Elasticsearch and Kibana +describes how to ingest cluster level logs into Elasticsearch and view them using Kibana.
+ +Cluster level logging only collects the standard output and standard error output of the applications +running in containers. The guide Collecting log files within containers with Fluentd explains how the log files of applications can also be ingested into Google Cloud logging.
+ +Kubernetes does log rotation for Kubernetes components and docker containers. The command kubectl logs
currently only read the latest logs, not all historical ones.
You’ve deployed your application and exposed it via a service. Now what? Kubernetes provides a number of tools to help you manage your application deployment, including scaling and updating. Among the features we’ll discuss in more depth are configuration files and labels.
+ +Table of Contents +
+ + + + + +Many applications require multiple resources to be created, such as a Replication Controller and a Service. Management of multiple resources can be simplified by grouping them together in the same file (separated by ---
in YAML). For example:
apiVersion: v1
+kind: Service
+metadata:
+ name: my-nginx-svc
+ labels:
+ app: nginx
+spec:
+ type: LoadBalancer
+ ports:
+ - port: 80
+ selector:
+ app: nginx
+---
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: my-nginx
+spec:
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+
+Multiple resources can be created the same way as a single resource:
+ +$ kubectl create -f ./nginx-app.yaml
+services/my-nginx-svc
+replicationcontrollers/my-nginx
+
+The resources will be created in the order they appear in the file. Therefore, it’s best to specify the service first, since that will ensure the scheduler can spread the pods associated with the service as they are created by the replication controller(s).
+ +kubectl create
also accepts multiple -f
arguments:
$ kubectl create -f ./nginx-svc.yaml -f ./nginx-rc.yaml
+
+And a directory can be specified rather than or in addition to individual files:
+ +$ kubectl create -f ./nginx/
+
+kubectl
will read any files with suffixes .yaml
, .yml
, or .json
.
It is a recommended practice to put resources related to the same microservice or application tier into the same file, and to group all of the files associated with your application in the same directory. If the tiers of your application bind to each other using DNS, then you can then simply deploy all of the components of your stack en masse.
+ +A URL can also be specified as a configuration source, which is handy for deploying directly from configuration files checked into github:
+ +$ kubectl create -f https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/docs/user-guide/replication.yaml
+replicationcontrollers/nginx
+
+Resource creation isn’t the only operation that kubectl
can perform in bulk. It can also extract resource names from configuration files in order to perform other operations, in particular to delete the same resources you created:
$ kubectl delete -f ./nginx/
+replicationcontrollers/my-nginx
+services/my-nginx-svc
+
+In the case of just two resources, it’s also easy to specify both on the command line using the resource/name syntax:
+ +$ kubectl delete replicationcontrollers/my-nginx services/my-nginx-svc
+
+For larger numbers of resources, one can use labels to filter resources. The selector is specified using -l
:
$ kubectl delete all -lapp=nginx
+replicationcontrollers/my-nginx
+services/my-nginx-svc
+
+Because kubectl
outputs resource names in the same syntax it accepts, it’s easy to chain operations using $()
or xargs
:
$ kubectl get $(kubectl create -f ./nginx/ | grep my-nginx)
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+my-nginx nginx nginx app=nginx 2
+NAME LABELS SELECTOR IP(S) PORT(S)
+my-nginx-svc app=nginx app=nginx 10.0.152.174 80/TCP
+
+The examples we’ve used so far apply at most a single label to any resource. There are many scenarios where multiple labels should be used to distinguish sets from one another.
+ +For instance, different applications would use different values for the app
label, but a multi-tier application, such as the guestbook example, would additionally need to distinguish each tier. The frontend could carry the following labels:
labels:
+ app: guestbook
+ tier: frontend
+
+while the Redis master and slave would have different tier
labels, and perhaps even an additional role
label:
labels:
+ app: guestbook
+ tier: backend
+ role: master
+
+and
+ +labels:
+ app: guestbook
+ tier: backend
+ role: slave
+
+The labels allow us to slice and dice our resources along any dimension specified by a label:
+ +$ kubectl create -f ./guestbook-fe.yaml -f ./redis-master.yaml -f ./redis-slave.yaml
+replicationcontrollers/guestbook-fe
+replicationcontrollers/guestbook-redis-master
+replicationcontrollers/guestbook-redis-slave
+$ kubectl get pods -Lapp -Ltier -Lrole
+NAME READY STATUS RESTARTS AGE APP TIER ROLE
+guestbook-fe-4nlpb 1/1 Running 0 1m guestbook frontend <n/a>
+guestbook-fe-ght6d 1/1 Running 0 1m guestbook frontend <n/a>
+guestbook-fe-jpy62 1/1 Running 0 1m guestbook frontend <n/a>
+guestbook-redis-master-5pg3b 1/1 Running 0 1m guestbook backend master
+guestbook-redis-slave-2q2yf 1/1 Running 0 1m guestbook backend slave
+guestbook-redis-slave-qgazl 1/1 Running 0 1m guestbook backend slave
+my-nginx-divi2 1/1 Running 0 29m nginx <n/a> <n/a>
+my-nginx-o0ef1 1/1 Running 0 29m nginx <n/a> <n/a>
+$ kubectl get pods -lapp=guestbook,role=slave
+NAME READY STATUS RESTARTS AGE
+guestbook-redis-slave-2q2yf 1/1 Running 0 3m
+guestbook-redis-slave-qgazl 1/1 Running 0 3m
+
+Another scenario where multiple labels are needed is to distinguish deployments of different releases or configurations of the same component. For example, it is common practice to deploy a canary of a new application release (specified via image tag) side by side with the previous release so that the new release can receive live production traffic before fully rolling it out. For instance, a new release of the guestbook frontend might carry the following labels:
+ +labels:
+ app: guestbook
+ tier: frontend
+ track: canary
+
+and the primary, stable release would have a different value of the track
label, so that the sets of pods controlled by the two replication controllers would not overlap:
labels:
+ app: guestbook
+ tier: frontend
+ track: stable
+
+The frontend service would span both sets of replicas by selecting the common subset of their labels, omitting the track
label:
selector:
+ app: guestbook
+ tier: frontend
+
+Sometimes existing pods and other resources need to be relabeled before creating new resources. This can be done with kubectl label
. For example:
$ kubectl label pods -lapp=nginx tier=fe
+NAME READY STATUS RESTARTS AGE
+my-nginx-v4-9gw19 1/1 Running 0 14m
+NAME READY STATUS RESTARTS AGE
+my-nginx-v4-hayza 1/1 Running 0 13m
+NAME READY STATUS RESTARTS AGE
+my-nginx-v4-mde6m 1/1 Running 0 17m
+NAME READY STATUS RESTARTS AGE
+my-nginx-v4-sh6m8 1/1 Running 0 18m
+NAME READY STATUS RESTARTS AGE
+my-nginx-v4-wfof4 1/1 Running 0 16m
+$ kubectl get pods -lapp=nginx -Ltier
+NAME READY STATUS RESTARTS AGE TIER
+my-nginx-v4-9gw19 1/1 Running 0 15m fe
+my-nginx-v4-hayza 1/1 Running 0 14m fe
+my-nginx-v4-mde6m 1/1 Running 0 18m fe
+my-nginx-v4-sh6m8 1/1 Running 0 19m fe
+my-nginx-v4-wfof4 1/1 Running 0 16m fe
+
+When load on your application grows or shrinks, it’s easy to scale with kubectl
. For instance, to increase the number of nginx replicas from 2 to 3, do:
$ kubectl scale rc my-nginx --replicas=3
+scaled
+$ kubectl get pods -lapp=nginx
+NAME READY STATUS RESTARTS AGE
+my-nginx-1jgkf 1/1 Running 0 3m
+my-nginx-divi2 1/1 Running 0 1h
+my-nginx-o0ef1 1/1 Running 0 1h
+
+At some point, you’ll eventually need to update your deployed application, typically by specifying a new image or image tag, as in the canary deployment scenario above. kubectl
supports several update operations, each of which is applicable to different scenarios.
To update a service without an outage, kubectl
supports what is called “rolling update”, which updates one pod at a time, rather than taking down the entire service at the same time. See the rolling update design document and the example of rolling update for more information.
Let’s say you were running version 1.7.9 of nginx:
+ +apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: my-nginx
+spec:
+ replicas: 5
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx:1.7.9
+ ports:
+ - containerPort: 80
+
+To update to version 1.9.1, you can use kubectl rolling-update --image
:
$ kubectl rolling-update my-nginx --image=nginx:1.9.1
+Creating my-nginx-ccba8fbd8cc8160970f63f9a2696fc46
+
+In another window, you can see that kubectl
added a deployment
label to the pods, whose value is a hash of the configuration, to distinguish the new pods from the old:
$ kubectl get pods -lapp=nginx -Ldeployment
+NAME READY STATUS RESTARTS AGE DEPLOYMENT
+my-nginx-1jgkf 1/1 Running 0 1h 2d1d7a8f682934a254002b56404b813e
+my-nginx-ccba8fbd8cc8160970f63f9a2696fc46-k156z 1/1 Running 0 1m ccba8fbd8cc8160970f63f9a2696fc46
+my-nginx-ccba8fbd8cc8160970f63f9a2696fc46-v95yh 1/1 Running 0 35s ccba8fbd8cc8160970f63f9a2696fc46
+my-nginx-divi2 1/1 Running 0 2h 2d1d7a8f682934a254002b56404b813e
+my-nginx-o0ef1 1/1 Running 0 2h 2d1d7a8f682934a254002b56404b813e
+my-nginx-q6all 1/1 Running 0 8m 2d1d7a8f682934a254002b56404b813e
+
+kubectl rolling-update
reports progress as it progresses:
Updating my-nginx replicas: 4, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 1
+At end of loop: my-nginx replicas: 4, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 1
+At beginning of loop: my-nginx replicas: 3, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 2
+Updating my-nginx replicas: 3, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 2
+At end of loop: my-nginx replicas: 3, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 2
+At beginning of loop: my-nginx replicas: 2, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 3
+Updating my-nginx replicas: 2, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 3
+At end of loop: my-nginx replicas: 2, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 3
+At beginning of loop: my-nginx replicas: 1, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 4
+Updating my-nginx replicas: 1, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 4
+At end of loop: my-nginx replicas: 1, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 4
+At beginning of loop: my-nginx replicas: 0, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 5
+Updating my-nginx replicas: 0, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 5
+At end of loop: my-nginx replicas: 0, my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 replicas: 5
+Update succeeded. Deleting old controller: my-nginx
+Renaming my-nginx-ccba8fbd8cc8160970f63f9a2696fc46 to my-nginx
+my-nginx
+
+If you encounter a problem, you can stop the rolling update midway and revert to the previous version using --rollback
:
$ kubectl kubectl rolling-update my-nginx --image=nginx:1.9.1 --rollback
+Found existing update in progress (my-nginx-ccba8fbd8cc8160970f63f9a2696fc46), resuming.
+Found desired replicas.Continuing update with existing controller my-nginx.
+Stopping my-nginx-02ca3e87d8685813dbe1f8c164a46f02 replicas: 1 -> 0
+Update succeeded. Deleting my-nginx-ccba8fbd8cc8160970f63f9a2696fc46
+my-nginx
+
+This is one example where the immutability of containers is a huge asset.
+ +If you need to update more than just the image (e.g., command arguments, environment variables), you can create a new replication controller, with a new name and distinguishing label value, such as:
+ +apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: my-nginx-v4
+spec:
+ replicas: 5
+ selector:
+ app: nginx
+ deployment: v4
+ template:
+ metadata:
+ labels:
+ app: nginx
+ deployment: v4
+ spec:
+ containers:
+ - name: nginx
+ image: nginx:1.9.2
+ args: [“nginx”,”-T”]
+ ports:
+ - containerPort: 80
+
+and roll it out:
+ +$ kubectl rolling-update my-nginx -f ./nginx-rc.yaml
+Creating my-nginx-v4
+At beginning of loop: my-nginx replicas: 4, my-nginx-v4 replicas: 1
+Updating my-nginx replicas: 4, my-nginx-v4 replicas: 1
+At end of loop: my-nginx replicas: 4, my-nginx-v4 replicas: 1
+At beginning of loop: my-nginx replicas: 3, my-nginx-v4 replicas: 2
+Updating my-nginx replicas: 3, my-nginx-v4 replicas: 2
+At end of loop: my-nginx replicas: 3, my-nginx-v4 replicas: 2
+At beginning of loop: my-nginx replicas: 2, my-nginx-v4 replicas: 3
+Updating my-nginx replicas: 2, my-nginx-v4 replicas: 3
+At end of loop: my-nginx replicas: 2, my-nginx-v4 replicas: 3
+At beginning of loop: my-nginx replicas: 1, my-nginx-v4 replicas: 4
+Updating my-nginx replicas: 1, my-nginx-v4 replicas: 4
+At end of loop: my-nginx replicas: 1, my-nginx-v4 replicas: 4
+At beginning of loop: my-nginx replicas: 0, my-nginx-v4 replicas: 5
+Updating my-nginx replicas: 0, my-nginx-v4 replicas: 5
+At end of loop: my-nginx replicas: 0, my-nginx-v4 replicas: 5
+Update succeeded. Deleting my-nginx
+my-nginx-v4
+
+You can also run the update demo to see a visual representation of the rolling update process.
+ +Sometimes it’s necessary to make narrow, non-disruptive updates to resources you’ve created. For instance, you might want to add an annotation with a description of your object. That’s easiest to do with kubectl patch
:
$ kubectl patch rc my-nginx-v4 -p '{"metadata": {"annotations": {"description": "my frontend running nginx"}}}'
+my-nginx-v4
+$ kubectl get rc my-nginx-v4 -o yaml
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ annotations:
+ description: my frontend running nginx
+...
+
+The patch is specified using json.
+ +For more significant changes, you can get
the resource, edit it, and then replace
the resource with the updated version:
$ kubectl get rc my-nginx-v4 -o yaml > /tmp/nginx.yaml
+$ vi /tmp/nginx.yaml
+$ kubectl replace -f /tmp/nginx.yaml
+replicationcontrollers/my-nginx-v4
+$ rm $TMP
+
+The system ensures that you don’t clobber changes made by other users or components by confirming that the resourceVersion
doesn’t differ from the version you edited. If you want to update regardless of other changes, remove the resourceVersion
field when you edit the resource. However, if you do this, don’t use your original configuration file as the source since additional fields most likely were set in the live state.
In some cases, you may need to update resource fields that cannot be updated once initialized, or you may just want to make a recursive change immediately, such as to fix broken pods created by a replication controller. To change such fields, use replace --force
, which deletes and re-creates the resource. In this case, you can simply modify your original configuration file:
$ kubectl replace -f ./nginx-rc.yaml --force
+replicationcontrollers/my-nginx-v4
+replicationcontrollers/my-nginx-v4
+
+Understanding how an application behaves when deployed is crucial to scaling the application and providing a reliable service. In a Kubernetes cluster, application performance can be examined at many different levels: containers, pods, services, and whole clusters. As part of Kubernetes we want to provide users with detailed resource usage information about their running applications at all these levels. This will give users deep insights into how their applications are performing and where possible application bottlenecks may be found. In comes Heapster, a project meant to provide a base monitoring platform on Kubernetes.
+ +Heapster is a cluster-wide aggregator of monitoring and event data. It currently supports Kubernetes natively and works on all Kubernetes setups. Heapster runs as a pod in the cluster, similar to how any Kubernetes application would run. The Heapster pod discovers all nodes in the cluster and queries usage information from the nodes’ Kubelets, the on-machine Kubernetes agent. The Kubelet itself fetches the data from cAdvisor. Heapster groups the information by pod along with the relevant labels. This data is then pushed to a configurable backend for storage and visualization. Currently supported backends include InfluxDB (with Grafana for visualization) and Google Cloud Monitoring. The overall architecture of the service can be seen below:
+ + + +Let’s look at some of the other components in more detail.
+ +cAdvisor is an open source container resource usage and performance analysis agent. It is purpose built for containers and supports Docker containers natively. In Kubernetes, cadvisor is integrated into the Kubelet binary. cAdvisor auto-discovers all containers in the machine and collects CPU, memory, filesystem, and network usage statistics. cAdvisor also provides the overall machine usage by analyzing the “root” container on the machine.
+ +On most Kubernetes clusters, cAdvisor exposes a simple UI for on-machine containers on port 4194. Here is a snapshot of part of cAdvisor’s UI that shows the overall machine usage:
+ + + +The Kubelet acts as a bridge between the Kubernetes master and the nodes. It manages the pods and containers running on a machine. Kubelet translates each pod into its constituent containers and fetches individual container usage statistics from cAdvisor. It then exposes the aggregated pod resource usage statistics via a REST API.
+ +A Grafana setup with InfluxDB is a very popular combination for monitoring in the open source world. InfluxDB exposes an easy to use API to write and fetch time series data. Heapster is setup to use this storage backend by default on most Kubernetes clusters. A detailed setup guide can be found here. InfluxDB and Grafana run in Pods. The pod exposes itself as a Kubernetes service which is how Heapster discovers it.
+ +The Grafana container serves Grafana’s UI which provides an easy to configure dashboard interface. The default dashboard for Kubernetes contains an example dashboard that monitors resource usage of the cluster and the pods inside of it. This dashboard can easily be customized and expanded. Take a look at the storage schema for InfluxDB here.
+ +Here is a video showing how to monitor a Kubernetes cluster using heapster, InfluxDB and Grafana:
+ + + +Here is a snapshot of the default Kubernetes Grafana dashboard that shows the CPU and Memory usage of the entire cluster, individual pods and containers:
+ + + +Google Cloud Monitoring is a hosted monitoring service that allows you to visualize and alert on important metrics in your application. Heapster can be setup to automatically push all collected metrics to Google Cloud Monitoring. These metrics are then available in the Cloud Monitoring Console. This storage backend is the easiest to setup and maintain. The monitoring console allows you to easily create and customize dashboards using the exported data.
+ +Here is a video showing how to setup and run a Google Cloud Monitoring backed Heapster:
+ + + +Here is a snapshot of the a Google Cloud Monitoring dashboard showing cluster-wide resource usage.
+ + + +Now that you’ve learned a bit about Heapster, feel free to try it out on your own clusters! The Heapster repository is available on GitHub. It contains detailed instructions to setup Heapster and its storage backends. Heapster runs by default on most Kubernetes clusters, so you may already have it! Feedback is always welcome. Please let us know if you run into any issues via the troubleshooting channels.
+ +Authors: Vishnu Kannan and Victor Marmol, Google Software Engineers. +This article was originally posted in Kubernetes blog.
+ + + + + + + + + +Kubernetes supports multiple virtual clusters backed by the same physical cluster. +These virtual clusters are called namespaces.
+ +Namespaces are intended for use in environments with many users spread across multiple +teams, or projects. For clusters with a few to tens of users, you should not +need to create or think about namespaces at all. Start using namespaces when you +need the features they provide.
+ +Namespaces provide a scope for names. Names of resources need to be unique within a namespace, but not across namespaces.
+ +Namespaces are a way to divide cluster resources between multiple uses (via resource quota).
+ +In future versions of Kubernetes, objects in the same namespace will have the same +access control policies by default.
+ +It is not necessary to use multiple namespaces just to separate slightly different +resources, such as different versions of the same software: use labels to distinguish +resources within the same namespace.
+ +Creation and deletion of namespaces is described in the Admin Guide documentation +for namespaces
+ +You can list the current namespaces in a cluster using:
+ +$ kubectl get namespaces
+NAME LABELS STATUS
+default <none> Active
+kube-system <none> Active
+
+Kubernetes starts with two initial namespaces:
+ * default
The default namespace for objects with no other namespace
+ * kube-system
The namespace for objects created by the Kubernetes system
To temporarily set the namespace for a request, use the --namespace
flag.
For example:
+ +$ kubectl --namespace=<insert-namespace-name-here> run nginx --image=nginx
+$ kubectl --namespace=<insert-namespace-name-here> get pods
+
+You can permanently save the namespace for all subsequent kubectl commands in that +context.
+ +First get your current context:
+ +$ export CONTEXT=$(kubectl config view | grep current-context | awk '{print $2}')
+
+Then update the default namespace:
+ +$ kubectl config set-context $(CONTEXT) --namespace=<insert-namespace-name-here>
+
+When you create a Service, it creates a corresponding DNS entry.
+This entry is of the form <service-name>.<namespace-name>.svc.cluster.local
, which means
+that if a container just uses <service-name>
it will resolve to the service which
+is local to a namespace. This is useful for using the same configuration across
+multiple namespaces such as Development, Staging and Production. If you want to reach
+across namespaces, you need to use the fully qualified domain name (FQDN).
Most kubernetes resources (e.g. pods, services, replication controllers, and others) are +in a some namespace. However namespace resources are not themselves in a namespace. +And, low-level resources, such as nodes and +persistentVolumes, are not in any namespace. Events are an exception: they may or may not +have a namespace, depending on the object the event is about.
+ + + + + + + + + +This example shows how to assign a pod to a specific node or to one of a set of nodes using node labels and the nodeSelector field in a pod specification. Generally this is unnecessary, as the scheduler will take care of things for you, but you may want to do so in certain circumstances like to ensure that your pod ends up on a machine with an SSD attached to it.
+ +This example assumes that you have a basic understanding of Kubernetes pods and that you have turned up a Kubernetes cluster.
+ +Run kubectl get nodes
to get the names of your cluster’s nodes. Pick out the one that you want to add a label to.
Then, to add a label to the node you’ve chosen, run kubectl label nodes <node-name> <label-key>=<label-value>
. For example, if my node name is ‘kubernetes-foo-node-1.c.a-robinson.internal’ and my desired label is ‘disktype=ssd’, then I can run kubectl label nodes kubernetes-foo-node-1.c.a-robinson.internal disktype=ssd
.
If this fails with an “invalid command” error, you’re likely using an older version of kubectl that doesn’t have the label
command. In that case, see the previous version of this guide for instructions on how to manually set labels on a node.
Also, note that label keys must be in the form of DNS labels (as described in the identifiers doc), meaning that they are not allowed to contain any upper-case letters.
+ +You can verify that it worked by re-running kubectl get nodes
and checking that the node now has a label.
Take whatever pod config file you want to run, and add a nodeSelector section to it, like this. For example, if this is my pod config:
+ +
+apiVersion: v1
+kind: Pod
+metadata:
+ name: nginx
+ labels:
+ env: test
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+
+
+Then add a nodeSelector like so:
+ +
+apiVersion: v1
+kind: Pod
+metadata:
+ name: nginx
+ labels:
+ env: test
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+ imagePullPolicy: IfNotPresent
+ nodeSelector:
+ disktype: ssd
+
+
+When you then run kubectl create -f pod.yaml
, the pod will get scheduled on the node that you attached the label to! You can verify that it worked by running kubectl get pods -o wide
and looking at the “NODE” that the pod was assigned to.
While this example only covered one node, you can attach labels to as many nodes as you want. Then when you schedule a pod with a nodeSelector, it can be scheduled on any of the nodes that satisfy that nodeSelector. Be careful that it will match at least one node, however, because if it doesn’t the pod won’t be scheduled at all.
+ + + + + + + + + +This example shows how to assign a pod to a specific node or to one of a set of nodes using node labels and the nodeSelector field in a pod specification. Generally this is unnecessary, as the scheduler will take care of things for you, but you may want to do so in certain circumstances like to ensure that your pod ends up on a machine with an SSD attached to it.
+ +This example assumes that you have a basic understanding of Kubernetes pods and that you have turned up a Kubernetes cluster.
+ +Run kubectl get nodes
to get the names of your cluster’s nodes. Pick out the one that you want to add a label to.
Then, to add a label to the node you’ve chosen, run kubectl label nodes <node-name> <label-key>=<label-value>
. For example, if my node name is ‘kubernetes-foo-node-1.c.a-robinson.internal’ and my desired label is ‘disktype=ssd’, then I can run kubectl label nodes kubernetes-foo-node-1.c.a-robinson.internal disktype=ssd
.
If this fails with an “invalid command” error, you’re likely using an older version of kubectl that doesn’t have the label
command. In that case, see the previous version of this guide for instructions on how to manually set labels on a node.
Also, note that label keys must be in the form of DNS labels (as described in the identifiers doc), meaning that they are not allowed to contain any upper-case letters.
+ +You can verify that it worked by re-running kubectl get nodes
and checking that the node now has a label.
Take whatever pod config file you want to run, and add a nodeSelector section to it, like this. For example, if this is my pod config:
+ +
+apiVersion: v1
+kind: Pod
+metadata:
+ name: nginx
+ labels:
+ env: test
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+
+
+Then add a nodeSelector like so:
+ +
+apiVersion: v1
+kind: Pod
+metadata:
+ name: nginx
+ labels:
+ env: test
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+ imagePullPolicy: IfNotPresent
+ nodeSelector:
+ disktype: ssd
+
+
+When you then run kubectl create -f pod.yaml
, the pod will get scheduled on the node that you attached the label to! You can verify that it worked by running kubectl get pods -o wide
and looking at the “NODE” that the pod was assigned to.
While this example only covered one node, you can attach labels to as many nodes as you want. Then when you schedule a pod with a nodeSelector, it can be scheduled on any of the nodes that satisfy that nodeSelector. Be careful that it will match at least one node, however, because if it doesn’t the pod won’t be scheduled at all.
+ + + + + + + + + +Kubernetes is an open-source system for managing containerized applications across multiple hosts in a cluster. Kubernetes is intended to make deploying containerized/microservice-based applications easy but powerful.
+ +Kubernetes provides mechanisms for application deployment, scheduling, updating, maintenance, and scaling. A key feature of Kubernetes is that it actively manages the containers to ensure that the state of the cluster continually matches the user’s intentions. An operations user should be able to launch a micro-service, letting the scheduler find the right placement. We also want to improve the tools and experience for how users can roll-out applications through patterns like canary deployments.
+ +Kubernetes supports Docker and Rocket containers, and other container image formats and container runtimes will be supported in the future.
+ +While Kubernetes currently focuses on continuously-running stateless (e.g. web server or in-memory object cache) and “cloud native” stateful applications (e.g. NoSQL datastores), in the near future it will support all the other workload types commonly found in production cluster environments, such as batch, stream processing, and traditional databases.
+ +In Kubernetes, all containers run inside pods. A pod can host a single container, or multiple cooperating containers; in the latter case, the containers in the pod are guaranteed to be co-located on the same machine and can share resources. A pod can also contain zero or more volumes, which are directories that are private to a container or shared across containers in a pod. For each pod the user creates, the system finds a machine that is healthy and that has sufficient available capacity, and starts up the corresponding container(s) there. If a container fails it can be automatically restarted by Kubernetes’ node agent, called the Kubelet. But if the pod or its machine fails, it is not automatically moved or restarted unless the user also defines a replication controller, which we discuss next.
+ +Users can create and manage pods themselves, but Kubernetes drastically simplifies system management by allowing users to delegate two common pod-related activities: deploying multiple pod replicas based on the same pod configuration, and creating replacement pods when a pod or its machine fails. The Kubernetes API object that manages these behaviors is called a replication controller. It defines a pod in terms of a template, that the system then instantiates as some number of pods (specified by the user). The replicated set of pods might constitute an entire application, a micro-service, or one layer in a multi-tier application. Once the pods are created, the system continually monitors their health and that of the machines they are running on; if a pod fails due to a software problem or machine failure, the replication controller automatically creates a new pod on a healthy machine, to maintain the set of pods at the desired replication level. Multiple pods from the same or different applications can share the same machine. Note that a replication controller is needed even in the case of a single non-replicated pod if the user wants it to be re-created when it or its machine fails.
+ +Frequently it is useful to refer to a set of pods, for example to limit the set of pods on which a mutating operation should be performed, or that should be queried for status. As a general mechanism, users can attach to most Kubernetes API objects arbitrary key-value pairs called labels, and then use a set of label selectors (key-value queries over labels) to constrain the target of API operations. Each resource also has a map of string keys and values that can be used by external tooling to store and retrieve arbitrary metadata about this object, called annotations.
+ +Kubernetes supports a unique networking model. Kubernetes encourages a flat address space and does not dynamically allocate ports, instead allowing users to select whichever ports are convenient for them. To achieve this, it allocates an IP address for each pod.
+ +Modern Internet applications are commonly built by layering micro-services, for example a set of web front-ends talking to a distributed in-memory key-value store talking to a replicated storage service. To facilitate this architecture, Kubernetes offers the service abstraction, which provides a stable IP address and DNS name that corresponds to a dynamic set of pods such as the set of pods constituting a micro-service. The set is defined using a label selector and thus can refer to any set of pods. When a container running in a Kubernetes pod connects to this address, the connection is forwarded by a local agent (called the kube proxy) running on the source machine, to one of the corresponding back-end containers. The exact back-end is chosen using a round-robin policy to balance load. The kube proxy takes care of tracking the dynamic set of back-ends as pods are replaced by new pods on new hosts, so that the service IP address (and DNS name) never changes.
+ +Every resource in Kubernetes, such as a pod, is identified by a URI and has a UID. Important components of the URI are the kind of object (e.g. pod), the object’s name, and the object’s namespace. For a certain object kind, every name is unique within its namespace. In contexts where an object name is provided without a namespace, it is assumed to be in the default namespace. UID is unique across time and space.
+ + + + + + + + + +The purpose of this guide is to help you become familiar with Kubernetes Persistent Volumes. By the end of the guide, we’ll have +nginx serving content from your persistent volume.
+ +This guide assumes knowledge of Kubernetes fundamentals and that you have a cluster up and running.
+ +See Persistent Storage design document for more information.
+ +A Persistent Volume (PV) in Kubernetes represents a real piece of underlying storage capacity in the infrastructure. Cluster administrators +must first create storage (create their Google Compute Engine (GCE) disks, export their NFS shares, etc.) in order for Kubernetes to mount it.
+ +PVs are intended for “network volumes” like GCE Persistent Disks, NFS shares, and AWS ElasticBlockStore volumes. HostPath
was included
+for ease of development and testing. You’ll create a local HostPath
for this example.
++ +IMPORTANT! For
+HostPath
to work, you will need to run a single node cluster. Kubernetes does not +support local storage on the host at this time. There is no guarantee your pod ends up on the correct node where theHostPath
resides.
# This will be nginx's webroot
+$ mkdir /tmp/data01
+$ echo 'I love Kubernetes storage!' > /tmp/data01/index.html
+
+PVs are created by posting them to the API server.
+ +$ kubectl create -f docs/user-guide/persistent-volumes/volumes/local-01.yaml
+NAME LABELS CAPACITY ACCESSMODES STATUS CLAIM REASON
+pv0001 type=local 10737418240 RWO Available
+
+Users of Kubernetes request persistent storage for their pods. They don’t know how the underlying cluster is provisioned. +They just know they can rely on their claim to storage and can manage its lifecycle independently from the many pods that may use it.
+ +Claims must be created in the same namespace as the pods that use them.
+ +$ kubectl create -f docs/user-guide/persistent-volumes/claims/claim-01.yaml
+
+$ kubectl get pvc
+NAME LABELS STATUS VOLUME
+myclaim-1 map[]
+
+
+# A background process will attempt to match this claim to a volume.
+# The eventual state of your claim will look something like this:
+
+$ kubectl get pvc
+NAME LABELS STATUS VOLUME
+myclaim-1 map[] Bound pv0001
+
+$ kubectl get pv
+NAME LABELS CAPACITY ACCESSMODES STATUS CLAIM REASON
+pv0001 type=local 10737418240 RWO Bound default/myclaim-1
+
+Claims are used as volumes in pods. Kubernetes uses the claim to look up its bound PV. The PV is then exposed to the pod.
+ +$ kubectl create -f docs/user-guide/persistent-volumes/simpletest/pod.yaml
+
+$ kubectl get pods
+NAME READY STATUS RESTARTS AGE
+mypod 1/1 Running 0 1h
+
+$ kubectl create -f docs/user-guide/persistent-volumes/simpletest/service.json
+$ kubectl get services
+NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE
+frontendservice 10.0.0.241 <none> 3000/TCP name=frontendhttp 1d
+kubernetes 10.0.0.2 <none> 443/TCP <none> 2d
+
+You should be able to query your service endpoint and see what content nginx is serving. A “forbidden” error might mean you +need to disable SELinux (setenforce 0).
+ +$ curl 10.0.0.241:3000
+I love Kubernetes storage!
+
+Hopefully this simple guide is enough to get you started with PersistentVolumes. If you have any questions, join the team on Slack and ask!
+ +Enjoy!
+ + + + + + + + + +This document describes the current state of PersistentVolumes
in Kubernetes. Familiarity with volumes is suggested.
Table of Contents +
+ +Managing storage is a distinct problem from managing compute. The PersistentVolume
subsystem provides an API for users and administrators that abstracts details of how storage is provided from how it is consumed. To do this we introduce two new API resources: PersistentVolume
and PersistentVolumeClaim
.
A PersistentVolume
(PV) is a piece of networked storage in the cluster that has been provisioned by an administrator. It is a resource in the cluster just like a node is a cluster resource. PVs are volume plugins like Volumes, but have a lifecycle independent of any individual pod that uses the PV. This API object captures the details of the implementation of the storage, be that NFS, iSCSI, or a cloud-provider-specific storage system.
A PersistentVolumeClaim
(PVC) is a request for storage by a user. It is similar to a pod. Pods consume node resources and PVCs consume PV resources. Pods can request specific levels of resources (CPU and Memory). Claims can request specific size and access modes (e.g, can be mounted once read/write or many times read-only).
Please see the detailed walkthrough with working examples.
+ +PVs are resources in the cluster. PVCs are requests for those resources and also act as claim checks to the resource. The interaction between PVs and PVCs follows this lifecycle:
+ +A cluster administrator will create a number of PVs. They carry the details of the real storage which is available for use by cluster users. They exist in the Kubernetes API and are available for consumption.
+ +A user creates a PersistentVolumeClaim
with a specific amount of storage requested and with certain access modes. A control loop in the master watches for new PVCs, finds a matching PV (if possible), and binds them together. The user will always get at least what they asked for, but the volume may be in excess of what was requested.
Claims will remain unbound indefinitely if a matching volume does not exist. Claims will be bound as matching volumes become available. For example, a cluster provisioned with many 50Gi PVs would not match a PVC requesting 100Gi. The PVC can be bound when a 100Gi PV is added to the cluster.
+ +Pods use claims as volumes. The cluster inspects the claim to find the bound volume and mounts that volume for a pod. For volumes which support multiple access modes, the user specifies which mode desired when using their claim as a volume in a pod.
+ +Once a user has a claim and that claim is bound, the bound PV belongs to the user for as long as they need it. Users schedule Pods and access their claimed PVs by including a persistentVolumeClaim in their Pod’s volumes block. See below for syntax details.
+ +When a user is done with their volume, they can delete the PVC objects from the API which allows reclamation of the resource. The volume is considered “released” when the claim is deleted, but it is not yet available for another claim. The previous claimant’s data remains on the volume which must be handled according to policy.
+ +The reclaim policy for a PersistentVolume
tells the cluster what to do with the volume after it has been released. Currently, volumes can either be Retained or Recycled. Retention allows for manual reclamation of the resource. For those volume plugins that support it, recycling performs a basic scrub (rm -rf /thevolume/*
) on the volume and makes it available again for a new claim.
PersistentVolume
types are implemented as plugins. Kubernetes currently supports the following plugins:
Each PV contains a spec and status, which is the specification and status of the volume.
+ +apiVersion: v1
+ kind: PersistentVolume
+ metadata:
+ name: pv0003
+ spec:
+ capacity:
+ storage: 5Gi
+ accessModes:
+ - ReadWriteOnce
+ persistentVolumeReclaimPolicy: Recycle
+ nfs:
+ path: /tmp
+ server: 172.17.0.2
+
+Generally, a PV will have a specific storage capacity. This is set using the PV’s capacity
attribute. See the Kubernetes Resource Model to understand the units expected by capacity
.
Currently, storage size is the only resource that can be set or requested. Future attributes may include IOPS, throughput, etc.
+ +A PersistentVolume
can be mounted on a host in any way supported by the resource provider. Providers will have different capabilities and each PV’s access modes are set to the specific modes supported by that particular volume. For example, NFS can support multiple read/write clients, but a specific NFS PV might be exported on the server as read-only. Each PV gets its own set of access modes describing that specific PV’s capabilities.
The access modes are:
+ +In the CLI, the access modes are abbreviated to:
+ +++ +Important! A volume can only be mounted using one access mode at a time, even if it supports many. For example, a GCEPersistentDisk can be mounted as ReadWriteOnce by a single node or ReadOnlyMany by many nodes, but not at the same time.
+
Current recycling policies are:
+ +Currently, NFS and HostPath support recycling.
+ +A volume will be in one of the following phases:
+ +The CLI will show the name of the PVC bound to the PV.
+ +Each PVC contains a spec and status, which is the specification and status of the claim.
+ +kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+ name: myclaim
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 8Gi
+
+Claims use the same conventions as volumes when requesting storage with specific access modes.
+ +Claims, like pods, can request specific quantities of a resource. In this case, the request is for storage. The same resource model applies to both volumes and claims.
+ +Pods access storage by using the claim as a volume. Claims must exist in the same namespace as the pod using the claim. The cluster finds the claim in the pod’s namespace and uses it to get the PersistentVolume
backing the claim. The volume is then mounted to the host and into the pod.
kind: Pod
+apiVersion: v1
+metadata:
+ name: mypod
+spec:
+ containers:
+ - name: myfrontend
+ image: dockerfile/nginx
+ volumeMounts:
+ - mountPath: "/var/www/html"
+ name: mypd
+ volumes:
+ - name: mypd
+ persistentVolumeClaim:
+ claimName: myclaim
+
+Updated: 4/14/2015
+ +This document covers the lifecycle of a pod. It is not an exhaustive document, but an introduction to the topic.
+ +As consistent with the overall API convention, phase is a simple, high-level summary of the phase of the lifecycle of a pod. It is not intended to be a comprehensive rollup of observations of container-level or even pod-level conditions or other state, nor is it intended to be a comprehensive state machine.
+ +The number and meanings of PodPhase
values are tightly guarded. Other than what is documented here, nothing should be assumed about pods with a given PodPhase
.
A pod containing containers that specify readiness probes will also report the Ready condition. Condition status values may be True
, False
, or Unknown
.
A Probe is a diagnostic performed periodically by the kubelet on a container. Specifically the diagnostic is one of three Handlers:
+ +ExecAction
: executes a specified command inside the container expecting on success that the command exits with status code 0.TCPSocketAction
: performs a tcp check against the container’s IP address on a specified port expecting on success that the port is open.HTTPGetAction
: performs an HTTP Get against the container’s IP address on a specified port and path expecting on success that the response has a status code greater than or equal to 200 and less than 400.Each probe will have one of three results:
+ +Success
: indicates that the container passed the diagnostic.Failure
: indicates that the container failed the diagnostic.Unknown
: indicates that the diagnostic failed so no action should be taken.Currently, the kubelet optionally performs two independent diagnostics on running containers which trigger action:
+ +LivenessProbe
: indicates whether the container is live, i.e. still running. The LivenessProbe hints to the kubelet when a container is unhealthy. If the LivenessProbe fails, the kubelet will kill the container and the container will be subjected to it’s RestartPolicy. The default state of Liveness before the initial delay is Success
. The state of Liveness for a container when no probe is provided is assumed to be Success
.ReadinessProbe
: indicates whether the container is ready to service requests. If the ReadinessProbe fails, the endpoints controller will remove the pod’s IP address from the endpoints of all services that match the pod. Thus, the ReadinessProbe is sometimes useful to signal to the endpoints controller that even though a pod may be running, it should not receive traffic from the proxy (e.g. the container has a long startup time before it starts listening or the container is down for maintenance). The default state of Readiness before the initial delay is Failure
. The state of Readiness for a container when no probe is provided is assumed to be Success
.More detailed information about the current (and previous) container statuses can be found in ContainerStatuses. The information reported depends on the current ContainerState, which may be Waiting, Running, or Terminated.
+ +The possible values for RestartPolicy are Always
, OnFailure
, or Never
. If RestartPolicy is not set, the default value is Always
. RestartPolicy applies to all containers in the pod. RestartPolicy only refers to restarts of the containers by the Kubelet on the same node. Failed containers that are restarted by Kubelet, are restarted with an exponential back-off delay, the delay is in multiples of sync-frequency 0, 1x, 2x, 4x, 8x … capped at 5 minutes and is reset after 10 minutes of successful execution. As discussed in the pods document, once bound to a node, a pod will never be rebound to another node. This means that some kind of controller is necessary in order for a pod to survive node failure, even if just a single pod at a time is desired.
Three types of controllers are currently available:
+ +Job
for pods which are expected to terminate (e.g. batch computations).ReplicationController
for pods which are not expected to
+terminate, and where (e.g. web servers).DaemonSet
: Use for pods which need to run 1 per machine because they provide a
+machine-specific system service.
+If you are unsure whether to use ReplicationController or Daemon, then see Daemon Set versus
+Replication Controller.ReplicationController
is only appropriate for pods with RestartPolicy = Always
.
+Job
is only appropriate for pods with RestartPolicy
equal to OnFailure
or Never
.
All 3 types of controllers contain a PodTemplate, which has all the same fields as a Pod. +It is recommended to create the appropriate controller and let it create pods, rather than to +directly create pods yourself. That is because pods alone are not resilient to machine failures, +but Controllers are.
+ +In general, pods which are created do not disappear until someone destroys them. This might be a human or a ReplicationController
, or another controller. The only exception to this rule is that pods with a PodPhase
of Succeeded
or Failed
for more than some duration (determined by the master) will expire and be automatically reaped.
If a node dies or is disconnected from the rest of the cluster, some entity within the system (call it the NodeController for now) is responsible for applying policy (e.g. a timeout) and marking any pods on the lost node as Failed
.
Running
, 1 container, container exits success
+ Running
Succeeded
Succeeded
Running
, 1 container, container exits failure
+ Running
Running
Failed
Running
, 2 containers, container 1 exits failure
+ Running
Running
Running
Running
Running
Failed
Running
, container becomes OOM
+ Running
Running
Failed
Running
, a disk dies
+ Failed
Running
, its node is segmented out
+ Failed
Table of Contents +
+ +In Kubernetes, rather than individual application containers, pods are the smallest deployable units that can be created, scheduled, and managed.
+ +A pod (as in a pod of whales or pea pod) corresponds to a colocated group of applications running with a shared context. Within that context, the applications may also have individual cgroup isolations applied. A pod models an application-specific “logical host” in a containerized environment. It may contain one or more applications which are relatively tightly coupled — in a pre-container world, they would have executed on the same physical or virtual host.
+ +The context of the pod can be defined as the conjunction of several Linux namespaces:
+ +Applications within a pod also have access to shared volumes, which are defined at the pod level and made available in each application’s filesystem. Additionally, a pod may define top-level cgroup isolations which form an outer bound to any individual isolation applied to constituent applications.
+ +In terms of Docker constructs, a pod consists of a colocated group of Docker containers with shared volumes. PID namespace sharing is not yet implemented with Docker.
+ +Like individual application containers, pods are considered to be relatively ephemeral rather than durable entities. As discussed in life of a pod, pods are scheduled to nodes and remain there until termination (according to restart policy) or deletion. When a node dies, the pods scheduled to that node are deleted. Specific pods are never rescheduled to new nodes; instead, they must be replaced (see replication controller for more details). (In the future, a higher-level API may support pod migration.)
+ +Pods facilitate data sharing and communication among their constituents.
+ +The applications in the pod all use the same network namespace/IP and port space, and can find and communicate with each other using localhost. Each pod has an IP address in a flat shared networking namespace that has full communication with other physical computers and containers across the network. The hostname is set to the pod’s Name for the application containers within the pod. More details on networking.
+ +In addition to defining the application containers that run in the pod, the pod specifies a set of shared storage volumes. Volumes enable data to survive container restarts and to be shared among the applications within the pod.
+ +Pods also simplify application deployment and management by providing a higher-level abstraction than the raw, low-level container interface. Pods serve as units of deployment and horizontal scaling/replication. Co-location (co-scheduling), fate sharing, coordinated replication, resource sharing, and dependency management are handled automatically.
+ +Pods can be used to host vertically integrated application stacks, but their primary motivation is to support co-located, co-managed helper programs, such as:
+ +Individual pods are not intended to run multiple instances of the same application, in general.
+ +For a longer explanation, see The Distributed System ToolKit: Patterns for Composite Containers.
+ +Why not just run multiple programs in a single (Docker) container?
+ +Why not support affinity-based co-scheduling of containers?
+ +That approach would provide co-location, but would not provide most of the benefits of pods, such as resource sharing, IPC, guaranteed fate sharing, and simplified management.
+ +Pods aren’t intended to be treated as durable pets. They won’t survive scheduling failures, node failures, or other evictions, such as due to lack of resources, or in the case of node maintenance.
+ +In general, users shouldn’t need to create pods directly. They should almost always use controllers (e.g., replication controller), even for singletons. Controllers provide self-healing with a cluster scope, as well as replication and rollout management.
+ +The use of collective APIs as the primary user-facing primitive is relatively common among cluster scheduling systems, including Borg, Marathon, Aurora, and Tupperware.
+ +Pod is exposed as a primitive in order to facilitate:
+ +The current best practice for pets is to create a replication controller with replicas
equal to 1
and a corresponding service. If you find this cumbersome, please comment on issue #260.
Because pods represent running processes on nodes in the cluster, it is important to allow those processes to gracefully terminate when they are no longer needed (vs being violently killed with a KILL signal and having no chance to clean up). Users should be able to request deletion and know when processes terminate, but also be able to ensure that deletes eventually complete. When a user requests deletion of a pod the system records the intended grace period before the pod is allowed to be forcefully killed, and a TERM signal is sent to the main process in each container. Once the grace period has expired the KILL signal is sent to those processes and the pod is then deleted from the API server. If the Kubelet or the container manager is restarted while waiting for processes to terminate, the termination will be retried with the full grace period.
+ +An example flow:
+ +preStop
hook is still running after the grace period expires, step 2 is then invoked with a small (2 second) extended grace period.By default, all deletes are graceful within 30 seconds. The kubectl delete
command supports the --grace-period=<seconds>
option which allows a user to override the default and specify their own value. The value 0
indicates that delete should be immediate, and removes the pod in the API immediately so a new pod can be created with the same name. On the node pods that are set to terminate immediately will still be given a small grace period before being force killed.
From kubernetes v1.1, any container in a pod can enable privileged mode, using the privileged
flag on the SecurityContext
of the container spec. This is useful for containers that want to use linux capabilities like manipulating the network stack and accessing devices. Processes within the container get almost the same privileges that are available to processes outside a container. With privileged mode, it should be easier to write network and volume plugins as seperate pods that don’t need to be compiled into the kubelet.
If the master is running kubernetes v1.1 or higher, and the nodes are running a version lower than v1.1, then new privileged pods will be accepted by api-server, but will not be launched. They will be pending state.
+If user calls kubectl describe pod FooPodName
, user can see the reason why the pod is in pending state. The events table in the describe command output will say:
+Error validating pod "FooPodName"."FooPodNamespace" from api, ignoring: spec.containers[0].securityContext.privileged: forbidden '<*>(0xc2089d3248)true'
If the master is running a version lower than v1.1, then privileged pods cannot be created. If user attempts to create a pod, that has a privileged container, the user will get the following error:
+The Pod "FooPodName" is invalid.
+spec.containers[0].securityContext.privileged: forbidden '<*>(0xc20b222db0)true'
Pod is a top-level resource in the kubernetes REST API. More details about the +API object can be found at: Pod API +object.
+ + + + + + + + + +To deploy and manage applications on Kubernetes, you’ll use the Kubernetes command-line tool, kubectl. It lets you inspect your cluster resources, create, delete, and update components, and much more. You will use it to look at your new cluster and bring up example apps.
+ +If you downloaded a pre-compiled release, kubectl should be under platforms/<os>/<arch>
from the tar bundle.
If you built from source, kubectl should be either under _output/local/bin/<os>/<arch>
or _output/dockerized/bin/<os>/<arch>
.
The kubectl binary doesn’t have to be installed to be executable, but the rest of the walkthrough will assume that it’s in your PATH.
+ +The simplest way to install is to copy or move kubectl into a dir already in PATH (e.g. /usr/local/bin
). For example:
# OS X
+$ sudo cp kubernetes/platforms/darwin/amd64/kubectl /usr/local/bin/kubectl
+# Linux
+$ sudo cp kubernetes/platforms/linux/amd64/kubectl /usr/local/bin/kubectl
+
+You also need to ensure it’s executable:
+ +$ sudo chmod +x /usr/local/bin/kubectl
+
+If you prefer not to copy kubectl, you need to ensure the tool is in your path:
+ +# OS X
+export PATH=<path/to/kubernetes-directory>/platforms/darwin/amd64:$PATH
+
+# Linux
+export PATH=<path/to/kubernetes-directory>/platforms/linux/amd64:$PATH
+
+In order for kubectl to find and access the Kubernetes cluster, it needs a kubeconfig file, which is created automatically when creating a cluster using kube-up.sh (see the getting started guides for more about creating clusters). If you need access to a cluster you didn’t create, see the Sharing Cluster Access document.
+By default, kubectl configuration lives at ~/.kube/config
.
Check that kubectl is properly configured by getting the cluster state:
+ +$ kubectl cluster-info
+
+If you see a url response, you are ready to go.
+ +Table of Contents +
+ +You’ve seen how to configure and deploy pods and containers, using some of the most common configuration parameters. This section dives into additional features that are especially useful for running applications in production.
+ +The container file system only lives as long as the container does, so when a container crashes and restarts, changes to the filesystem will be lost and the container will restart from a clean slate. To access more-persistent storage, outside the container file system, you need a volume. This is especially important to stateful applications, such as key-value stores and databases.
+ +For example, Redis is a key-value cache and store, which we use in the guestbook and other examples. We can add a volume to it to store persistent data as follows:
+ +apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: redis
+spec:
+ template:
+ metadata:
+ labels:
+ app: redis
+ tier: backend
+ spec:
+ # Provision a fresh volume for the pod
+ volumes:
+ - name: data
+ emptyDir: {}
+ containers:
+ - name: redis
+ image: kubernetes/redis:v1
+ ports:
+ - containerPort: 6379
+ # Mount the volume into the pod
+ volumeMounts:
+ - mountPath: /redis-master-data
+ name: data # must match the name of the volume, above
+
+emptyDir
volumes live for the lifespan of the pod, which is longer than the lifespan of any one container, so if the container fails and is restarted, our storage will live on.
In addition to the local disk storage provided by emptyDir
, Kubernetes supports many different network-attached storage solutions, including PD on GCE and EBS on EC2, which are preferred for critical data, and will handle details such as mounting and unmounting the devices on the nodes. See the volumes doc for more details.
Many applications need credentials, such as passwords, OAuth tokens, and TLS keys, to authenticate with other applications, databases, and services. Storing these credentials in container images or environment variables is less than ideal, since the credentials can then be copied by anyone with access to the image, pod/container specification, host file system, or host Docker daemon.
+ +Kubernetes provides a mechanism, called secrets, that facilitates delivery of sensitive credentials to applications. A Secret
is a simple resource containing a map of data. For instance, a simple secret with a username and password might look as follows:
apiVersion: v1
+kind: Secret
+metadata:
+ name: mysecret
+type: Opaque
+data:
+ password: dmFsdWUtMg0K
+ username: dmFsdWUtMQ0K
+
+As with other resources, this secret can be instantiated using create
and can be viewed with get
:
$ kubectl create -f ./secret.yaml
+secrets/mysecret
+$ kubectl get secrets
+NAME TYPE DATA
+default-token-v9pyz kubernetes.io/service-account-token 2
+mysecret Opaque 2
+
+To use the secret, you need to reference it in a pod or pod template. The secret
volume source enables you to mount it as an in-memory directory into your containers.
apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: redis
+spec:
+ template:
+ metadata:
+ labels:
+ app: redis
+ tier: backend
+ spec:
+ volumes:
+ - name: data
+ emptyDir: {}
+ - name: supersecret
+ secret:
+ secretName: mysecret
+ containers:
+ - name: redis
+ image: kubernetes/redis:v1
+ ports:
+ - containerPort: 6379
+ # Mount the volume into the pod
+ volumeMounts:
+ - mountPath: /redis-master-data
+ name: data # must match the name of the volume, above
+ - mountPath: /var/run/secrets/super
+ name: supersecret
+
+For more details, see the secrets document, example and design doc.
+ +Secrets can also be used to pass image registry credentials.
+ +First, create a .dockercfg
file, such as running docker login <registry.domain>
.
+Then put the resulting .dockercfg
file into a secret resource. For example:
$ docker login
+Username: janedoe
+Password: ●●●●●●●●●●●
+Email: jdoe@example.com
+WARNING: login credentials saved in /Users/jdoe/.dockercfg.
+Login Succeeded
+
+$ echo $(cat ~/.dockercfg)
+{ "https://index.docker.io/v1/": { "auth": "ZmFrZXBhc3N3b3JkMTIK", "email": "jdoe@example.com" } }
+
+$ cat ~/.dockercfg | base64
+eyAiaHR0cHM6Ly9pbmRleC5kb2NrZXIuaW8vdjEvIjogeyAiYXV0aCI6ICJabUZyWlhCaGMzTjNiM0prTVRJSyIsICJlbWFpbCI6ICJqZG9lQGV4YW1wbGUuY29tIiB9IH0K
+
+$ cat > /tmp/image-pull-secret.yaml <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+ name: myregistrykey
+data:
+ .dockercfg: eyAiaHR0cHM6Ly9pbmRleC5kb2NrZXIuaW8vdjEvIjogeyAiYXV0aCI6ICJabUZyWlhCaGMzTjNiM0prTVRJSyIsICJlbWFpbCI6ICJqZG9lQGV4YW1wbGUuY29tIiB9IH0K
+type: kubernetes.io/dockercfg
+EOF
+
+$ kubectl create -f ./image-pull-secret.yaml
+secrets/myregistrykey
+
+Now, you can create pods which reference that secret by adding an imagePullSecrets
+section to a pod definition.
apiVersion: v1
+kind: Pod
+metadata:
+ name: foo
+spec:
+ containers:
+ - name: foo
+ image: janedoe/awesomeapp:v1
+ imagePullSecrets:
+ - name: myregistrykey
+
+Pods support running multiple containers co-located together. They can be used to host vertically integrated application stacks, but their primary motivation is to support auxiliary helper programs that assist the primary application. Typical examples are data pullers, data pushers, and proxies.
+ +Such containers typically need to communicate with one another, often through the file system. This can be achieved by mounting the same volume into both containers. An example of this pattern would be a web server with a program that polls a git repository for new updates:
+ +apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: my-nginx
+spec:
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ volumes:
+ - name: www-data
+ emptyDir: {}
+ containers:
+ - name: nginx
+ image: nginx
+ # This container reads from the www-data volume
+ volumeMounts:
+ - mountPath: /srv/www
+ name: www-data
+ readOnly: true
+ - name: git-monitor
+ image: myrepo/git-monitor
+ env:
+ - name: GIT_REPO
+ value: http://github.com/some/repo.git
+ # This container writes to the www-data volume
+ volumeMounts:
+ - mountPath: /data
+ name: www-data
+
+More examples can be found in our blog article and presentation slides.
+ +Kubernetes’s scheduler will place applications only where they have adequate CPU and memory, but it can only do so if it knows how much resources they require. The consequence of specifying too little CPU is that the containers could be starved of CPU if too many other containers were scheduled onto the same node. Similarly, containers could die unpredictably due to running out of memory if no memory were requested, which can be especially likely for large-memory applications.
+ +If no resource requirements are specified, a nominal amount of resources is assumed. (This default is applied via a LimitRange for the default Namespace. It can be viewed with kubectl describe limitrange limits
.) You may explicitly specify the amount of resources required as follows:
apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: my-nginx
+spec:
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+ resources:
+ limits:
+ # cpu units are cores
+ cpu: 500m
+ # memory units are bytes
+ memory: 64Mi
+ requests:
+ # cpu units are cores
+ cpu: 500m
+ # memory units are bytes
+ memory: 64Mi
+
+The container will die due to OOM (out of memory) if it exceeds its specified limit, so specifying a value a little higher than expected generally improves reliability. By specifying request, pod is guaranteed to be able to use that much of resource when needed. See Resource QoS for the difference between resource limits and requests.
+ +If you’re not sure how much resources to request, you can first launch the application without specifying resources, and use resource usage monitoring to determine appropriate values.
+ +Many applications running for long periods of time eventually transition to broken states, and cannot recover except by restarting them. Kubernetes provides liveness probes to detect and remedy such situations.
+ +A common way to probe an application is using HTTP, which can be specified as follows:
+ +apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: my-nginx
+spec:
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+ livenessProbe:
+ httpGet:
+ # Path to probe; should be cheap, but representative of typical behavior
+ path: /index.html
+ port: 80
+ initialDelaySeconds: 30
+ timeoutSeconds: 1
+
+Other times, applications are only temporarily unable to serve, and will recover on their own. Typically in such cases you’d prefer not to kill the application, but don’t want to send it requests, either, since the application won’t respond correctly or at all. A common such scenario is loading large data or configuration files during application startup. Kubernetes provides readiness probes to detect and mitigate such situations. Readiness probes are configured similarly to liveness probes, just using the readinessProbe
field. A pod with containers reporting that they are not ready will not receive traffic through Kubernetes services.
For more details (e.g., how to specify command-based probes), see the example in the walkthrough, the standalone example, and the documentation.
+ +Of course, nodes and applications may fail at any time, but many applications benefit from clean shutdown, such as to complete in-flight requests, when the termination of the application is deliberate. To support such cases, Kubernetes supports two kinds of notifications:
+ +spec.terminationGracePeriodSeconds
).The specification of a pre-stop hook is similar to that of probes, but without the timing-related parameters. For example:
+ +apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: my-nginx
+spec:
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+ lifecycle:
+ preStop:
+ exec:
+ # SIGTERM triggers a quick exit; gracefully terminate instead
+ command: ["/usr/sbin/nginx","-s","quit"]
+
+In order to achieve a reasonably high level of availability, especially for actively developed applications, it’s important to debug failures quickly. Kubernetes can speed debugging by surfacing causes of fatal errors in a way that can be display using kubectl
or the UI, in addition to general log collection. It is possible to specify a terminationMessagePath
where a container will write its “death rattle”, such as assertion failure messages, stack traces, exceptions, and so on. The default path is /dev/termination-log
.
Here is a toy example:
+ +apiVersion: v1
+kind: Pod
+metadata:
+ name: pod-w-message
+spec:
+ containers:
+ - name: messager
+ image: "ubuntu:14.04"
+ command: ["/bin/sh","-c"]
+ args: ["sleep 60 && /bin/echo Sleep expired > /dev/termination-log"]
+
+The message is recorded along with the other state of the last (i.e., most recent) termination:
+ +$ kubectl create -f ./pod.yaml
+pods/pod-w-message
+$ sleep 70
+$ kubectl get pods/pod-w-message -o go-template="{{range .status.containerStatuses}}{{.lastState.terminated.message}}{{end}}"
+Sleep expired
+$ kubectl get pods/pod-w-message -o go-template="{{range .status.containerStatuses}}{{.lastState.terminated.exitCode}}{{end}}"
+0
+
+Table of Contents +
+ + + + + +This guide will help you get oriented to Kubernetes and running your first containers on the cluster. If you are already familiar with the docker-cli, you can also checkout the docker-cli to kubectl migration guide here.
+ +Once your application is packaged into a container and pushed to an image registry, you’re ready to deploy it to Kubernetes.
+ +For example, nginx is a popular HTTP server, with a pre-built container on Docker hub. The kubectl run
command below will create two nginx replicas, listening on port 80.
$ kubectl run my-nginx --image=nginx --replicas=2 --port=80
+CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS
+my-nginx my-nginx nginx run=my-nginx 2
+
+You can see that they are running by:
+ +$ kubectl get po
+NAME READY STATUS RESTARTS AGE
+my-nginx-l8n3i 1/1 Running 0 29m
+my-nginx-q7jo3 1/1 Running 0 29m
+
+Kubernetes will ensure that your application keeps running, by automatically restarting containers that fail, spreading containers across nodes, and recreating containers on new nodes when nodes fail.
+ +Through integration with some cloud providers (for example Google Compute Engine and AWS EC2), Kubernetes enables you to request that it provision a public IP address for your application. To do this run:
+ +$ kubectl expose rc my-nginx --port=80 --type=LoadBalancer
+service "my-nginx" exposed
+
+To find the public IP address assigned to your application, execute:
+ +$ kubectl get svc my-nginx
+NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE
+my-nginx 10.179.240.1 25.1.2.3 80/TCP run=nginx 8d
+
+You may need to wait for a minute or two for the external ip address to be provisioned.
+ +In order to access your nginx landing page, you also have to make sure that traffic from external IPs is allowed. Do this by opening a firewall to allow traffic on port 80.
+ +To kill the application and delete its containers and public IP address, do:
+ +$ kubectl delete rc my-nginx
+replicationcontrollers/my-nginx
+$ kubectl delete svc my-nginx
+services/my-nginx
+
+Table of Contents +
+ +A replication controller ensures that a specified number of pod “replicas” are running at any one time. If there are too many, it will kill some. If there are too few, it will start more. Unlike in the case where a user directly created pods, a replication controller replaces pods that are deleted or terminated for any reason, such as in the case of node failure or disruptive node maintenance, such as a kernel upgrade. For this reason, we recommend that you use a replication controller even if your application requires only a single pod. Think of it similarly to a process supervisor, only it supervises multiple pods across multiple nodes instead of individual processes on a single node. A replication controller delegates local container restarts to some agent on the node (e.g., Kubelet or Docker).
+ +As discussed in life of a pod, ReplicationController
is only appropriate for pods with RestartPolicy = Always
. (Note: If RestartPolicy
is not set, the default value is Always
.) ReplicationController
should refuse to instantiate any pod that has a different restart policy. As discussed in issue #503, we expect other types of controllers to be added to Kubernetes to handle other types of workloads, such as build/test and batch workloads, in the future.
A replication controller will never terminate on its own, but it isn’t expected to be as long-lived as services. Services may be composed of pods controlled by multiple replication controllers, and it is expected that many replication controllers may be created and destroyed over the lifetime of a service (for instance, to perform an update of pods that run the service). Both services themselves and their clients should remain oblivious to the replication controllers that maintain the pods of the services.
+ +A replication controller creates new pods from a template, which is currently inline in the ReplicationController
object, but which we plan to extract into its own resource #170.
Rather than specifying the current desired state of all replicas, pod templates are like cookie cutters. Once a cookie has been cut, the cookie has no relationship to the cutter. There is no quantum entanglement. Subsequent changes to the template or even switching to a new template has no direct effect on the pods already created. Similarly, pods created by a replication controller may subsequently be updated directly. This is in deliberate contrast to pods, which do specify the current desired state of all containers belonging to the pod. This approach radically simplifies system semantics and increases the flexibility of the primitive, as demonstrated by the use cases explained below.
+ +Pods created by a replication controller are intended to be fungible and semantically identical, though their configurations may become heterogeneous over time. This is an obvious fit for replicated stateless servers, but replication controllers can also be used to maintain availability of master-elected, sharded, and worker-pool applications. Such applications should use dynamic work assignment mechanisms, such as the etcd lock module or RabbitMQ work queues, as opposed to static/one-time customization of the configuration of each pod, which is considered an anti-pattern. Any pod customization performed, such as vertical auto-sizing of resources (e.g., cpu or memory), should be performed by another online controller process, not unlike the replication controller itself.
+ +The population of pods that a replication controller is monitoring is defined with a label selector, which creates a loosely coupled relationship between the controller and the pods controlled, in contrast to pods, which are more tightly coupled to their definition. We deliberately chose not to represent the set of pods controlled using a fixed-length array of pod specifications, because our experience is that approach increases complexity of management operations, for both clients and the system.
+ +The replication controller should verify that the pods created from the specified template have labels that match its label selector. Though it isn’t verified yet, you should also ensure that only one replication controller controls any given pod, by ensuring that the label selectors of replication controllers do not target overlapping sets. If you do end up with multiple controllers that have overlapping selectors, you will have to manage the deletion yourself with –cascade=false until there are no controllers with an overlapping superset of selectors.
+ +Note that replication controllers may themselves have labels and would generally carry the labels their corresponding pods have in common, but these labels do not affect the behavior of the replication controllers.
+ +Pods may be removed from a replication controller’s target set by changing their labels. This technique may be used to remove pods from service for debugging, data recovery, etc. Pods that are removed in this way will be replaced automatically (assuming that the number of replicas is not also changed).
+ +Similarly, deleting a replication controller using the API does not affect the pods it created. Its replicas
field must first be set to 0
in order to delete the pods controlled. (Note that the client tool, kubectl
, provides a single operation, delete to delete both the replication controller and the pods it controls. If you want to leave the pods running when deleting a replication controller, specify --cascade=false
. However, there is no such operation in the API at the moment)
The replication controller simply ensures that the desired number of pods matches its label selector and are operational. Currently, only terminated pods are excluded from its count. In the future, readiness and other information available from the system may be taken into account, we may add more controls over the replacement policy, and we plan to emit events that could be used by external clients to implement arbitrarily sophisticated replacement and/or scale-down policies.
+ +The replication controller is forever constrained to this narrow responsibility. It itself will not perform readiness nor liveness probes. Rather than performing auto-scaling, it is intended to be controlled by an external auto-scaler (as discussed in #492), which would change its replicas
field. We will not add scheduling policies (e.g., spreading) to the replication controller. Nor should it verify that the pods controlled match the currently specified template, as that would obstruct auto-sizing and other automated processes. Similarly, completion deadlines, ordering dependencies, configuration expansion, and other features belong elsewhere. We even plan to factor out the mechanism for bulk pod creation (#170).
The replication controller is intended to be a composable building-block primitive. We expect higher-level APIs and/or tools to be built on top of it and other complementary primitives for user convenience in the future. The “macro” operations currently supported by kubectl (run, stop, scale, rolling-update) are proof-of-concept examples of this. For instance, we could imagine something like [Spinnaker] (http://spinnaker.io/) managing replication controllers, auto-scalers, services, scheduling policies, canaries, etc.
+ +As mentioned above, whether you have 1 pod you want to keep running, or 1000, a replication controller will ensure that the specified number of pods exists, even in the event of node failure or pod termination (e.g., due to an action by another control agent).
+ +The replication controller makes it easy to scale the number of replicas up or down, either manually or by an auto-scaling control agent, by simply updating the replicas
field.
The replication controller is designed to facilitate rolling updates to a service by replacing pods one-by-one.
+ +As explained in #1353, the recommended approach is to create a new replication controller with 1 replica, scale the new (+1) and old (-1) controllers one by one, and then delete the old controller after it reaches 0 replicas. This predictably updates the set of pods regardless of unexpected failures.
+ +Ideally, the rolling update controller would take application readiness into account, and would ensure that a sufficient number of pods were productively serving at any given time.
+ +The two replication controllers would need to create pods with at least one differentiating label, such as the image tag of the primary container of the pod, since it is typically image updates that motivate rolling updates.
+ +Rolling update is implemented in the client tool +kubectl
+ +In addition to running multiple releases of an application while a rolling update is in progress, it’s common to run multiple releases for an extended period of time, or even continuously, using multiple release tracks. The tracks would be differentiated by labels.
+ +For instance, a service might target all pods with tier in (frontend), environment in (prod)
. Now say you have 10 replicated pods that make up this tier. But you want to be able to ‘canary’ a new version of this component. You could set up a replication controller with replicas
set to 9 for the bulk of the replicas, with labels tier=frontend, environment=prod, track=stable
, and another replication controller with replicas
set to 1 for the canary, with labels tier=frontend, environment=prod, track=canary
. Now the service is covering both the canary and non-canary pods. But you can mess with the replication controllers separately to test things out, monitor the results, etc.
Replication controller is a top-level resource in the kubernetes REST API. More details about the +API object can be found at: ReplicationController API +object.
+ + + + + + + + + +Following this example, you will create a secret and a pod that consumes that secret in a volume. See Secrets design document for more information.
+ +This example assumes you have a Kubernetes cluster installed and running, and that you have
+installed the kubectl
command line tool somewhere in your path. Please see the getting
+started for installation instructions for your platform.
A secret contains a set of named byte arrays.
+ +Use the examples/secrets/secret.yaml
file to create a secret:
$ kubectl create -f docs/user-guide/secrets/secret.yaml
+
+You can use kubectl
to see information about the secret:
$ kubectl get secrets
+NAME TYPE DATA
+test-secret Opaque 2
+
+$ kubectl describe secret test-secret
+Name: test-secret
+Labels: <none>
+Annotations: <none>
+
+Type: Opaque
+
+Data
+====
+data-1: 9 bytes
+data-2: 11 bytes
+
+Pods consume secrets in volumes. Now that you have created a secret, you can create a pod that +consumes it.
+ +Use the examples/secrets/secret-pod.yaml
file to create a Pod that consumes the secret.
$ kubectl create -f docs/user-guide/secrets/secret-pod.yaml
+
+This pod runs a binary that displays the content of one of the pieces of secret data in the secret +volume:
+ +$ kubectl logs secret-test-pod
+2015-04-29T21:17:24.712206409Z content of file "/etc/secret-volume/data-1": value-1
+
+Objects of type secret
are intended to hold sensitive information, such as
+passwords, OAuth tokens, and ssh keys. Putting this information in a secret
+is safer and more flexible than putting it verbatim in a pod
definition or in
+a docker image. See Secrets design document for more information.
Table of Contents +
+ +A Secret is an object that contains a small amount of sensitive data such as +a password, a token, or a key. Such information might otherwise be put in a +Pod specification or in an image; putting it in a Secret object allows for +more control over how it is used, and reduces the risk of accidental exposure.
+ +Users can create secrets, and the system also creates some secrets.
+ +To use a secret, a pod needs to reference the secret. +A secret can be used with a pod in two ways: either as files in a volume mounted on one or more of +its containers, or used by kubelet when pulling images for the pod.
+ +Kubernetes automatically creates secrets which contain credentials for +accessing the API and it automatically modifies your pods to use this type of +secret.
+ +The automatic creation and use of API credentials can be disabled or overridden +if desired. However, if all you need to do is securely access the apiserver, +this is the recommended workflow.
+ +See the Service Account documentation for more +information on how Service Accounts work.
+ +This is an example of a simple secret, in yaml format:
+ +apiVersion: v1
+kind: Secret
+metadata:
+ name: mysecret
+type: Opaque
+data:
+ password: dmFsdWUtMg0K
+ username: dmFsdWUtMQ0K
+
+The data field is a map. Its keys must match
+DNS_SUBDOMAIN
, except that leading dots are also
+allowed. The values are arbitrary data, encoded using base64. The values of
+username and password in the example above, before base64 encoding,
+are value-1
and value-2
, respectively, with carriage return and newline characters at the end.
Create the secret using kubectl create
.
Once the secret is created, you can need to modify your pod to specify +that it should use the secret.
+ +This is an example of a pod that mounts a secret in a volume:
+ +{
+ "apiVersion": "v1",
+ "kind": "Pod",
+ "metadata": {
+ "name": "mypod",
+ "namespace": "myns"
+ },
+ "spec": {
+ "containers": [{
+ "name": "mypod",
+ "image": "redis",
+ "volumeMounts": [{
+ "name": "foo",
+ "mountPath": "/etc/foo",
+ "readOnly": true
+ }]
+ }],
+ "volumes": [{
+ "name": "foo",
+ "secret": {
+ "secretName": "mysecret"
+ }
+ }]
+ }
+}
+
+Each secret you want to use needs its own spec.volumes
.
If there are multiple containers in the pod, then each container needs its
+own volumeMounts
block, but only one spec.volumes
is needed per secret.
You can package many files into one secret, or use many secrets, +whichever is convenient.
+ +See another example of creating a secret and a pod that consumes that secret in a volume here.
+ +Use of imagePullSecrets is described in the images documentation
+ +You can manually create an imagePullSecret, and reference it from +a serviceAccount. Any pods created with that serviceAccount +or that default to use that serviceAccount, will get have the imagePullSecret of the +field set to that of the service account. +See here + for a detailed explanation of that process.
+ +We plan to extend the service account behavior so that manually created +secrets (e.g. one containing a token for accessing a github account) +can be automatically attached to pods based on their service account. +This is not implemented yet. See issue 9902.
+ +Secret volume sources are validated to ensure that the specified object
+reference actually points to an object of type Secret
. Therefore, a secret
+needs to be created before any pods that depend on it.
Secret API objects reside in a namespace. They can only be referenced by pods +in that same namespace.
+ +Individual secrets are limited to 1MB in size. This is to discourage creation +of very large secrets which would exhaust apiserver and kubelet memory. +However, creation of many smaller secrets could also exhaust memory. More +comprehensive limits on memory usage due to secrets is a planned feature.
+ +Kubelet only supports use of secrets for Pods it gets from the API server.
+This includes any pods created using kubectl, or indirectly via a replication
+controller. It does not include pods created via the kubelets
+--manifest-url
flag, its --config
flag, or its REST API (these are
+not common ways to create pods.)
Inside the container that mounts a secret volume, the secret keys appear as +files and the secret values are base-64 decoded and stored inside these files. +This is the result of commands +executed inside the container from the example above:
+ +$ ls /etc/foo/
+username
+password
+$ cat /etc/foo/username
+value-1
+$ cat /etc/foo/password
+value-2
+
+The program in a container is responsible for reading the secret(s) from the +files. Currently, if a program expects a secret to be stored in an environment +variable, then the user needs to modify the image to populate the environment +variable from the file as an step before running the main program. Future +versions of Kubernetes are expected to provide more automation for populating +environment variables from files.
+ +When a pod is created via the API, there is no check whether a referenced +secret exists. Once a pod is scheduled, the kubelet will try to fetch the +secret value. If the secret cannot be fetched because it does not exist or +because of a temporary lack of connection to the API server, kubelet will +periodically retry. It will report an event about the pod explaining the +reason it is not started yet. Once the a secret is fetched, the kubelet will +create and mount a volume containing it. None of the pod’s containers will +start until all the pod’s volumes are mounted.
+ +Once the kubelet has started a pod’s containers, its secret volumes will not
+change, even if the secret resource is modified. To change the secret used,
+the original pod must be deleted, and a new pod (perhaps with an identical
+PodSpec
) must be created. Therefore, updating a secret follows the same
+workflow as deploying a new container image. The kubectl rolling-update
+command can be used (man page).
The resourceVersion
+of the secret is not specified when it is referenced.
+Therefore, if a secret is updated at about the same time as pods are starting,
+then it is not defined which version of the secret will be used for the pod. It
+is not possible currently to check what resource version of a secret object was
+used when a pod was created. It is planned that pods will report this
+information, so that a replication controller restarts ones using an old
+resourceVersion
. In the interim, if this is a concern, it is recommended to not
+update the data of existing secrets, but to create new ones with distinct names.
To create a pod that uses an ssh key stored as a secret, we first need to create a secret:
+ +{
+ "kind": "Secret",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "ssh-key-secret"
+ },
+ "data": {
+ "id-rsa": "dmFsdWUtMg0KDQo=",
+ "id-rsa.pub": "dmFsdWUtMQ0K"
+ }
+}
+
+Note: The serialized JSON and YAML values of secret data are encoded as +base64 strings. Newlines are not valid within these strings and must be +omitted.
+ +Now we can create a pod which references the secret with the ssh key and +consumes it in a volume:
+ +{
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "secret-test-pod",
+ "labels": {
+ "name": "secret-test"
+ }
+ },
+ "spec": {
+ "volumes": [
+ {
+ "name": "secret-volume",
+ "secret": {
+ "secretName": "ssh-key-secret"
+ }
+ }
+ ],
+ "containers": [
+ {
+ "name": "ssh-test-container",
+ "image": "mySshImage",
+ "volumeMounts": [
+ {
+ "name": "secret-volume",
+ "readOnly": true,
+ "mountPath": "/etc/secret-volume"
+ }
+ ]
+ }
+ ]
+ }
+}
+
+When the container’s command runs, the pieces of the key will be available in:
+ +/etc/secret-volume/id-rsa.pub
+/etc/secret-volume/id-rsa
+
+
+The container is then free to use the secret data to establish an ssh connection.
+ +This example illustrates a pod which consumes a secret containing prod +credentials and another pod which consumes a secret with test environment +credentials.
+ +The secrets:
+ +{
+ "apiVersion": "v1",
+ "kind": "List",
+ "items":
+ [{
+ "kind": "Secret",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "prod-db-secret"
+ },
+ "data": {
+ "password": "dmFsdWUtMg0KDQo=",
+ "username": "dmFsdWUtMQ0K"
+ }
+ },
+ {
+ "kind": "Secret",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "test-db-secret"
+ },
+ "data": {
+ "password": "dmFsdWUtMg0KDQo=",
+ "username": "dmFsdWUtMQ0K"
+ }
+ }]
+}
+
+The pods:
+ +{
+ "apiVersion": "v1",
+ "kind": "List",
+ "items":
+ [{
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "prod-db-client-pod",
+ "labels": {
+ "name": "prod-db-client"
+ }
+ },
+ "spec": {
+ "volumes": [
+ {
+ "name": "secret-volume",
+ "secret": {
+ "secretName": "prod-db-secret"
+ }
+ }
+ ],
+ "containers": [
+ {
+ "name": "db-client-container",
+ "image": "myClientImage",
+ "volumeMounts": [
+ {
+ "name": "secret-volume",
+ "readOnly": true,
+ "mountPath": "/etc/secret-volume"
+ }
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "kind": "Pod",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "test-db-client-pod",
+ "labels": {
+ "name": "test-db-client"
+ }
+ },
+ "spec": {
+ "volumes": [
+ {
+ "name": "secret-volume",
+ "secret": {
+ "secretName": "test-db-secret"
+ }
+ }
+ ],
+ "containers": [
+ {
+ "name": "db-client-container",
+ "image": "myClientImage",
+ "volumeMounts": [
+ {
+ "name": "secret-volume",
+ "readOnly": true,
+ "mountPath": "/etc/secret-volume"
+ }
+ ]
+ }
+ ]
+ }
+ }]
+}
+
+Both containers will have the following files present on their filesystems:
+ +/etc/secret-volume/username
+ /etc/secret-volume/password
+
+Note how the specs for the two pods differ only in one field; this facilitates +creating pods with different capabilities from a common pod config template.
+ +You could further simplify the base pod specification by using two Service Accounts:
+one called, say, prod-user
with the prod-db-secret
, and one called, say,
+test-user
with the test-db-secret
. Then, the pod spec can be shortened to, for example:
{
+"kind": "Pod",
+"apiVersion": "v1",
+"metadata": {
+ "name": "prod-db-client-pod",
+ "labels": {
+ "name": "prod-db-client"
+ }
+},
+"spec": {
+ "serviceAccount": "prod-db-client",
+ "containers": [
+ {
+ "name": "db-client-container",
+ "image": "myClientImage",
+ }
+ ]
+}
+
+Consider a program that needs to handle HTTP requests, do some complex business +logic, and then sign some messages with an HMAC. Because it has complex +application logic, there might be an unnoticed remote file reading exploit in +the server, which could expose the private key to an attacker.
+ +This could be divided into two processes in two containers: a frontend container +which handles user interaction and business logic, but which cannot see the +private key; and a signer container that can see the private key, and responds +to simple signing requests from the frontend (e.g. over localhost networking).
+ +With this partitioned approach, an attacker now has to trick the application +server into doing something rather arbitrary, which may be harder than getting +it to read a file.
+ + + +Because secret
objects can be created independently of the pods
that use
+them, there is less risk of the secret being exposed during the workflow of
+creating, viewing, and editing pods. The system can also take additional
+precautions with secret
objects, such as avoiding writing them to disk where
+possible.
A secret is only sent to a node if a pod on that node requires it. It is not +written to disk. It is stored in a tmpfs. It is deleted once the pod that +depends on it is deleted.
+ +On most Kubernetes-project-maintained distributions, communication between user +to the apiserver, and from apiserver to the kubelets, is protected by SSL/TLS. +Secrets are protected when transmitted over these channels.
+ +Secret data on nodes is stored in tmpfs volumes and thus does not come to rest +on the node.
+ +There may be secrets for several pods on the same node. However, only the +secrets that a pod requests are potentially visible within its containers. +Therefore, one Pod does not have access to the secrets of another pod.
+ +There may be several containers in a pod. However, each container in a pod has
+to request the secret volume in its volumeMounts
for it to be visible within
+the container. This can be used to construct useful security partitions at the
+Pod level.
A security context defines the operating system security settings (uid, gid, capabilities, SELinux role, etc..) applied to a container. See security context design for more details.
+ + + + + + + + + +A service account provides an identity for processes that run in a Pod.
+ +This is a user introduction to Service Accounts. See also the +Cluster Admin Guide to Service Accounts.
+ +Note: This document describes how service accounts behave in a cluster set up +as recommended by the Kubernetes project. Your cluster administrator may have +customized the behavior in your cluster, in which case this documentation may +not apply.
+ +When you (a human) access the cluster (e.g. using kubectl
), you are
+authenticated by the apiserver as a particular User Account (currently this is
+usually admin
, unless your cluster administrator has customized your
+cluster). Processes in containers inside pods can also contact the apiserver.
+When they do, they are authenticated as a particular Service Account (e.g.
+default
).
When you create a pod, you do not need to specify a service account. It is
+automatically assigned the default
service account of the same namespace. If
+you get the raw json or yaml for a pod you have created (e.g. kubectl get
+pods/podname -o yaml
), you can see the spec.serviceAccount
field has been
+automatically set.
You can access the API using a proxy or with a client library, as described in +Accessing the Cluster.
+ +Every namespace has a default service account resource called default
.
+You can list this and any other serviceAccount resources in the namespace with this command:
$ kubectl get serviceAccounts
+NAME SECRETS
+default 1
+
+You can create additional serviceAccounts like this:
+ +$ cat > /tmp/serviceaccount.yaml <<EOF
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: build-robot
+EOF
+$ kubectl create -f /tmp/serviceaccount.yaml
+serviceaccounts/build-robot
+
+If you get a complete dump of the service account object, like this:
+ +$ kubectl get serviceaccounts/build-robot -o yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ creationTimestamp: 2015-06-16T00:12:59Z
+ name: build-robot
+ namespace: default
+ resourceVersion: "272500"
+ selfLink: /api/v1/namespaces/default/serviceaccounts/build-robot
+ uid: 721ab723-13bc-11e5-aec2-42010af0021e
+secrets:
+- name: build-robot-token-bvbk5
+
+then you will see that a token has automatically been created and is referenced by the service account.
+ +In the future, you will be able to configure different access policies for each service account.
+ +To use a non-default service account, simply set the spec.serviceAccount
+field of a pod to the name of the service account you wish to use.
The service account has to exist at the time the pod is created, or it will be rejected.
+ +You cannot update the service account of an already created pod.
+ +You can clean up the service account from this example like this:
+ +$ kubectl delete serviceaccount/build-robot
+
+Note that if a pod does not have a ServiceAccount
set, the ServiceAccount
will be set to default
.
Suppose we have an existing service account named “build-robot” as mentioned above, and we create +a new secret manually.
+ +$ cat > /tmp/build-robot-secret.yaml <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+ name: build-robot-secret
+ annotations:
+ kubernetes.io/service-account.name: build-robot
+type: kubernetes.io/service-account-token
+EOF
+$ kubectl create -f /tmp/build-robot-secret.yaml
+secrets/build-robot-secret
+
+Now you can confirm that the newly built secret is populated with an API token for the “build-robot” service account.
+ +$ kubectl describe secrets/build-robot-secret
+Name: build-robot-secret
+Namespace: default
+Labels: <none>
+Annotations: kubernetes.io/service-account.name=build-robot,kubernetes.io/service-account.uid=870ef2a5-35cf-11e5-8d06-005056b45392
+
+Type: kubernetes.io/service-account-token
+
+Data
+====
+ca.crt: 1220 bytes
+token:
+
+++ +Note that the content of
+token
is elided here.
First, create an imagePullSecret, as described here +Next, verify it has been created. For example:
+ +$ kubectl get secrets myregistrykey
+NAME TYPE DATA
+myregistrykey kubernetes.io/dockercfg 1
+
+Next, read/modify/write the service account for the namespace to use this secret as an imagePullSecret
+ +$ kubectl get serviceaccounts default -o yaml > ./sa.yaml
+$ cat sa.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ creationTimestamp: 2015-08-07T22:02:39Z
+ name: default
+ namespace: default
+ resourceVersion: "243024"
+ selfLink: /api/v1/namespaces/default/serviceaccounts/default
+ uid: 052fb0f4-3d50-11e5-b066-42010af0d7b6
+secrets:
+- name: default-token-uudge
+$ vi sa.yaml
+[editor session not shown]
+[delete line with key "resourceVersion"]
+[add lines with "imagePullSecret:"]
+$ cat sa.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ creationTimestamp: 2015-08-07T22:02:39Z
+ name: default
+ namespace: default
+ selfLink: /api/v1/namespaces/default/serviceaccounts/default
+ uid: 052fb0f4-3d50-11e5-b066-42010af0d7b6
+secrets:
+- name: default-token-uudge
+imagePullSecrets:
+- name: myregistrykey
+$ kubectl replace serviceaccount default -f ./sa.yaml
+serviceaccounts/default
+
+Now, any new pods created in the current namespace will have this added to their spec:
+ +spec:
+ imagePullSecrets:
+ - name: myregistrykey
+
+TODO: Test and explain how to use additional non-K8s secrets with an existing service account.
+ +TODO explain: + - The token goes to: “/var/run/secrets/kubernetes.io/serviceaccount/$WHATFILENAME”
+ + + + + + + + + +Many cloud providers (e.g. Google Compute Engine) define firewalls that help prevent inadvertent +exposure to the internet. When exposing a service to the external world, you may need to open up +one or more ports in these firewalls to serve traffic. This document describes this process, as +well as any provider specific details that may be necessary.
+ +When using a Service with spec.type: LoadBalancer
, the firewall will be
+opened automatically. When using spec.type: NodePort
, however, the firewall
+is not opened by default.
Google Compute Engine firewalls are documented elsewhere.
+ +You can add a firewall with the gcloud
command line tool:
$ gcloud compute firewall-rules create my-rule --allow=tcp:<port>
+
+Note +There is one important security note when using firewalls on Google Compute Engine:
+ +as of Kubernetes v1.0.0, GCE firewalls are defined per-vm, rather than per-ip +address. This means that when you open a firewall for a service’s ports, +anything that serves on that port on that VM’s host IP address may potentially +serve traffic. Note that this is not a problem for other Kubernetes services, +as they listen on IP addresses that are different than the host node’s external +IP address.
+ +Consider: + * You create a Service with an external load balancer (IP Address 1.2.3.4) + and port 80 + * You open the firewall for port 80 for all nodes in your cluster, so that + the external Service actually can deliver packets to your Service + * You start an nginx server, running on port 80 on the host virtual machine + (IP Address 2.3.4.5). This nginx is also exposed to the internet on + the VM’s external IP address.
+ +Consequently, please be careful when opening firewalls in Google Compute Engine +or Google Container Engine. You may accidentally be exposing other services to +the wilds of the internet.
+ +This will be fixed in an upcoming release of Kubernetes.
+ +Coming soon.
+ + + + + + + + + +Table of Contents +
+ +Kubernetes Pods
are mortal. They are born and they die, and they
+are not resurrected. ReplicationControllers
in
+particular create and destroy Pods
dynamically (e.g. when scaling up or down
+or when doing rolling updates). While each Pod
gets its own IP address, even
+those IP addresses cannot be relied upon to be stable over time. This leads to
+a problem: if some set of Pods
(let’s call them backends) provides
+functionality to other Pods
(let’s call them frontends) inside the Kubernetes
+cluster, how do those frontends find out and keep track of which backends are
+in that set?
Enter Services
.
A Kubernetes Service
is an abstraction which defines a logical set of Pods
+and a policy by which to access them - sometimes called a micro-service. The
+set of Pods
targeted by a Service
is (usually) determined by a Label
+Selector
(see below for why you might want a
+Service
without a selector).
As an example, consider an image-processing backend which is running with 3
+replicas. Those replicas are fungible - frontends do not care which backend
+they use. While the actual Pods
that compose the backend set may change, the
+frontend clients should not need to be aware of that or keep track of the list
+of backends themselves. The Service
abstraction enables this decoupling.
For Kubernetes-native applications, Kubernetes offers a simple Endpoints
API
+that is updated whenever the set of Pods
in a Service
changes. For
+non-native applications, Kubernetes offers a virtual-IP-based bridge to Services
+which redirects to the backend Pods
.
A Service
in Kubernetes is a REST object, similar to a Pod
. Like all of the
+REST objects, a Service
definition can be POSTed to the apiserver to create a
+new instance. For example, suppose you have a set of Pods
that each expose
+port 9376 and carry a label "app=MyApp"
.
{
+ "kind": "Service",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "my-service"
+ },
+ "spec": {
+ "selector": {
+ "app": "MyApp"
+ },
+ "ports": [
+ {
+ "protocol": "TCP",
+ "port": 80,
+ "targetPort": 9376
+ }
+ ]
+ }
+}
+
+This specification will create a new Service
object named “my-service” which
+targets TCP port 9376 on any Pod
with the "app=MyApp"
label. This Service
+will also be assigned an IP address (sometimes called the “cluster IP”), which
+is used by the service proxies (see below). The Service
’s selector will be
+evaluated continuously and the results will be POSTed to an Endpoints
object
+also named “my-service”.
Note that a Service
can map an incoming port to any targetPort
. By default
+the targetPort
will be set to the same value as the port
field. Perhaps
+more interesting is that targetPort
can be a string, referring to the name of
+a port in the backend Pods
. The actual port number assigned to that name can
+be different in each backend Pod
. This offers a lot of flexibility for
+deploying and evolving your Services
. For example, you can change the port
+number that pods expose in the next version of your backend software, without
+breaking clients.
Kubernetes Services
support TCP
and UDP
for protocols. The default
+is TCP
.
Services generally abstract access to Kubernetes Pods
, but they can also
+abstract other kinds of backends. For example:
Namespace
or on another cluster.In any of these scenarios you can define a service without a selector:
+ +{
+ "kind": "Service",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "my-service"
+ },
+ "spec": {
+ "ports": [
+ {
+ "protocol": "TCP",
+ "port": 80,
+ "targetPort": 9376
+ }
+ ]
+ }
+}
+
+Because this has no selector, the corresponding Endpoints
object will not be
+created. You can manually map the service to your own specific endpoints:
{
+ "kind": "Endpoints",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "my-service"
+ },
+ "subsets": [
+ {
+ "addresses": [
+ { "IP": "1.2.3.4" }
+ ],
+ "ports": [
+ { "port": 80 }
+ ]
+ }
+ ]
+}
+
+NOTE: Endpoint IPs may not be loopback (127.0.0.0/8), link-local +(169.254.0.0/16), or link-local multicast ((224.0.0.0/24).
+ +Accessing a Service
without a selector works the same as if it had selector.
+The traffic will be routed to endpoints defined by the user (1.2.3.4:80
in
+this example).
Every node in a Kubernetes cluster runs a kube-proxy
. This application
+watches the Kubernetes master for the addition and removal of Service
+and Endpoints
objects. For each Service
it opens a port (randomly chosen)
+on the local node. Any connections made to that port will be proxied to one of
+the corresponding backend Pods
. Which backend to use is decided based on the
+SessionAffinity
of the Service
. Lastly, it installs iptables rules which
+capture traffic to the Service
’s cluster IP (which is virtual) and Port
and
+redirects that traffic to the previously described port.
The net result is that any traffic bound for the Service
is proxied to an
+appropriate backend without the clients knowing anything about Kubernetes or
+Services
or Pods
.
By default, the choice of backend is round robin. Client-IP based session affinity
+can be selected by setting service.spec.sessionAffinity
to "ClientIP"
(the
+default is "None"
).
As of Kubernetes 1.0, Services
are a “layer 3” (TCP/UDP over IP) construct. We do not
+yet have a concept of “layer 7” (HTTP) services.
Many Services
need to expose more than one port. For this case, Kubernetes
+supports multiple port definitions on a Service
object. When using multiple
+ports you must give all of your ports names, so that endpoints can be
+disambiguated. For example:
{
+ "kind": "Service",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "my-service"
+ },
+ "spec": {
+ "selector": {
+ "app": "MyApp"
+ },
+ "ports": [
+ {
+ "name": "http",
+ "protocol": "TCP",
+ "port": 80,
+ "targetPort": 9376
+ },
+ {
+ "name": "https",
+ "protocol": "TCP",
+ "port": 443,
+ "targetPort": 9377
+ }
+ ]
+ }
+}
+
+You can specify your own cluster IP address as part of a Service
creation
+request. To do this, set the spec.clusterIP
field. For example, if you
+already have an existing DNS entry that you wish to replace, or legacy systems
+that are configured for a specific IP address and difficult to re-configure.
+The IP address that a user chooses must be a valid IP address and within the
+service-cluster-ip-range
CIDR range that is specified by flag to the API
+server. If the IP address value is invalid, the apiserver returns a 422 HTTP
+status code to indicate that the value is invalid.
A question that pops up every now and then is why we do all this stuff with +virtual IPs rather than just use standard round-robin DNS. There are a few +reasons:
+ +We try to discourage users from doing things that hurt themselves. That said, +if enough people ask for this, we may implement it as an alternative.
+ +Kubernetes supports 2 primary modes of finding a Service
- environment
+variables and DNS.
When a Pod
is run on a Node
, the kubelet adds a set of environment variables
+for each active Service
. It supports both Docker links
+compatible variables (see
+makeLinkVariables)
+and simpler {SVCNAME}_SERVICE_HOST
and {SVCNAME}_SERVICE_PORT
variables,
+where the Service name is upper-cased and dashes are converted to underscores.
For example, the Service "redis-master"
which exposes TCP port 6379 and has been
+allocated cluster IP address 10.0.0.11 produces the following environment
+variables:
REDIS_MASTER_SERVICE_HOST=10.0.0.11
+REDIS_MASTER_SERVICE_PORT=6379
+REDIS_MASTER_PORT=tcp://10.0.0.11:6379
+REDIS_MASTER_PORT_6379_TCP=tcp://10.0.0.11:6379
+REDIS_MASTER_PORT_6379_TCP_PROTO=tcp
+REDIS_MASTER_PORT_6379_TCP_PORT=6379
+REDIS_MASTER_PORT_6379_TCP_ADDR=10.0.0.11
+
+This does imply an ordering requirement - any Service
that a Pod
wants to
+access must be created before the Pod
itself, or else the environment
+variables will not be populated. DNS does not have this restriction.
An optional (though strongly recommended) cluster
+add-on is a DNS server. The
+DNS server watches the Kubernetes API for new Services
and creates a set of
+DNS records for each. If DNS has been enabled throughout the cluster then all
+Pods
should be able to do name resolution of Services
automatically.
For example, if you have a Service
called "my-service"
in Kubernetes
+Namespace
"my-ns"
a DNS record for "my-service.my-ns"
is created. Pods
+which exist in the "my-ns"
Namespace
should be able to find it by simply doing
+a name lookup for "my-service"
. Pods
which exist in other Namespaces
must
+qualify the name as "my-service.my-ns"
. The result of these name lookups is the
+cluster IP.
Kubernetes also supports DNS SRV (service) records for named ports. If the
+"my-service.my-ns"
Service
has a port named "http"
with protocol TCP
, you
+can do a DNS SRV query for "_http._tcp.my-service.my-ns"
to discover the port
+number for "http"
.
Sometimes you don’t need or want load-balancing and a single service IP. In
+this case, you can create “headless” services by specifying "None"
for the
+cluster IP (spec.clusterIP
).
For such Services
, a cluster IP is not allocated. DNS is configured to return
+multiple A records (addresses) for the Service
name, which point directly to
+the Pods
backing the Service
. Additionally, the kube proxy does not handle
+these services and there is no load balancing or proxying done by the platform
+for them. The endpoints controller will still create Endpoints
records in
+the API.
This option allows developers to reduce coupling to the Kubernetes system, if +they desire, but leaves them freedom to do discovery in their own way. +Applications can still use a self-registration pattern and adapters for other +discovery systems could easily be built upon this API.
+ +For some parts of your application (e.g. frontends) you may want to expose a +Service onto an external (outside of your cluster, maybe public internet) IP +address, other services should be visible only from inside of the cluster.
+ +Kubernetes ServiceTypes
allow you to specify what kind of service you want.
+The default and base type is ClusterIP
, which exposes a service to connection
+from inside the cluster. NodePort
and LoadBalancer
are two types that expose
+services to external traffic.
Valid values for the ServiceType
field are:
ClusterIP
: use a cluster-internal IP only - this is the default and is
+discussed above. Choosing this value means that you want this service to be
+reachable only from inside of the cluster.NodePort
: on top of having a cluster-internal IP, expose the service on a
+port on each node of the cluster (the same port on each node). You’ll be able
+to contact the service on any <NodeIP>:NodePort
address.LoadBalancer
: on top of having a cluster-internal IP and exposing service
+on a NodePort also, ask the cloud provider for a load balancer
+which forwards to the Service
exposed as a <NodeIP>:NodePort
+for each Node.Note that while NodePort
s can be TCP or UDP, LoadBalancer
s only support TCP
+as of Kubernetes 1.0.
If you set the type
field to "NodePort"
, the Kubernetes master will
+allocate a port from a flag-configured range (default: 30000-32767), and each
+Node will proxy that port (the same port number on every Node) into your Service
.
+That port will be reported in your Service
’s spec.ports[*].nodePort
field.
If you want a specific port number, you can specify a value in the nodePort
+field, and the system will allocate you that port or else the API transaction
+will fail (i.e. you need to take care about possible port collisions yourself).
+The value you specify must be in the configured range for node ports.
This gives developers the freedom to set up their own load balancers, to +configure cloud environments that are not fully supported by Kubernetes, or +even to just expose one or more nodes’ IPs directly.
+ +Note that this Service will be visible as both <NodeIP>:spec.ports[*].nodePort
+and spec.clusterIp:spec.ports[*].port
.
On cloud providers which support external load balancers, setting the type
+field to "LoadBalancer"
will provision a load balancer for your Service
.
+The actual creation of the load balancer happens asynchronously, and
+information about the provisioned balancer will be published in the Service
’s
+status.loadBalancer
field. For example:
{
+ "kind": "Service",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "my-service"
+ },
+ "spec": {
+ "selector": {
+ "app": "MyApp"
+ },
+ "ports": [
+ {
+ "protocol": "TCP",
+ "port": 80,
+ "targetPort": 9376,
+ "nodePort": 30061
+ }
+ ],
+ "clusterIP": "10.0.171.239",
+ "loadBalancerIP": "78.11.24.19",
+ "type": "LoadBalancer"
+ },
+ "status": {
+ "loadBalancer": {
+ "ingress": [
+ {
+ "ip": "146.148.47.155"
+ }
+ ]
+ }
+ }
+}
+
+Traffic from the external load balancer will be directed at the backend Pods
,
+though exactly how that works depends on the cloud provider. Some cloud providers allow
+the loadBalancerIP
to be specified. In those cases, the load-balancer will be created
+with the user-specified loadBalancerIP
. If the loadBalancerIP
field is not specified,
+an ephemeral IP will be assigned to the loadBalancer. If the loadBalancerIP
is specified, but the
+cloud provider does not support the feature, the field will be ignored.
If there are external IPs that route to one or more cluster nodes, Kubernetes services can be exposed on those
+externalIPs
. Traffic that ingresses into the cluster with the external IP (as destination IP), on the service port,
+will be routed to one of the service endpoints. externalIPs
are not managed by Kubernetes and are the responsibility
+of the cluster administrator.
In the ServiceSpec, externalIPs
can be specified along with any of the ServiceTypes
.
+In the example below, my-service can be accessed by clients on 80.11.12.10:80 (externalIP:port)
{
+ "kind": "Service",
+ "apiVersion": "v1",
+ "metadata": {
+ "name": "my-service"
+ },
+ "spec": {
+ "selector": {
+ "app": "MyApp"
+ },
+ "ports": [
+ {
+ "name": "http",
+ "protocol": "TCP",
+ "port": 80,
+ "targetPort": 9376
+ }
+ ],
+ "externalIPs" : [
+ "80.11.12.10"
+ ]
+ }
+}
+
+We expect that using iptables and userspace proxies for VIPs will work at +small to medium scale, but may not scale to very large clusters with thousands +of Services. See the original design proposal for +portals for more +details.
+ +Using the kube-proxy obscures the source-IP of a packet accessing a Service
.
+This makes some kinds of firewalling impossible.
LoadBalancers only support TCP, not UDP.
+ +The Type
field is designed as nested functionality - each level adds to the
+previous. This is not strictly required on all cloud providers (e.g. Google Compute Engine does
+not need to allocate a NodePort
to make LoadBalancer
work, but AWS does)
+but the current API requires it.
In the future we envision that the proxy policy can become more nuanced than
+simple round robin balancing, for example master-elected or sharded. We also
+envision that some Services
will have “real” load balancers, in which case the
+VIP will simply transport the packets there.
There’s a +proposal to +eliminate userspace proxying in favor of doing it all in iptables. This should +perform better and fix the source-IP obfuscation, though is less flexible than +arbitrary userspace code.
+ +We intend to have first-class support for L7 (HTTP) Services
.
We intend to have more flexible ingress modes for Services
which encompass
+the current ClusterIP
, NodePort
, and LoadBalancer
modes and more.
The previous information should be sufficient for many people who just want to
+use Services
. However, there is a lot going on behind the scenes that may be
+worth understanding.
One of the primary philosophies of Kubernetes is that users should not be +exposed to situations that could cause their actions to fail through no fault +of their own. In this situation, we are looking at network ports - users +should not have to choose a port number if that choice might collide with +another user. That is an isolation failure.
+ +In order to allow users to choose a port number for their Services
, we must
+ensure that no two Services
can collide. We do that by allocating each
+Service
its own IP address.
To ensure each service receives a unique IP, an internal allocator atomically +updates a global allocation map in etcd prior to each service. The map object +must exist in the registry for services to get IPs, otherwise creations will +fail with a message indicating an IP could not be allocated. A background +controller is responsible for creating that map (to migrate from older versions +of Kubernetes that used in memory locking) as well as checking for invalid +assignments due to administrator intervention and cleaning up any IPs +that were allocated but which no service currently uses.
+ +Unlike Pod
IP addresses, which actually route to a fixed destination,
+Service
IPs are not actually answered by a single host. Instead, we use
+iptables
(packet processing logic in Linux) to define virtual IP addresses
+which are transparently redirected as needed. When clients connect to the
+VIP, their traffic is automatically transported to an appropriate endpoint.
+The environment variables and DNS for Services
are actually populated in
+terms of the Service
’s VIP and port.
As an example, consider the image processing application described above.
+When the backend Service
is created, the Kubernetes master assigns a virtual
+IP address, for example 10.0.0.1. Assuming the Service
port is 1234, the
+Service
is observed by all of the kube-proxy
instances in the cluster.
+When a proxy sees a new Service
, it opens a new random port, establishes an
+iptables redirect from the VIP to this new port, and starts accepting
+connections on it.
When a client connects to the VIP the iptables rule kicks in, and redirects
+the packets to the Service proxy
’s own port. The Service proxy
chooses a
+backend, and starts proxying traffic from the client to the backend.
This means that Service
owners can choose any port they want without risk of
+collision. Clients can simply connect to an IP and port, without being aware
+of which Pods
they are actually accessing.
Service is a top-level resource in the kubernetes REST API. More details about the +API object can be found at: Service API +object.
+ + + + + + + + + +Client access to a running Kubernetes cluster can be shared by copying
+the kubectl
client config bundle (kubeconfig).
+This config bundle lives in $HOME/.kube/config
, and is generated
+by cluster/kube-up.sh
. Sample steps for sharing kubeconfig
below.
1. Create a cluster
+ +$ cluster/kube-up.sh
+
+2. Copy kubeconfig
to new host
$ scp $HOME/.kube/config user@remotehost:/path/to/.kube/config
+
+3. On new host, make copied config
available to kubectl
$ mv /path/to/.kube/config $HOME/.kube/config
+
+$ mv /path/to/.kube/config $PWD
+
+kubeconfig
location to kubectl
# via environment variable
+$ export KUBECONFIG=/path/to/.kube/config
+
+# via commandline flag
+$ kubectl ... --kubeconfig=/path/to/.kube/config
+
+kubeconfig
kubeconfig
is generated by kube-up
but you can generate your own
+using (any desired subset of) the following commands.
# create kubeconfig entry
+$ kubectl config set-cluster $CLUSTER_NICK \
+ --server=https://1.1.1.1 \
+ --certificate-authority=/path/to/apiserver/ca_file \
+ --embed-certs=true \
+ # Or if tls not needed, replace --certificate-authority and --embed-certs with
+ --insecure-skip-tls-verify=true \
+ --kubeconfig=/path/to/standalone/.kube/config
+
+# create user entry
+$ kubectl config set-credentials $USER_NICK \
+ # bearer token credentials, generated on kube master
+ --token=$token \
+ # use either username|password or token, not both
+ --username=$username \
+ --password=$password \
+ --client-certificate=/path/to/crt_file \
+ --client-key=/path/to/key_file \
+ --embed-certs=true \
+ --kubeconfig=/path/to/standalone/.kube/config
+
+# create context entry
+$ kubectl config set-context $CONTEXT_NAME --cluster=$CLUSTER_NICKNAME --user=$USER_NICK
+
+Notes:
+* The --embed-certs
flag is needed to generate a standalone
+kubeconfig
, that will work as-is on another host.
+* --kubeconfig
is both the preferred file to load config from and the file to
+save config too. In the above commands the --kubeconfig
file could be
+omitted if you first run
$ export KUBECONFIG=/path/to/standalone/.kube/config
+
+/srv/kubernetes
. Bearer token/basic auth are also generated on the kube master.For more details on kubeconfig
see kubeconfig-file.md,
+and/or run kubectl config -h
.
kubeconfig
Examplekubectl
loads and merges config from the following locations (in order)
--kubeconfig=/path/to/.kube/config
command line flagKUBECONFIG=/path/to/.kube/config
env variable$PWD/.kube/config
$HOME/.kube/config
If you create clusters A, B on host1, and clusters C, D on host2, you can +make all four clusters available on both hosts by running
+ +# on host2, copy host1's default kubeconfig, and merge it from env
+$ scp host1:/path/to/home1/.kube/config /path/to/other/.kube/config
+
+$ export $KUBECONFIG=/path/to/other/.kube/config
+
+# on host1, copy host2's default kubeconfig and merge it from env
+$ scp host2:/path/to/home2/.kube/config /path/to/other/.kube/config
+
+$ export $KUBECONFIG=/path/to/other/.kube/config
+
+Detailed examples and explanation of kubeconfig
loading/merging rules can be found in kubeconfig-file.md.
Ok, you’ve run one of the getting started guides and you have +successfully turned up a Kubernetes cluster. Now what? This guide will help you get oriented +to Kubernetes and running your first containers on the cluster.
+ +From this point onwards, it is assumed that kubectl
is on your path from one of the getting started guides.
The kubectl run
line below will create two nginx pods listening on port 80. It will also create a replication controller named my-nginx
to ensure that there are always two pods running.
kubectl run my-nginx --image=nginx --replicas=2 --port=80
+
+Once the pods are created, you can list them to see what is up and running:
+ +kubectl get pods
+
+You can also see the replication controller that was created:
+ +kubectl get rc
+
+To stop the two replicated containers, stop the replication controller:
+ +kubectl stop rc my-nginx
+
+On some platforms (for example Google Compute Engine) the kubectl command can integrate with your cloud provider to add a public IP address for the pods, +to do this run:
+ +kubectl expose rc my-nginx --port=80 --type=LoadBalancer
+
+This should print the service that has been created, and map an external IP address to the service. Where to find this external IP address will depend on the environment you run in. For instance, for Google Compute Engine the external IP address is listed as part of the newly created service and can be retrieved by running
+ +kubectl get services
+
+In order to access your nginx landing page, you also have to make sure that traffic from external IPs is allowed. Do this by opening a firewall to allow traffic on port 80.
+ +Most people will eventually want to use declarative configuration files for creating/modifying their applications. A simplified introduction +is given in a different document.
+ + + + + + + + + +In addition to the imperative style commands described elsewhere, Kubernetes +supports declarative YAML or JSON configuration files. Often times config files are preferable +to imperative commands, since they can be checked into version control and changes to the files +can be code reviewed, producing a more robust, reliable and archival system.
+ +$ cd kubernetes
+$ kubectl create -f ./pod.yaml
+
+Where pod.yaml contains something like:
+ + + +apiVersion: v1
+kind: Pod
+metadata:
+ name: nginx
+ labels:
+ app: nginx
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+
+You can see your cluster’s pods:
+ +$ kubectl get pods
+
+and delete the pod you just created:
+ +$ kubectl delete pods nginx
+
+To run replicated containers, you need a Replication Controller. +A replication controller is responsible for ensuring that a specific number of pods exist in the +cluster.
+ +$ cd kubernetes
+$ kubectl create -f ./replication.yaml
+
+Where replication.yaml
contains:
apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: nginx
+spec:
+ replicas: 3
+ selector:
+ app: nginx
+ template:
+ metadata:
+ name: nginx
+ labels:
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+
+To delete the replication controller (and the pods it created):
+ +$ kubectl delete rc nginx
+
+Kubernetes has a web-based user interface that displays the current cluster state graphically.
+ +By default, the Kubernetes UI is deployed as a cluster addon. To access it, visit https://<kubernetes-master>/ui
, which redirects to https://<kubernetes-master>/api/v1/proxy/namespaces/kube-system/services/kube-ui/#/dashboard/
.
If you find that you’re not able to access the UI, it may be because the kube-ui service has not been started on your cluster. In that case, you can start it manually with:
+ +kubectl create -f cluster/addons/kube-ui/kube-ui-rc.yaml --namespace=kube-system
+kubectl create -f cluster/addons/kube-ui/kube-ui-svc.yaml --namespace=kube-system
+
+Normally, this should be taken care of automatically by the kube-addons.sh
script that runs on the master.
The Kubernetes UI can be used to introspect your current cluster, such as checking how resources are used, or looking at error messages. You cannot, however, use the UI to modify your cluster.
+ +After accessing Kubernetes UI, you’ll see a homepage dynamically listing out all nodes in your current cluster, with related information including internal IP addresses, CPU usage, memory usage, and file systems usage. +
+ +Click on the “Views” button in the top-right of the page to see other views available, which include: Explore, Pods, Nodes, Replication Controllers, Services, and Events.
+ +The “Explore” view allows your to see the pods, replication controllers, and services in current cluster easily. + +The “Group by” dropdown list allows you to group these resources by a number of factors, such as type, name, host, etc. + +You can also create filters by clicking on the down triangle of any listed resource instances and choose which filters you want to add. + +To see more details of each resource instance, simply click on it. +
+ +Other views (Pods, Nodes, Replication Controllers, Services, and Events) simply list information about each type of resource. You can also click on any instance for more details. +
+ +For more information, see the Kubernetes UI development document in the www directory.
+ + + + + + + + + +This example demonstrates the usage of Kubernetes to perform a rolling update on a running group of pods. See here to understand why you need a rolling update. Also check rolling update design document for more information.
+ +This example assumes that you have forked the repository and turned up a Kubernetes cluster:
+ +$ cd kubernetes
+$ ./cluster/kube-up.sh
+
+You can use bash job control to run this in the background (note that you must use the default port – 8001 – for the following demonstration to work properly).
+This can sometimes spew to the output so you could also run it in a different terminal. You have to run kubectl proxy
in the root of the
+Kubernetes repository. Otherwise you will get “404 page not found” errors as the paths will not match. You can find more information about kubectl proxy
+here.
$ kubectl proxy --www=docs/user-guide/update-demo/local/ &
+I0218 15:18:31.623279 67480 proxy.go:36] Starting to serve on localhost:8001
+
+Now visit the the demo website. You won’t see anything much quite yet.
+ +Now we will turn up two replicas of an image. They all serve on internal port 80.
+ +$ kubectl create -f docs/user-guide/update-demo/nautilus-rc.yaml
+
+After pulling the image from the Docker Hub to your worker nodes (which may take a minute or so) you’ll see a couple of squares in the UI detailing the pods that are running along with the image that they are serving up. A cute little nautilus.
+ +Now we will increase the number of replicas from two to four:
+ +$ kubectl scale rc update-demo-nautilus --replicas=4
+
+If you go back to the demo website you should eventually see four boxes, one for each pod.
+ +We will now update the docker image to serve a different image by doing a rolling update to a new Docker image.
+ +$ kubectl rolling-update update-demo-nautilus --update-period=10s -f docs/user-guide/update-demo/kitten-rc.yaml
+
+The rolling-update command in kubectl will do 2 things:
+ +gcr.io/google_containers/update-demo:kitten
)Watch the demo website, it will update one pod every 10 seconds until all of the pods have the new image. +Note that the new replication controller definition does not include the replica count, so the current replica count of the old replication controller is preserved. +But if the replica count had been specified, the final replica count of the new replication controller will be equal this number.
+ +$ kubectl delete rc update-demo-kitten
+
+This first stops the replication controller by turning the target number of replicas to 0 and then deletes the controller.
+ +To turn down a Kubernetes cluster:
+ +$ ./cluster/kube-down.sh
+
+Kill the proxy running in the background: +After you are done running this demo make sure to kill it:
+ +$ jobs
+[1]+ Running ./kubectl proxy --www=local/ &
+$ kill %1
+[1]+ Terminated: 15 ./kubectl proxy --www=local/
+
+If you want to build your own docker images, you can set $DOCKER_HUB_USER
to your Docker user id and run the included shell script. It can take a few minutes to download/upload stuff.
$ export DOCKER_HUB_USER=my-docker-id
+$ ./docs/user-guide/update-demo/build-images.sh
+
+To use your custom docker image in the above examples, you will need to change the image name in docs/user-guide/update-demo/nautilus-rc.yaml
and docs/user-guide/update-demo/kitten-rc.yaml
.
Note that the images included here are public domain.
+ + + + + + + + + + + +This example demonstrates the usage of Kubernetes to perform a rolling update on a running group of pods. See here to understand why you need a rolling update. Also check rolling update design document for more information.
+ +This example assumes that you have forked the repository and turned up a Kubernetes cluster:
+ +$ cd kubernetes
+$ ./cluster/kube-up.sh
+
+You can use bash job control to run this in the background (note that you must use the default port – 8001 – for the following demonstration to work properly).
+This can sometimes spew to the output so you could also run it in a different terminal. You have to run kubectl proxy
in the root of the
+Kubernetes repository. Otherwise you will get “404 page not found” errors as the paths will not match. You can find more information about kubectl proxy
+here.
$ kubectl proxy --www=docs/user-guide/update-demo/local/ &
+I0218 15:18:31.623279 67480 proxy.go:36] Starting to serve on localhost:8001
+
+Now visit the the demo website. You won’t see anything much quite yet.
+ +Now we will turn up two replicas of an image. They all serve on internal port 80.
+ +$ kubectl create -f docs/user-guide/update-demo/nautilus-rc.yaml
+
+After pulling the image from the Docker Hub to your worker nodes (which may take a minute or so) you’ll see a couple of squares in the UI detailing the pods that are running along with the image that they are serving up. A cute little nautilus.
+ +Now we will increase the number of replicas from two to four:
+ +$ kubectl scale rc update-demo-nautilus --replicas=4
+
+If you go back to the demo website you should eventually see four boxes, one for each pod.
+ +We will now update the docker image to serve a different image by doing a rolling update to a new Docker image.
+ +$ kubectl rolling-update update-demo-nautilus --update-period=10s -f docs/user-guide/update-demo/kitten-rc.yaml
+
+The rolling-update command in kubectl will do 2 things:
+ +gcr.io/google_containers/update-demo:kitten
)Watch the demo website, it will update one pod every 10 seconds until all of the pods have the new image. +Note that the new replication controller definition does not include the replica count, so the current replica count of the old replication controller is preserved. +But if the replica count had been specified, the final replica count of the new replication controller will be equal this number.
+ +$ kubectl delete rc update-demo-kitten
+
+This first stops the replication controller by turning the target number of replicas to 0 and then deletes the controller.
+ +To turn down a Kubernetes cluster:
+ +$ ./cluster/kube-down.sh
+
+Kill the proxy running in the background: +After you are done running this demo make sure to kill it:
+ +$ jobs
+[1]+ Running ./kubectl proxy --www=local/ &
+$ kill %1
+[1]+ Terminated: 15 ./kubectl proxy --www=local/
+
+If you want to build your own docker images, you can set $DOCKER_HUB_USER
to your Docker user id and run the included shell script. It can take a few minutes to download/upload stuff.
$ export DOCKER_HUB_USER=my-docker-id
+$ ./docs/user-guide/update-demo/build-images.sh
+
+To use your custom docker image in the above examples, you will need to change the image name in docs/user-guide/update-demo/nautilus-rc.yaml
and docs/user-guide/update-demo/kitten-rc.yaml
.
Note that the images included here are public domain.
+ + + + + + + + + + + +=h&&(n.resolve(r),m(p.$$intervalId),delete e[p.$$intervalId]);z||b.$apply()},f);e[p.$$intervalId]=n;return p}var e={};d.cancel=function(a){return a&&a.$$intervalId in e?(e[a.$$intervalId].reject("canceled"),clearInterval(a.$$intervalId),delete e[a.$$intervalId],
+!0):!1};return d}]}function ad(){this.$get=function(){return{id:"en-us",NUMBER_FORMATS:{DECIMAL_SEP:".",GROUP_SEP:",",PATTERNS:[{minInt:1,minFrac:0,maxFrac:3,posPre:"",posSuf:"",negPre:"-",negSuf:"",gSize:3,lgSize:3},{minInt:1,minFrac:2,maxFrac:2,posPre:"\u00a4",posSuf:"",negPre:"(\u00a4",negSuf:")",gSize:3,lgSize:3}],CURRENCY_SYM:"$"},DATETIME_FORMATS:{MONTH:"January February March April May June July August September October November December".split(" "),SHORTMONTH:"Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec".split(" "),
+DAY:"Sunday Monday Tuesday Wednesday Thursday Friday Saturday".split(" "),SHORTDAY:"Sun Mon Tue Wed Thu Fri Sat".split(" "),AMPMS:["AM","PM"],medium:"MMM d, y h:mm:ss a","short":"M/d/yy h:mm a",fullDate:"EEEE, MMMM d, y",longDate:"MMMM d, y",mediumDate:"MMM d, y",shortDate:"M/d/yy",mediumTime:"h:mm:ss a",shortTime:"h:mm a"},pluralCat:function(b){return 1===b?"one":"other"}}}}function wc(b){b=b.split("/");for(var a=b.length;a--;)b[a]=wb(b[a]);return b.join("/")}function xc(b,a,c){b=sa(b,c);a.$$protocol=
+b.protocol;a.$$host=b.hostname;a.$$port=Y(b.port)||we[b.protocol]||null}function yc(b,a,c){var d="/"!==b.charAt(0);d&&(b="/"+b);b=sa(b,c);a.$$path=decodeURIComponent(d&&"/"===b.pathname.charAt(0)?b.pathname.substring(1):b.pathname);a.$$search=Yb(b.search);a.$$hash=decodeURIComponent(b.hash);a.$$path&&"/"!=a.$$path.charAt(0)&&(a.$$path="/"+a.$$path)}function oa(b,a){if(0===a.indexOf(b))return a.substr(b.length)}function Ya(b){var a=b.indexOf("#");return-1==a?b:b.substr(0,a)}function Jb(b){return b.substr(0,
+Ya(b).lastIndexOf("/")+1)}function zc(b,a){this.$$html5=!0;a=a||"";var c=Jb(b);xc(b,this,b);this.$$parse=function(a){var e=oa(c,a);if(!w(e))throw Kb("ipthprfx",a,c);yc(e,this,b);this.$$path||(this.$$path="/");this.$$compose()};this.$$compose=function(){var a=Zb(this.$$search),b=this.$$hash?"#"+wb(this.$$hash):"";this.$$url=wc(this.$$path)+(a?"?"+a:"")+b;this.$$absUrl=c+this.$$url.substr(1)};this.$$rewrite=function(d){var e;if((e=oa(b,d))!==s)return d=e,(e=oa(a,e))!==s?c+(oa("/",e)||e):b+d;if((e=oa(c,
+d))!==s)return c+e;if(c==d+"/")return c}}function Lb(b,a){var c=Jb(b);xc(b,this,b);this.$$parse=function(d){var e=oa(b,d)||oa(c,d),e="#"==e.charAt(0)?oa(a,e):this.$$html5?e:"";if(!w(e))throw Kb("ihshprfx",d,a);yc(e,this,b);d=this.$$path;var g=/^\/?.*?:(\/.*)/;0===e.indexOf(b)&&(e=e.replace(b,""));g.exec(e)||(d=(e=g.exec(d))?e[1]:d);this.$$path=d;this.$$compose()};this.$$compose=function(){var c=Zb(this.$$search),e=this.$$hash?"#"+wb(this.$$hash):"";this.$$url=wc(this.$$path)+(c?"?"+c:"")+e;this.$$absUrl=
+b+(this.$$url?a+this.$$url:"")};this.$$rewrite=function(a){if(Ya(b)==Ya(a))return a}}function Ac(b,a){this.$$html5=!0;Lb.apply(this,arguments);var c=Jb(b);this.$$rewrite=function(d){var e;if(b==Ya(d))return d;if(e=oa(c,d))return b+a+e;if(c===d+"/")return c}}function nb(b){return function(){return this[b]}}function Bc(b,a){return function(c){if(E(c))return this[b];this[b]=a(c);this.$$compose();return this}}function Vd(){var b="",a=!1;this.hashPrefix=function(a){return B(a)?(b=a,this):b};this.html5Mode=
+function(b){return B(b)?(a=b,this):a};this.$get=["$rootScope","$browser","$sniffer","$rootElement",function(c,d,e,g){function f(a){c.$broadcast("$locationChangeSuccess",h.absUrl(),a)}var h,l=d.baseHref(),k=d.url();a?(l=k.substring(0,k.indexOf("/",k.indexOf("//")+2))+(l||"/"),e=e.history?zc:Ac):(l=Ya(k),e=Lb);h=new e(l,"#"+b);h.$$parse(h.$$rewrite(k));g.on("click",function(a){if(!a.ctrlKey&&!a.metaKey&&2!=a.which){for(var b=y(a.target);"a"!==K(b[0].nodeName);)if(b[0]===g[0]||!(b=b.parent())[0])return;
+var e=b.prop("href");X(e)&&"[object SVGAnimatedString]"===e.toString()&&(e=sa(e.animVal).href);var f=h.$$rewrite(e);e&&(!b.attr("target")&&f&&!a.isDefaultPrevented())&&(a.preventDefault(),f!=d.url()&&(h.$$parse(f),c.$apply(),O.angular["ff-684208-preventDefault"]=!0))}});h.absUrl()!=k&&d.url(h.absUrl(),!0);d.onUrlChange(function(a){h.absUrl()!=a&&(c.$evalAsync(function(){var b=h.absUrl();h.$$parse(a);c.$broadcast("$locationChangeStart",a,b).defaultPrevented?(h.$$parse(b),d.url(b)):f(b)}),c.$$phase||
+c.$digest())});var m=0;c.$watch(function(){var a=d.url(),b=h.$$replace;m&&a==h.absUrl()||(m++,c.$evalAsync(function(){c.$broadcast("$locationChangeStart",h.absUrl(),a).defaultPrevented?h.$$parse(a):(d.url(h.absUrl(),b),f(a))}));h.$$replace=!1;return m});return h}]}function Wd(){var b=!0,a=this;this.debugEnabled=function(a){return B(a)?(b=a,this):b};this.$get=["$window",function(c){function d(a){a instanceof Error&&(a.stack?a=a.message&&-1===a.stack.indexOf(a.message)?"Error: "+a.message+"\n"+a.stack:
+a.stack:a.sourceURL&&(a=a.message+"\n"+a.sourceURL+":"+a.line));return a}function e(a){var b=c.console||{},e=b[a]||b.log||C;a=!1;try{a=!!e.apply}catch(l){}return a?function(){var a=[];q(arguments,function(b){a.push(d(b))});return e.apply(b,a)}:function(a,b){e(a,null==b?"":b)}}return{log:e("log"),info:e("info"),warn:e("warn"),error:e("error"),debug:function(){var c=e("debug");return function(){b&&c.apply(a,arguments)}}()}}]}function fa(b,a){if("constructor"===b)throw Ba("isecfld",a);return b}function Za(b,
+a){if(b){if(b.constructor===b)throw Ba("isecfn",a);if(b.document&&b.location&&b.alert&&b.setInterval)throw Ba("isecwindow",a);if(b.children&&(b.nodeName||b.prop&&b.attr&&b.find))throw Ba("isecdom",a);}return b}function ob(b,a,c,d,e){e=e||{};a=a.split(".");for(var g,f=0;1 On-disk files in a container are ephemeral, which presents some problems for
+non-trivial applications when running in containers. First, when a container
+crashes kubelet will restart it, but the files will be lost - the
+container starts with a clean slate. Second, when running containers together
+in a Familiarity with pods is suggested. Table of Contents
+ Docker also has a concept of
+volumes, though it is
+somewhat looser and less managed. In Docker, a volume is simply a directory on
+disk or in another container. Lifetimes are not managed and until very
+recently there were only local-disk-backed volumes. Docker now provides volume
+drivers, but the functionality is very limited for now (e.g. as of Docker 1.7
+only one volume driver is allowed per container and there is no way to pass
+parameters to volumes). A Kubernetes volume, on the other hand, has an explicit lifetime - the same as
+the pod that encloses it. Consequently, a volume outlives any containers that run
+within the Pod, and data is preserved across Container restarts. Of course, when a
+Pod ceases to exist, the volume will cease to exist, too. Perhaps more
+importantly than this, Kubernetes supports many type of volumes, and a Pod can
+use any number of them simultaneously. At its core, a volume is just a directory, possibly with some data in it, which
+is accessible to the containers in a pod. How that directory comes to be, the
+medium that backs it, and the contents of it are determined by the particular
+volume type used. To use a volume, a pod specifies what volumes to provide for the pod (the
+ A process in a container sees a filesystem view composed from their Docker
+image and volumes. The Docker
+image is at the root of the
+filesystem hierarchy, and any volumes are mounted at the specified paths within
+the image. Volumes can not mount onto other volumes or have hard links to
+other volumes. Each container in the Pod must independently specify where to
+mount each volume. Kubernetes supports several types of Volumes:
+ * We welcome additional contributions. An Some uses for an By default, A For example, some uses for a Watch out when using this type of volume, because: A Important: You must create a PD using There are some restrictions when using a A feature of PD is that they can be mounted as read-only by multiple consumers
+simultaneously. This means that you can pre-populate a PD with your dataset
+and then serve it in parallel from as many pods as you need. Unfortunately,
+PDs can only be mounted by a single consumer in read-write mode - no
+simultaneous readers allowed. Using a PD on a pod controlled by a ReplicationController will fail unless
+the PD is read-only or the replica count is 0 or 1. Before you can use a GCE PD with a pod, you need to create it. An Important: You must create an EBS volume using There are some restrictions when using an awsElasticBlockStore volume: Before you can use a EBS volume with a pod, you need to create it. Make sure the zone matches the zone you brought up your cluster in. (And also check that the size and EBS volume
+type are suitable for your use!) (Note: the syntax of volumeID is currently awkward; #10181 fixes it) An Important: You must have your own NFS server running with the share exported
+before you can use it See the NFS example for more details. An Important: You must have your own iSCSI server running with the volume
+created before you can use it A feature of iSCSI is that it can be mounted as read-only by multiple consumers
+simultaneously. This means that you can pre-populate a volume with your dataset
+and then serve it in parallel from as many pods as you need. Unfortunately,
+iSCSI volumes can only be mounted by a single consumer in read-write mode - no
+simultaneous readers allowed. See the iSCSI example for more details. Flocker is an open-source clustered container data volume manager. It provides management
+and orchestration of data volumes backed by a variety of storage backends. A Important: You must have your own Flocker installation running before you can use it See the Flocker example for more details. A Important: You must have your own GlusterFS installation running before you
+can use it See the GlusterFS example for more details. An Important: You must have your own Ceph installation running before you
+can use RBD A feature of RBD is that it can be mounted as read-only by multiple consumers
+simultaneously. This means that you can pre-populate a volume with your dataset
+and then serve it in parallel from as many pods as you need. Unfortunately,
+RBD volumes can only be mounted by a single consumer in read-write mode - no
+simultaneous writers allowed. See the RBD example for more details. A Here is a example for gitRepo volume: A Important: You must create a secret in the Kubernetes API before you can use
+it Secrets are described in more detail here. A See the PersistentVolumes example for more
+details. A See the The storage media (Disk, SSD, etc) of an In the future, we expect that For Kubernetes 101, we will cover kubectl, pods, volumes, and multiple containers In order for the kubectl usage examples to work, make sure you have an examples directory locally, either from a release or the source. Table of Contents
+ The easiest way to interact with Kubernetes is via the kubectl command-line interface. For more info about kubectl, including its usage, commands, and parameters, see the kubectl CLI reference. If you haven’t installed and configured kubectl, finish the prerequisites before continuing. In Kubernetes, a group of one or more containers is called a pod. Containers in a pod are deployed together, and are started, stopped, and replicated as a group. See pods for more details. The simplest pod definition describes the deployment of a single container. For example, an nginx web server pod might be defined as such: A pod definition is a declaration of a desired state. Desired state is a very important concept in the Kubernetes model. Many things present a desired state to the system, and it is Kubernetes’ responsibility to make sure that the current state matches the desired state. For example, when you create a Pod, you declare that you want the containers in it to be running. If the containers happen to not be running (e.g. program failure, …), Kubernetes will continue to (re-)create them for you in order to drive them to the desired state. This process continues until the Pod is deleted. See the design document for more details. Create a pod containing an nginx server (pod-nginx.yaml): List all pods: On most providers, the pod IPs are not externally accessible. The easiest way to test that the pod is working is to create a busybox pod and exec commands on it remotely. See the command execution documentation for details. Provided the pod IP is accessible, you should be able to access its http endpoint with curl on port 80: Delete the pod by name: That’s great for a simple static web server, but what about persistent storage? The container file system only lives as long as the container does. So if your app’s state needs to survive relocation, reboots, and crashes, you’ll need to configure some persistent storage. For this example we’ll be creating a Redis pod with a named volume and volume mount that defines the path to mount the volume. Example Redis pod definition with a persistent storage volume (pod-redis.yaml): Notes:
+- The volume mount name is a reference to a specific empty dir volume.
+- The volume mount path is the path to mount the empty dir volume within the container. See volumes for more details. Note:
+The examples below are syntactically correct, but some of the images (e.g. kubernetes/git-monitor) don’t exist yet. We’re working on turning these into working examples. However, often you want to have two different containers that work together. An example of this would be a web server, and a helper job that polls a git repository for new updates: Note that we have also added a volume here. In this case, the volume is mounted into both containers. It is marked Finally, we have also introduced an environment variable to the Continue on to Kubernetes 201 or
+for a complete application see the guestbook example For Kubernetes 101, we will cover kubectl, pods, volumes, and multiple containers In order for the kubectl usage examples to work, make sure you have an examples directory locally, either from a release or the source. Table of Contents
+ The easiest way to interact with Kubernetes is via the kubectl command-line interface. For more info about kubectl, including its usage, commands, and parameters, see the kubectl CLI reference. If you haven’t installed and configured kubectl, finish the prerequisites before continuing. In Kubernetes, a group of one or more containers is called a pod. Containers in a pod are deployed together, and are started, stopped, and replicated as a group. See pods for more details. The simplest pod definition describes the deployment of a single container. For example, an nginx web server pod might be defined as such: A pod definition is a declaration of a desired state. Desired state is a very important concept in the Kubernetes model. Many things present a desired state to the system, and it is Kubernetes’ responsibility to make sure that the current state matches the desired state. For example, when you create a Pod, you declare that you want the containers in it to be running. If the containers happen to not be running (e.g. program failure, …), Kubernetes will continue to (re-)create them for you in order to drive them to the desired state. This process continues until the Pod is deleted. See the design document for more details. Create a pod containing an nginx server (pod-nginx.yaml): List all pods: On most providers, the pod IPs are not externally accessible. The easiest way to test that the pod is working is to create a busybox pod and exec commands on it remotely. See the command execution documentation for details. Provided the pod IP is accessible, you should be able to access its http endpoint with curl on port 80: Delete the pod by name: That’s great for a simple static web server, but what about persistent storage? The container file system only lives as long as the container does. So if your app’s state needs to survive relocation, reboots, and crashes, you’ll need to configure some persistent storage. For this example we’ll be creating a Redis pod with a named volume and volume mount that defines the path to mount the volume. Example Redis pod definition with a persistent storage volume (pod-redis.yaml): Notes:
+- The volume mount name is a reference to a specific empty dir volume.
+- The volume mount path is the path to mount the empty dir volume within the container. See volumes for more details. Note:
+The examples below are syntactically correct, but some of the images (e.g. kubernetes/git-monitor) don’t exist yet. We’re working on turning these into working examples. However, often you want to have two different containers that work together. An example of this would be a web server, and a helper job that polls a git repository for new updates: Note that we have also added a volume here. In this case, the volume is mounted into both containers. It is marked Finally, we have also introduced an environment variable to the Continue on to Kubernetes 201 or
+for a complete application see the guestbook example If you went through Kubernetes 101, you learned about kubectl, pods, volumes, and multiple containers.
+For Kubernetes 201, we will pick up where 101 left off and cover some slightly more advanced topics in Kubernetes, related to application productionization, deployment and
+scaling. In order for the kubectl usage examples to work, make sure you have an examples directory locally, either from a release or the source. Table of Contents
+ Having already learned about Pods and how to create them, you may be struck by an urge to create many, many pods. Please do! But eventually you will need a system to organize these pods into groups. The system for achieving this in Kubernetes is Labels. Labels are key-value pairs that are attached to each object in Kubernetes. Label selectors can be passed along with a RESTful To add a label, add a labels section under metadata in the pod definition: For example, here is the nginx pod definition with labels (pod-nginx-with-label.yaml): Create the labeled pod (pod-nginx-with-label.yaml): List all pods with the label For more information, see Labels.
+They are a core concept used by two additional Kubernetes building blocks: Replication Controllers and Services. OK, now you know how to make awesome, multi-container, labeled pods and you want to use them to build an application, you might be tempted to just start building a whole bunch of individual pods, but if you do that, a whole host of operational concerns pop up. For example: how will you scale the number of pods up or down and how will you ensure that all pods are homogenous? Replication controllers are the objects to answer these questions. A replication controller combines a template for pod creation (a “cookie-cutter” if you will) and a number of desired replicas, into a single Kubernetes object. The replication controller also contains a label selector that identifies the set of objects managed by the replication controller. The replication controller constantly measures the size of this set relative to the desired size, and takes action by creating or deleting pods. For example, here is a replication controller that instantiates two nginx pods (replication-controller.yaml): Create an nginx replication controller (replication-controller.yaml): List all replication controllers: Delete the replication controller by name: For more information, see Replication Controllers. Once you have a replicated set of pods, you need an abstraction that enables connectivity between the layers of your application. For example, if you have a replication controller managing your backend jobs, you don’t want to have to reconfigure your front-ends whenever you re-scale your backends. Likewise, if the pods in your backends are scheduled (or rescheduled) onto different machines, you can’t be required to re-configure your front-ends. In Kubernetes, the service abstraction achieves these goals. A service provides a way to refer to a set of pods (selected by labels) with a single static IP address. It may also provide load balancing, if supported by the provider. For example, here is a service that balances across the pods created in the previous nginx replication controller example (service.yaml): Create an nginx service (service.yaml): List all services: On most providers, the service IPs are not externally accessible. The easiest way to test that the service is working is to create a busybox pod and exec commands on it remotely. See the command execution documentation for details. Provided the service IP is accessible, you should be able to access its http endpoint with curl on port 80: To delete the service by name: When created, each service is assigned a unique IP address. This address is tied to the lifespan of the Service, and will not change while the Service is alive. Pods can be configured to talk to the service, and know that communication to the service will be automatically load-balanced out to some pod that is a member of the set identified by the label selector in the Service. For more information, see Services. When I write code it never crashes, right? Sadly the Kubernetes issues list indicates otherwise… Rather than trying to write bug-free code, a better approach is to use a management system to perform periodic health checking
+and repair of your application. That way a system outside of your application itself is responsible for monitoring the
+application and taking action to fix it. It’s important that the system be outside of the application, since if
+your application fails and the health checking agent is part of your application, it may fail as well and you’ll never know.
+In Kubernetes, the health check monitor is the Kubelet agent. The simplest form of health-checking is just process level health checking. The Kubelet constantly asks the Docker daemon
+if the container process is still running, and if not, the container process is restarted. In all of the Kubernetes examples
+you have run so far, this health checking was actually already enabled. It’s on for every single container that runs in
+Kubernetes. However, in many cases this low-level health checking is insufficient. Consider, for example, the following code: This is a classic example of a problem in computer science known as “Deadlock”. From Docker’s perspective your application is
+still operating and the process is still running, but from your application’s perspective your code is locked up and will never respond correctly. To address this problem, Kubernetes supports user implemented application health-checks. These checks are performed by the
+Kubelet to ensure that your application is operating correctly for a definition of “correctly” that you provide. Currently, there are three types of application health checks that you can choose from: In all cases, if the Kubelet discovers a failure the container is restarted. The container health checks are configured in the Here is an example config for a pod with an HTTP health check (pod-with-http-healthcheck.yaml): For more information about health checking, see Container Probes. For a complete application see the guestbook example. This document is aimed at users who have worked through some of the examples,
+and who want to learn more about using kubectl to manage resources such
+as pods and services. Users who want to access the REST API directly,
+and developers who want to extend the Kubernetes API should
+refer to the api conventions and
+the api document. When you create a resource such as pod, and then retrieve the created
+resource, a number of the fields of the resource are added.
+You can see this at work in the following example: The resource we posted had only 9 lines, but the one we got back had 51 lines.
+If you You can browse auto-generated API documentation at the project website or on github. Kubernetes is an open-source platform for automating deployment, scaling, and operations of application containers across clusters of hosts. With Kubernetes, you are able to quickly and efficiently respond to customer demand: Our goal is to foster an ecosystem of components and tools that relieve the burden of running applications in public and private clouds. The Kubernetes project was started by Google in 2014. Kubernetes builds upon a decade and a half of experience that Google has with running production workloads at scale, combined with best-of-breed ideas and practices from the community. Looking for reasons why you should be using containers? Here are some key points: Kubernetes can schedule and run application containers on clusters of physical or virtual machines. It can also do much more than that. Kubernetes satisfies a number of common needs of applications running in production, such as:
+* co-locating helper processes,
+* mounting storage systems,
+* distributing secrets,
+* application health checking,
+* replicating application instances,
+* horizontal auto-scaling,
+* load balancing,
+* rolling updates, and
+* resource monitoring. For more details, see the user guide. Even though Kubernetes provides a lot of functionality, there are always new scenarios that would benefit from new features. Ad hoc orchestration that is acceptable initially often requires robust automation at scale. Application-specific workflows can be streamlined to accelerate developer velocity. This is why Kubernetes was also designed to serve as a platform for building an ecosystem of components and tools to make it easier to deploy, scale, and manage applications. Labels empower users to organize their resources however they please. Annotations enable users to decorate resources with custom information to facilitate their workflows and provide an easy way for management tools to checkpoint state. Additionally, the Kubernetes control plane is built upon the same APIs that are available to developers and users. Users can write their own controllers, schedulers, etc., if they choose, with their own APIs that can be targeted by a general-purpose command-line tool. This design has enabled a number of other systems to build atop Kubernetes. Kubernetes is not a PaaS (Platform as a Service).
+* Kubernetes does not limit the types of applications supported. It does not dictate application frameworks, restrict the set of supported language runtimes, nor cater to only 12-factor applications. Kubernetes aims to support an extremely diverse variety of workloads: if an application can run in a container, it should run great on Kubernetes.
+* Kubernetes is unopinionated in the source-to-image space. It does not build your application. Continuous Integration (CI) workflow is an area where different users and projects have their own requirements and preferences, so we support layering CI workflows on Kubernetes but don’t dictate how it should work.
+* On the other hand, a number of PaaS systems run on Kubernetes, such as Openshift and Deis. You could also roll your own custom PaaS, integrate with a CI system of your choice, or get along just fine with just Kubernetes: bring your container images and deploy them on Kubernetes.
+* Since Kubernetes operates at the application level rather than at just the hardware level, it provides some generally applicable features common to PaaS offerings, such as deployment, scaling, load balancing, logging, monitoring, etc. However, Kubernetes is not monolithic, and these default solutions are optional and pluggable. Kubernetes is not a mere “orchestration system”; it eliminates the need for orchestration:
+* The technical definition of “orchestration” is execution of a defined workflow: do A, then B, then C. In contrast, Kubernetes is comprised of a set of control processes that continuously drive current state towards the provided desired state. It shouldn’t matter how you get from A to C: make it so. This results in a system that is easier to use and more powerful, robust, and resilient. The name Kubernetes originates from Greek, meaning “helmsman” or “pilot”, and is the root of “governor” and “cybernetic”. K8s is an abbreviation derived by replacing the 8 letters “ubernete” with 8. Use these solutions to create Kubernetes clusters in your custom environment. First time here? See Picking the Right Solution for more information. Table of Contents: Use these solutions to create a single cluster with one or more Kubernetes nodes on a single physical machine. First time here? See Picking the Right Solution for more information. Table of Contents: Use these solutions to create Kubernetes clusters in your Cloud. First time here? See Picking the Right Solution for more information. Table of Contents: Testing version v1.1 Test of heading 2 is above Test of heading 3 is above Test of heading 4 is above Testing Okeedoke. Testing version v1.1 Test of heading 2 is above Test of heading 3 is above Test of heading 4 is above Testing Okeedoke. Test documentation here.","
"],col:[2,"
"],tr:[2,"","
"],td:[3,"
"],_default:[0,"",""]};ea.optgroup=ea.option;ea.tbody=ea.tfoot=ea.colgroup=ea.caption=ea.thead;ea.th=ea.td;var Ja=N.prototype={ready:function(b){function a(){c||(c=!0,b())}var c=!1;"complete"===U.readyState?setTimeout(a):(this.on("DOMContentLoaded",a),N(O).on("load",a))},toString:function(){var b=
+[];q(this,function(a){b.push(""+a)});return"["+b.join(", ")+"]"},eq:function(b){return 0<=b?y(this[b]):y(this[this.length+b])},length:0,push:Ke,sort:[].sort,splice:[].splice},kb={};q("multiple selected checked disabled readOnly required open".split(" "),function(b){kb[K(b)]=b});var nc={};q("input select option textarea button form details".split(" "),function(b){nc[Fa(b)]=!0});q({data:jc,inheritedData:jb,scope:function(b){return y(b).data("$scope")||jb(b.parentNode||b,["$isolateScope","$scope"])},
+isolateScope:function(b){return y(b).data("$isolateScope")||y(b).data("$isolateScopeNoTemplate")},controller:kc,injector:function(b){return jb(b,"$injector")},removeAttr:function(b,a){b.removeAttribute(a)},hasClass:Gb,css:function(b,a,c){a=Ta(a);if(B(c))b.style[a]=c;else{var d;8>=S&&(d=b.currentStyle&&b.currentStyle[a],""===d&&(d="auto"));d=d||b.style[a];8>=S&&(d=""===d?s:d);return d}},attr:function(b,a,c){var d=K(a);if(kb[d])if(B(c))c?(b[a]=!0,b.setAttribute(a,d)):(b[a]=!1,b.removeAttribute(d));
+else return b[a]||(b.attributes.getNamedItem(a)||C).specified?d:s;else if(B(c))b.setAttribute(a,c);else if(b.getAttribute)return b=b.getAttribute(a,2),null===b?s:b},prop:function(b,a,c){if(B(c))b[a]=c;else return b[a]},text:function(){function b(b,d){var e=a[b.nodeType];if(E(d))return e?b[e]:"";b[e]=d}var a=[];9>S?(a[1]="innerText",a[3]="nodeValue"):a[1]=a[3]="textContent";b.$dv="";return b}(),val:function(b,a){if(E(a)){if("SELECT"===Ka(b)&&b.multiple){var c=[];q(b.options,function(a){a.selected&&
+c.push(a.value||a.text)});return 0===c.length?null:c}return b.value}b.value=a},html:function(b,a){if(E(a))return b.innerHTML;for(var c=0,d=b.childNodes;c","
+ Host: {{server.host}}
+ Status: {{server.status}}
+ Image: {{server.dockerImage}}
+ Labels:
+
+
+ Volumes
+
+
+
+
+Volumes
+
+Pod
it is often necessary to share files between those containers. The
+Kubernetes Volume
abstraction solves both of these problems.
+
+
+
+
+
+
+
+
+
+
+ Background
+
+spec.volumes
+field) and where to mount those into containers(the
+spec.containers.volumeMounts
+field).Types of Volumes
+
+emptyDir
+ * hostPath
+ * gcePersistentDisk
+ * awsElasticBlockStore
+ * nfs
+ * iscsi
+ * flocker
+ * glusterfs
+ * rbd
+ * gitRepo
+ * secret
+ * persistentVolumeClaim
emptyDir
+
+emptyDir
volume is first created when a Pod is assigned to a Node, and
+exists as long as that Pod is running on that node. As the name says, it is
+initially empty. Containers in the pod can all read and write the same
+files in the emptyDir
volume, though that volume can be mounted at the same
+or different paths in each container. When a Pod is removed from a node for
+any reason, the data in the emptyDir
is deleted forever. NOTE: a container
+crashing does NOT remove a pod from a node, so the data in an emptyDir
+volume is safe across container crashes.emptyDir
are:
+
+
+emptyDir
volumes are stored on whatever medium is backing the
+machine - that might be disk or SSD or network storage, depending on your
+environment. However, you can set the emptyDir.medium
field to "Memory"
+to tell Kubernetes to mount a tmpfs (RAM-backed filesystem) for you instead.
+While tmpfs is very fast, be aware that unlike disks, tmpfs is cleared on
+machine reboot and any files you write will count against your container’s
+memory limit.hostPath
+
+hostPath
volume mounts a file or directory from the host node’s filesystem
+into your pod. This is not something that most Pods will need, but it offers a
+powerful escape hatch for some applications.hostPath
are:
+
+
+hostPath
+of /var/lib/docker
hostPath
of /dev/cgroups
+
+
+hostPath
gcePersistentDisk
+
+gcePersistentDisk
volume mounts a Google Compute Engine (GCE) Persistent
+Disk into your pod. Unlike
+emptyDir
, which is erased when a Pod is removed, the contents of a PD are
+preserved and the volume is merely unmounted. This means that a PD can be
+pre-populated with data, and that data can be “handed off” between pods.gcloud
or the GCE API or UI
+before you can use itgcePersistentDisk
:
+
+
+Creating a PD
+
+
+gcloud compute disks create --size=500GB --zone=us-central1-a my-data-disk
+
Example pod
+
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: test-pd
+spec:
+ containers:
+ - image: gcr.io/google_containers/test-webserver
+ name: test-container
+ volumeMounts:
+ - mountPath: /test-pd
+ name: test-volume
+ volumes:
+ - name: test-volume
+ # This GCE PD must already exist.
+ gcePersistentDisk:
+ pdName: my-data-disk
+ fsType: ext4
+
awsElasticBlockStore
+
+awsElasticBlockStore
volume mounts an Amazon Web Services (AWS) EBS
+Volume into your pod. Unlike
+emptyDir
, which is erased when a Pod is removed, the contents of an EBS
+volume are preserved and the volume is merely unmounted. This means that an
+EBS volume can be pre-populated with data, and that data can be “handed off”
+between pods.aws ec2 create-volume
or
+the AWS API before you can use it
+
+
+Creating an EBS volume
+
+
+aws ec2 create-volume --availability-zone eu-west-1a --size 10 --volume-type gp2
+
AWS EBS Example configuration
+
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: test-ebs
+spec:
+ containers:
+ - image: gcr.io/google_containers/test-webserver
+ name: test-container
+ volumeMounts:
+ - mountPath: /test-ebs
+ name: test-volume
+ volumes:
+ - name: test-volume
+ # This AWS EBS volume must already exist.
+ awsElasticBlockStore:
+ volumeID: aws://<availability-zone>/<volume-id>
+ fsType: ext4
+
nfs
+
+nfs
volume allows an existing NFS (Network File System) share to be
+mounted into your pod. Unlike emptyDir
, which is erased when a Pod is
+removed, the contents of an nfs
volume are preserved and the volume is merely
+unmounted. This means that an NFS volume can be pre-populated with data, and
+that data can be “handed off” between pods. NFS can be mounted by multiple
+writers simultaneously.iscsi
+
+iscsi
volume allows an existing iSCSI (SCSI over IP) volume to be mounted
+into your pod. Unlike emptyDir
, which is erased when a Pod is removed, the
+contents of an iscsi
volume are preserved and the volume is merely
+unmounted. This means that an iscsi volume can be pre-populated with data, and
+that data can be “handed off” between pods.flocker
+
+flocker
volume allows a Flocker dataset to be mounted into a pod. If the
+dataset does not already exist in Flocker, it needs to be created with Flocker
+CLI or the using the Flocker API. If the dataset already exists it will
+reattached by Flocker to the node that the pod is scheduled. This means data
+can be “handed off” between pods as required.glusterfs
+
+glusterfs
volume allows a Glusterfs (an open
+source networked filesystem) volume to be mounted into your pod. Unlike
+emptyDir
, which is erased when a Pod is removed, the contents of a
+glusterfs
volume are preserved and the volume is merely unmounted. This
+means that a glusterfs volume can be pre-populated with data, and that data can
+be “handed off” between pods. GlusterFS can be mounted by multiple writers
+simultaneously.rbd
+
+rbd
volume allows a Rados Block
+Device volume to be mounted into your
+pod. Unlike emptyDir
, which is erased when a Pod is removed, the contents of
+a rbd
volume are preserved and the volume is merely unmounted. This
+means that a RBD volume can be pre-populated with data, and that data can
+be “handed off” between pods.gitRepo
+
+gitRepo
volume is an example of what can be done as a volume plugin. It
+mounts an empty directory and clones a git repository into it for your pod to
+use. In the future, such volumes may be moved to an even more decoupled model,
+rather than extending the Kubernetes API for every such use case.
+apiVersion: v1
+kind: Pod
+metadata:
+ name: server
+spec:
+ containers:
+ - image: nginx
+ name: nginx
+ volumeMounts:
+ - mountPath: /mypath
+ name: git-volume
+ volumes:
+ - name: git-volume
+ gitRepo:
+ repository: "git@somewhere:me/my-git-repository.git"
+ revision: "22f1d8406d464b0c0874075539c1f2e96c253775"
+
secret
+
+secret
volume is used to pass sensitive information, such as passwords, to
+pods. You can store secrets in the Kubernetes API and mount them as files for
+use by pods without coupling to Kubernetes directly. secret
volumes are
+backed by tmpfs (a RAM-backed filesystem) so they are never written to
+non-volatile storage.persistentVolumeClaim
+
+persistentVolumeClaim
volume is used to mount a
+PersistentVolume into a pod. PersistentVolumes are a
+way for users to “claim” durable storage (such as a GCE PersistentDisk or an
+iSCSI volume) without knowing the details of the particular cloud environment.downwardAPI
+
+downwardAPI
volume is used to make downward API data available to applications.
+It mounts a directory and writes the requested data in plain text files.downwardAPI
volume example for more details.Resources
+
+emptyDir
volume is determined by the
+medium of the filesystem holding the kubelet root dir (typically
+/var/lib/kubelet
). There is no limit on how much space an emptyDir
or
+hostPath
volume can consume, and no isolation between containers or between
+pods.emptyDir
and hostPath
volumes will be able to
+request a certain amount of space using a resource
+specification, and to select the type of media to use, for clusters that have
+several media types.Kubernetes 101 - Kubectl CLI and Pods
+
+
+
+
+Kubernetes 101 - Kubectl CLI and Pods
+
+
+
+
+
+
+
+
+
+
+
+
+ Kubectl CLI
+
+Pods
+
+Pod Definition
+
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: nginx
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+
Pod Management
+
+
+$ kubectl create -f docs/user-guide/walkthrough/pod-nginx.yaml
+
+$ kubectl get pods
+
+$ curl http://$(kubectl get pod nginx -o go-template={{.status.podIP}})
+
+$ kubectl delete pod nginx
+
Volumes
+
+
+
+
+
+volumes:
+ - name: redis-persistent-storage
+ emptyDir: {}
+
+
+
+
+volumeMounts:
+ # name must match the volume name below
+ - name: redis-persistent-storage
+ # mount path within the container
+ mountPath: /data/redis
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: redis
+spec:
+ containers:
+ - name: redis
+ image: redis
+ volumeMounts:
+ - name: redis-persistent-storage
+ mountPath: /data/redis
+ volumes:
+ - name: redis-persistent-storage
+ emptyDir: {}
+
Volume Types
+
+
+
+
+/var/logs
).Multiple Containers
+
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: www
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+ volumeMounts:
+ - mountPath: /srv/www
+ name: www-data
+ readOnly: true
+ - name: git-monitor
+ image: kubernetes/git-monitor
+ env:
+ - name: GIT_REPO
+ value: http://github.com/some/repo.git
+ volumeMounts:
+ - mountPath: /data
+ name: www-data
+ volumes:
+ - name: www-data
+ emptyDir: {}
+
readOnly
in the web server’s case, since it doesn’t need to write to the directory.git-monitor
container, which allows us to parameterize that container with the particular git repository that we want to track.What’s Next?
+
+Kubernetes 101 - Kubectl CLI and Pods
+
+
+
+
+Kubernetes 101 - Kubectl CLI and Pods
+
+
+
+
+
+
+
+
+
+
+
+
+ Kubectl CLI
+
+Pods
+
+Pod Definition
+
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: nginx
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+
Pod Management
+
+
+$ kubectl create -f docs/user-guide/walkthrough/pod-nginx.yaml
+
+$ kubectl get pods
+
+$ curl http://$(kubectl get pod nginx -o go-template={{.status.podIP}})
+
+$ kubectl delete pod nginx
+
Volumes
+
+
+
+
+
+volumes:
+ - name: redis-persistent-storage
+ emptyDir: {}
+
+
+
+
+volumeMounts:
+ # name must match the volume name below
+ - name: redis-persistent-storage
+ # mount path within the container
+ mountPath: /data/redis
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: redis
+spec:
+ containers:
+ - name: redis
+ image: redis
+ volumeMounts:
+ - name: redis-persistent-storage
+ mountPath: /data/redis
+ volumes:
+ - name: redis-persistent-storage
+ emptyDir: {}
+
Volume Types
+
+
+
+
+/var/logs
).Multiple Containers
+
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: www
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+ volumeMounts:
+ - mountPath: /srv/www
+ name: www-data
+ readOnly: true
+ - name: git-monitor
+ image: kubernetes/git-monitor
+ env:
+ - name: GIT_REPO
+ value: http://github.com/some/repo.git
+ volumeMounts:
+ - mountPath: /data
+ name: www-data
+ volumes:
+ - name: www-data
+ emptyDir: {}
+
readOnly
in the web server’s case, since it doesn’t need to write to the directory.git-monitor
container, which allows us to parameterize that container with the particular git repository that we want to track.What’s Next?
+
+Kubernetes 201 - Labels, Replication Controllers, Services and Health Checking
+
+
+
+
+Kubernetes 201 - Labels, Replication Controllers, Services and Health Checking
+
+
+
+
+
+
+
+
+ Labels
+
+list
request to the apiserver to retrieve a list of objects which match that label selector.
+labels:
+ app: nginx
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: nginx
+ labels:
+ app: nginx
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+
+$ kubectl create -f docs/user-guide/walkthrough/pod-nginx-with-label.yaml
+
app=nginx
:
+$ kubectl get pods -l app=nginx
+
Replication Controllers
+
+
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: nginx-controller
+spec:
+ replicas: 2
+ # selector identifies the set of Pods that this
+ # replication controller is responsible for managing
+ selector:
+ app: nginx
+ # podTemplate defines the 'cookie cutter' used for creating
+ # new pods when necessary
+ template:
+ metadata:
+ labels:
+ # Important: these labels need to match the selector above
+ # The api server enforces this constraint.
+ app: nginx
+ spec:
+ containers:
+ - name: nginx
+ image: nginx
+ ports:
+ - containerPort: 80
+
Replication Controller Management
+
+
+$ kubectl create -f docs/user-guide/walkthrough/replication-controller.yaml
+
+$ kubectl get rc
+
+$ kubectl delete rc nginx-controller
+
Services
+
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: nginx-service
+spec:
+ ports:
+ - port: 8000 # the port that this service should serve on
+ # the container on each pod to connect to, can be a name
+ # (e.g. 'www') or a number (e.g. 80)
+ targetPort: 80
+ protocol: TCP
+ # just like the selector in the replication controller,
+ # but this time it identifies the set of pods to load balance
+ # traffic to.
+ selector:
+ app: nginx
+
Service Management
+
+
+$ kubectl create -f docs/user-guide/walkthrough/service.yaml
+
+$ kubectl get services
+
+$ export SERVICE_IP=$(kubectl get service nginx-service -o go-template={{.spec.clusterIP}})
+$ export SERVICE_PORT=$(kubectl get service nginx-service -o go-template'={{(index .spec.ports 0).port}}')
+$ curl http://${SERVICE_IP}:${SERVICE_PORT}
+
+$ kubectl delete service nginx-service
+
Health Checking
+
+Process Health Checking
+
+Application Health Checking
+
+
+lockOne := sync.Mutex{}
+lockTwo := sync.Mutex{}
+
+go func() {
+ lockOne.Lock();
+ lockTwo.Lock();
+ ...
+}()
+
+lockTwo.Lock();
+lockOne.Lock();
+
+
+
+livenessProbe
section of your container config. There you can also specify an initialDelaySeconds
that is a grace period from when the container is started to when health checks are performed, to enable your container to perform any necessary initialization.
+apiVersion: v1
+kind: Pod
+metadata:
+ name: pod-with-healthcheck
+spec:
+ containers:
+ - name: nginx
+ image: nginx
+ # defines the health checking
+ livenessProbe:
+ # an http probe
+ httpGet:
+ path: /_status/healthz
+ port: 80
+ # length of time to wait for a pod to initialize
+ # after pod startup, before applying health checking
+ initialDelaySeconds: 30
+ timeoutSeconds: 1
+ ports:
+ - containerPort: 80
+
What’s Next?
+
+Working with Resources
+
+
+
+
+Working with Resources
+
+Resources are Automatically Modified
+
+
+$ cat > /tmp/original.yaml <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+ name: foo
+spec:
+ containers:
+ - name: foo
+ image: busybox
+ restartPolicy: Never
+EOF
+$ kubectl create -f /tmp/original.yaml
+pods/original
+$ kubectl get pods/original -o yaml > /tmp/current.yaml
+pods/original
+$ wc -l /tmp/original.yaml /tmp/current.yaml
+ 51 /tmp/current.yaml
+ 9 /tmp/original.yaml
+ 60 total
+
diff -u /tmp/original.yaml /tmp/current.yaml
, you can see the fields added to the pod.
+The system adds fields in several ways:
+ - Some fields are added synchronously with creation of the resource and some are set asynchronously.
+ - For example: metadata.uid
is set synchronously. (Read more about metadata).
+ - For example, status.hostIP
is set only after the pod has been scheduled. This often happens fast, but you may notice pods which do not have this set yet. This is called Late Initialization. (Read mode about status and late initialization ).
+ - Some fields are set to default values. Some defaults vary by cluster and some are fixed for the API at a certain version. (Read more about defaulting).
+ - For example, spec.containers[0].imagePullPolicy
always defaults to IfNotPresent
in api v1.
+ - For example, spec.containers[0].resources.limits.cpu
may be defaulted to 100m
on some clusters, to some other value on others, and not defaulted at all on others.
+The API will generally not modify fields that you have set; it just sets ones which were unspecified.Finding Documentation on Resource Fields
+
+What is Kubernetes?
+
+
+
+
+What is Kubernetes?
+
+
+
+
+Kubernetes is:
+
+
+
+
+Ready to Get Started?
+
+
+
+Why containers?
+
+
+
+
+What can Kubernetes do?
+
+Why and how is Kubernetes a platform?
+
+Kubernetes is not:
+
+What does Kubernetes mean? K8s?
+
+Getting Started: Custom Solutions
+ Getting Started: Custom Solutions
+
+
+
+
+
+
+
+ Getting Started: Local-machine Solutions
+ Getting Started: Local-machine Solutions
+
+
+
+
+
+
+
+ Getting Started: Turn-key Solutions
+ Getting Started: Turn-key Solutions
+
+
+
+
+
+
+
+ User Guides
- How to get started, and acheive tasks, using Kubernetes
- Doc test
- This is a test of the documentation layout use
-
-Heading 2
-
-Heading 3
-
-Heading 4
-
-Code tests
-
-inline code
and below is a code example:
-
-console.log("Hello, world!");
-
Doc test
- This is a test of the documentation layout use
-
-Heading 2
-
-Heading 3
-
-Heading 4
-
-Code tests
-
-inline code
and below is a code example:
-
-console.log("Hello, world!");
-
Documentation
+
Use the following examples to learn how to setup and configure simple multi-tier applications in a Kubernetes cluster.
+ +Table of Contents:
+Testing version v1.1
- -Test of heading 2 is above
- -Test of heading 3 is above
- -Test of heading 4 is above
- -Testing inline code
and below is a code example:
console.log("Hello, world!");
-
-
-Okeedoke.
- -Use the following examples to learn how to perform some of the basic setup and configuration tasks in Kubernetes.
+ +Table of Contents:
+