Portainer
/
portainer
mirror of https://github.com/portainer/portainer.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(stacks): use docker-compose-wrapper library (#4979)

fix/EE-1152/importing-images
Chaim Lev-Ari 2021-07-21 04:56:28 +03:00 committed by GitHub
parent 592f7024e1
commit f9ffb1a712
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 269 additions and 342 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
api/cmd/portainer/main.go
View File

@ -6,6 +6,7 @@ import (
"os"
"strings"
wrapper "github.com/portainer/docker-compose-wrapper"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/bolt"
"github.com/portainer/portainer/api/chisel"
@ -77,12 +78,17 @@ func initDataStore(dataStorePath string, fileService portainer.FileService) port
}
func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
composeWrapper := exec.NewComposeWrapper(assetsPath, dataStorePath, proxyManager)
if composeWrapper != nil {
return composeWrapper
composeWrapper, err := exec.NewComposeStackManager(assetsPath, dataStorePath, proxyManager)
if err != nil {
if err == wrapper.ErrBinaryNotFound {
log.Printf("[INFO] [message: docker-compose binary not found, falling back to libcompose]")
return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
}
log.Fatalf("failed initalizing compose stack manager; err=%s", err)
}
return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
return composeWrapper
}
func initSwarmStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (portainer.SwarmStackManager, error) {

120
api/exec/compose_stack.go Normal file
View File

@ -0,0 +1,120 @@
package exec
import (
"fmt"
"os"
"path"
"strings"
wrapper "github.com/portainer/docker-compose-wrapper"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/http/proxy"
"github.com/portainer/portainer/api/http/proxy/factory"
)
// ComposeStackManager is a wrapper for docker-compose binary
type ComposeStackManager struct {
wrapper *wrapper.ComposeWrapper
configPath string
proxyManager *proxy.Manager
}
// NewComposeStackManager returns a docker-compose wrapper if corresponding binary present, otherwise nil
func NewComposeStackManager(binaryPath string, configPath string, proxyManager *proxy.Manager) (*ComposeStackManager, error) {
wrap, err := wrapper.NewComposeWrapper(binaryPath)
if err != nil {
return nil, err
}
return &ComposeStackManager{
wrapper: wrap,
proxyManager: proxyManager,
configPath: configPath,
}, nil
}
// NormalizeStackName returns a new stack name with unsupported characters replaced
func (w *ComposeStackManager) NormalizeStackName(name string) string {
return name
}
// ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string {
return portainer.ComposeSyntaxMaxVersion
}
// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
url, proxy, err := w.fetchEndpointProxy(endpoint)
if err != nil {
return err
}
if proxy != nil {
defer proxy.Close()
}
envFilePath, err := createEnvFile(stack)
if err != nil {
return err
}
filePath := stackFilePath(stack)
_, err = w.wrapper.Up([]string{filePath}, url, stack.Name, envFilePath, w.configPath)
return err
}
// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
func (w *ComposeStackManager) Down(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
url, proxy, err := w.fetchEndpointProxy(endpoint)
if err != nil {
return err
}
if proxy != nil {
defer proxy.Close()
}
filePath := stackFilePath(stack)
_, err = w.wrapper.Down([]string{filePath}, url, stack.Name)
return err
}
func stackFilePath(stack *portainer.Stack) string {
return path.Join(stack.ProjectPath, stack.EntryPoint)
}
func (w *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
return "", nil, nil
}
proxy, err := w.proxyManager.CreateComposeProxyServer(endpoint)
if err != nil {
return "", nil, err
}
return fmt.Sprintf("http://127.0.0.1:%d", proxy.Port), proxy, nil
}
func createEnvFile(stack *portainer.Stack) (string, error) {
if stack.Env == nil || len(stack.Env) == 0 {
return "", nil
}
envFilePath := path.Join(stack.ProjectPath, "stack.env")
envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return "", err
}
for _, v := range stack.Env {
envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
}
envfile.Close()
return envFilePath, nil
}

19
api/exec/compose_wrapper_integration_test.go → api/exec/compose_stack_integration_test.go
View File

@ -33,7 +33,9 @@ func setup(t *testing.T) (*portainer.Stack, *portainer.Endpoint) {
Name: "project-name",
}
endpoint := &portainer.Endpoint{}
endpoint := &portainer.Endpoint{
URL: "unix://",
}
return stack, endpoint
}
@ -42,14 +44,17 @@ func Test_UpAndDown(t *testing.T) {
stack, endpoint := setup(t)
w := NewComposeWrapper("", "", nil)
w, err := NewComposeStackManager("", "", nil)
if err != nil {
t.Fatalf("Failed creating manager: %s", err)
}
err := w.Up(stack, endpoint)
err = w.Up(stack, endpoint)
if err != nil {
t.Fatalf("Error calling docker-compose up: %s", err)
}
if containerExists(composedContainerName) == false {
if !containerExists(composedContainerName) {
t.Fatal("container should exist")
}
@ -63,13 +68,13 @@ func Test_UpAndDown(t *testing.T) {
}
}
func containerExists(contaierName string) bool {
cmd := exec.Command(osProgram("docker"), "ps", "-a", "-f", fmt.Sprintf("name=%s", contaierName))
func containerExists(containerName string) bool {
cmd := exec.Command("docker", "ps", "-a", "-f", fmt.Sprintf("name=%s", containerName))
out, err := cmd.Output()
if err != nil {
log.Fatalf("failed to list containers: %s", err)
}
return strings.Contains(string(out), contaierName)
return strings.Contains(string(out), containerName)
}

112
api/exec/compose_stack_test.go Normal file
View File

@ -0,0 +1,112 @@
package exec
import (
"io/ioutil"
"os"
"path"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
)
func Test_stackFilePath(t *testing.T) {
tests := []struct {
name string
stack *portainer.Stack
expected string
}{
// {
// name: "should return empty result if stack is missing",
// stack: nil,
// expected: "",
// },
// {
// name: "should return empty result if stack don't have entrypoint",
// stack: &portainer.Stack{},
// expected: "",
// },
{
name: "should allow file name and dir",
stack: &portainer.Stack{
ProjectPath: "dir",
EntryPoint: "file",
},
expected: path.Join("dir", "file"),
},
{
name: "should allow file name only",
stack: &portainer.Stack{
EntryPoint: "file",
},
expected: "file",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := stackFilePath(tt.stack)
assert.Equal(t, tt.expected, result)
})
}
}
func Test_createEnvFile(t *testing.T) {
dir := t.TempDir()
tests := []struct {
name string
stack *portainer.Stack
expected string
expectedFile bool
}{
// {
// name: "should not add env file option if stack is missing",
// stack: nil,
// expected: "",
// },
{
name: "should not add env file option if stack doesn't have env variables",
stack: &portainer.Stack{
ProjectPath: dir,
},
expected: "",
},
{
name: "should not add env file option if stack's env variables are empty",
stack: &portainer.Stack{
ProjectPath: dir,
Env: []portainer.Pair{},
},
expected: "",
},
{
name: "should add env file option if stack has env variables",
stack: &portainer.Stack{
ProjectPath: dir,
Env: []portainer.Pair{
{Name: "var1", Value: "value1"},
{Name: "var2", Value: "value2"},
},
},
expected: "var1=value1\nvar2=value2\n",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, _ := createEnvFile(tt.stack)
if tt.expected != "" {
assert.Equal(t, path.Join(tt.stack.ProjectPath, "stack.env"), result)
f, _ := os.Open(path.Join(dir, "stack.env"))
content, _ := ioutil.ReadAll(f)
assert.Equal(t, tt.expected, string(content))
} else {
assert.Equal(t, "", result)
}
})
}
}

141
api/exec/compose_wrapper.go
View File

@ -1,141 +0,0 @@
package exec
import (
"bytes"
"errors"
"fmt"
"os"
"os/exec"
"path"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/http/proxy"
)
// ComposeWrapper is a wrapper for docker-compose binary
type ComposeWrapper struct {
binaryPath string
dataPath string
proxyManager *proxy.Manager
}
// NewComposeWrapper returns a docker-compose wrapper if corresponding binary present, otherwise nil
func NewComposeWrapper(binaryPath, dataPath string, proxyManager *proxy.Manager) *ComposeWrapper {
if !IsBinaryPresent(programPath(binaryPath, "docker-compose")) {
return nil
}
return &ComposeWrapper{
binaryPath: binaryPath,
dataPath: dataPath,
proxyManager: proxyManager,
}
}
// ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax
func (w *ComposeWrapper) ComposeSyntaxMaxVersion() string {
return portainer.ComposeSyntaxMaxVersion
}
// NormalizeStackName returns a new stack name with unsupported characters replaced
func (w *ComposeWrapper) NormalizeStackName(name string) string {
return name
}
// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
func (w *ComposeWrapper) Up(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
_, err := w.command([]string{"up", "-d"}, stack, endpoint)
return err
}
// Down stops and removes containers, networks, images, and volumes. Wraps `docker-compose down --remove-orphans` command
func (w *ComposeWrapper) Down(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
_, err := w.command([]string{"down", "--remove-orphans"}, stack, endpoint)
return err
}
func (w *ComposeWrapper) command(command []string, stack *portainer.Stack, endpoint *portainer.Endpoint) ([]byte, error) {
if endpoint == nil {
return nil, errors.New("cannot call a compose command on an empty endpoint")
}
program := programPath(w.binaryPath, "docker-compose")
options := setComposeFile(stack)
options = addProjectNameOption(options, stack)
options, err := addEnvFileOption(options, stack)
if err != nil {
return nil, err
}
if !(endpoint.URL == "" || strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://")) {
proxy, err := w.proxyManager.CreateComposeProxyServer(endpoint)
if err != nil {
return nil, err
}
defer proxy.Close()
options = append(options, "-H", fmt.Sprintf("http://127.0.0.1:%d", proxy.Port))
}
args := append(options, command...)
var stderr bytes.Buffer
cmd := exec.Command(program, args...)
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, fmt.Sprintf("DOCKER_CONFIG=%s", w.dataPath))
cmd.Stderr = &stderr
out, err := cmd.Output()
if err != nil {
return out, errors.New(stderr.String())
}
return out, nil
}
func setComposeFile(stack *portainer.Stack) []string {
options := make([]string, 0)
if stack == nil || stack.EntryPoint == "" {
return options
}
composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
options = append(options, "-f", composeFilePath)
return options
}
func addProjectNameOption(options []string, stack *portainer.Stack) []string {
if stack == nil || stack.Name == "" {
return options
}
options = append(options, "-p", stack.Name)
return options
}
func addEnvFileOption(options []string, stack *portainer.Stack) ([]string, error) {
if stack == nil || stack.Env == nil || len(stack.Env) == 0 {
return options, nil
}
envFilePath := path.Join(stack.ProjectPath, "stack.env")
envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return options, err
}
for _, v := range stack.Env {
envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
}
envfile.Close()
options = append(options, "--env-file", envFilePath)
return options, nil
}

143
api/exec/compose_wrapper_test.go
View File

@ -1,143 +0,0 @@
package exec
import (
"io/ioutil"
"os"
"path"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
)
func Test_setComposeFile(t *testing.T) {
tests := []struct {
name string
stack *portainer.Stack
expected []string
}{
{
name: "should return empty result if stack is missing",
stack: nil,
expected: []string{},
},
{
name: "should return empty result if stack don't have entrypoint",
stack: &portainer.Stack{},
expected: []string{},
},
{
name: "should allow file name and dir",
stack: &portainer.Stack{
ProjectPath: "dir",
EntryPoint: "file",
},
expected: []string{"-f", path.Join("dir", "file")},
},
{
name: "should allow file name only",
stack: &portainer.Stack{
EntryPoint: "file",
},
expected: []string{"-f", "file"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := setComposeFile(tt.stack)
assert.ElementsMatch(t, tt.expected, result)
})
}
}
func Test_addProjectNameOption(t *testing.T) {
tests := []struct {
name string
stack *portainer.Stack
expected []string
}{
{
name: "should not add project option if stack is missing",
stack: nil,
expected: []string{},
},
{
name: "should not add project option if stack doesn't have name",
stack: &portainer.Stack{},
expected: []string{},
},
{
name: "should add project name option if stack has a name",
stack: &portainer.Stack{
Name: "project-name",
},
expected: []string{"-p", "project-name"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
options := []string{"-a", "b"}
result := addProjectNameOption(options, tt.stack)
assert.ElementsMatch(t, append(options, tt.expected...), result)
})
}
}
func Test_addEnvFileOption(t *testing.T) {
dir := t.TempDir()
tests := []struct {
name string
stack *portainer.Stack
expected []string
expectedContent string
}{
{
name: "should not add env file option if stack is missing",
stack: nil,
expected: []string{},
},
{
name: "should not add env file option if stack doesn't have env variables",
stack: &portainer.Stack{},
expected: []string{},
},
{
name: "should not add env file option if stack's env variables are empty",
stack: &portainer.Stack{
ProjectPath: dir,
Env: []portainer.Pair{},
},
expected: []string{},
},
{
name: "should add env file option if stack has env variables",
stack: &portainer.Stack{
ProjectPath: dir,
Env: []portainer.Pair{
{Name: "var1", Value: "value1"},
{Name: "var2", Value: "value2"},
},
},
expected: []string{"--env-file", path.Join(dir, "stack.env")},
expectedContent: "var1=value1\nvar2=value2\n",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
options := []string{"-a", "b"}
result, _ := addEnvFileOption(options, tt.stack)
assert.ElementsMatch(t, append(options, tt.expected...), result)
if tt.expectedContent != "" {
f, _ := os.Open(path.Join(dir, "stack.env"))
content, _ := ioutil.ReadAll(f)
assert.Equal(t, tt.expectedContent, string(content))
}
})
}
}

24
api/exec/utils.go
View File

@ -1,24 +0,0 @@
package exec
import (
"os/exec"
"path/filepath"
"runtime"
)
func osProgram(program string) string {
if runtime.GOOS == "windows" {
program += ".exe"
}
return program
}
func programPath(rootPath, program string) string {
return filepath.Join(rootPath, osProgram(program))
}
// IsBinaryPresent returns true if corresponding program exists on PATH
func IsBinaryPresent(program string) bool {
_, err := exec.LookPath(program)
return err == nil
}

16
api/exec/utils_test.go
View File

@ -1,16 +0,0 @@
package exec
import (
"testing"
)
func Test_isBinaryPresent(t *testing.T) {
if !IsBinaryPresent("docker") {
t.Error("expect docker binary to exist on the path")
}
if IsBinaryPresent("executable-with-this-name-should-not-exist") {
t.Error("expect binary with a random name to be missing on the path")
}
}

3
api/go.mod
View File

@ -27,10 +27,11 @@ require (
github.com/mitchellh/mapstructure v1.1.2 // indirect
github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
github.com/pkg/errors v0.9.1
github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92
github.com/portainer/libcompose v0.5.3
github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2
github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33
github.com/stretchr/testify v1.6.1
github.com/stretchr/testify v1.7.0
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
gopkg.in/alecthomas/kingpin.v2 v2.2.6

6
api/go.sum
View File

@ -242,6 +242,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92 h1:Hh7SHCf3SJblVywU0TTn5lpTKsH5W23LAKH5sqWggig=
github.com/portainer/docker-compose-wrapper v0.0.0-20210527221011-0a1418224b92/go.mod h1:PF2O2O4UNYWdtPcp6n/mIKpKk+f1jhFTezS8txbf+XM=
github.com/portainer/libcompose v0.5.3 h1:tE4WcPuGvo+NKeDkDWpwNavNLZ5GHIJ4RvuZXsI9uI8=
github.com/portainer/libcompose v0.5.3/go.mod h1:7SKd/ho69rRKHDFSDUwkbMcol2TMKU5OslDsajr8Ro8=
github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2 h1:0PfgGLys9yHr4rtnirg0W0Cjvv6/DzxBIZk5sV59208=
@ -277,8 +279,8 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc=
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ=

13
api/http/proxy/factory/docker_compose.go
View File

@ -49,13 +49,18 @@ func (factory *ProxyFactory) NewDockerComposeAgentProxy(endpoint *portainer.Endp
proxy.Transport = dockercompose.NewAgentTransport(factory.signatureService, httpTransport)
proxyServer := &ProxyServer{
&http.Server{
server: &http.Server{
Handler: proxy,
},
0,
Port: 0,
}
return proxyServer, proxyServer.start()
err = proxyServer.start()
if err != nil {
return nil, err
}
return proxyServer, err
}
func (proxy *ProxyServer) start() error {
@ -72,7 +77,7 @@ func (proxy *ProxyServer) start() error {
err := proxy.server.Serve(listener)
log.Printf("Exiting Proxy server %s\n", proxyHost)
if err != http.ErrServerClosed {
if err != nil && err != http.ErrServerClosed {
log.Printf("Proxy server %s exited with an error: %s\n", proxyHost, err)
}
}()