fix(compose): merge default and in-place stack env vars [EE-2860] (#7076)

api/exec/compose_stack.go

@@ -6,7 +6,6 @@ import (
 	"io"
 	"os"
 	"path"
-	"regexp"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -14,7 +13,6 @@ import (
 	libstack "github.com/portainer/docker-compose-wrapper"
 	"github.com/portainer/docker-compose-wrapper/compose"
 
-	"github.com/docker/cli/cli/compose/loader"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/proxy/factory"
@@ -56,13 +54,13 @@ func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Sta
 		defer proxy.Close()
 	}
 
-	envFilePath, err := createEnvFile(stack)
+	envFile, err := createEnvFile(stack)
 	if err != nil {
 		return errors.Wrap(err, "failed to create env file")
 	}
 
 	filePaths := stackutils.GetStackFilePaths(stack)
-	err = manager.deployer.Deploy(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFilePath, forceRereate)
+	err = manager.deployer.Deploy(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile, forceRereate)
 	return errors.Wrap(err, "failed to deploy a stack")
 }
 
@@ -76,12 +74,14 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S
 		defer proxy.Close()
 	}
 
-	if err := updateNetworkEnvFile(stack); err != nil {
+	envFile, err := createEnvFile(stack)
-		return err
+	if err != nil {
+		return errors.Wrap(err, "failed to create env file")
 	}
 
 	filePaths := stackutils.GetStackFilePaths(stack)
-	err = manager.deployer.Remove(ctx, stack.ProjectPath, url, stack.Name, filePaths)
+
+	err = manager.deployer.Remove(ctx, stack.ProjectPath, url, stack.Name, filePaths, envFile)
 	return errors.Wrap(err, "failed to remove a stack")
 }
 
@@ -103,200 +103,42 @@ func (manager *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpo
 	return fmt.Sprintf("tcp://127.0.0.1:%d", proxy.Port), proxy, nil
 }
 
+// createEnvFile creates a file that would hold both "in-place" and default environment variables.
+// It will return the name of the file if the stack has "in-place" env vars, otherwise empty string.
 func createEnvFile(stack *portainer.Stack) (string, error) {
-	// workaround for EE-1862. It will have to be removed when
-	// docker/compose upgraded to v2.x.
-	if err := createNetworkEnvFile(stack); err != nil {
-		return "", errors.Wrap(err, "failed to create network env file")
-	}
-
 	if stack.Env == nil || len(stack.Env) == 0 {
 		return "", nil
 	}
 
 	envFilePath := path.Join(stack.ProjectPath, "stack.env")
-
 	envfile, err := os.OpenFile(envFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return "", err
 	}
+	defer envfile.Close()
+
+	copyDefaultEnvFile(stack, envfile)
 
 	for _, v := range stack.Env {
 		envfile.WriteString(fmt.Sprintf("%s=%s\n", v.Name, v.Value))
 	}
-	envfile.Close()
 
 	return "stack.env", nil
 }
 
-func fileNotExist(filePath string) bool {
+// copyDefaultEnvFile copies the default .env file if it exists to the provided writer
-	if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) {
+func copyDefaultEnvFile(stack *portainer.Stack, w io.Writer) {
-		return true
+	defaultEnvFile, err := os.Open(path.Join(path.Join(stack.ProjectPath, path.Dir(stack.EntryPoint)), ".env"))
-	}
-
-	return false
-}
-
-func updateNetworkEnvFile(stack *portainer.Stack) error {
-	envFilePath := path.Join(stack.ProjectPath, ".env")
-	stackFilePath := path.Join(stack.ProjectPath, "stack.env")
-	if fileNotExist(envFilePath) {
-		if fileNotExist(stackFilePath) {
-			return nil
-		}
-
-		flags := os.O_WRONLY | os.O_SYNC | os.O_CREATE
-		envFile, err := os.OpenFile(envFilePath, flags, 0666)
-		if err != nil {
-			return err
-		}
-
-		defer envFile.Close()
-
-		stackFile, err := os.Open(stackFilePath)
-		if err != nil {
-			return err
-		}
-
-		defer stackFile.Close()
-
-		_, err = io.Copy(envFile, stackFile)
-		return err
-	}
-
-	return nil
-}
-
-func createNetworkEnvFile(stack *portainer.Stack) error {
-	networkNameSet := NewStringSet()
-
-	for _, filePath := range stackutils.GetStackFilePaths(stack) {
-		networkNames, err := extractNetworkNames(filePath)
-		if err != nil {
-			return errors.Wrap(err, "failed to extract network name")
-		}
-
-		if networkNames == nil || networkNames.Len() == 0 {
-			continue
-		}
-
-		networkNameSet.Union(networkNames)
-	}
-
-	for _, s := range networkNameSet.List() {
-		if _, ok := os.LookupEnv(s); ok {
-			networkNameSet.Remove(s)
-		}
-	}
-
-	if networkNameSet.Len() == 0 && stack.Env == nil {
-		return nil
-	}
-
-	envfile, err := os.OpenFile(path.Join(stack.ProjectPath, ".env"),
-		os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
-		return errors.Wrap(err, "failed to open env file")
+		// If cannot open a default file, then don't need to copy it.
+		// We could as well stat it and check if it exists, but this is more efficient.
+		return
 	}
 
-	defer envfile.Close()
+	defer defaultEnvFile.Close()
 
-	var scanEnvSettingFunc = func(name string) (string, bool) {
+	if _, err = io.Copy(w, defaultEnvFile); err == nil {
-		if stack.Env != nil {
+		io.WriteString(w, "\n")
-			for _, v := range stack.Env {
-				if name == v.Name {
-					return v.Value, true
-				}
-			}
-		}
-
-		return "", false
 	}
-
+	// If couldn't copy the .env file, then ignore the error and try to continue
-	for _, s := range networkNameSet.List() {
-		if _, ok := scanEnvSettingFunc(s); !ok {
-			stack.Env = append(stack.Env, portainer.Pair{
-				Name:  s,
-				Value: "None",
-			})
-		}
-	}
-
-	if stack.Env != nil {
-		for _, v := range stack.Env {
-			envfile.WriteString(
-				fmt.Sprintf("%s=%s\n", v.Name, v.Value))
-		}
-	}
-
-	return nil
-}
-
-func extractNetworkNames(filePath string) (StringSet, error) {
-	if info, err := os.Stat(filePath); errors.Is(err,
-		os.ErrNotExist) || info.IsDir() {
-		return nil, nil
-	}
-
-	stackFileContent, err := os.ReadFile(filePath)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to open yaml file")
-	}
-
-	config, err := loader.ParseYAML(stackFileContent)
-	if err != nil {
-		// invalid stack file
-		return nil, errors.Wrap(err, "invalid stack file")
-	}
-
-	var version string
-	if _, ok := config["version"]; ok {
-		version, _ = config["version"].(string)
-	}
-
-	var networks map[string]interface{}
-	if value, ok := config["networks"]; ok {
-		if value == nil {
-			return nil, nil
-		}
-
-		if networks, ok = value.(map[string]interface{}); !ok {
-			return nil, nil
-		}
-	} else {
-		return nil, nil
-	}
-
-	networkContent, err := loader.LoadNetworks(networks, version)
-	if err != nil {
-		return nil, nil // skip the error
-	}
-
-	re := regexp.MustCompile(`^\$\{?([^\}]+)\}?$`)
-	networkNames := NewStringSet()
-
-	for _, v := range networkContent {
-		matched := re.FindAllStringSubmatch(v.Name, -1)
-		if matched != nil && matched[0] != nil {
-			if strings.Contains(matched[0][1], ":-") {
-				continue
-			}
-
-			if strings.Contains(matched[0][1], "?") {
-				continue
-			}
-
-			if strings.Contains(matched[0][1], "-") {
-				continue
-			}
-
-			networkNames.Add(matched[0][1])
-		}
-	}
-
-	if networkNames.Len() == 0 {
-		return nil, nil
-	}
-
-	return networkNames, nil
 }

api/exec/compose_stack_test.go

@@ -65,56 +65,22 @@ func Test_createEnvFile(t *testing.T) {
 	}
 }
 
-func Test_createNetworkEnvFile(t *testing.T) {
+func Test_createEnvFile_mergesDefultAndInplaceEnvVars(t *testing.T) {
 	dir := t.TempDir()
-	buf := []byte(`
+	os.WriteFile(path.Join(dir, ".env"), []byte("VAR1=VAL1\nVAR2=VAL2\n"), 0600)
-version: '3.6'
+	stack := &portainer.Stack{
-services:
-  nginx-example:
-    image: nginx:latest
-networks:
-  default:
-    name: ${test}
-    driver: bridge
-`)
-	if err := ioutil.WriteFile(path.Join(dir,
-		"docker-compose.yml"), buf, 0644); err != nil {
-		t.Fatalf("Failed to create yaml file: %s", err)
-	}
-
-	stackWithoutEnv := &portainer.Stack{
 		ProjectPath: dir,
-		EntryPoint:  "docker-compose.yml",
-		Env:         []portainer.Pair{},
-	}
-
-	if err := createNetworkEnvFile(stackWithoutEnv); err != nil {
-		t.Fatalf("Failed to create network env file: %s", err)
-	}
-
-	content, err := ioutil.ReadFile(path.Join(dir, ".env"))
-	if err != nil {
-		t.Fatalf("Failed to read network env file: %s", err)
-	}
-
-	assert.Equal(t, "test=None\n", string(content))
-
-	stackWithEnv := &portainer.Stack{
-		ProjectPath: dir,
-		EntryPoint:  "docker-compose.yml",
 		Env: []portainer.Pair{
-			{Name: "test", Value: "test-value"},
+			{Name: "VAR1", Value: "NEW_VAL1"},
+			{Name: "VAR3", Value: "VAL3"},
 		},
 	}
+	result, err := createEnvFile(stack)
+	assert.Equal(t, "stack.env", result)
+	assert.NoError(t, err)
+	assert.FileExists(t, path.Join(dir, "stack.env"))
+	f, _ := os.Open(path.Join(dir, "stack.env"))
+	content, _ := ioutil.ReadAll(f)
 
-	if err := createNetworkEnvFile(stackWithEnv); err != nil {
+	assert.Equal(t, []byte("VAR1=VAL1\nVAR2=VAL2\n\nVAR1=NEW_VAL1\nVAR3=VAL3\n"), content)
-		t.Fatalf("Failed to create network env file: %s", err)
-	}
-
-	content, err = ioutil.ReadFile(path.Join(dir, ".env"))
-	if err != nil {
-		t.Fatalf("Failed to read network env file: %s", err)
-	}
-
-	assert.Equal(t, "test=test-value\n", string(content))
 }

api/go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c
 	github.com/orcaman/concurrent-map v0.0.0-20190826125027-8c72a8bb44f6
 	github.com/pkg/errors v0.9.1
-	github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410
+	github.com/portainer/docker-compose-wrapper v0.0.0-20220703222411-e3cf664b39c6
 	github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a
 	github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108
 	github.com/portainer/libhttp v0.0.0-20211208103139-07a5f798eb3f

api/go.sum

@@ -809,8 +809,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410 h1:LjxLd8UGR8ae73ov/vLrt/0jedj/nh98XnONkr8DJj8=
+github.com/portainer/docker-compose-wrapper v0.0.0-20220703222411-e3cf664b39c6 h1:6VQZsYaJGfEq1LSKiNQ8HIW3olB04MpnW6HTnLnpMSQ=
-github.com/portainer/docker-compose-wrapper v0.0.0-20220531190153-c597b853e410/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
+github.com/portainer/docker-compose-wrapper v0.0.0-20220703222411-e3cf664b39c6/go.mod h1:WxDlJWZxCnicdLCPnLNEv7/gRhjeIVuCGmsv+iOPH3c=
 github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a h1:qY8TbocN75n5PDl16o0uVr5MevtM5IhdwSelXEd4nFM=
 github.com/portainer/libcrypto v0.0.0-20210422035235-c652195c5c3a/go.mod h1:n54EEIq+MM0NNtqLeCby8ljL+l275VpolXO0ibHegLE=
 github.com/portainer/libhelm v0.0.0-20210929000907-825e93d62108 h1:5e8KAnDa2G3cEHK7aV/ue8lOaoQwBZUzoALslwWkR04=