feat(upgrade): upgrade security

pull/19745/head
Ales Pour 2020-10-12 18:01:00 +02:00 committed by Stuart Carnie
parent a171c4e2cb
commit 2308e6c50e
3 changed files with 581 additions and 4 deletions

View File

@ -1,11 +1,315 @@
package upgrade
// Security upgrade implementation.
// Generates script for upgrading 1.x users into tokens in 2.x.
import (
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"sort"
"strings"
"text/template"
"github.com/influxdata/influxdb/v2/v1/services/meta"
"github.com/influxdata/influxql"
"go.uber.org/zap"
)
// generateSecurityScript generates security upgrade script.
func generateSecurityScript(v1 *influxDBv1, dbBuckets map[string][]string, log *zap.Logger) error {
return errors.New("not implemented")
// generateSecurityScript creates security upgrade script with 1.x users mapped to 2.x tokens.
func generateSecurityScript(v1 *influxDBv1, targetOptions optionsV2, dbBuckets map[string][]string, log *zap.Logger) error {
// check if there any 1.x users at all
v1meta := v1.meta
if len(v1meta.Users()) == 0 {
log.Info("There are no users in 1.x, no script will be created.")
return nil
}
// get helper instance
helper := newSecurityScriptHelper(log)
// check if target buckets exists in 2.x
proceed := helper.checkDbBuckets(v1meta, dbBuckets)
if !proceed {
return errors.New("upgrade: there were errors/warnings, please fix them and run the command again")
}
// create output
var output *os.File
var isFileOutput bool
if targetOptions.securityScriptPath == "" {
output = os.Stdout
} else {
file, err := os.OpenFile(targetOptions.securityScriptPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0777)
if err != nil {
return err
}
defer file.Close()
output = file
isFileOutput = true
}
// get `influx` path
influxExe, err := helper.getInflux2ExePath()
if err != nil {
return err
}
// template data type
type dataObject struct {
AdminUsers []string
IgnoreUsers []string
UpgradeUsers []string
UserArgs map[string][]string
Exe string
TargetAdmin string
TargetOrg string
TargetToken string
IsFileOutput bool
InfluxExe string
}
// data
data := dataObject{
UserArgs: make(map[string][]string),
TargetAdmin: targetOptions.userName,
TargetOrg: targetOptions.orgName,
TargetToken: targetOptions.token,
IsFileOutput: isFileOutput,
InfluxExe: influxExe,
}
// fill data with users and their permissions
for _, row := range helper.sortUserInfo(v1meta.Users()) {
username := row.Name
if row.Admin {
data.AdminUsers = append(data.AdminUsers, username)
} else if len(row.Privileges) == 0 {
data.IgnoreUsers = append(data.IgnoreUsers, username)
} else {
data.UpgradeUsers = append(data.UpgradeUsers, username)
dbList := make([]string, 0)
for database := range row.Privileges {
dbList = append(dbList, database)
}
sort.Strings(dbList)
accessArgs := make([]string, 0)
for _, database := range dbList {
permission := row.Privileges[database]
for _, id := range dbBuckets[database] {
switch permission {
case influxql.ReadPrivilege:
accessArgs = append(accessArgs, fmt.Sprintf("--read-bucket=%s", id))
case influxql.WritePrivilege:
accessArgs = append(accessArgs, fmt.Sprintf("--write-bucket=%s", id))
case influxql.AllPrivileges:
accessArgs = append(accessArgs, fmt.Sprintf("--read-bucket=%s", id))
accessArgs = append(accessArgs, fmt.Sprintf("--write-bucket=%s", id))
}
}
}
if len(accessArgs) > 0 { // should always be true
data.UserArgs[username] = accessArgs
}
}
}
// load template
var tmpl string
if runtime.GOOS == "win" {
tmpl = securityScriptCmdTemplate
} else {
tmpl = securityScriptShTemplate
}
t, err := template.New("script").Funcs(template.FuncMap{
"shUserVar": func(name string) string {
return helper.shUserVar(name)
},
"userArgs": func(args []string) string {
return strings.Join(args, " ")
},
}).Parse(tmpl)
if err != nil {
return err
}
// generate the script
err = t.Execute(output, data)
if err != nil {
return err
}
if isFileOutput {
log.Info(fmt.Sprintf("Security upgrade script saved to %s.", targetOptions.securityScriptPath))
}
return nil
}
// securityScriptHelper is a helper used by `generate-security-script` command.
type securityScriptHelper struct {
shReg *regexp.Regexp
log *zap.Logger
}
// newSecurityScriptHelper returns new security script helper instance for `generate-security-script` command.
func newSecurityScriptHelper(log *zap.Logger) *securityScriptHelper {
helper := &securityScriptHelper{
log: log,
}
helper.shReg = regexp.MustCompile("[^a-zA-Z0-9]+")
return helper
}
func (h *securityScriptHelper) checkDbBuckets(meta *meta.Client, databases map[string][]string) bool {
ok := true
for _, row := range meta.Users() {
for database := range row.Privileges {
if database == "_internal" {
continue
}
ids := databases[database]
if len(ids) == 0 {
h.log.Warn(fmt.Sprintf("No buckets for database [%s] exist in 2.x.", database))
ok = false
}
}
}
return ok
}
func (h *securityScriptHelper) getInflux2ExePath() (string, error) {
var exeName string
if runtime.GOOS == "win" {
exeName = "influx.exe"
} else {
exeName = "influx"
}
exePath, err := exec.LookPath(exeName)
if err == nil {
if h.checkInflux2Version(exePath) {
return exePath, nil
}
}
exePath, err = os.Executable()
if err == nil {
exePath = filepath.Join(filepath.Dir(exePath), exeName)
if h.checkInflux2Version(exePath) {
return exePath, nil
}
h.log.Error("Version 2.x influx executable not found.")
err = errors.New("upgrade: version 2.x influx executable not found")
}
return "", err
}
func (h *securityScriptHelper) checkInflux2Version(path string) bool {
return exec.Command(path, "version").Run() == nil
}
func (h *securityScriptHelper) sortUserInfo(info []meta.UserInfo) []meta.UserInfo {
sort.Slice(info, func(i, j int) bool {
return info[i].Name < info[j].Name
})
return info
}
func (h *securityScriptHelper) shUserVar(name string) string {
return "UPGRADE_USER_" + h.shReg.ReplaceAllString(name, "_")
}
// script templates
var securityScriptShTemplate = `#!/bin/sh
{{- range $u := $.UpgradeUsers}}
{{shUserVar $u}}=yes # user {{$u}}
{{- end}}
{{- range $u := $.AdminUsers}}
# user {{$u}} is 1.x admin and will not be upgraded automatically
{{- end}}
{{- range $u := $.IgnoreUsers}}
# user {{$u}} has no privileges and will be skipped
{{- end}}
#
# SCRIPT VARS
#
INFLUX={{$.InfluxExe}}
INFLUX_TOKEN={{$.TargetToken}}
{{- if $.IsFileOutput}}
LOG="${0%.*}.$(date +%Y%m%d-%H%M%S).log"
{{end}}
#
# USERS UPGRADES
#
{{range $u := $.UpgradeUsers}}
if [ "${{shUserVar $u}}" = "yes" ]; then
echo Creating authorization token for user {{$u}}...
env INFLUX_TOKEN=$INFLUX_TOKEN $INFLUX auth create --user={{$.TargetAdmin}} --org={{$.TargetOrg}} --description="{{$u}}" {{index $.UserArgs $u | userArgs}}
fi {{- if $.IsFileOutput}} 2>&1 | tee -a $LOG{{- end}}
{{- end}}
{{- if $.IsFileOutput}}
echo
echo Output saved to $LOG
{{- end}}
`
var securityScriptCmdTemplate = `@ECHO OFF
{{- range $u := $.UpgradeUsers}}
REM user {{$u}}
set {{shUserVar $u}}=yes
{{- end}}
{{- range $u := $.AdminUsers}}
REM user {{$u}} is 1.x admin and will not be upgraded automatically
{{- end}}
{{- range $u := $.IgnoreUsers}}
REM user {{$u}} has no privileges will be skipped
{{- end}}
REM
REM SCRIPT VARS
REM
set INFLUX="{{$.InfluxExe}}.exe"
set INFLUX_TOKEN={{$.TargetToken}}
{{- if $.IsFileOutput}}
set PATH=%PATH%;C:\WINDOWS\system32\wbem
for /f %%x in ('wmic os get localdatetime ^| findstr /b [0-9]') do @set X=%%x && set LOG=%~dpn0.%X:~0,8%-%X:~8,6%.log
{{end}}
REM
REM INDIVIDUAL USERS UPGRADES
REM
{{range $u := $.UpgradeUsers}}
IF /I "%{{shUserVar $u}}%" == "yes" (
echo Creating authorization token for user {{$u}}...
%INFLUX% auth create --user={{$.TargetAdmin}} --org={{$.TargetOrg}} --description="{{$u}}" {{index $.UserArgs $u | userArgs}}
) {{- if $.IsFileOutput}} >> %LOG% 2>&1 {{- end}}
{{- end}}
{{- if $.IsFileOutput}}
type %LOG%
echo.
echo Output saved to %LOG%
{{- end}}
`

View File

@ -0,0 +1,273 @@
package upgrade
import (
"bufio"
"errors"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"testing"
"unsafe"
"github.com/google/go-cmp/cmp"
"github.com/influxdata/influxdb/v2/v1/services/meta"
"github.com/influxdata/influxql"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)
func TestGenerateScript(t *testing.T) {
type testCase struct {
name string
users []meta.UserInfo
db2ids map[string][]string
skipExe bool
want string
wantErr error
}
var testCases = []testCase{
{
name: "ordinary",
users: []meta.UserInfo{
{ // not upgraded because admin
Name: "superman",
Admin: true,
},
{ // not upgraded because no privileges
Name: "loser",
Admin: false,
},
{
Name: "weatherman",
Admin: false,
Privileges: map[string]influxql.Privilege{
"water": influxql.AllPrivileges,
"air": influxql.AllPrivileges,
},
},
{
Name: "hitgirl",
Admin: false,
Privileges: map[string]influxql.Privilege{
"hits": influxql.WritePrivilege,
},
},
{
Name: "boss@hits.org", // special name
Admin: false,
Privileges: map[string]influxql.Privilege{
"hits": influxql.AllPrivileges,
},
},
{
Name: "viewer",
Admin: false,
Privileges: map[string]influxql.Privilege{
"water": influxql.ReadPrivilege,
"air": influxql.ReadPrivilege,
},
},
},
db2ids: map[string][]string{
"water": {"33f9d67bc9cbc5b7", "33f9d67bc9cbc5b8", "33f9d67bc9cbc5b9"},
"air": {"43f9d67bc9cbc5b7", "43f9d67bc9cbc5b8", "43f9d67bc9cbc5b9"},
"hits": {"53f9d67bc9cbc5b7"},
},
want: testScriptShOrdinary,
},
{
name: "missing buckets",
users: []meta.UserInfo{
{
Name: "weatherman",
Admin: false,
Privileges: map[string]influxql.Privilege{
"water": influxql.AllPrivileges,
"air": influxql.AllPrivileges,
},
},
},
db2ids: nil,
wantErr: errors.New("upgrade: there were errors/warnings, please fix them and run the command again"),
},
{
name: "no users",
users: []meta.UserInfo{},
want: "",
},
{
name: "influx 2.x not found",
users: []meta.UserInfo{
{
Name: "dummy",
Admin: false,
},
},
skipExe: true,
wantErr: errors.New("upgrade: version 2.x influx executable not found"),
},
}
var shSuffix, mockInfluxCode string
if runtime.GOOS == "win" {
shSuffix = ".cmd"
mockInfluxCode = testMockInfluxWin
} else {
shSuffix = ".sh"
mockInfluxCode = testMockInfluxUnix
}
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) { // better do not run in parallel
// mock v1 meta
v1 := &influxDBv1{
meta: &meta.Client{},
}
data := &meta.Data{
Users: tc.users,
}
// inject users into mock v1 meta client
f := reflect.ValueOf(v1.meta).Elem().Field(4)
f = reflect.NewAt(f.Type(), unsafe.Pointer(f.UnsafeAddr())).Elem()
f.Set(reflect.ValueOf(data))
// script output file
tmpfile, err := ioutil.TempFile(t.TempDir(), "upgrade-security-*"+shSuffix)
require.NoError(t, err)
require.NoError(t, tmpfile.Close())
// options passed on cmdline
targetOptions := optionsV2{
userName: "admin",
orgName: "demo",
token: "ABC007==",
securityScriptPath: tmpfile.Name(),
}
// create mock v2.x influx executable
if !tc.skipExe {
testExePath, err := os.Executable()
require.NoError(t, err)
mockInfluxExePath := filepath.Join(filepath.Dir(testExePath), "influx")
file, err := os.OpenFile(mockInfluxExePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0777)
require.NoError(t, err)
_, err = file.WriteString(mockInfluxCode)
require.NoError(t, err)
file.Close()
defer os.Remove(mockInfluxExePath)
}
// command execution
err = generateSecurityScript(v1, targetOptions, tc.db2ids, zaptest.NewLogger(t))
if err != nil {
if tc.wantErr != nil {
if diff := cmp.Diff(tc.wantErr.Error(), err.Error()); diff != "" {
t.Fatal(diff)
}
} else {
t.Fatal(err)
}
} else if tc.wantErr != nil {
t.Fatalf("should have with %v", tc.wantErr)
}
// validate result by comparing arrays of non-empty lines of wanted vs actual output
parse := func(content string) []string {
var lines []string
scanner := bufio.NewScanner(strings.NewReader(content))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" {
lines = append(lines, line)
}
}
return lines
}
bs, err := ioutil.ReadFile(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
if strings.Contains(tc.want, "?") {
// Generated security script contains path to `influx` executable
// and this must be updated with test executable build path in the wanted result.
exePath, err := os.Executable()
if err != nil {
t.Fatal(err)
}
tc.want = strings.Replace(tc.want, "?", filepath.Dir(exePath), -1)
}
expected := parse(tc.want)
actual := parse(string(bs))
if diff := cmp.Diff(expected, actual); diff != "" {
t.Fatal(diff)
}
})
}
}
var testScriptShOrdinary = `#!/bin/sh
UPGRADE_USER_boss_hits_org=yes # user boss@hits.org
UPGRADE_USER_hitgirl=yes # user hitgirl
UPGRADE_USER_viewer=yes # user viewer
UPGRADE_USER_weatherman=yes # user weatherman
# user superman is 1.x admin and will not be upgraded automatically
# user loser has no privileges and will be skipped
#
# SCRIPT VARS
#
INFLUX=?/influx
INFLUX_TOKEN=ABC007==
LOG="${0%.*}.$(date +%Y%m%d-%H%M%S).log"
#
# USERS UPGRADES
#
if [ "$UPGRADE_USER_boss_hits_org" = "yes" ]; then
echo Creating authorization token for user boss@hits.org...
env INFLUX_TOKEN=$INFLUX_TOKEN $INFLUX auth create --user=admin --org=demo --description="boss@hits.org" --read-bucket=53f9d67bc9cbc5b7 --write-bucket=53f9d67bc9cbc5b7
fi 2>&1 | tee -a $LOG
if [ "$UPGRADE_USER_hitgirl" = "yes" ]; then
echo Creating authorization token for user hitgirl...
env INFLUX_TOKEN=$INFLUX_TOKEN $INFLUX auth create --user=admin --org=demo --description="hitgirl" --write-bucket=53f9d67bc9cbc5b7
fi 2>&1 | tee -a $LOG
if [ "$UPGRADE_USER_viewer" = "yes" ]; then
echo Creating authorization token for user viewer...
env INFLUX_TOKEN=$INFLUX_TOKEN $INFLUX auth create --user=admin --org=demo --description="viewer" --read-bucket=43f9d67bc9cbc5b7 --read-bucket=43f9d67bc9cbc5b8 --read-bucket=43f9d67bc9cbc5b9 --read-bucket=33f9d67bc9cbc5b7 --read-bucket=33f9d67bc9cbc5b8 --read-bucket=33f9d67bc9cbc5b9
fi 2>&1 | tee -a $LOG
if [ "$UPGRADE_USER_weatherman" = "yes" ]; then
echo Creating authorization token for user weatherman...
env INFLUX_TOKEN=$INFLUX_TOKEN $INFLUX auth create --user=admin --org=demo --description="weatherman" --read-bucket=43f9d67bc9cbc5b7 --write-bucket=43f9d67bc9cbc5b7 --read-bucket=43f9d67bc9cbc5b8 --write-bucket=43f9d67bc9cbc5b8 --read-bucket=43f9d67bc9cbc5b9 --write-bucket=43f9d67bc9cbc5b9 --read-bucket=33f9d67bc9cbc5b7 --write-bucket=33f9d67bc9cbc5b7 --read-bucket=33f9d67bc9cbc5b8 --write-bucket=33f9d67bc9cbc5b8 --read-bucket=33f9d67bc9cbc5b9 --write-bucket=33f9d67bc9cbc5b9
fi 2>&1 | tee -a $LOG
echo
echo Output saved to $LOG
`
var testMockInfluxUnix = `#!/bin/sh
if [ "$1" = "version" ]; then
echo Influx CLI 2.0.0
else
exit 1
fi
`
var testMockInfluxWin = `@echo off
IF /I "%1" == "version" (
echo "Influx CLI 2.0.0"
) else (
exit 1
)
`

View File

@ -360,7 +360,7 @@ func runUpgradeE(*cobra.Command, []string) error {
return err
}
if err = generateSecurityScript(v1, db2BucketIds, log); err != nil {
if err = generateSecurityScript(v1, options.target, db2BucketIds, log); err != nil {
return err
}