fix(cmd): Resolve build issues with influx command and tests
Stuart Carnie
parent
f3501aa338
commit
8cfcfb0300
|
|
@ -1,126 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/influxdata/influxdb/v2"
|
||||
"github.com/influxdata/influxdb/v2/internal/fs"
|
||||
"github.com/influxdata/influxdb/v2/v1/tsdb/engine/tsm1"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var _ = debugCmd
|
||||
|
||||
func debugCmd() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "debug",
|
||||
Short: "commands for debugging InfluxDB",
|
||||
}
|
||||
cmd.AddCommand(initInspectReportTSMCommand()) // Add report-tsm command
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
var inspectReportTSMFlags struct {
|
||||
pattern string
|
||||
exact bool
|
||||
detailed bool
|
||||
organization
|
||||
bucketID string
|
||||
dataDir string
|
||||
}
|
||||
|
||||
func initInspectReportTSMCommand() *cobra.Command {
|
||||
inspectReportTSMCommand := &cobra.Command{
|
||||
Use: "report-tsm",
|
||||
Short: "Run a TSM report",
|
||||
Long: `This command will analyze TSM files within a storage engine
|
||||
directory, reporting the cardinality within the files as well as the time range that
|
||||
the point data covers.
|
||||
|
||||
This command only interrogates the index within each file, and does not read any
|
||||
block data. To reduce heap requirements, by default report-tsm estimates the overall
|
||||
cardinality in the file set by using the HLL++ algorithm. Exact cardinalities can
|
||||
be determined by using the --exact flag.
|
||||
|
||||
For each file, the following is output:
|
||||
|
||||
* The full filename;
|
||||
* The series cardinality within the file;
|
||||
* The number of series first encountered within the file;
|
||||
* The minimum and maximum timestamp associated with any TSM data in the file; and
|
||||
* The time taken to load the TSM index and apply any tombstones.
|
||||
|
||||
The summary section then outputs the total time range and series cardinality for
|
||||
the fileset. Depending on the --detailed flag, series cardinality is segmented
|
||||
in the following ways:
|
||||
|
||||
* Series cardinality for each organization;
|
||||
* Series cardinality for each bucket;
|
||||
* Series cardinality for each measurement;
|
||||
* Number of field keys for each measurement; and
|
||||
* Number of tag values for each tag key.
|
||||
`,
|
||||
RunE: inspectReportTSMF,
|
||||
}
|
||||
|
||||
inspectReportTSMCommand.Flags().StringVarP(&inspectReportTSMFlags.pattern, "pattern", "", "", "only process TSM files containing pattern")
|
||||
inspectReportTSMCommand.Flags().BoolVarP(&inspectReportTSMFlags.exact, "exact", "", false, "calculate and exact cardinality count. Warning, may use significant memory...")
|
||||
inspectReportTSMCommand.Flags().BoolVarP(&inspectReportTSMFlags.detailed, "detailed", "", false, "emit series cardinality segmented by measurements, tag keys and fields. Warning, may take a while.")
|
||||
|
||||
inspectReportTSMFlags.organization.register(inspectReportTSMCommand, false)
|
||||
inspectReportTSMCommand.Flags().StringVarP(&inspectReportTSMFlags.bucketID, "bucket-id", "", "", "process only data belonging to bucket ID. Requires org flag to be set.")
|
||||
|
||||
dir, err := fs.InfluxDir()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
inspectReportTSMCommand.Flags().StringVarP(&inspectReportTSMFlags.dataDir, "data-dir", "", "", fmt.Sprintf("use provided data directory (defaults to %s).", filepath.Join(dir, "engine/data")))
|
||||
return inspectReportTSMCommand
|
||||
}
|
||||
|
||||
// inspectReportTSMF runs the report-tsm tool.
|
||||
func inspectReportTSMF(cmd *cobra.Command, args []string) error {
|
||||
if err := inspectReportTSMFlags.organization.validOrgFlags(&flags); err != nil {
|
||||
return err
|
||||
}
|
||||
report := &tsm1.Report{
|
||||
Stderr: os.Stderr,
|
||||
Stdout: os.Stdout,
|
||||
Dir: inspectReportTSMFlags.dataDir,
|
||||
Pattern: inspectReportTSMFlags.pattern,
|
||||
Detailed: inspectReportTSMFlags.detailed,
|
||||
Exact: inspectReportTSMFlags.exact,
|
||||
}
|
||||
|
||||
if (inspectReportTSMFlags.organization.name == "" || inspectReportTSMFlags.organization.id == "") && inspectReportTSMFlags.bucketID != "" {
|
||||
return errors.New("org-id must be set for non-empty bucket-id")
|
||||
}
|
||||
|
||||
orgSvc, err := newOrganizationService()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
id, err := inspectReportTSMFlags.organization.getID(orgSvc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.OrgID = &id
|
||||
|
||||
if inspectReportTSMFlags.bucketID != "" {
|
||||
bucketID, err := influxdb.IDFromString(inspectReportTSMFlags.bucketID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.BucketID = bucketID
|
||||
}
|
||||
|
||||
_, err = report.Run(true)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
@ -2,7 +2,6 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/influxdata/flux"
|
||||
"github.com/influxdata/flux/repl"
|
||||
|
|
@ -29,10 +28,6 @@ func cmdREPL(f *globalFlags, opt genericCLIOpts) *cobra.Command {
|
|||
}
|
||||
|
||||
func replF(cmd *cobra.Command, args []string) error {
|
||||
if flags.local {
|
||||
return fmt.Errorf("local flag not supported for repl command")
|
||||
}
|
||||
|
||||
if err := replFlags.org.validOrgFlags(&flags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,16 +4,12 @@ import (
|
|||
"fmt"
|
||||
"io/ioutil"
|
||||
nethttp "net/http"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/influxdata/influxdb/v2"
|
||||
"github.com/influxdata/influxdb/v2/cmd/influxd/launcher"
|
||||
"github.com/influxdata/influxdb/v2/http"
|
||||
"github.com/influxdata/influxdb/v2/toml"
|
||||
"github.com/influxdata/influxdb/v2/v1/tsdb/engine/tsm1"
|
||||
)
|
||||
|
||||
func TestStorage_WriteAndQuery(t *testing.T) {
|
||||
|
|
@ -156,94 +152,3 @@ func TestLauncher_BucketDelete(t *testing.T) {
|
|||
t.Fatalf("after bucket delete got %d, exp %d", got, exp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorage_CacheSnapshot_Size(t *testing.T) {
|
||||
l := launcher.NewTestLauncher(nil)
|
||||
l.StorageConfig.Engine.Cache.SnapshotMemorySize = 10
|
||||
l.StorageConfig.Engine.Cache.SnapshotAgeDuration = toml.Duration(time.Hour)
|
||||
defer l.ShutdownOrFail(t, ctx)
|
||||
|
||||
if err := l.Run(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
l.SetupOrFail(t)
|
||||
|
||||
org1 := l.OnBoardOrFail(t, &influxdb.OnboardingRequest{
|
||||
User: "USER-1",
|
||||
Password: "PASSWORD-1",
|
||||
Org: "ORG-01",
|
||||
Bucket: "BUCKET",
|
||||
})
|
||||
|
||||
// Execute single write against the server.
|
||||
l.WriteOrFail(t, org1, `m,k=v1 f=100i 946684800000000000`)
|
||||
l.WriteOrFail(t, org1, `m,k=v2 f=101i 946684800000000000`)
|
||||
l.WriteOrFail(t, org1, `m,k=v3 f=102i 946684800000000000`)
|
||||
l.WriteOrFail(t, org1, `m,k=v4 f=103i 946684800000000000`)
|
||||
l.WriteOrFail(t, org1, `m,k=v5 f=104i 946684800000000000`)
|
||||
|
||||
// Wait for cache to snapshot. This should take no longer than one second.
|
||||
time.Sleep(time.Second * 5)
|
||||
|
||||
// Check there is TSM data.
|
||||
report := tsm1.Report{
|
||||
Dir: filepath.Join(l.Path, "/engine/data"),
|
||||
Exact: true,
|
||||
}
|
||||
|
||||
summary, err := report.Run(false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Five series should be in the snapshot
|
||||
if got, exp := summary.Total, uint64(5); got != exp {
|
||||
t.Fatalf("got %d series in TSM files, expected %d", got, exp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorage_CacheSnapshot_Age(t *testing.T) {
|
||||
l := launcher.NewTestLauncher(nil)
|
||||
l.StorageConfig.Engine.Cache.SnapshotAgeDuration = toml.Duration(time.Second)
|
||||
defer l.ShutdownOrFail(t, ctx)
|
||||
|
||||
if err := l.Run(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
l.SetupOrFail(t)
|
||||
|
||||
org1 := l.OnBoardOrFail(t, &influxdb.OnboardingRequest{
|
||||
User: "USER-1",
|
||||
Password: "PASSWORD-1",
|
||||
Org: "ORG-01",
|
||||
Bucket: "BUCKET",
|
||||
})
|
||||
|
||||
// Execute single write against the server.
|
||||
l.WriteOrFail(t, org1, `m,k=v1 f=100i 946684800000000000`)
|
||||
l.WriteOrFail(t, org1, `m,k=v2 f=101i 946684800000000000`)
|
||||
l.WriteOrFail(t, org1, `m,k=v3 f=102i 946684800000000000`)
|
||||
l.WriteOrFail(t, org1, `m,k=v4 f=102i 946684800000000000`)
|
||||
l.WriteOrFail(t, org1, `m,k=v5 f=102i 946684800000000000`)
|
||||
|
||||
// Wait for cache to snapshot. This should take no longer than one second.
|
||||
time.Sleep(time.Second * 5)
|
||||
|
||||
// Check there is TSM data.
|
||||
report := tsm1.Report{
|
||||
Dir: filepath.Join(l.Path, "/engine/data"),
|
||||
Exact: true,
|
||||
}
|
||||
|
||||
summary, err := report.Run(false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Five series should be in the snapshot
|
||||
if got, exp := summary.Total, uint64(5); got != exp {
|
||||
t.Fatalf("got %d series in TSM files, expected %d", got, exp)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,10 +8,8 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/influxdata/influxdb/v2/bolt"
|
||||
"github.com/influxdata/influxdb/v2/cmd/influxd/inspect"
|
||||
"github.com/influxdata/influxdb/v2/internal/fs"
|
||||
"github.com/influxdata/influxdb/v2/kit/cli"
|
||||
"github.com/influxdata/influxdb/v2/storage"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
|
@ -123,12 +121,14 @@ func restoreE(cmd *cobra.Command, args []string) error {
|
|||
}
|
||||
|
||||
if flags.rebuildTSI {
|
||||
sFilePath := filepath.Join(flags.enginePath, storage.DefaultSeriesFileDirectoryName)
|
||||
indexPath := filepath.Join(flags.enginePath, storage.DefaultIndexDirectoryName)
|
||||
// FIXME: Implement rebuildTSI
|
||||
panic("not implemented")
|
||||
//sFilePath := filepath.Join(flags.enginePath, storage.DefaultSeriesFileDirectoryName)
|
||||
//indexPath := filepath.Join(flags.enginePath, storage.DefaultIndexDirectoryName)
|
||||
|
||||
rebuild := inspect.NewBuildTSICommand()
|
||||
rebuild.SetArgs([]string{"--sfile-path", sFilePath, "--tsi-path", indexPath})
|
||||
rebuild.Execute()
|
||||
//rebuild := inspect.NewBuildTSICommand()
|
||||
//rebuild.SetArgs([]string{"--sfile-path", sFilePath, "--tsi-path", indexPath})
|
||||
//rebuild.Execute()
|
||||
}
|
||||
|
||||
if err := removeTmpBolt(); err != nil {
|
||||
|
|
|
|||
Loading…
Reference in New Issue