build: using libflux in OSS

2019-12-16 11:09:39 -05:00 · 2019-12-16 11:09:39 -05:00 · 3f9c2783b9
parent 88468822e2 1c327904c9
commit 3f9c2783b9
4 changed files with 45 additions and 17 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -1,4 +1,19 @@
-version: 2
+version: '2.1'
+
+commands:
+  install_rust_compiler:
+    description: >
+      This will install the rust compiler with the rust tools.
+    steps:
+      - run:
+          name: Install clang
+          command: sudo apt-get install -y --no-install-recommends clang
+      - run:
+          name: Install rust compiler
+          command: |
+            curl https://sh.rustup.rs -sSf | \
+            sh -s -- --default-toolchain stable -y
+            echo 'source $HOME/.cargo/env' >> $BASH_ENV

 jobs:
  litmus_daily:
@ -65,6 +80,7 @@ jobs:
            - influxdb-gomod-{{ checksum "go.sum" }} # Just match the go.sum checksum cache.
      - run: sudo apt-get install -y netcat-openbsd
      - run: sudo apt-get install -y bzr
+      - install_rust_compiler
      - run: make protoc
      - run: make build
      - run:
@ -154,6 +170,7 @@ jobs:
          keys:
            - influxdb-gomod-{{ checksum "go.sum" }} # Matches based on go.sum checksum.
      - run: sudo apt-get install -y bzr
+      - install_rust_compiler
      - run: mkdir -p $TEST_RESULTS
      - run: make test-go # This uses the test cache so it may succeed or fail quickly.
      - run: make vet
@ -162,7 +179,7 @@ jobs:
      - run: GO111MODULE=on go mod vendor # staticcheck looks in vendor for dependencies.
      - run: GO111MODULE=on go install honnef.co/go/tools/cmd/staticcheck # Install staticcheck from the version we specify in go.mod.
      - run: GO111MODULE=on staticcheck ./...
-      - run: GOTRACEBACK=all GO111MODULE=on gotestsum --format standard-verbose --junitfile /tmp/test-results/gotestsum.xml -- -race -count=1 ./...
+      - run: GOTRACEBACK=all GO111MODULE=on FLUX_PARSER_TYPE=rust CGO_LDFLAGS="$(cat .cgo_ldflags)" gotestsum --format standard-verbose --junitfile /tmp/test-results/gotestsum.xml -- -race -count=1 -tags 'libflux' ./...
      # TODO add these checks to the Makefile
      # - run: go get -v -t -d ./...

@ -193,6 +210,7 @@ jobs:
    steps:
      - checkout
      - run: sudo apt-get install -y bzr
+      - install_rust_compiler
      - run: make checkcommit

      # Speed up `make build` by restoring caches from previous runs.
--- a/.gitignore
+++ b/.gitignore
@ -8,6 +8,7 @@ vendor
 .tern-project
 .DS_Store
 .idea
+.cgo_ldflags

 # binary databases
 influxd.bolt
--- a/37
+++ b/37
@ -1,7 +1,7 @@
 # Top level Makefile for the entire project
 #
-# This Makefile encodes the "go generate" prerequeisites ensuring that the proper tooling is installed and
-# that the generate steps are executed when their prerequeisites files change.
+# This Makefile encodes the "go generate" prerequisites ensuring that the proper tooling is installed and
+# that the generate steps are executed when their prerequisite files change.
 #
 # This Makefile follows a few conventions:
 #
@ -12,18 +12,20 @@
 #

 # SUBDIRS are directories that have their own Makefile.
-# It is required that all subdirs have the `all` and `clean` targets.
+# It is required that all SUBDIRS have the `all` and `clean` targets.
 SUBDIRS := http ui chronograf query storage
+# The 'libflux' tag is required for instructing the flux to be compiled with the Rust parser
+GO_TAGS=libflux
 GO_ARGS=-tags '$(GO_TAGS)'

 # Test vars can be used by all recursive Makefiles
 export GOOS=$(shell go env GOOS)
-export GO_BUILD=env GO111MODULE=on go build $(GO_ARGS)
-export GO_INSTALL=env GO111MODULE=on go install $(GO_ARGS)
-export GO_TEST=env GOTRACEBACK=all GO111MODULE=on go test $(GO_ARGS)
+export GO_BUILD=env GO111MODULE=on CGO_LDFLAGS="$$(cat .cgo_ldflags)" go build $(GO_ARGS)
+export GO_INSTALL=env GO111MODULE=on CGO_LDFLAGS="$$(cat .cgo_ldflags)" go install $(GO_ARGS)
+export GO_TEST=env FLUX_PARSER_TYPE=rust GOTRACEBACK=all GO111MODULE=on CGO_LDFLAGS="$$(cat .cgo_ldflags)" go test $(GO_ARGS)
 # Do not add GO111MODULE=on to the call to go generate so it doesn't pollute the environment.
 export GO_GENERATE=go generate $(GO_ARGS)
-export GO_VET=env GO111MODULE=on go vet $(GO_ARGS)
+export GO_VET=env GO111MODULE=on CGO_LDFLAGS="$$(cat .cgo_ldflags)" go vet $(GO_ARGS)
 export GO_RUN=env GO111MODULE=on go run $(GO_ARGS)
 export PATH := $(PWD)/bin/$(GOOS):$(PATH)

@ -50,7 +52,7 @@ CMDS := \
 # This target sets up the dependencies to correctly build all go commands.
 # Other targets must depend on this target to correctly builds CMDS.
 ifeq ($(GOARCH), arm64)
-    all: GO_ARGS=-tags ' assets noasm $(GO_TAGS)'
+    all: GO_ARGS=-tags 'assets noasm $(GO_TAGS)'
 else
    all: GO_ARGS=-tags 'assets $(GO_TAGS)'
 endif
@ -64,7 +66,7 @@ subdirs: $(SUBDIRS)
 #
 # Define targets for commands
 #
-$(CMDS): $(SOURCES)
+$(CMDS): $(SOURCES) libflux
 	$(GO_BUILD) -o $@ ./cmd/$(shell basename "$@")

 # Ease of use build for just the go binary
@ -96,6 +98,12 @@ ui_client:
 # Define action only targets
 #

+libflux: .cgo_ldflags
+
+.cgo_ldflags: go.mod
+	$(GO_RUN) github.com/influxdata/flux/internal/cmd/flux-config --libs --verbose > .cgo_ldflags.tmp
+	mv .cgo_ldflags.tmp .cgo_ldflags
+
 fmt: $(SOURCES_NO_VENDOR)
 	gofmt -w -s $^

@ -120,7 +128,7 @@ generate: subdirs
 test-js: node_modules
 	make -C ui test

-test-go:
+test-go: libflux
 	$(GO_TEST) ./...

 test-promql-e2e:
@ -132,13 +140,13 @@ test-integration:

 test: test-go test-js

-test-go-race:
+test-go-race: libflux
 	$(GO_TEST) -v -race -count=1 ./...

-vet:
+vet: libflux
 	$(GO_VET) -v ./...

-bench:
+bench: libflux
 	$(GO_TEST) -bench=. -run=^$$ ./...

 build: all
@ -158,6 +166,7 @@ clean:
 	@for d in $(SUBDIRS); do $(MAKE) -C $$d clean; done
 	$(RM) -r bin
 	$(RM) -r dist
+	$(RM) .cgo_ldflags

 define CHRONOGIRAFFE
             ._ o o
@ -186,4 +195,4 @@ protoc:
 	chmod +x /go/bin/protoc

 # .PHONY targets represent actions that do not create an actual file.
-.PHONY: all subdirs $(SUBDIRS) run fmt checkfmt tidy checktidy checkgenerate test test-go test-js test-go-race bench clean node_modules vet nightly chronogiraffe dist ping protoc e2e run-e2e influxd
+.PHONY: all subdirs $(SUBDIRS) run fmt checkfmt tidy checktidy checkgenerate test test-go test-js test-go-race bench clean node_modules vet nightly chronogiraffe dist ping protoc e2e run-e2e influxd libflux
--- a/http/query_handler_test.go
+++ b/http/query_handler_test.go
@ -245,7 +245,7 @@ func TestFluxHandler_postFluxAST(t *testing.T) {
 			name: "get ast from()",
 			w:    httptest.NewRecorder(),
 			r:    httptest.NewRequest("POST", "/api/v2/query/ast", bytes.NewBufferString(`{"query": "from()"}`)),
-			want: `{"ast":{"type":"Package","package":"main","files":[{"type":"File","location":{"start":{"line":1,"column":1},"end":{"line":1,"column":7},"source":"from()"},"metadata":"parser-type=go","package":null,"imports":null,"body":[{"type":"ExpressionStatement","location":{"start":{"line":1,"column":1},"end":{"line":1,"column":7},"source":"from()"},"expression":{"type":"CallExpression","location":{"start":{"line":1,"column":1},"end":{"line":1,"column":7},"source":"from()"},"callee":{"type":"Identifier","location":{"start":{"line":1,"column":1},"end":{"line":1,"column":5},"source":"from"},"name":"from"}}}]}]}}
+			want: `{"ast":{"type":"Package","package":"main","files":[{"type":"File","location":{"start":{"line":1,"column":1},"end":{"line":1,"column":7},"source":"from()"},"metadata":"parser-type=rust","package":null,"imports":null,"body":[{"type":"ExpressionStatement","location":{"start":{"line":1,"column":1},"end":{"line":1,"column":7},"source":"from()"},"expression":{"type":"CallExpression","location":{"start":{"line":1,"column":1},"end":{"line":1,"column":7},"source":"from()"},"callee":{"type":"Identifier","location":{"start":{"line":1,"column":1},"end":{"line":1,"column":5},"source":"from"},"name":"from"}}}]}]}}
 `,
 			status: http.StatusOK,
 		},