feat: allowing notebooks to submit queries (#18257)

pull/18321/head
Alex Boatwright 2020-05-29 09:16:20 -07:00 committed by GitHub
parent 4eb53314e1
commit 494d6d4f76
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 942 additions and 2708 deletions

File diff suppressed because it is too large Load Diff

View File

@ -1,32 +1,18 @@
import loader from 'src/external/monaco.onigasm'
import {Registry} from 'monaco-textmate' // peer dependency
import {wireTmGrammars} from 'monaco-editor-textmate'
import register from 'src/external/monaco.onigasm'
import {MonacoType} from 'src/types'
const LANGID = 'flux'
const FLUXLANGID = 'flux'
async function addSyntax() {
await register(LANGID, async () => ({
format: 'json',
content: await import(/* webpackPrefetch: 0 */ 'src/external/flux.tmLanguage.json').then(
data => {
return JSON.stringify(data)
}
),
}))
async function addSyntax(monaco: MonacoType) {
monaco.languages.register({id: FLUXLANGID})
await loader()
const registry = new Registry({
getGrammarDefinition: async () => ({
format: 'json',
content: await import(/* webpackPrefetch: 0 */ 'src/external/flux.tmLanguage.json').then(
data => {
return JSON.stringify(data)
}
),
}),
})
// map of monaco "language id's" to TextMate scopeNames
const grammars = new Map()
grammars.set(FLUXLANGID, FLUXLANGID)
monaco.languages.setLanguageConfiguration(FLUXLANGID, {
window.monaco.languages.setLanguageConfiguration(LANGID, {
autoClosingPairs: [
{open: '"', close: '"'},
{open: '[', close: ']'},
@ -35,10 +21,8 @@ async function addSyntax(monaco: MonacoType) {
{open: '(', close: ')'},
],
})
await wireTmGrammars(monaco, registry, grammars)
}
addSyntax(window.monaco)
addSyntax()
export default FLUXLANGID
export default LANGID

View File

@ -1,34 +1,14 @@
import loader from 'src/external/monaco.onigasm'
import {Registry} from 'monaco-textmate' // peer dependency
import {wireTmGrammars} from 'monaco-editor-textmate'
import {MonacoType} from 'src/types'
import register from 'src/external/monaco.onigasm'
const LANGID = 'markdown'
async function addSyntax(monaco: MonacoType) {
monaco.languages.register({id: LANGID})
await loader()
const registry = new Registry({
getGrammarDefinition: async () => ({
format: 'json',
content: await import(/* webpackPrefetch: 0 */ 'src/external/markdown.tmLanguage.json').then(
data => {
return JSON.stringify(data)
}
),
}),
})
// map of monaco "language id's" to TextMate scopeNames
const grammars = new Map()
grammars.set(LANGID, LANGID)
await wireTmGrammars(monaco, registry, grammars)
}
addSyntax(window.monaco)
register(LANGID, async () => ({
format: 'json',
content: await import(/* webpackPrefetch: 0 */ 'src/external/markdown.tmLanguage.json').then(
data => {
return JSON.stringify(data)
}
),
}))
export default LANGID

View File

@ -1,25 +0,0 @@
import {MonacoType} from 'src/types'
const THEME_NAME = 'markdownTheme'
function addTheme(monaco: MonacoType) {
monaco.editor.defineTheme(THEME_NAME, {
base: 'vs-dark',
inherit: false,
rules: [],
colors: {
'editor.foreground': '#F8F8F8',
'editor.background': '#202028',
'editorGutter.background': '#25252e',
'editor.selectionBackground': '#353640',
'editorLineNumber.foreground': '#666978',
'editor.lineHighlightBackground': '#353640',
'editorCursor.foreground': '#ffffff',
'editorActiveLineNumber.foreground': '#bec2cc',
},
})
}
addTheme(window.monaco)
export default THEME_NAME

View File

@ -1,10 +1,47 @@
import {loadWASM} from 'onigasm' // peer dependency of 'monaco-textmate'
import {Registry, StackElement, INITIAL} from 'monaco-textmate' // peer dependency
let wasm: boolean = false
let loading: boolean = false
const queue: Array<() => void> = []
const grammars = new Map()
const grammarDefs = {}
export default function loader() {
const DEFAULT_DEF = async () => ({
format: 'json',
content: await import(/* webpackPrefetch: 0 */ 'src/external/plaintext.tmLanguage.json').then(
data => {
return JSON.stringify(data)
}
),
})
// NOTE: this comes from the monaco-editor-textmate package
class TokenizerState {
constructor(private _ruleStack: StackElement) {}
public get ruleStack(): StackElement {
return this._ruleStack
}
public clone(): TokenizerState {
return new TokenizerState(this._ruleStack)
}
public equals(other): boolean {
if (
!other ||
!(other instanceof TokenizerState) ||
other !== this ||
other._ruleStack !== this._ruleStack
) {
return false
}
return true
}
}
async function loader() {
return new Promise(resolve => {
if (wasm) {
resolve()
@ -21,7 +58,47 @@ export default function loader() {
loadWASM(require(`onigasm/lib/onigasm.wasm`)).then(() => {
wasm = true
queue.forEach(c => c())
const registry = new Registry({
getGrammarDefinition: async scope => {
if (!grammarDefs.hasOwnProperty(scope)) {
return await DEFAULT_DEF()
}
return await grammarDefs[scope]()
},
})
Promise.all(
Array.from(grammars.keys()).map(async lang => {
const grammar = await registry.loadGrammar(grammars.get(lang))
window.monaco.languages.setTokensProvider(lang, {
getInitialState: () => new TokenizerState(INITIAL),
tokenize: (line: string, state: TokenizerState) => {
const res = grammar.tokenizeLine(line, state.ruleStack)
return {
endState: new TokenizerState(res.ruleStack),
tokens: res.tokens.map(token => ({
...token,
scopes: token.scopes[token.scopes.length - 1],
})),
}
},
})
})
).then(() => {
queue.forEach(c => c())
})
})
})
}
export default async function register(scope, definition) {
window.monaco.languages.register({id: scope})
grammars.set(scope, scope)
grammarDefs[scope] = definition
await loader()
}

View File

@ -1,34 +1,14 @@
import loader from 'src/external/monaco.onigasm'
import {Registry} from 'monaco-textmate' // peer dependency
import {wireTmGrammars} from 'monaco-editor-textmate'
import register from 'src/external/monaco.onigasm'
import {MonacoType} from 'src/types'
const LANGID = 'toml'
const TOMLLANGID = 'toml'
register(LANGID, async () => ({
format: 'json',
content: await import(/* webpackPrefetch: 0 */ 'src/external/toml.tmLanguage.json').then(
data => {
return JSON.stringify(data)
}
),
}))
export async function addSyntax(monaco: MonacoType) {
monaco.languages.register({id: TOMLLANGID})
await loader()
const registry = new Registry({
getGrammarDefinition: async () => ({
format: 'json',
content: await import(/* webpackPrefetch: 0 */ 'src/external/toml.tmLanguage.json').then(
data => {
return JSON.stringify(data)
}
),
}),
})
// map of monaco "language id's" to TextMate scopeNames
const grammars = new Map()
grammars.set(TOMLLANGID, TOMLLANGID)
await wireTmGrammars(monaco, registry, grammars)
}
addSyntax(window.monaco)
export default TOMLLANGID
export default LANGID

View File

@ -0,0 +1 @@
{"scopeName":"text.plain","fileTypes":["txt"],"patterns":[{"match":"^\\s*(•).*$\\n?","name":"meta.bullet-point.strong.text","captures":{"1":{"name":"punctuation.definition.item.text"}}},{"match":"^\\s*(·).*$\\n?","name":"meta.bullet-point.light.text","captures":{"1":{"name":"punctuation.definition.item.text"}}},{"match":"^\\s*(\\*).*$\\n?","name":"meta.bullet-point.star.text","captures":{"1":{"name":"punctuation.definition.item.text"}}},{"begin":"^([ \\t]*)(?=\\S)","contentName":"meta.paragraph.text","end":"^(?!\\1(?=\\S))"}],"name":"Plain Text","keyEquivalent":"^~P","uuid":"3130E4FA-B10E-11D9-9F75-000D93589AF6"}

View File

@ -1,16 +1,21 @@
import React, {FC, createElement, useMemo} from 'react'
import {PipeContextProps, PipeData} from 'src/notebooks'
import {PipeContextProps, PipeData, PipeProp} from 'src/notebooks'
import Pipe from 'src/notebooks/components/Pipe'
import NotebookPanel from 'src/notebooks/components/panel/NotebookPanel'
export interface NotebookPipeProps {
export interface NotebookPipeProps
extends Omit<Omit<PipeProp, 'Context'>, 'onUpdate'> {
index: number
data: PipeData
onUpdate: (index: number, pipe: PipeData) => void
onUpdate: (idx: number, data: PipeData) => void
}
const NotebookPipe: FC<NotebookPipeProps> = ({index, data, onUpdate}) => {
const NotebookPipe: FC<NotebookPipeProps> = ({
index,
data,
onUpdate,
results,
}) => {
const panel: FC<PipeContextProps> = useMemo(
() => props => {
const _props = {
@ -27,7 +32,9 @@ const NotebookPipe: FC<NotebookPipeProps> = ({index, data, onUpdate}) => {
onUpdate(index, data)
}
return <Pipe data={data} onUpdate={_onUpdate} Context={panel} />
return (
<Pipe data={data} onUpdate={_onUpdate} results={results} Context={panel} />
)
}
export default NotebookPipe

View File

@ -10,11 +10,7 @@ import AppSettingProvider from 'src/notebooks/context/app'
import TimeZoneDropdown from 'src/notebooks/components/header/TimeZoneDropdown'
import TimeRangeDropdown from 'src/notebooks/components/header/TimeRangeDropdown'
import AutoRefreshDropdown from 'src/notebooks/components/header/AutoRefreshDropdown'
import {SubmitQueryButton} from 'src/timeMachine/components/SubmitQueryButton'
import {IconFont} from '@influxdata/clockface'
// Types
import {RemoteDataState} from 'src/types'
import Submit from 'src/notebooks/components/header/Submit'
export interface TimeContextProps {
context: TimeBlock
@ -34,31 +30,25 @@ const Buttons: FC = () => {
[id]
)
function submit() {} // eslint-disable-line @typescript-eslint/no-empty-function
if (!timeContext.hasOwnProperty(id)) {
addTimeContext(id)
return null
}
return (
<TimeProvider>
<AppSettingProvider>
<div className="notebook-header--buttons">
<TimeZoneDropdown />
<TimeRangeDropdown context={timeContext[id]} update={update} />
<AutoRefreshDropdown context={timeContext[id]} update={update} />
<SubmitQueryButton
text="Run Notebook"
icon={IconFont.Play}
submitButtonDisabled={false}
queryStatus={RemoteDataState.NotStarted}
onSubmit={submit}
/>
</div>
</AppSettingProvider>
</TimeProvider>
<div className="notebook-header--buttons">
<TimeZoneDropdown />
<TimeRangeDropdown context={timeContext[id]} update={update} />
<AutoRefreshDropdown context={timeContext[id]} update={update} />
<Submit />
</div>
)
}
export default Buttons
export default () => (
<TimeProvider>
<AppSettingProvider>
<Buttons />
</AppSettingProvider>
</TimeProvider>
)

View File

@ -0,0 +1,86 @@
// Libraries
import React, {FC, useContext, useEffect} from 'react'
import {SubmitQueryButton} from 'src/timeMachine/components/SubmitQueryButton'
import QueryProvider, {QueryContext} from 'src/notebooks/context/query'
import {NotebookContext, PipeMeta} from 'src/notebooks/context/notebook'
import {TimeContext} from 'src/notebooks/context/time'
import {IconFont} from '@influxdata/clockface'
// Types
import {RemoteDataState} from 'src/types'
const PREVIOUS_REGEXP = /__PREVIOUS_RESULT__/g
const COMMENT_REMOVER = /(\/\*([\s\S]*?)\*\/)|(\/\/(.*)$)/gm
export const Submit: FC = () => {
const {query} = useContext(QueryContext)
const {id, pipes, updateResult, updateMeta} = useContext(NotebookContext)
const {timeContext} = useContext(TimeContext)
const time = timeContext[id]
useEffect(() => {
submit()
}, [!!time && time.range])
const submit = () => {
pipes
.reduce((stages, pipe, index) => {
updateMeta(index, {loading: RemoteDataState.Loading} as PipeMeta)
if (pipe.type === 'query') {
let text = pipe.queries[pipe.activeQuery].text.replace(
COMMENT_REMOVER,
''
)
let requirements = {}
if (PREVIOUS_REGEXP.test(text)) {
requirements = {
...(index === 0 ? {} : stages[stages.length - 1].requirements),
[`prev_${index}`]: stages[stages.length - 1].text,
}
text = text.replace(PREVIOUS_REGEXP, `prev_${index}`)
}
stages.push({
text,
instances: [index],
requirements,
})
} else {
stages[stages.length - 1].instances.push(index)
}
return stages
}, [])
.map(queryStruct => {
const queryText =
Object.entries(queryStruct.requirements)
.map(([key, value]) => `${key} = (\n${value}\n)\n\n`)
.join('') + queryStruct.text
return query(queryText).then(response => {
queryStruct.instances.forEach(index => {
updateMeta(index, {loading: RemoteDataState.Done} as PipeMeta)
updateResult(index, response)
})
})
})
}
return (
<SubmitQueryButton
text="Run Notebook"
icon={IconFont.Play}
submitButtonDisabled={false}
queryStatus={RemoteDataState.NotStarted}
onSubmit={submit}
/>
)
}
export default () => (
<QueryProvider>
<Submit />
</QueryProvider>
)

View File

@ -1,78 +1,115 @@
{
"id": "testing",
"pipes": [
{
"type": "query",
"activeQuery": 0,
"queries": [
"id": "testing",
"meta": [
{
"text": "from(bucket: \"project\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")",
"editMode": "advanced",
"builderConfig": {
"buckets": [],
"tags": [],
"functions": []
}
}
]
},
{
"type": "query",
"activeQuery": 0,
"queries": [
"title": "0001",
"visible": true
},
{
"text": "// Tip: Use the __PREVIOUS_RESULT__ variable to link your queries\n\n__PREVIOUS_RESULT__\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")",
"editMode": "advanced",
"builderConfig": {
"buckets": [],
"tags": [],
"functions": []
}
}
]
},
{
"type": "visualization"
},
{
"type": "visualization"
},
{
"type": "query",
"activeQuery": 0,
"queries": [
"title": "0002",
"visible": true
},
{
"text": "// Tip: Use the __PREVIOUS_RESULT__ variable to link your queries\n\n__PREVIOUS_RESULT__\n |> sum()",
"editMode": "advanced",
"builderConfig": {
"buckets": [],
"tags": [],
"functions": []
}
}
]
},
{
"type": "visualization"
},
{
"type": "query",
"activeQuery": 0,
"queries": [
"title": "0003",
"visible": true
},
{
"text": "// Tip: Use the __PREVIOUS_RESULT__ variable to link your queries\n\nfrom(bucket: \"project\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")",
"editMode": "advanced",
"builderConfig": {
"buckets": [],
"tags": [],
"functions": []
}
"title": "0004",
"visible": true
},
{
"title": "0005",
"visible": true
},
{
"title": "0006",
"visible": true
},
{
"title": "0007",
"visible": true
},
{
"title": "0008",
"visible": true
}
]
},
{
"type": "visualization"
}
]
],
"pipes": [
{
"activeQuery": 0,
"queries": [
{
"builderConfig": {
"buckets": [],
"functions": [],
"tags": []
},
"editMode": "advanced",
"text": "from(bucket: \"project\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")"
}
],
"type": "query"
},
{
"activeQuery": 0,
"queries": [
{
"builderConfig": {
"buckets": [],
"functions": [],
"tags": []
},
"editMode": "advanced",
"text": "// Tip: Use the __PREVIOUS_RESULT__ variable to link your queries\n\n__PREVIOUS_RESULT__\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")"
}
],
"type": "query"
},
{
"text": "not flux",
"type": "markdown"
},
{
"text": "not flux",
"type": "markdown"
},
{
"activeQuery": 0,
"queries": [
{
"builderConfig": {
"buckets": [],
"functions": [],
"tags": []
},
"editMode": "advanced",
"text": "// Tip: Use the __PREVIOUS_RESULT__ variable to link your queries\n\n__PREVIOUS_RESULT__\n |> sum()"
}
],
"type": "query"
},
{
"text": "not flux",
"type": "markdown"
},
{
"activeQuery": 0,
"queries": [
{
"builderConfig": {
"buckets": [],
"functions": [],
"tags": []
},
"editMode": "advanced",
"text": "// Tip: Use the __PREVIOUS_RESULT__ variable to link your queries\n\nfrom(bucket: \"project\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")"
}
],
"type": "query"
},
{
"text": "not flux",
"type": "markdown"
}
]
}

View File

@ -1,20 +1,26 @@
import React, {FC, useState, useCallback, RefObject} from 'react'
import {RemoteDataState} from 'src/types'
import {PipeData} from 'src/notebooks'
import {FromFluxResult} from '@influxdata/giraffe'
export interface PipeMeta {
title: string
visible: boolean
loading: RemoteDataState
focus: boolean
panelRef: RefObject<HTMLDivElement>
}
// TODO: this is screaming for normalization. figure out frontend uuids for cells
export interface NotebookContextType {
id: string
pipes: PipeData[]
meta: PipeMeta[] // data only used for the view layer for Notebooks
results: FromFluxResult[]
addPipe: (pipe: PipeData) => void
updatePipe: (idx: number, pipe: PipeData) => void
updateMeta: (idx: number, pipe: PipeMeta) => void
updateResult: (idx: number, result: FromFluxResult) => void
movePipe: (currentIdx: number, newIdx: number) => void
removePipe: (idx: number) => void
}
@ -23,9 +29,11 @@ export const DEFAULT_CONTEXT: NotebookContextType = {
id: 'new',
pipes: [],
meta: [],
results: [],
addPipe: () => {},
updatePipe: () => {},
updateMeta: () => {},
updateResult: () => {},
movePipe: () => {},
removePipe: () => {},
}
@ -38,6 +46,8 @@ if (TEST_MODE) {
const TEST_NOTEBOOK = require('src/notebooks/context/notebook.mock.json')
DEFAULT_CONTEXT.id = TEST_NOTEBOOK.id
DEFAULT_CONTEXT.pipes = TEST_NOTEBOOK.pipes
DEFAULT_CONTEXT.meta = TEST_NOTEBOOK.meta
DEFAULT_CONTEXT.results = new Array(TEST_NOTEBOOK.pipes.length)
}
export const NotebookContext = React.createContext<NotebookContextType>(
@ -50,9 +60,11 @@ export const NotebookProvider: FC = ({children}) => {
const [id] = useState(DEFAULT_CONTEXT.id)
const [pipes, setPipes] = useState(DEFAULT_CONTEXT.pipes)
const [meta, setMeta] = useState(DEFAULT_CONTEXT.meta)
const [results, setResults] = useState(DEFAULT_CONTEXT.results)
const _setPipes = useCallback(setPipes, [id])
const _setMeta = useCallback(setMeta, [id])
const _setResults = useCallback(setResults, [id])
const addPipe = useCallback(
(pipe: PipeData) => {
@ -63,10 +75,13 @@ export const NotebookProvider: FC = ({children}) => {
}
}
_setPipes(add(pipe))
_setResults(add({}))
_setMeta(
add({
title: `Cell_${++GENERATOR_INDEX}`,
visible: true,
loading: RemoteDataState.NotStarted,
focus: false,
})
)
},
@ -99,6 +114,18 @@ export const NotebookProvider: FC = ({children}) => {
[id]
)
const updateResult = useCallback(
(idx: number, results: FromFluxResult) => {
_setResults(pipes => {
pipes[idx] = {
...results,
} as FromFluxResult
return pipes.slice()
})
},
[id]
)
const movePipe = useCallback(
(currentIdx: number, newIdx: number) => {
const move = list => {
@ -116,6 +143,7 @@ export const NotebookProvider: FC = ({children}) => {
}
_setPipes(move)
_setMeta(move)
_setResults(move)
},
[id]
)
@ -128,6 +156,7 @@ export const NotebookProvider: FC = ({children}) => {
}
_setPipes(remove)
_setMeta(remove)
_setResults(remove)
},
[id]
)
@ -138,8 +167,10 @@ export const NotebookProvider: FC = ({children}) => {
id,
pipes,
meta,
results,
updatePipe,
updateMeta,
updateResult,
movePipe,
addPipe,
removePipe,

View File

@ -0,0 +1,81 @@
import React, {FC, useContext, useMemo} from 'react'
import {connect} from 'react-redux'
import {AppState, Variable, Organization} from 'src/types'
import {runQuery} from 'src/shared/apis/query'
import {getWindowVars} from 'src/variables/utils/getWindowVars'
import {buildVarsOption} from 'src/variables/utils/buildVarsOption'
import {getTimeRangeVars} from 'src/variables/utils/getTimeRangeVars'
import {getVariables, asAssignment} from 'src/variables/selectors'
import {getOrg} from 'src/organizations/selectors'
import {NotebookContext} from 'src/notebooks/context/notebook'
import {TimeContext} from 'src/notebooks/context/time'
import {fromFlux as parse, FromFluxResult} from '@influxdata/giraffe'
export interface QueryContextType {
query: (text: string) => Promise<FromFluxResult>
}
export const DEFAULT_CONTEXT: QueryContextType = {
query: () => Promise.resolve({} as FromFluxResult),
}
export const QueryContext = React.createContext<QueryContextType>(
DEFAULT_CONTEXT
)
type Props = StateProps
export const QueryProvider: FC<Props> = ({children, variables, org}) => {
const {id} = useContext(NotebookContext)
const {timeContext} = useContext(TimeContext)
const time = timeContext[id]
const vars = useMemo(
() =>
variables.map(v => asAssignment(v)).concat(getTimeRangeVars(time.range)),
[variables, time]
)
const query = (text: string) => {
const windowVars = getWindowVars(text, vars)
const extern = buildVarsOption([...vars, ...windowVars])
return runQuery(org.id, text, extern)
.promise.then(raw => {
if (raw.type !== 'SUCCESS') {
// TODO actually pipe this somewhere
throw new Error('Unable to fetch results')
}
return raw
})
.then(raw => {
return parse(raw.csv)
})
}
if (!time) {
return null
}
return (
<QueryContext.Provider value={{query}}>{children}</QueryContext.Provider>
)
}
interface StateProps {
variables: Variable[]
org: Organization
}
const mstp = (state: AppState): StateProps => {
const variables = getVariables(state)
const org = getOrg(state)
return {
org,
variables,
}
}
const ConnectedQueryProvider = connect<StateProps>(mstp)(QueryProvider)
export default ConnectedQueryProvider

View File

@ -1,4 +1,5 @@
import {FunctionComponent, ComponentClass, ReactNode} from 'react'
import {FromFluxResult} from '@influxdata/giraffe'
export interface PipeContextProps {
children?: ReactNode
@ -10,6 +11,8 @@ export type PipeData = any
export interface PipeProp {
data: PipeData
onUpdate: (data: PipeData) => void
results?: FromFluxResult
Context:
| FunctionComponent<PipeContextProps>
| ComponentClass<PipeContextProps>

View File

@ -6,7 +6,7 @@ import MonacoEditor from 'react-monaco-editor'
// Utils
import LANGID from 'src/external/monaco.markdown.syntax'
import THEME_NAME from 'src/external/monaco.markdown.theme'
import THEME_NAME from 'src/external/monaco.flux.theme'
import {registerAutogrow} from 'src/external/monaco.autogrow'
import {isFlagEnabled} from 'src/shared/utils/featureFlag'