Add support for interval template variable in Flux

ui/package.json

@@ -39,7 +39,7 @@
     "@types/d3-scale": "^2.0.1",
     "@types/dygraphs": "^1.1.6",
     "@types/enzyme": "^3.1.13",
-    "@types/jest": "^22.1.4",
+    "@types/jest": "^23.3.5",
     "@types/levelup": "^0.0.30",
     "@types/lodash": "^4.14.104",
     "@types/node": "^9.4.6",

ui/src/flux/apis/index.ts

@@ -6,7 +6,7 @@ import {
 } from 'src/shared/parsing/flux/response'
 import {MAX_RESPONSE_BYTES} from 'src/flux/constants'
 import {manager} from 'src/worker/JobManager'
-import {addTemplatesToScript} from 'src/shared/parsing/flux/templates'
+import {renderTemplatesInScript} from 'src/flux/helpers/templates'
 import _ from 'lodash'
 
 export const getSuggestions = async (url: string) => {
@@ -58,17 +58,25 @@ export interface GetRawTimeSeriesResult {
 export const getRawTimeSeries = async (
   source: Source,
   script: string,
+  uuid: string,
   timeRange: TimeRange,
-  uuid?: string
+  fluxASTLink: string,
+  maxSideLength: number
 ): Promise<GetRawTimeSeriesResult> => {
   const path = encodeURIComponent(`/v2/query?organization=defaultorgname`)
   const url = `${window.basepath}${source.links.flux}?path=${path}`
 
-  const scriptWithTemplates = addTemplatesToScript(script, timeRange)
+  const renderedScript = await renderTemplatesInScript(
+    script,
+    timeRange,
+    fluxASTLink,
+    maxSideLength
+  )
+
   try {
     const {body, byteLength, uuid: responseUUID} = await manager.fetchFluxData(
       url,
-      scriptWithTemplates,
+      renderedScript,
       uuid
     )
 
@@ -99,14 +107,18 @@ export interface GetTimeSeriesResult {
 export const getTimeSeries = async (
   source,
   script: string,
+  uuid: string,
   timeRange: TimeRange,
-  uuid?: string
+  fluxASTLink: string,
+  maxSideLength: number
 ): Promise<GetTimeSeriesResult> => {
   const {csv, didTruncate, uuid: responseUUID} = await getRawTimeSeries(
     source,
     script,
+    uuid,
     timeRange,
-    uuid
+    fluxASTLink,
+    maxSideLength
   )
 
   const tables = parseResponse(csv)

ui/src/flux/helpers/templates.ts

@@ -0,0 +1,52 @@
+import {TimeRange} from 'src/types'
+import {getMinDuration} from 'src/shared/parsing/flux/durations'
+import {
+  DEFAULT_PIXELS,
+  DEFAULT_DURATION_MS,
+  RESOLUTION_SCALE_FACTOR,
+} from 'src/shared/constants'
+
+// For now we only support these template variables in Flux queries
+const DASHBOARD_TIME = 'dashboardTime'
+const UPPER_DASHBOARD_TIME = 'upperDashboardTime'
+const INTERVAL = 'autoInterval'
+
+const INTERVAL_REGEX = /autoInterval/g
+
+export const renderTemplatesInScript = async (
+  script: string,
+  timeRange: TimeRange,
+  astLink: string,
+  maxSideLength: number = DEFAULT_PIXELS
+): Promise<string> => {
+  let dashboardTime: string
+  let upperDashboardTime: string
+
+  if (timeRange.upper) {
+    dashboardTime = timeRange.lower
+    upperDashboardTime = timeRange.upper
+  } else {
+    dashboardTime = timeRange.lowerFlux || '1h'
+    upperDashboardTime = new Date().toISOString()
+  }
+
+  let rendered = `${DASHBOARD_TIME} = ${dashboardTime}\n\n${UPPER_DASHBOARD_TIME} = ${upperDashboardTime}\n\n${script}`
+
+  if (!script.match(INTERVAL_REGEX)) {
+    return rendered
+  }
+
+  let duration: number
+
+  try {
+    duration = await getMinDuration(astLink, rendered)
+  } catch (error) {
+    duration = DEFAULT_DURATION_MS
+  }
+
+  const interval = duration / (maxSideLength * RESOLUTION_SCALE_FACTOR)
+
+  rendered = `${INTERVAL} = ${Math.floor(interval)}ms\n\n${rendered}`
+
+  return rendered
+}

ui/src/shared/apis/query.ts

@@ -51,7 +51,7 @@ const replace = async (
   const durationMs = await duration(templateReplacedQuery, source)
   const replacedQuery = replaceInterval(
     templateReplacedQuery,
-    Math.floor(resolution / 3),
+    resolution,
     durationMs
   )
 

ui/src/shared/components/RefreshingGraph.tsx

@@ -83,6 +83,7 @@ interface Props {
   autoRefresher: AutoRefresher
   manualRefresh: number
   resizerTopHeight: number
+  fluxASTLink: string
   onZoom: () => void
   editQueryStatus: () => void
   onSetResolution: () => void
@@ -121,6 +122,7 @@ class RefreshingGraph extends Component<Props> {
       onNotify,
       timeRange,
       templates,
+      fluxASTLink,
       grabFluxData,
       manualRefresh,
       autoRefresher,
@@ -152,6 +154,7 @@ class RefreshingGraph extends Component<Props> {
             queries={this.queries}
             timeRange={timeRange}
             templates={templates}
+            fluxASTLink={fluxASTLink}
             editQueryStatus={editQueryStatus}
             onNotify={onNotify}
             grabDataForDownload={grabDataForDownload}
@@ -469,8 +472,9 @@ class RefreshingGraph extends Component<Props> {
   }
 }
 
-const mapStateToProps = ({annotations: {mode}}) => ({
+const mapStateToProps = ({links, annotations: {mode}}) => ({
   mode,
+  fluxASTLink: links.flux.ast,
 })
 
 const mdtp = {

ui/src/shared/components/TimeMachine/CSVExporter.tsx

@@ -23,6 +23,7 @@ interface Props {
   script: string
   source: Source
   timeRange: TimeRange
+  fluxASTLink: string
 
   isFluxSelected: boolean
   onNotify: typeof notify
@@ -78,9 +79,14 @@ class CSVExporter extends PureComponent<Props, State> {
   }
 
   private downloadFluxCSV = async (): Promise<void> => {
-    const {source, script, onNotify, timeRange} = this.props
+    const {source, script, onNotify, timeRange, fluxASTLink} = this.props
 
-    const {didTruncate} = await downloadFluxCSV(source, script, timeRange)
+    const {didTruncate} = await downloadFluxCSV(
+      source,
+      script,
+      timeRange,
+      fluxASTLink
+    )
 
     if (didTruncate) {
       onNotify(fluxResponseTruncatedError())
@@ -90,8 +96,12 @@ class CSVExporter extends PureComponent<Props, State> {
   }
 }
 
+const mstp = state => ({
+  fluxASTLink: state.links.flux.ast,
+})
+
 const mdtp = {
   onNotify: notify,
 }
 
-export default connect(null, mdtp)(CSVExporter)
+export default connect(mstp, mdtp)(CSVExporter)

ui/src/shared/components/time_series/TimeSeries.tsx

@@ -50,6 +50,7 @@ interface Props {
   children: (r: RenderProps) => JSX.Element
   inView?: boolean
   templates?: Template[]
+  fluxASTLink?: string
   editQueryStatus?: (queryID: string, status: Status) => void
   grabDataForDownload?: GrabDataForDownloadHandler
   grabFluxData?: (data: FluxTable[]) => void
@@ -66,6 +67,8 @@ interface State {
   latestUUID: string
 }
 
+const TEMP_RES = 300 // FIXME
+
 const GraphLoadingDots = () => (
   <div className="graph-panel__refreshing">
     <div />
@@ -255,14 +258,16 @@ class TimeSeries extends PureComponent<Props, State> {
   private executeFluxQuery = async (
     latestUUID: string
   ): Promise<GetTimeSeriesResult> => {
-    const {queries, onNotify, source, timeRange} = this.props
+    const {queries, onNotify, source, timeRange, fluxASTLink} = this.props
 
     const script: string = _.get(queries, '0.text', '')
     const results = await fetchFluxTimeSeries(
       source,
       script,
+      latestUUID,
       timeRange,
-      latestUUID
+      fluxASTLink,
+      TEMP_RES
     )
 
     if (results.didTruncate && onNotify) {
@@ -288,7 +293,6 @@ class TimeSeries extends PureComponent<Props, State> {
     latestUUID: string
   ): Promise<TimeSeriesServerResponse> => {
     const {source, templates, editQueryStatus} = this.props
-    const TEMP_RES = 300 // FIXME
 
     editQueryStatus(query.id, {loading: true})
 

ui/src/shared/constants/index.ts

@@ -8,6 +8,7 @@ export const GIT_SHA = process.env.GIT_SHA
 
 export const DEFAULT_DURATION_MS = 1000
 export const DEFAULT_PIXELS = 333
+export const RESOLUTION_SCALE_FACTOR = 0.5
 
 export const NO_CELL = 'none'
 

ui/src/shared/parsing/flux/durations.ts

@@ -0,0 +1,253 @@
+import {get, isObject, isArray} from 'lodash'
+import {getAST} from 'src/flux/apis'
+
+export async function getMinDuration(
+  astLink: string,
+  fluxQuery: string
+): Promise<number> {
+  const ast = await getAST({url: astLink, body: fluxQuery})
+  const result = getMinDurationFromAST(ast)
+
+  return result
+}
+
+export function getMinDurationFromAST(ast: any) {
+  // We can't take the minimum of durations of each range individually, since
+  // seperate ranges are potentially combined via an inner `join` call. So the
+  // approach is to:
+  //
+  // 1. Find every possible `[start, stop]` combination for all start and stop
+  //    times across every `range` call
+  // 2. Map each combination to a duration via `stop - start`
+  // 3. Filter out the non-positive durations
+  // 4. Take the minimum duration
+  //
+  const times = allRangeTimes(ast)
+  const starts = times.map(t => t[0])
+  const stops = times.map(t => t[1])
+  const crossProduct = starts.map(start => stops.map(stop => [start, stop]))
+
+  const durations = []
+    .concat(...crossProduct)
+    .map(([start, stop]) => stop - start)
+    .filter(d => d > 0)
+
+  const result = Math.min(...durations)
+
+  return result
+}
+
+// The following interfaces only represent AST structs as they appear
+// in the context of a `range` call
+
+interface RangeCallExpression {
+  type: 'CallExpression'
+  callee: {
+    type: 'Identifier'
+    name: 'range'
+  }
+  arguments: [{properties: RangeCallProperty[]}]
+}
+
+interface RangeCallProperty {
+  type: 'Property'
+  key: {
+    name: 'start' | 'stop'
+  }
+  value: RangeCallPropertyValue
+}
+
+type RangeCallPropertyValue =
+  | MinusUnaryExpression<DurationLiteral>
+  | DurationLiteral
+  | DateTimeLiteral
+  | Identifier
+  | DurationBinaryExpression
+
+interface MinusUnaryExpression<T> {
+  type: 'UnaryExpression'
+  operator: '-'
+  argument: T
+}
+
+interface DurationLiteral {
+  type: 'DurationLiteral'
+  values: Array<{
+    magnitude: number
+    unit: DurationUnit
+  }>
+}
+
+type DurationUnit =
+  | 'y'
+  | 'mo'
+  | 'w'
+  | 'd'
+  | 'h'
+  | 'm'
+  | 's'
+  | 'ms'
+  | 'us'
+  | 'µs'
+  | 'ns'
+
+interface DateTimeLiteral {
+  type: 'DateTimeLiteral'
+  value: string
+}
+
+interface Identifier {
+  type: 'Identifier'
+  name: string
+}
+
+interface DurationBinaryExpression {
+  type: 'BinaryExpression'
+  left: DateTimeLiteral
+  right: DurationLiteral
+  operator: '+' | '-'
+}
+
+export function allRangeTimes(ast: any): Array<[number, number]> {
+  return findNodes(isRangeNode, ast).map(node => rangeTimes(ast, node))
+}
+
+/*
+  Given a `range` call in an AST, reports the `start` and `stop` arguments the
+  the call as absolute instants in time. If the `start` or `stop` argument is a
+  relative duration literal, it is interpreted as relative to the current
+  instant (`Date.now()`).
+*/
+function rangeTimes(
+  ast: any,
+  rangeNode: RangeCallExpression
+): [number, number] {
+  const properties = rangeNode.arguments[0].properties
+  const now = Date.now()
+
+  // The `start` argument is required
+  const startProperty = properties.find(p => p.key.name === 'start')
+  const start = propertyTime(ast, startProperty.value, now)
+
+  // The `end` argument to a `range` call is optional, and defaults to now
+  const endProperty = properties.find(p => p.key.name === 'stop')
+  const end = endProperty ? propertyTime(ast, endProperty.value, now) : now
+
+  if (isNaN(start) || isNaN(end)) {
+    throw new Error('failed to analyze query')
+  }
+
+  return [start, end]
+}
+
+function propertyTime(
+  ast: any,
+  value: RangeCallPropertyValue,
+  now: number
+): number {
+  switch (value.type) {
+    case 'UnaryExpression':
+      return now - durationDuration(value.argument)
+    case 'DurationLiteral':
+      return now + durationDuration(value)
+    case 'DateTimeLiteral':
+      return Date.parse(value.value)
+    case 'Identifier':
+      return propertyTime(ast, resolveDeclaration(ast, value.name), now)
+    case 'BinaryExpression':
+      const leftTime = Date.parse(value.left.value)
+      const rightDuration = durationDuration(value.right)
+
+      switch (value.operator) {
+        case '+':
+          return leftTime + rightDuration
+        case '-':
+          return leftTime - rightDuration
+      }
+  }
+}
+
+const UNIT_TO_APPROX_DURATION = {
+  ns: 1 / 1000000,
+  µs: 1 / 1000,
+  us: 1 / 1000,
+  ms: 1,
+  s: 1000,
+  m: 1000 * 60,
+  h: 1000 * 60 * 60,
+  d: 1000 * 60 * 60 * 24,
+  w: 1000 * 60 * 60 * 24 * 7,
+  mo: 1000 * 60 * 60 * 24 * 30,
+  y: 1000 * 60 * 60 * 24 * 365,
+}
+
+function durationDuration(durationLiteral: DurationLiteral): number {
+  const duration = durationLiteral.values.reduce(
+    (sum, {magnitude, unit}) => sum + magnitude * UNIT_TO_APPROX_DURATION[unit],
+    0
+  )
+
+  return duration
+}
+
+/*
+  Find the node in the `ast` that defines the value of the variable with the
+  given `name`.
+*/
+function resolveDeclaration(ast: any, name: string): RangeCallPropertyValue {
+  const isDeclarator = node => {
+    return (
+      get(node, 'type') === 'VariableDeclarator' &&
+      get(node, 'id.name') === name
+    )
+  }
+
+  const declarator = findNodes(isDeclarator, ast)
+
+  if (!declarator.length) {
+    throw new Error(`unable to resolve identifier "${name}"`)
+  }
+
+  if (declarator.length > 1) {
+    throw new Error('cannot resolve identifier with duplicate declarations')
+  }
+
+  const init = declarator[0].init
+
+  return init
+}
+
+function isRangeNode(node: any) {
+  return (
+    get(node, 'type') === 'CallExpression' &&
+    get(node, 'callee.type') === 'Identifier' &&
+    get(node, 'callee.name') === 'range'
+  )
+}
+
+/*
+  Find all nodes in a tree matching the `predicate` function. Each node in the
+  tree is an object, which may contain objects or arrays of objects as children
+  under any key.
+*/
+function findNodes(
+  predicate: (node: any[]) => boolean,
+  node: any,
+  acc: any[] = []
+) {
+  if (predicate(node)) {
+    acc.push(node)
+  }
+
+  for (const value of Object.values(node)) {
+    if (isObject(value)) {
+      findNodes(predicate, value, acc)
+    } else if (isArray(value)) {
+      for (const innerValue of value) {
+        findNodes(predicate, innerValue, acc)
+      }
+    }
+  }
+
+  return acc
+}

ui/src/shared/utils/downloadTimeseriesCSV.ts

@@ -1,12 +1,16 @@
 // Libraries
 import moment from 'moment'
 import {unparse} from 'papaparse'
+import uuid from 'uuid'
 
 // Utils
 import {timeSeriesToTableGraph} from 'src/utils/timeSeriesTransformers'
 import {executeQuery as executeInfluxQLQuery} from 'src/shared/apis/query'
 import {getRawTimeSeries as executeFluxQuery} from 'src/flux/apis/index'
 
+// Constants
+import {DEFAULT_PIXELS} from 'src/shared/constants'
+
 // Types
 import {Query, Template, Source, TimeRange} from 'src/types'
 import {TimeSeriesResponse} from 'src/types/series'
@@ -29,9 +33,17 @@ export const downloadInfluxQLCSV = async (
 export const downloadFluxCSV = async (
   source: Source,
   script: string,
-  timeRange: TimeRange
+  timeRange: TimeRange,
+  fluxASTLink: string
 ): Promise<{didTruncate: boolean}> => {
-  const {csv, didTruncate} = await executeFluxQuery(source, script, timeRange)
+  const {csv, didTruncate} = await executeFluxQuery(
+    source,
+    script,
+    uuid.v4(),
+    timeRange,
+    fluxASTLink,
+    DEFAULT_PIXELS
+  )
 
   downloadCSV(csv, csvName())
 

ui/src/tempVars/utils/replace.ts

@@ -10,6 +10,7 @@ import {
   TEMP_VAR_INTERVAL,
   DEFAULT_PIXELS,
   DEFAULT_DURATION_MS,
+  RESOLUTION_SCALE_FACTOR,
 } from 'src/shared/constants'
 
 function sortTemplates(templates: Template[]): Template[] {
@@ -35,8 +36,7 @@ export const replaceInterval = (
     durationMs = DEFAULT_DURATION_MS
   }
 
-  // duration / width of visualization in pixels
+  const msPerPixel = Math.floor(durationMs / (pixels * RESOLUTION_SCALE_FACTOR))
-  const msPerPixel = Math.floor(durationMs / pixels)
 
   return replaceAll(query, TEMP_VAR_INTERVAL, `${msPerPixel}ms`)
 }

ui/test/shared/parsing/flux/durations.test.ts

@@ -0,0 +1,8 @@
+import {getMinDurationFromAST} from 'src/shared/parsing/flux/durations'
+import {AST_TESTS} from 'test/shared/parsing/flux/durations'
+
+describe('getMinDurationFromAST', () => {
+  test.each(AST_TESTS)('%s:\n\n```\n%s\n```', (__, ___, expected, ast) => {
+    expect(getMinDurationFromAST(ast)).toEqual(expected)
+  })
+})

ui/test/tempVars/utils/replace.test.ts

@@ -288,7 +288,7 @@ describe('templates.utils.replace', () => {
   describe('replaceInterval', () => {
     it('can replace :interval:', () => {
       const query = `SELECT mean(usage_idle) from "cpu" where time > now() - 4320h group by time(:interval:)`
-      const expected = `SELECT mean(usage_idle) from "cpu" where time > now() - 4320h group by time(46702702ms)`
+      const expected = `SELECT mean(usage_idle) from "cpu" where time > now() - 4320h group by time(93405405ms)`
       const pixels = 333
       const durationMs = 15551999999
       const actual = replaceInterval(query, pixels, durationMs)
@@ -298,7 +298,7 @@ describe('templates.utils.replace', () => {
 
     it('can replace multiple intervals', () => {
       const query = `SELECT NON_NEGATIVE_DERIVATIVE(mean(usage_idle), :interval:) from "cpu" where time > now() - 4320h group by time(:interval:)`
-      const expected = `SELECT NON_NEGATIVE_DERIVATIVE(mean(usage_idle), 46702702ms) from "cpu" where time > now() - 4320h group by time(46702702ms)`
+      const expected = `SELECT NON_NEGATIVE_DERIVATIVE(mean(usage_idle), 93405405ms) from "cpu" where time > now() - 4320h group by time(93405405ms)`
 
       const pixels = 333
       const durationMs = 15551999999
@@ -338,7 +338,7 @@ describe('templates.utils.replace', () => {
         const query = `SELECT mean(usage_idle) from "cpu" WHERE time > :dashboardTime: group by time(:interval:)`
         let actual = templateReplace(query, vars)
         actual = replaceInterval(actual, pixels, durationMs)
-        const expected = `SELECT mean(usage_idle) from "cpu" WHERE time > now() - 24h group by time(259459ms)`
+        const expected = `SELECT mean(usage_idle) from "cpu" WHERE time > now() - 24h group by time(518918ms)`
 
         expect(actual).toBe(expected)
       })
@@ -373,7 +373,7 @@ describe('templates.utils.replace', () => {
         const query = `SELECT mean(usage_idle) from "cpu" WHERE time > :dashboardTime: group by time(:interval:)`
         let actual = templateReplace(query, vars)
         actual = replaceInterval(actual, pixels, durationMs)
-        const expected = `SELECT mean(usage_idle) from "cpu" WHERE time > now() - 1h group by time(94736ms)`
+        const expected = `SELECT mean(usage_idle) from "cpu" WHERE time > now() - 1h group by time(189473ms)`
 
         expect(actual).toBe(expected)
       })

ui/yarn.lock

@@ -794,10 +794,10 @@
   resolved "https://registry.yarnpkg.com/@types/history/-/history-3.2.2.tgz#b6affa240cb10b5f841c6443d8a24d7f3fc8bb0c"
   integrity sha512-DMvBzeA2dp1uZZftXkoqPC4TrdHlyuuTabCOxHY6EAKOJRMaPVu8b6lvX0QxEGKZq3cK/h3JCSxgfKmbDOYmRw==
 
-"@types/jest@^22.1.4":
+"@types/jest@^23.3.5":
-  version "22.2.3"
+  version "23.3.5"
-  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-22.2.3.tgz#0157c0316dc3722c43a7b71de3fdf3acbccef10d"
+  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-23.3.5.tgz#870a1434208b60603745bfd214fc3fc675142364"
-  integrity sha512-e74sM9W/4qqWB6D4TWV9FQk0WoHtX1X4FJpbjxucMSVJHtFjbQOH3H6yp+xno4br0AKG0wz/kPtaN599GUOvAg==
+  integrity sha512-3LI+vUC3Wju28vbjIjsTKakhMB8HC4l+tMz+Z8WRzVK+kmvezE5jcOvKtBpznWSI5KDLFo+FouUhpTKoekadCA==
 
 "@types/levelup@^0.0.30":
   version "0.0.30"