diff --git a/ui/src/logs/actions/index.ts b/ui/src/logs/actions/index.ts index 28a5313fdc..7b9713293d 100644 --- a/ui/src/logs/actions/index.ts +++ b/ui/src/logs/actions/index.ts @@ -2,11 +2,7 @@ import _ from 'lodash' import {getDeep} from 'src/utils/wrappers' import {serverToUIConfig, uiToServerConfig} from 'src/logs/utils/config' -import { - buildInfiniteScrollLogQuery, - buildTableQueryConfig, -} from 'src/logs/utils/queryBuilder' -import {transformFluxLogsResponse} from 'src/logs/utils' +import {getTableData, buildTableQueryConfig} from 'src/logs/utils/logQuery' // APIs import { @@ -438,25 +434,13 @@ export const fetchTailAsync = () => async ( const upperUTC = Date.parse(upper) dispatch(setCurrentTailUpperBound(upperUTC)) - const query = buildInfiniteScrollLogQuery( + const logSeries = await getTableData(executeQueryAsync, { lower, upper, - tableQueryConfig, - filters - ) - const { - links: {query: queryLink}, - } = currentSource - const response = await executeQueryAsync(queryLink, query) - - if (response.status !== SearchStatus.Loaded) { - return - } - const columnNames: string[] = tableQueryConfig.fields.map(f => f.alias) - const logSeries: TableData = transformFluxLogsResponse( - response.tables, - columnNames - ) + filters, + source: currentSource, + config: tableQueryConfig, + }) const currentForwardBufferDuration = upperUTC - tailLowerBound const maxTailBufferDurationMs = getMaxTailBufferDurationMs(state) diff --git a/ui/src/logs/api/v2/index.ts b/ui/src/logs/api/v2/index.ts index 56f60657a7..dfc185a951 100644 --- a/ui/src/logs/api/v2/index.ts +++ b/ui/src/logs/api/v2/index.ts @@ -14,7 +14,8 @@ export interface QueryResponse { export const executeQueryAsync = async ( link: string, - query: string + query: string, + type: InfluxLanguage = InfluxLanguage.Flux ): Promise => { try { const dialect = { @@ -27,7 +28,7 @@ export const executeQueryAsync = async ( method: 'POST', url: link, data: { - type: InfluxLanguage.Flux, + type, query, dialect, }, diff --git a/ui/src/logs/components/loading_status/LoadingStatus.tsx b/ui/src/logs/components/loading_status/LoadingStatus.tsx index 827cb9a38b..b16fa9eb84 100644 --- a/ui/src/logs/components/loading_status/LoadingStatus.tsx +++ b/ui/src/logs/components/loading_status/LoadingStatus.tsx @@ -8,7 +8,7 @@ import { } from 'src/clockface' import {SearchStatus} from 'src/types/logs' -import {formatTime} from 'src/logs/utils' +import {formatTime} from 'src/logs/utils/v2' interface Props { status: SearchStatus diff --git a/ui/src/logs/utils/helpers/formatting.ts b/ui/src/logs/utils/helpers/formatting.ts new file mode 100644 index 0000000000..dbfa80344a --- /dev/null +++ b/ui/src/logs/utils/helpers/formatting.ts @@ -0,0 +1,2 @@ +export const oneline = ({raw: [template]}: TemplateStringsArray) => + template.trim().replace(/\n(\s|\t)*/g, ' ') diff --git a/ui/src/logs/utils/logQuery.test.ts b/ui/src/logs/utils/logQuery.test.ts new file mode 100644 index 0000000000..ef74c5ef76 --- /dev/null +++ b/ui/src/logs/utils/logQuery.test.ts @@ -0,0 +1,91 @@ +import {buildTableQueryConfig, buildLogQuery} from 'src/logs/utils/logQuery' + +import {oneline} from 'src/logs/utils/helpers/formatting' + +import {QueryConfig} from 'src/types' +import {Filter} from 'src/types/logs' +import {InfluxLanguage} from 'src/types/v2/dashboards' + +describe('Logs.LogQuery', () => { + let config: QueryConfig + let filters: Filter[] + let lower: string + let upper: string + + beforeEach(() => { + config = buildTableQueryConfig({ + id: '1', + organization: 'default', + organizationID: '1', + name: 'telegraf', + rp: 'autogen', + retentionRules: [], + links: { + self: '', + org: '', + }, + }) + + filters = [] + lower = '2018-10-10T22:46:24.859Z' + upper = '2018-10-10T22:46:54.859Z' + }) + + it('can build a flux query', () => { + const actual = buildLogQuery(InfluxLanguage.Flux, { + lower, + upper, + config, + filters, + }) + + const expected = oneline` + from(bucket: "telegraf/autogen") + |> range(start: 2018-10-10T22:46:24.859Z, stop: 2018-10-10T22:46:54.859Z) + |> filter(fn: (r) => r._measurement == "syslog") + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + |> group(none: true) + |> sort(columns: ["_time"]) + |> map(fn: (r) => ({time: r._time, + severity: r.severity, + timestamp: r.timestamp, + message: r.message, + facility: r.facility, + procid: r.procid, + appname: r.appname, + host: r.host})) + ` + + expect(actual).toEqual(expected) + }) + + it('can build an influxql query', () => { + filters = [{key: 'severity', operator: '==', value: 'notice', id: '1'}] + const actual = buildLogQuery(InfluxLanguage.InfluxQL, { + lower, + upper, + config, + filters, + }) + + const expected = oneline` + SELECT + "_time" AS "time", + "severity" AS "severity", + "timestamp" AS "timestamp", + "message" AS "message", + "facility" AS "facility", + "procid" AS "procid", + "appname" AS "appname", + "host" AS "host" + FROM + "telegraf"."autogen"."syslog" + WHERE + time >= '2018-10-10T22:46:24.859Z' AND + time < '2018-10-10T22:46:54.859Z' AND + "severity" = 'notice' + ` + + expect(actual).toEqual(expected) + }) +}) diff --git a/ui/src/logs/utils/logQuery.ts b/ui/src/logs/utils/logQuery.ts new file mode 100644 index 0000000000..d81a571797 --- /dev/null +++ b/ui/src/logs/utils/logQuery.ts @@ -0,0 +1,127 @@ +// Libraries +import uuid from 'uuid' + +// APIs +import {executeQueryAsync} from 'src/logs/api/v2' + +// Utils +import {fluxToTableData} from 'src/logs/utils/v2' +import {buildFluxQuery} from 'src/logs/utils/v2/queryBuilder' +import {buildInfluxQLQuery} from 'src/logs/utils/v1/queryBuilder' + +// Types +import {Bucket} from 'src/types/v2/buckets' +import {InfluxLanguage} from 'src/types/v2/dashboards' +import {QueryConfig} from 'src/types' +import {Source} from 'src/types/v2' +import {LogSearchParams, SearchStatus, TableData} from 'src/types/logs' + +type FetchSeries = typeof executeQueryAsync + +const defaultQueryConfig = { + areTagsAccepted: false, + fill: '0', + measurement: 'syslog', + rawText: null, + shifts: [], + tags: {}, +} + +const tableFields = [ + { + alias: 'time', + type: 'field', + value: '_time', + }, + { + alias: 'severity', + type: 'field', + value: 'severity', + }, + { + alias: 'timestamp', + type: 'field', + value: 'timestamp', + }, + { + alias: 'message', + type: 'field', + value: 'message', + }, + { + alias: 'facility', + type: 'field', + value: 'facility', + }, + { + alias: 'procid', + type: 'field', + value: 'procid', + }, + { + alias: 'appname', + type: 'field', + value: 'appname', + }, + { + alias: 'host', + type: 'field', + value: 'host', + }, +] + +export const buildTableQueryConfig = (bucket: Bucket): QueryConfig => { + const id = uuid.v4() + const {name, rp} = bucket + + return { + ...defaultQueryConfig, + id, + database: name, + retentionPolicy: rp, + groupBy: {tags: []}, + fields: tableFields, + fill: null, + } +} + +export const buildLogQuery = ( + type: InfluxLanguage, + searchParams: LogSearchParams +): string => { + switch (type) { + case InfluxLanguage.InfluxQL: + return buildInfluxQLQuery(searchParams) + case InfluxLanguage.Flux: + return buildFluxQuery(searchParams) + } +} + +interface LogQuery extends LogSearchParams { + source: Source +} + +export const getTableData = async ( + executeQuery: FetchSeries, + logQuery: LogQuery +): Promise => { + const {source, ...searchParams} = logQuery + const { + links: {query: queryLink}, + } = source + // TODO: get type from source + const type = InfluxLanguage.Flux + const query = buildLogQuery(type, searchParams) + + const response = await executeQuery(queryLink, query, type) + + if (response.status !== SearchStatus.Loaded) { + return + } + + const {config} = searchParams + const columnNames: string[] = config.fields.map(f => f.alias) + const logSeries: TableData = fluxToTableData(response.tables, columnNames) + + return logSeries +} diff --git a/ui/src/logs/utils/queryBuilder.test.ts b/ui/src/logs/utils/queryBuilder.test.ts deleted file mode 100644 index ac2376c923..0000000000 --- a/ui/src/logs/utils/queryBuilder.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { - buildTableQueryConfig, - buildInfiniteScrollLogQuery, -} from 'src/logs/utils/queryBuilder' - -import {QueryConfig} from 'src/types' -import {Filter} from 'src/types/logs' - -describe('Logs.queryBuilder', () => { - let queryConfig: QueryConfig - let filters: Filter[] - let lower: string - let upper: string - - beforeEach(() => { - queryConfig = buildTableQueryConfig({ - id: '1', - organization: 'default', - organizationID: '1', - name: 'telegraf', - rp: 'autogen', - retentionRules: [], - links: { - self: '', - org: '', - }, - }) - - filters = [] - lower = '2018-10-10T22:46:24.859Z' - upper = '2018-10-10T22:46:54.859Z' - }) - - it('can build a query config into a query', () => { - const actual = buildInfiniteScrollLogQuery( - lower, - upper, - queryConfig, - filters - ) - const expected = [ - `from(bucket: "telegraf/autogen")`, - `range(start: 2018-10-10T22:46:24.859Z, stop: 2018-10-10T22:46:54.859Z)`, - `filter(fn: (r) => r._measurement == "syslog")`, - `pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")`, - `group(none: true)`, - `sort(columns: ["_time"])`, - `map(fn: (r) => ({time: r._time, severity: r.severity, timestamp: r.timestamp, message: r.message, facility: r.facility, procid: r.procid, appname: r.appname, host: r.host}))`, - ].join('\n |> ') - - expect(actual).toEqual(expected) - }) - - it('can build a query config into a query with a filter', () => { - filters = [{key: 'severity', operator: '!=', value: 'notice', id: '1'}] - const actual = buildInfiniteScrollLogQuery( - lower, - upper, - queryConfig, - filters - ) - - const expected = [ - `from(bucket: "telegraf/autogen")`, - `range(start: 2018-10-10T22:46:24.859Z, stop: 2018-10-10T22:46:54.859Z)`, - `filter(fn: (r) => r._measurement == "syslog")`, - `pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")`, - `group(none: true)`, - `filter(fn: (r) => r.severity != "notice")`, - `sort(columns: ["_time"])`, - `map(fn: (r) => ({time: r._time, severity: r.severity, timestamp: r.timestamp, message: r.message, facility: r.facility, procid: r.procid, appname: r.appname, host: r.host}))`, - ].join('\n |> ') - - expect(actual).toEqual(expected) - }) - - it('can build a query config into a query with multiple filters', () => { - filters = [ - {key: 'severity', operator: '==', value: 'notice', id: '1'}, - {key: 'appname', operator: '!~', value: 'beep', id: '1'}, - {key: 'appname', operator: '=~', value: 'o_trace_id=broken', id: '1'}, - ] - - const actual = buildInfiniteScrollLogQuery( - lower, - upper, - queryConfig, - filters - ) - - const expected = [ - `from(bucket: "telegraf/autogen")`, - `range(start: 2018-10-10T22:46:24.859Z, stop: 2018-10-10T22:46:54.859Z)`, - `filter(fn: (r) => r._measurement == "syslog")`, - `pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")`, - `group(none: true)`, - `filter(fn: (r) => r.severity == "notice" and r.appname !~ /beep/ and r.appname =~ /o_trace_id=broken/)`, - `sort(columns: ["_time"])`, - `map(fn: (r) => ({time: r._time, severity: r.severity, timestamp: r.timestamp, message: r.message, facility: r.facility, procid: r.procid, appname: r.appname, host: r.host}))`, - ].join('\n |> ') - - expect(actual).toEqual(expected) - }) -}) diff --git a/ui/src/logs/utils/v1/queryBuilder.test.ts b/ui/src/logs/utils/v1/queryBuilder.test.ts new file mode 100644 index 0000000000..130b57c741 --- /dev/null +++ b/ui/src/logs/utils/v1/queryBuilder.test.ts @@ -0,0 +1,127 @@ +import {buildInfluxQLQuery} from 'src/logs/utils/v1/queryBuilder' +import {buildTableQueryConfig} from 'src/logs/utils/logQuery' +import {oneline} from 'src/logs/utils/helpers/formatting' + +import {QueryConfig} from 'src/types' +import {Filter} from 'src/types/logs' + +describe('Logs.V1.queryBuilder', () => { + let config: QueryConfig + let filters: Filter[] + let lower: string + let upper: string + + beforeEach(() => { + config = buildTableQueryConfig({ + id: '1', + organization: 'default', + organizationID: '1', + name: 'telegraf', + rp: 'autogen', + retentionRules: [], + links: { + self: '', + org: '', + }, + }) + + filters = [] + lower = '2018-10-10T22:46:24.859Z' + upper = '2018-10-10T22:46:54.859Z' + }) + + it('can build a query config into a query', () => { + const actual = buildInfluxQLQuery({ + lower, + upper, + filters, + config, + }) + + const expected = oneline` + SELECT + "_time" AS "time", + "severity" AS "severity", + "timestamp" AS "timestamp", + "message" AS "message", + "facility" AS "facility", + "procid" AS "procid", + "appname" AS "appname", + "host" AS "host" + FROM + "telegraf"."autogen"."syslog" + WHERE + time >= '2018-10-10T22:46:24.859Z' AND + time < '2018-10-10T22:46:54.859Z' + ` + + expect(actual).toEqual(expected) + }) + + it('can build a query config into a query with a filter', () => { + filters = [{key: 'severity', operator: '!=', value: 'notice', id: '1'}] + const actual = buildInfluxQLQuery({ + lower, + upper, + filters, + config, + }) + + const expected = oneline` + SELECT + "_time" AS "time", + "severity" AS "severity", + "timestamp" AS "timestamp", + "message" AS "message", + "facility" AS "facility", + "procid" AS "procid", + "appname" AS "appname", + "host" AS "host" + FROM + "telegraf"."autogen"."syslog" + WHERE + time >= '2018-10-10T22:46:24.859Z' AND + time < '2018-10-10T22:46:54.859Z' AND + "severity" != 'notice' + ` + + expect(actual).toEqual(expected) + }) + + it('can build a query config into a query with multiple filters', () => { + filters = [ + {key: 'severity', operator: '==', value: 'notice', id: '1'}, + {key: 'appname', operator: '!~', value: 'beep', id: '1'}, + {key: 'appname', operator: '=~', value: 'o_trace_id=broken', id: '1'}, + ] + + const actual = buildInfluxQLQuery({ + lower, + upper, + filters, + config, + }) + + const expected = oneline` + SELECT + "_time" AS "time", + "severity" AS "severity", + "timestamp" AS "timestamp", + "message" AS "message", + "facility" AS "facility", + "procid" AS "procid", + "appname" AS "appname", + "host" AS "host" + FROM + "telegraf"."autogen"."syslog" + WHERE + time >= '2018-10-10T22:46:24.859Z' AND + time < '2018-10-10T22:46:54.859Z' AND + "severity" = 'notice' AND + "appname" !~ /beep/ AND + "appname" =~ /o_trace_id=broken/ + ` + + expect(actual).toEqual(expected) + }) +}) diff --git a/ui/src/logs/utils/v1/queryBuilder.ts b/ui/src/logs/utils/v1/queryBuilder.ts new file mode 100644 index 0000000000..55bc2d7488 --- /dev/null +++ b/ui/src/logs/utils/v1/queryBuilder.ts @@ -0,0 +1,151 @@ +import _ from 'lodash' +import moment from 'moment' +import {Filter, LogSearchParams} from 'src/types/logs' +import {TimeRange, QueryConfig} from 'src/types' +import {NULL_STRING} from 'src/shared/constants/queryFillOptions' + +import { + quoteIfTimestamp, + buildSelect, + buildWhereClause, + buildGroupBy, + buildFill, +} from 'src/utils/influxql' + +import {DEFAULT_TIME_FORMAT} from 'src/logs/constants' + +const keyMapping = (key: string): string => { + switch (key) { + case 'severity_1': + return 'severity' + default: + return key + } +} + +const operatorMapping = (operator: string): string => { + switch (operator) { + case '==': + return '=' + default: + return operator + } +} + +const valueMapping = (operator: string, value): string => { + if (operator === '=~' || operator === '!~') { + return `${new RegExp(value)}` + } else { + return `'${value}'` + } +} + +export const filtersClause = (filters: Filter[]): string => { + return _.map( + filters, + (filter: Filter) => + `"${keyMapping(filter.key)}" ${operatorMapping( + filter.operator + )} ${valueMapping(filter.operator, filter.value)}` + ).join(' AND ') +} + +export function buildInfiniteScrollWhereClause({ + lower, + upper, + tags, + areTagsAccepted, +}: QueryConfig): string { + const timeClauses = [] + + if (lower) { + timeClauses.push(`time >= '${lower}'`) + } + + if (upper) { + timeClauses.push(`time < '${upper}'`) + } + + const tagClauses = _.keys(tags).map(k => { + const operator = areTagsAccepted ? '=' : '!=' + + if (tags[k].length > 1) { + const joinedOnOr = tags[k] + .map(v => `"${k}"${operator}'${v}'`) + .join(' OR ') + return `(${joinedOnOr})` + } + + return `"${k}"${operator}'${tags[k]}'` + }) + + const subClauses = timeClauses.concat(tagClauses) + if (!subClauses.length) { + return '' + } + + return ` WHERE ${subClauses.join(' AND ')}` +} + +export function buildGeneralLogQuery( + condition: string, + config: QueryConfig, + filters: Filter[] +) { + const {groupBy, fill = NULL_STRING} = config + const select = buildSelect(config, '') + const dimensions = buildGroupBy(groupBy) + const fillClause = groupBy.time ? buildFill(fill) : '' + + if (!_.isEmpty(filters)) { + condition = `${condition} AND ${filtersClause(filters)}` + } + + return `${select}${condition}${dimensions}${fillClause}` +} + +export function buildInfluxQLQuery({ + lower, + upper, + config, + filters, +}: LogSearchParams) { + const {tags, areTagsAccepted} = config + + const condition = buildInfiniteScrollWhereClause({ + lower, + upper, + tags, + areTagsAccepted, + }) + + return buildGeneralLogQuery(condition, config, filters) +} + +export function buildLogQuery( + timeRange: TimeRange, + config: QueryConfig, + filters: Filter[], + searchTerm: string | null = null +): string { + const {groupBy, fill = NULL_STRING, tags, areTagsAccepted} = config + const {upper, lower} = quoteIfTimestamp(timeRange) + const select = buildSelect(config, '') + const dimensions = buildGroupBy(groupBy) + const fillClause = groupBy.time ? buildFill(fill) : '' + + let condition = buildWhereClause({lower, upper, tags, areTagsAccepted}) + if (!_.isEmpty(searchTerm)) { + condition = `${condition} AND message =~ ${new RegExp(searchTerm)}` + } + + if (!_.isEmpty(filters)) { + condition = `${condition} AND ${filtersClause(filters)}` + } + + return `${select}${condition}${dimensions}${fillClause}` +} + +export const formatTime = (time: number): string => { + return moment(time).format(DEFAULT_TIME_FORMAT) +} diff --git a/ui/src/logs/utils/fluxResponse.test.ts b/ui/src/logs/utils/v2/fluxResponse.test.ts similarity index 88% rename from ui/src/logs/utils/fluxResponse.test.ts rename to ui/src/logs/utils/v2/fluxResponse.test.ts index e9de4a489e..aa2d57d017 100644 --- a/ui/src/logs/utils/fluxResponse.test.ts +++ b/ui/src/logs/utils/v2/fluxResponse.test.ts @@ -1,7 +1,7 @@ -import {transformFluxLogsResponse} from 'src/logs/utils' +import {fluxToTableData} from 'src/logs/utils/v2' import {fluxResponse} from 'src/logs/utils/fixtures/fluxResponse' -describe('Logs.transformFluxLogsResponse', () => { +describe('Logs.fluxToTableData', () => { const {tables: fluxResponseTables} = fluxResponse it('can transform a Flux server response to a TableData shape', () => { @@ -15,10 +15,7 @@ describe('Logs.transformFluxLogsResponse', () => { 'timestamp', ] - const actual = transformFluxLogsResponse( - fluxResponseTables, - columnNamesToExtract - ) + const actual = fluxToTableData(fluxResponseTables, columnNamesToExtract) const expected = { columns: [ 'appname', @@ -91,10 +88,7 @@ describe('Logs.transformFluxLogsResponse', () => { 'timestamp', ] - const actual = transformFluxLogsResponse( - fluxResponseTables, - columnNamesToExtract - ) + const actual = fluxToTableData(fluxResponseTables, columnNamesToExtract) const expected = { columns: [ 'facility', @@ -154,10 +148,7 @@ describe('Logs.transformFluxLogsResponse', () => { it('can extract in the specified column ordering', () => { const columnNamesToExtract = ['host', 'facility'] - const actual = transformFluxLogsResponse( - fluxResponseTables, - columnNamesToExtract - ) + const actual = fluxToTableData(fluxResponseTables, columnNamesToExtract) const expected = { columns: ['host', 'facility'], values: [ diff --git a/ui/src/logs/utils/index.ts b/ui/src/logs/utils/v2/index.ts similarity index 96% rename from ui/src/logs/utils/index.ts rename to ui/src/logs/utils/v2/index.ts index 430a650a77..e656e4fb4d 100644 --- a/ui/src/logs/utils/index.ts +++ b/ui/src/logs/utils/v2/index.ts @@ -16,7 +16,7 @@ export const formatTime = (time: number): string => { return moment(time).format(DEFAULT_TIME_FORMAT) } -export const transformFluxLogsResponse = ( +export const fluxToTableData = ( tables: FluxTable[], columnNames: string[] ): TableData => { diff --git a/ui/src/logs/utils/v2/queryBuilder.test.ts b/ui/src/logs/utils/v2/queryBuilder.test.ts new file mode 100644 index 0000000000..1519364938 --- /dev/null +++ b/ui/src/logs/utils/v2/queryBuilder.test.ts @@ -0,0 +1,112 @@ +import {buildFluxQuery} from 'src/logs/utils/v2/queryBuilder' +import {buildTableQueryConfig} from 'src/logs/utils/logQuery' + +import {QueryConfig} from 'src/types' +import {Filter} from 'src/types/logs' +import {oneline} from 'src/logs/utils/helpers/formatting' + +describe('Logs.V2.queryBuilder', () => { + let config: QueryConfig + let filters: Filter[] + let lower: string + let upper: string + + beforeEach(() => { + config = buildTableQueryConfig({ + id: '1', + organization: 'default', + organizationID: '1', + name: 'telegraf', + rp: 'autogen', + retentionRules: [], + links: { + self: '', + org: '', + }, + }) + + filters = [] + lower = '2018-10-10T22:46:24.859Z' + upper = '2018-10-10T22:46:54.859Z' + }) + + it('can build a query config into a query', () => { + const actual = buildFluxQuery({lower, upper, config, filters}) + const expected = oneline` + from(bucket: "telegraf/autogen") + |> range(start: 2018-10-10T22:46:24.859Z, stop: 2018-10-10T22:46:54.859Z) + |> filter(fn: (r) => r._measurement == "syslog") + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + |> group(none: true) + |> sort(columns: ["_time"]) + |> map(fn: (r) => ({time: r._time, + severity: r.severity, + timestamp: r.timestamp, + message: r.message, + facility: r.facility, + procid: r.procid, + appname: r.appname, + host: r.host})) + ` + + expect(actual).toEqual(expected) + }) + + it('can build a query config into a query with a filter', () => { + filters = [{key: 'severity', operator: '!=', value: 'notice', id: '1'}] + const actual = buildFluxQuery({lower, upper, config, filters}) + + const expected = oneline` + from(bucket: "telegraf/autogen") + |> range(start: 2018-10-10T22:46:24.859Z, stop: 2018-10-10T22:46:54.859Z) + |> filter(fn: (r) => r._measurement == "syslog") + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + |> group(none: true) + |> filter(fn: (r) => r.severity != "notice") + |> sort(columns: ["_time"]) + |> map(fn: (r) => ({time: r._time, + severity: r.severity, + timestamp: r.timestamp, + message: r.message, + facility: r.facility, + procid: r.procid, + appname: r.appname, + host: r.host})) + ` + + expect(actual).toEqual(expected) + }) + + it('can build a query config into a query with multiple filters', () => { + filters = [ + {key: 'severity', operator: '==', value: 'notice', id: '1'}, + {key: 'appname', operator: '!~', value: 'beep', id: '1'}, + {key: 'appname', operator: '=~', value: 'o_trace_id=broken', id: '1'}, + ] + + const actual = buildFluxQuery({lower, upper, config, filters}) + + const expected = oneline` + from(bucket: "telegraf/autogen") + |> range(start: 2018-10-10T22:46:24.859Z, stop: 2018-10-10T22:46:54.859Z) + |> filter(fn: (r) => r._measurement == "syslog") + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") + |> group(none: true) + |> filter(fn: (r) => + r.severity == "notice" and + r.appname !~ /beep/ and + r.appname =~ /o_trace_id=broken/) + |> sort(columns: ["_time"]) + |> map(fn: (r) => ({time: r._time, + severity: r.severity, + timestamp: r.timestamp, + message: r.message, + facility: r.facility, + procid: r.procid, + appname: r.appname, + host: r.host})) + ` + + expect(actual).toEqual(expected) + }) +}) diff --git a/ui/src/logs/utils/queryBuilder.ts b/ui/src/logs/utils/v2/queryBuilder.ts similarity index 53% rename from ui/src/logs/utils/queryBuilder.ts rename to ui/src/logs/utils/v2/queryBuilder.ts index 306b7a13ae..962d544104 100644 --- a/ui/src/logs/utils/queryBuilder.ts +++ b/ui/src/logs/utils/v2/queryBuilder.ts @@ -1,88 +1,17 @@ -// Libraries -import uuid from 'uuid' - // Types -import {Bucket} from 'src/types/v2/buckets' -import {QueryConfig, Field} from 'src/types' -import {Filter} from 'src/types/logs' +import {Field} from 'src/types' +import {Filter, LogSearchParams} from 'src/types/logs' -const defaultQueryConfig = { - areTagsAccepted: false, - fill: '0', - measurement: 'syslog', - rawText: null, - shifts: [], - tags: {}, -} - -const tableFields = [ - { - alias: 'time', - type: 'field', - value: '_time', - }, - { - alias: 'severity', - type: 'field', - value: 'severity', - }, - { - alias: 'timestamp', - type: 'field', - value: 'timestamp', - }, - { - alias: 'message', - type: 'field', - value: 'message', - }, - { - alias: 'facility', - type: 'field', - value: 'facility', - }, - { - alias: 'procid', - type: 'field', - value: 'procid', - }, - { - alias: 'appname', - type: 'field', - value: 'appname', - }, - { - alias: 'host', - type: 'field', - value: 'host', - }, -] - -export const buildTableQueryConfig = (bucket: Bucket): QueryConfig => { - const id = uuid.v4() - const {name, rp} = bucket - - return { - ...defaultQueryConfig, - id, - database: name, - retentionPolicy: rp, - groupBy: {tags: []}, - fields: tableFields, - fill: null, - } -} - -const PIPE = '\n |> ' +const PIPE = ' |> ' const ROW_NAME = 'r' const SORT_FUNC = ['sort(columns: ["_time"])'] -export function buildInfiniteScrollLogQuery( - lower: string, - upper: string, - config: QueryConfig, - filters: Filter[] -) { +export function buildFluxQuery({ + lower, + upper, + config, + filters, +}: LogSearchParams) { const {database, retentionPolicy, fields, measurement} = config const bucketName = `"${database}/${retentionPolicy}"` diff --git a/ui/src/types/logs.ts b/ui/src/types/logs.ts index 9d818c9c44..d069b2ffe5 100644 --- a/ui/src/types/logs.ts +++ b/ui/src/types/logs.ts @@ -5,6 +5,13 @@ import {QueryConfig} from 'src/types' import {FieldOption, TimeSeriesValue} from 'src/types/v2/dashboards' +export interface LogSearchParams { + lower: string + upper: string + config: QueryConfig + filters: Filter[] +} + export enum SearchStatus { None = 'None', Loading = 'Loading', diff --git a/ui/src/utils/defaultQueryConfig.ts b/ui/src/utils/defaultQueryConfig.ts new file mode 100644 index 0000000000..f3c47e6728 --- /dev/null +++ b/ui/src/utils/defaultQueryConfig.ts @@ -0,0 +1,38 @@ +import uuid from 'uuid' + +import {NULL_STRING} from 'src/shared/constants/queryFillOptions' +import {QueryConfig} from 'src/types' +import {TEMPLATE_RANGE} from 'src/tempVars/constants' + +interface DefaultQueryArgs { + id?: string + isKapacitorRule?: boolean +} + +const defaultQueryConfig = ( + {id, isKapacitorRule = false}: DefaultQueryArgs = {id: uuid.v4()} +): QueryConfig => { + const queryConfig = { + id, + database: null, + measurement: null, + retentionPolicy: null, + fields: [], + tags: {}, + groupBy: { + time: null, + tags: [], + }, + areTagsAccepted: true, + rawText: null, + status: null, + shifts: [], + fill: null, + range: TEMPLATE_RANGE, + originalQuery: null, + } + + return isKapacitorRule ? queryConfig : {...queryConfig, fill: NULL_STRING} +} + +export default defaultQueryConfig diff --git a/ui/src/utils/influxql.test.ts b/ui/src/utils/influxql.test.ts new file mode 100644 index 0000000000..6c48956390 --- /dev/null +++ b/ui/src/utils/influxql.test.ts @@ -0,0 +1,353 @@ +import buildInfluxQLQuery, {buildQuery} from 'src/utils/influxql' +import defaultQueryConfig from 'src/utils/defaultQueryConfig' + +import {NONE, NULL_STRING} from 'src/shared/constants/queryFillOptions' +import {TYPE_QUERY_CONFIG} from 'src/dashboards/constants' + +import {QueryConfig} from 'src/types' + +function mergeConfig(options: Partial) { + return {...defaultQueryConfig({id: '123'}), ...options} +} + +describe('buildInfluxQLQuery', () => { + let config + let timeBounds + describe('when information is missing', () => { + it('returns a null select statement', () => { + timeBounds = {lower: 'now() - 15m', upper: null} + expect(buildInfluxQLQuery(timeBounds, mergeConfig({}))).toBe(null) + + let merged = mergeConfig({database: 'db1'}) + let actual = buildInfluxQLQuery(timeBounds, merged) + expect(actual).toBe(null) // no measurement + + merged = mergeConfig({database: 'db1', measurement: 'm1'}) + actual = buildInfluxQLQuery(timeBounds, merged) + expect(actual).toBe(null) // no fields + }) + }) + + describe('with a database, measurement, field, and NO retention policy', () => { + beforeEach(() => { + config = mergeConfig({ + database: 'db1', + measurement: 'm1', + fields: [{value: 'f1', type: 'field'}], + }) + timeBounds = {lower: null, upper: null} + }) + + it('builds the right query', () => { + expect(buildInfluxQLQuery(timeBounds, config)).toBe( + 'SELECT "f1" FROM "db1".."m1"' + ) + }) + }) + + describe('with a database, measurement, retention policy, and field', () => { + beforeEach(() => { + config = mergeConfig({ + database: 'db1', + measurement: 'm1', + retentionPolicy: 'rp1', + fields: [{value: 'f1', type: 'field'}], + }) + }) + + it('builds the right query', () => { + const actual = buildInfluxQLQuery(timeBounds, config) + const expected = 'SELECT "f1" FROM "db1"."rp1"."m1"' + expect(actual).toBe(expected) + }) + + it('builds the right query with a time range', () => { + timeBounds = {lower: 'now() - 1hr', upper: null} + const actual = buildInfluxQLQuery(timeBounds, config) + const expected = + 'SELECT "f1" FROM "db1"."rp1"."m1" WHERE time > now() - 1hr' + + expect(actual).toBe(expected) + }) + }) + + describe('when the field is *', () => { + beforeEach(() => { + config = mergeConfig({ + database: 'db1', + measurement: 'm1', + retentionPolicy: 'rp1', + fields: [{value: '*', type: 'field'}], + }) + timeBounds = {lower: null, upper: null} + }) + + it('does not quote the star', () => { + expect(buildInfluxQLQuery(timeBounds, config)).toBe( + 'SELECT * FROM "db1"."rp1"."m1"' + ) + }) + }) + + describe('with a measurement and one field, an aggregate, and a GROUP BY time()', () => { + beforeEach(() => { + config = mergeConfig({ + database: 'db1', + measurement: 'm0', + retentionPolicy: 'rp1', + fields: [ + { + value: 'min', + type: 'func', + alias: 'min_value', + args: [{value: 'value', type: 'field'}], + }, + ], + groupBy: {time: '10m', tags: []}, + fill: NULL_STRING, + }) + timeBounds = {lower: 'now() - 12h'} + }) + + it('builds the right query', () => { + const expected = + 'SELECT min("value") AS "min_value" FROM "db1"."rp1"."m0" WHERE time > now() - 12h GROUP BY time(10m) FILL(null)' + expect(buildInfluxQLQuery(timeBounds, config)).toBe(expected) + }) + }) + + describe('with a measurement and one field, an aggregate, and a GROUP BY tags', () => { + beforeEach(() => { + config = mergeConfig({ + database: 'db1', + measurement: 'm0', + retentionPolicy: 'rp1', + fields: [ + { + value: 'min', + type: 'func', + alias: 'min_value', + args: [{value: 'value', type: 'field'}], + }, + ], + groupBy: {time: null, tags: ['t1', 't2']}, + }) + timeBounds = {lower: 'now() - 12h'} + }) + + it('builds the right query', () => { + const expected = `SELECT min("value") AS "min_value" FROM "db1"."rp1"."m0" WHERE time > now() - 12h GROUP BY "t1", "t2"` + expect(buildInfluxQLQuery(timeBounds, config)).toBe(expected) + }) + }) + + describe('with a measurement, one field, and an upper / lower absolute time range', () => { + beforeEach(() => { + config = mergeConfig({ + database: 'db1', + retentionPolicy: 'rp1', + measurement: 'm0', + fields: [{value: 'value', type: 'field'}], + }) + timeBounds = { + lower: "'2015-07-23T15:52:24.447Z'", + upper: "'2015-07-24T15:52:24.447Z'", + } + }) + + it('builds the right query', () => { + const expected = + 'SELECT "value" FROM "db1"."rp1"."m0" WHERE time > \'2015-07-23T15:52:24.447Z\' AND time < \'2015-07-24T15:52:24.447Z\'' + expect(buildInfluxQLQuery(timeBounds, config)).toBe(expected) + }) + }) + + describe('with a measurement and one field, an aggregate, and a GROUP BY time(), and tags', () => { + beforeEach(() => { + config = mergeConfig({ + database: 'db1', + retentionPolicy: 'rp1', + measurement: 'm0', + fields: [ + { + value: 'min', + type: 'func', + alias: 'min_value', + args: [{value: 'value', type: 'field'}], + }, + ], + groupBy: {time: '10m', tags: ['t1', 't2']}, + fill: NULL_STRING, + }) + timeBounds = {lower: 'now() - 12h'} + }) + + it('builds the right query', () => { + const expected = + 'SELECT min("value") AS "min_value" FROM "db1"."rp1"."m0" WHERE time > now() - 12h GROUP BY time(10m), "t1", "t2" FILL(null)' + expect(buildInfluxQLQuery(timeBounds, config)).toBe(expected) + }) + }) + + describe('with a measurement and two fields', () => { + beforeEach(() => { + config = mergeConfig({ + database: 'db1', + retentionPolicy: 'rp1', + measurement: 'm0', + fields: [{value: 'f0', type: 'field'}, {value: 'f1', type: 'field'}], + }) + timeBounds = {upper: "'2015-02-24T00:00:00Z'"} + }) + + it('builds the right query', () => { + timeBounds = {lower: null, upper: null} + expect(buildInfluxQLQuery(timeBounds, config)).toBe( + 'SELECT "f0", "f1" FROM "db1"."rp1"."m0"' + ) + }) + + it('builds the right query with a time range', () => { + const expected = `SELECT "f0", "f1" FROM "db1"."rp1"."m0" WHERE time < '2015-02-24T00:00:00Z'` + expect(buildInfluxQLQuery(timeBounds, config)).toBe(expected) + }) + + describe('with multiple tag pairs', () => { + beforeEach(() => { + config = mergeConfig({ + database: 'db1', + measurement: 'm0', + retentionPolicy: 'rp1', + fields: [{value: 'f0', type: 'field'}], + tags: { + k1: ['v1', 'v3', 'v4'], + k2: ['v2'], + }, + }) + timeBounds = {lower: 'now() - 6h'} + }) + + it('correctly uses AND/OR to combine pairs', () => { + const expected = `SELECT "f0" FROM "db1"."rp1"."m0" WHERE time > now() - 6h AND ("k1"='v1' OR "k1"='v3' OR "k1"='v4') AND "k2"='v2'` + expect(buildInfluxQLQuery(timeBounds, config)).toBe(expected) + }) + }) + }) + + describe('with GROUP BY time()', () => { + describe('and no explicit fill', () => { + it('makes fill(null) explicit', () => { + config = mergeConfig({ + database: 'db1', + retentionPolicy: 'rp1', + measurement: 'm0', + fields: [ + { + value: 'min', + type: 'func', + alias: 'min_value', + args: [{value: 'value', type: 'field'}], + }, + ], + groupBy: {time: '10m', tags: []}, + }) + timeBounds = {lower: 'now() - 12h'} + + const expected = + 'SELECT min("value") AS "min_value" FROM "db1"."rp1"."m0" WHERE time > now() - 12h GROUP BY time(10m) FILL(null)' + expect(buildInfluxQLQuery(timeBounds, config)).toBe(expected) + }) + }) + + describe('and explicit fills', () => { + it('includes those explicit fills', () => { + // Test fill null + config = mergeConfig({ + database: 'db1', + retentionPolicy: 'rp1', + measurement: 'm0', + fields: [ + { + value: 'min', + type: 'func', + alias: 'min_value', + args: [{value: 'value', type: 'field'}], + }, + ], + groupBy: {time: '10m', tags: []}, + fill: NULL_STRING, + }) + timeBounds = {lower: 'now() - 12h'} + + let expected = + 'SELECT min("value") AS "min_value" FROM "db1"."rp1"."m0" WHERE time > now() - 12h GROUP BY time(10m) FILL(null)' + expect(buildInfluxQLQuery(timeBounds, config)).toBe(expected) + + // Test fill another option + config = mergeConfig({ + database: 'db1', + retentionPolicy: 'rp1', + measurement: 'm0', + fields: [ + { + value: 'min', + type: 'func', + alias: 'min_value', + args: [{value: 'value', type: 'field'}], + }, + ], + groupBy: {time: '10m', tags: []}, + fill: NONE, + }) + timeBounds = {lower: 'now() - 12h'} + + expected = + 'SELECT min("value") AS "min_value" FROM "db1"."rp1"."m0" WHERE time > now() - 12h GROUP BY time(10m) FILL(none)' + expect(buildInfluxQLQuery(timeBounds, config)).toBe(expected) + + // Test fill number + config = mergeConfig({ + database: 'db1', + retentionPolicy: 'rp1', + measurement: 'm0', + fields: [ + { + value: 'min', + type: 'func', + alias: 'min_value', + args: [{value: 'value', type: 'field'}], + }, + ], + groupBy: {time: '10m', tags: ['t1', 't2']}, + fill: '1337', + }) + timeBounds = {lower: 'now() - 12h'} + + expected = + 'SELECT min("value") AS "min_value" FROM "db1"."rp1"."m0" WHERE time > now() - 12h GROUP BY time(10m), "t1", "t2" FILL(1337)' + expect(buildInfluxQLQuery(timeBounds, config)).toBe(expected) + }) + }) + }) + + describe('build query', () => { + beforeEach(() => { + config = mergeConfig({ + database: 'db1', + measurement: 'm1', + retentionPolicy: 'rp1', + fields: [{value: 'f1', type: 'field'}], + groupBy: {time: '10m', tags: []}, + }) + }) + + it('builds an influxql relative time bound query', () => { + const timeRange = {upper: null, lower: 'now() - 15m'} + const expected = + 'SELECT "f1" FROM "db1"."rp1"."m1" WHERE time > now() - 15m GROUP BY time(10m) FILL(null)' + const actual = buildQuery(TYPE_QUERY_CONFIG, timeRange, config) + + expect(actual).toBe(expected) + }) + }) +}) diff --git a/ui/src/utils/influxql.ts b/ui/src/utils/influxql.ts new file mode 100644 index 0000000000..5b05e5a70e --- /dev/null +++ b/ui/src/utils/influxql.ts @@ -0,0 +1,205 @@ +import _ from 'lodash' + +import {TEMP_VAR_INTERVAL, AUTO_GROUP_BY} from 'src/shared/constants' +import {NULL_STRING} from 'src/shared/constants/queryFillOptions' +import { + TYPE_QUERY_CONFIG, + TYPE_SHIFTED, + TYPE_FLUX, +} from 'src/dashboards/constants' +import {shiftTimeRange} from 'src/shared/query/helpers' +import {QueryConfig, Field, GroupBy, TimeShift, TimeRange} from 'src/types' + +export const quoteIfTimestamp = ({lower, upper}: TimeRange): TimeRange => { + if (lower && lower.includes('Z') && !lower.includes("'")) { + lower = `'${lower}'` + } + + if (upper && upper.includes('Z') && !upper.includes("'")) { + upper = `'${upper}'` + } + + return {lower, upper} +} + +export default function buildInfluxQLQuery( + timeRange: TimeRange, + config: QueryConfig, + shift: string = '' +): string { + const {groupBy, fill = NULL_STRING, tags, areTagsAccepted} = config + const {upper, lower} = quoteIfTimestamp(timeRange) + + const select = buildSelect(config, shift) + if (select === null) { + return null + } + + const condition = buildWhereClause({lower, upper, tags, areTagsAccepted}) + const dimensions = buildGroupBy(groupBy) + const fillClause = groupBy.time ? buildFill(fill) : '' + + return `${select}${condition}${dimensions}${fillClause}` +} + +export function buildSelect( + {fields, database, retentionPolicy, measurement}: QueryConfig, + shift: string | null = null +): string { + if (!database || !measurement || _.isEmpty(fields)) { + return null + } + + const rpSegment = retentionPolicy ? `"${retentionPolicy}"` : '' + const fieldsClause = buildFields(fields, shift) + const fullyQualifiedMeasurement = `"${database}".${rpSegment}."${measurement}"` + const statement = `SELECT ${fieldsClause} FROM ${fullyQualifiedMeasurement}` + return statement +} + +// type arg will reason about new query types i.e. Flux, GraphQL, or queryConfig +export const buildQuery = ( + type: string, + timeRange: TimeRange, + config: QueryConfig, + shift: TimeShift | null = null +): string => { + switch (type) { + case TYPE_QUERY_CONFIG: { + return buildInfluxQLQuery(timeRange, config) + } + case TYPE_SHIFTED: { + const {quantity, unit} = shift + return buildInfluxQLQuery( + shiftTimeRange(timeRange, shift), + config, + `_shifted__${quantity}__${unit}` + ) + } + + case TYPE_FLUX: { + // build query usining FLUX here + } + } + + return buildInfluxQLQuery(timeRange, config) +} + +export function buildSelectStatement(config: QueryConfig): string { + return buildSelect(config) +} + +function buildFields(fieldFuncs: Field[], shift = '', useAlias = true): string { + if (!fieldFuncs) { + return '' + } + + return fieldFuncs + .map(f => { + switch (f.type) { + case 'field': { + const quoted = f.value === '*' ? '*' : `"${f.value}"` + const aliased = + useAlias && f.alias ? `${quoted} AS "${f.alias}"` : quoted + + return aliased + } + case 'wildcard': { + return '*' + } + case 'regex': { + return `/${f.value}/` + } + case 'number': { + return `${f.value}` + } + case 'integer': { + return `${f.value}` + } + case 'func': { + const args = buildFields(f.args, '', false) + const alias = f.alias ? ` AS "${f.alias}${shift}"` : '' + return `${f.value}(${args})${alias}` + } + } + }) + .join(', ') +} + +export function buildWhereClause({ + lower, + upper, + tags, + areTagsAccepted, +}: QueryConfig): string { + const timeClauses = [] + + const timeClause = quoteIfTimestamp({lower, upper}) + + if (timeClause.lower) { + timeClauses.push(`time > ${lower}`) + } + + if (timeClause.upper) { + timeClauses.push(`time < ${upper}`) + } + + // If a tag key has more than one value, * e.g. cpu=cpu1, cpu=cpu2, combine + // them with OR instead of AND for the final query. + const tagClauses = _.keys(tags).map(k => { + const operator = areTagsAccepted ? '=' : '!=' + + if (tags[k].length > 1) { + const joinedOnOr = tags[k] + .map(v => `"${k}"${operator}'${v}'`) + .join(' OR ') + return `(${joinedOnOr})` + } + + return `"${k}"${operator}'${tags[k]}'` + }) + + const subClauses = timeClauses.concat(tagClauses) + if (!subClauses.length) { + return '' + } + + return ` WHERE ${subClauses.join(' AND ')}` +} + +export function buildGroupBy(groupBy: GroupBy): string { + return `${buildGroupByTime(groupBy)}${buildGroupByTags(groupBy)}` +} + +function buildGroupByTime(groupBy: GroupBy): string { + if (!groupBy || !groupBy.time) { + return '' + } + + return ` GROUP BY time(${ + groupBy.time === AUTO_GROUP_BY ? TEMP_VAR_INTERVAL : `${groupBy.time}` + })` +} + +function buildGroupByTags(groupBy: GroupBy): string { + if (!groupBy || !groupBy.tags.length) { + return '' + } + + const tags = groupBy.tags.map(t => `"${t}"`).join(', ') + + if (groupBy.time) { + return `, ${tags}` + } + + return ` GROUP BY ${tags}` +} + +export function buildFill(fill: string): string { + return ` FILL(${fill})` +} + +export const buildRawText = ( + config: QueryConfig, + timeRange: TimeRange +): string => config.rawText || buildInfluxQLQuery(timeRange, config) || ''