diff --git a/ui/src/logs/actions/index.ts b/ui/src/logs/actions/index.ts index 002b630be..e9516a0c7 100644 --- a/ui/src/logs/actions/index.ts +++ b/ui/src/logs/actions/index.ts @@ -9,9 +9,10 @@ import { buildHistogramQueryConfig, buildTableQueryConfig, buildLogQuery, - buildForwardLogQuery, - buildBackwardLogQuery, + buildInfiniteScrollLogQuery, parseHistogramQueryResponse, + findOlderLowerTimeBounds, + findNewerUpperTimeBounds, } from 'src/logs/utils' import {logConfigServerToUI, logConfigUIToServer} from 'src/logs/utils/config' import {getDeep} from 'src/utils/wrappers' @@ -347,27 +348,43 @@ export const setTimeBounds = (timeBounds: TimeBounds): SetTimeBoundsAction => ({ payload: {timeBounds}, }) -export const executeTableForwardQueryAsync = () => async ( +export const executeTableNewerQueryAsync = () => async ( dispatch, getState: GetState ) => { const state = getState() - const time = getTableSelectedTime(state) + const startTime = getTableSelectedTime(state) const queryConfig = getTableQueryConfig(state) const namespace = getNamespace(state) const proxyLink = getProxyLink(state) const searchTerm = getSearchTerm(state) const filters = getFilters(state) - if (!_.every([queryConfig, time, namespace, proxyLink])) { + if (!_.every([queryConfig, startTime, namespace, proxyLink])) { return } try { dispatch(incrementQueryCount()) - const query = buildForwardLogQuery(time, queryConfig, filters, searchTerm) + const endTime = await findNewerUpperTimeBounds( + startTime, + queryConfig, + filters, + searchTerm, + proxyLink, + namespace + ) + + const query: string = await buildInfiniteScrollLogQuery( + startTime, + endTime, + queryConfig, + filters, + searchTerm + ) + const response = await executeQueryAsync( proxyLink, namespace, @@ -387,7 +404,7 @@ export const executeTableForwardQueryAsync = () => async ( } } -export const executeTableBackwardQueryAsync = () => async ( +export const executeTableOlderQueryAsync = () => async ( dispatch, getState: GetState ) => { @@ -407,7 +424,23 @@ export const executeTableBackwardQueryAsync = () => async ( try { dispatch(incrementQueryCount()) - const query = buildBackwardLogQuery(time, queryConfig, filters, searchTerm) + const lower: string = await findOlderLowerTimeBounds( + time, + queryConfig, + filters, + searchTerm, + proxyLink, + namespace + ) + + const query: string = await buildInfiniteScrollLogQuery( + lower, + time, + queryConfig, + filters, + searchTerm + ) + const response = await executeQueryAsync( proxyLink, namespace, @@ -489,8 +522,8 @@ export const executeHistogramQueryAsync = () => async ( export const executeTableQueryAsync = () => async (dispatch): Promise => { await Promise.all([ - dispatch(executeTableForwardQueryAsync()), - dispatch(executeTableBackwardQueryAsync()), + dispatch(executeTableNewerQueryAsync()), + dispatch(executeTableOlderQueryAsync()), dispatch(clearRowsAdded()), ]) } @@ -577,7 +610,7 @@ export const setTableQueryConfigAsync = () => async ( } } -export const fetchMoreAsync = (queryTimeEnd: string) => async ( +export const fetchOlderLogsAsync = (queryTimeEnd: string) => async ( dispatch, getState ): Promise => { @@ -595,7 +628,17 @@ export const fetchMoreAsync = (queryTimeEnd: string) => async ( const params = [namespace, proxyLink, tableQueryConfig] if (_.every(params)) { - const query = buildBackwardLogQuery( + const queryTimeStart = await findOlderLowerTimeBounds( + queryTimeEnd, + newQueryConfig, + filters, + searchTerm, + proxyLink, + namespace + ) + + const query = await buildInfiniteScrollLogQuery( + queryTimeStart, queryTimeEnd, newQueryConfig, filters, @@ -613,7 +656,7 @@ export const fetchMoreAsync = (queryTimeEnd: string) => async ( } } -export const fetchNewerAsync = (queryTimeStart: string) => async ( +export const fetchNewerLogsAsync = (queryTimeStart: string) => async ( dispatch, getState ): Promise => { @@ -631,10 +674,20 @@ export const fetchNewerAsync = (queryTimeStart: string) => async ( const params = [namespace, proxyLink, tableQueryConfig] if (_.every(params)) { - const query = buildForwardLogQuery( + const queryTimeEnd = await findNewerUpperTimeBounds( queryTimeStart, newQueryConfig, filters, + searchTerm, + proxyLink, + namespace + ) + + const query: string = await buildInfiniteScrollLogQuery( + queryTimeStart, + queryTimeEnd, + newQueryConfig, + filters, searchTerm ) diff --git a/ui/src/logs/containers/LogsPage.tsx b/ui/src/logs/containers/LogsPage.tsx index 060942e45..bbc96250b 100644 --- a/ui/src/logs/containers/LogsPage.tsx +++ b/ui/src/logs/containers/LogsPage.tsx @@ -22,8 +22,8 @@ import { addFilter, removeFilter, changeFilter, - fetchMoreAsync, - fetchNewerAsync, + fetchOlderLogsAsync, + fetchNewerLogsAsync, getLogConfigAsync, updateLogConfigAsync, } from 'src/logs/actions' @@ -81,8 +81,8 @@ interface Props { setSearchTermAsync: (searchTerm: string) => void setTableRelativeTime: (time: number) => void setTableCustomTime: (time: string) => void - fetchMoreAsync: (queryTimeEnd: string) => Promise - fetchNewerAsync: (queryTimeEnd: string) => Promise + fetchOlderLogsAsync: (queryTimeEnd: string) => Promise + fetchNewerLogsAsync: (queryTimeEnd: string) => Promise addFilter: (filter: Filter) => void removeFilter: (id: string) => void changeFilter: (id: string, operator: string, value: string) => void @@ -200,7 +200,7 @@ class LogsPage extends Component { onScrolledToTop={this.handleScrollToTop} isScrolledToTop={false} onTagSelection={this.handleTagSelection} - fetchMore={this.props.fetchMoreAsync} + fetchMore={this.props.fetchOlderLogsAsync} fetchNewer={this.fetchNewer} timeRange={timeRange} scrollToRow={this.tableScrollToRow} @@ -220,7 +220,7 @@ class LogsPage extends Component { private fetchNewer = (time: string) => { this.loadingNewer = true - this.props.fetchNewerAsync(time) + this.props.fetchNewerLogsAsync(time) } private get tableScrollToRow() { @@ -511,7 +511,6 @@ class LogsPage extends Component { private handleSubmitSearch = (value: string): void => { this.props.setSearchTermAsync(value) - this.setState({liveUpdating: LiveUpdating.Play}) } private handleFilterDelete = (id: string): void => { @@ -694,8 +693,8 @@ const mapDispatchToProps = { addFilter, removeFilter, changeFilter, - fetchMoreAsync, - fetchNewerAsync, + fetchOlderLogsAsync, + fetchNewerLogsAsync, setTableCustomTime: setTableCustomTimeAsync, setTableRelativeTime: setTableRelativeTimeAsync, getConfig: getLogConfigAsync, diff --git a/ui/src/logs/utils/index.ts b/ui/src/logs/utils/index.ts index 5037f096e..6ff6988c0 100644 --- a/ui/src/logs/utils/index.ts +++ b/ui/src/logs/utils/index.ts @@ -14,8 +14,10 @@ import { } from 'src/utils/influxql' import {HistogramData} from 'src/types/histogram' +import {executeQueryAsync} from 'src/logs/api' const BIN_COUNT = 30 +const SECONDS_AWAY_LIMIT = 2592000 const histogramFields = [ { @@ -115,7 +117,7 @@ export const filtersClause = (filters: Filter[]): string => { ).join(' AND ') } -export function buildInfiniteWhereClause({ +export function buildInfiniteScrollWhereClause({ lower, upper, tags, @@ -174,7 +176,119 @@ export function buildGeneralLogQuery( return `${select}${condition}${dimensions}${fillClause}` } -export function buildBackwardLogQuery( +export async function getQueryCountForBounds( + lower: string, + upper: string, + config: QueryConfig, + filters: Filter[], + searchTerm: string, + proxyLink: string, + namespace: Namespace +): Promise { + const {database, retentionPolicy, measurement} = config + + let rpSegment = '' + if (retentionPolicy) { + rpSegment = `"${retentionPolicy}"` + } + + const fullyQualifiedMeasurement = `"${database}".${rpSegment}."${measurement}"` + const select = `SELECT count(message) FROM ${fullyQualifiedMeasurement}` + let condition = `WHERE time >= '${lower}' AND time <='${upper}'` + + if (!_.isEmpty(searchTerm)) { + condition = `${condition} AND message =~ ${new RegExp(searchTerm)}` + } + + if (!_.isEmpty(filters)) { + condition = `${condition} AND ${filtersClause(filters)}` + } + + const query = `${select} ${condition} FILL(0)` + const result = await executeQueryAsync(proxyLink, namespace, query) + return getDeep(result, 'results.0.series.0.values.0.1', 0) +} + +export async function findOlderLowerTimeBounds( + upper: string, + config: QueryConfig, + filters: Filter[], + searchTerm: string | null = null, + proxyLink: string, + namespace: Namespace +): Promise { + const parsedUpper = moment(upper) + + let secondsBack = 30 + let currentLower = parsedUpper.subtract(secondsBack, 'seconds') + + while (true) { + if (secondsBack > SECONDS_AWAY_LIMIT) { + break + } + + const count = await getQueryCountForBounds( + currentLower.toISOString(), + upper, + config, + filters, + searchTerm, + proxyLink, + namespace + ) + + if (count >= 400) { + break + } + + secondsBack *= secondsBack // exponential backoff + currentLower = parsedUpper.subtract(secondsBack, 'seconds') + } + + return currentLower.toISOString() +} + +export async function findNewerUpperTimeBounds( + lower: string, + config: QueryConfig, + filters: Filter[], + searchTerm: string | null = null, + proxyLink: string, + namespace: Namespace +): Promise { + const parsedLower = moment(lower) + + let secondsForward = 30 + let currentUpper = parsedLower.add(secondsForward, 'seconds') + + while (true) { + if (secondsForward > SECONDS_AWAY_LIMIT) { + break + } + + const count = await getQueryCountForBounds( + lower, + currentUpper.toISOString(), + config, + filters, + searchTerm, + proxyLink, + namespace + ) + + if (count >= 400) { + break + } + + secondsForward *= secondsForward // exponential backoff + currentUpper = parsedLower.add(secondsForward, 'seconds') + } + + return currentUpper.toISOString() +} + +export async function buildInfiniteScrollLogQuery( + lower: string, upper: string, config: QueryConfig, filters: Filter[], @@ -182,25 +296,9 @@ export function buildBackwardLogQuery( ) { const {tags, areTagsAccepted} = config - const condition = buildInfiniteWhereClause({ - upper, - tags, - areTagsAccepted, - }) - - return buildGeneralLogQuery(condition, config, filters, searchTerm) -} - -export function buildForwardLogQuery( - lower: string, - config: QueryConfig, - filters: Filter[], - searchTerm: string | null = null -) { - const {tags, areTagsAccepted} = config - - const condition = buildInfiniteWhereClause({ + const condition = buildInfiniteScrollWhereClause({ lower, + upper, tags, areTagsAccepted, }) diff --git a/ui/src/logs/utils/table.ts b/ui/src/logs/utils/table.ts index ca7956478..e897094af 100644 --- a/ui/src/logs/utils/table.ts +++ b/ui/src/logs/utils/table.ts @@ -10,6 +10,7 @@ import { export const ROW_HEIGHT = 18 const CHAR_WIDTH = 9 +const DEFAULT_COLUMN_WIDTH = 200 export const getValuesFromData = (data: TableData): string[][] => getDeep(data, 'values', []) @@ -37,6 +38,9 @@ export const formatColumnValue = ( switch (column) { case 'timestamp': return moment(+value / 1000000).format('YYYY/MM/DD HH:mm:ss') + case 'appname': + const length = Math.floor(DEFAULT_COLUMN_WIDTH / CHAR_WIDTH) - 2 + return _.truncate(value || '', {length}) case 'message': value = (value || 'No Message Provided').replace('\\n', '') if (value.indexOf(' ') > charLimit - 5) { @@ -70,7 +74,7 @@ export const getColumnWidth = (column: string): number => { host: 300, }, column, - 200 + DEFAULT_COLUMN_WIDTH ) }