From 61d5c1766ee04c2c70083721a128475595966095 Mon Sep 17 00:00:00 2001 From: Brandon Farmer Date: Fri, 6 Jul 2018 10:12:24 -0700 Subject: [PATCH 1/5] Allow selection of single time from log viewer --- ui/src/logs/actions/index.ts | 15 ++- ui/src/logs/components/LogsTable.tsx | 120 +++++++++++------- ui/src/logs/components/TimeRangeDropdown.tsx | 14 +- ui/src/logs/containers/LogsPage.tsx | 14 +- ui/src/logs/utils/index.ts | 2 + ui/src/logs/utils/table.ts | 22 ++++ .../shared/components/CustomSingularTime.tsx | 98 ++++++++++++++ ui/src/shared/components/CustomTimeRange.js | 17 +-- ui/src/shared/utils/time.ts | 22 ++++ 9 files changed, 247 insertions(+), 77 deletions(-) create mode 100644 ui/src/shared/components/CustomSingularTime.tsx create mode 100644 ui/src/shared/utils/time.ts diff --git a/ui/src/logs/actions/index.ts b/ui/src/logs/actions/index.ts index 3ef168c56..60bffbf38 100644 --- a/ui/src/logs/actions/index.ts +++ b/ui/src/logs/actions/index.ts @@ -62,7 +62,6 @@ export enum ActionTypes { ConcatMoreLogs = 'LOGS_CONCAT_MORE_LOGS', SetConfig = 'SET_CONFIG', } - export interface ConcatMoreLogsAction { type: ActionTypes.ConcatMoreLogs payload: { @@ -445,15 +444,17 @@ export const setNamespaces = ( }, }) +export const setTimeRange = timeRange => ({ + type: ActionTypes.SetTimeRange, + payload: { + timeRange, + }, +}) + export const setTimeRangeAsync = (timeRange: TimeRange) => async ( dispatch ): Promise => { - dispatch({ - type: ActionTypes.SetTimeRange, - payload: { - timeRange, - }, - }) + dispatch(setTimeRange(timeRange)) dispatch(setHistogramQueryConfigAsync()) dispatch(setTableQueryConfigAsync()) } diff --git a/ui/src/logs/components/LogsTable.tsx b/ui/src/logs/components/LogsTable.tsx index 6dd37f4a2..db878a03f 100644 --- a/ui/src/logs/components/LogsTable.tsx +++ b/ui/src/logs/components/LogsTable.tsx @@ -10,6 +10,9 @@ import {getDeep} from 'src/utils/wrappers' import {colorForSeverity} from 'src/logs/utils/colors' import { + ROW_HEIGHT, + calculateRowCharWidth, + calculateMessageHeight, getColumnFromData, getValueFromData, getValuesFromData, @@ -36,8 +39,6 @@ import { SeverityLevelColor, } from 'src/types/logs' -const ROW_HEIGHT = 26 -const CHAR_WIDTH = 9 interface Props { data: TableData isScrolledToTop: boolean @@ -50,6 +51,8 @@ interface Props { tableColumns: LogsTableColumn[] severityFormat: SeverityFormat severityLevelColors: SeverityLevelColor[] + scrollToRow?: number + hasScrolled: boolean } interface State { @@ -64,13 +67,35 @@ interface State { class LogsTable extends Component { public static getDerivedStateFromProps(props, state): State { - const {isScrolledToTop} = props + const { + isScrolledToTop, + scrollToRow, + data, + tableColumns, + severityFormat, + hasScrolled, + } = props + const currentMessageWidth = getMessageWidth( + data, + tableColumns, + severityFormat + ) let lastQueryTime = _.get(state, 'lastQueryTime', null) let scrollTop = _.get(state, 'scrollTop', 0) if (isScrolledToTop) { lastQueryTime = null scrollTop = 0 + } else if (scrollToRow && !hasScrolled) { + const rowCharLimit = calculateRowCharWidth(currentMessageWidth) + + scrollTop = _.reduce( + _.range(0, scrollToRow), + (acc, index) => { + return acc + calculateMessageHeight(index, data, rowCharLimit) + }, + 0 + ) } const scrollLeft = _.get(state, 'scrollLeft', 0) @@ -90,11 +115,7 @@ class LogsTable extends Component { scrollTop, scrollLeft, currentRow: -1, - currentMessageWidth: getMessageWidth( - props.data, - props.tableColumns, - props.severityFormat - ), + currentMessageWidth, isMessageVisible, visibleColumnsCount, } @@ -204,21 +225,13 @@ class LogsTable extends Component { autoHide={false} > { - registerChild(ref) - this.grid = ref - }} + {...this.gridProperties( + width, + height, + onRowsRendered, + columnCount, + registerChild + )} style={{ height: this.calculateTotalHeight(), overflowY: 'hidden', @@ -233,6 +246,40 @@ class LogsTable extends Component { ) } + private gridProperties = ( + width: number, + height: number, + onRowsRendered: (params: {startIndex: number; stopIndex: number}) => void, + columnCount: number, + registerChild: (g: Grid) => void + ) => { + const {hasScrolled, scrollToRow} = this.props + const {scrollLeft, scrollTop} = this.state + const result: {scrollToRow?: number} & any = { + width, + height, + rowHeight: this.calculateRowHeight, + rowCount: getValuesFromData(this.props.data).length, + scrollLeft, + scrollTop, + cellRenderer: this.cellRenderer, + onSectionRendered: this.handleRowRender(onRowsRendered), + onScroll: this.handleGridScroll, + columnCount, + columnWidth: this.getColumnWidth, + ref: (ref: Grid) => { + registerChild(ref) + this.grid = ref + }, + } + + if (!hasScrolled && scrollToRow) { + result.scrollToRow = scrollToRow + } + + return result + } + private handleGridScroll = ({scrollLeft}) => { this.handleScroll({scrollLeft}) } @@ -346,8 +393,7 @@ class LogsTable extends Component { } private get rowCharLimit(): number { - const {currentMessageWidth} = this.state - return Math.floor(currentMessageWidth / CHAR_WIDTH) + return calculateRowCharWidth(this.state.currentMessageWidth) } private calculateTotalHeight = (): number => { @@ -356,29 +402,17 @@ class LogsTable extends Component { return _.reduce( data, (acc, __, index) => { - return acc + this.calculateMessageHeight(index) + return ( + acc + + calculateMessageHeight(index, this.props.data, this.rowCharLimit) + ) }, 0 ) } - private calculateMessageHeight = (index: number): number => { - const columns = getColumnsFromData(this.props.data) - const columnIndex = columns.indexOf('message') - const value = getValueFromData(this.props.data, index, columnIndex) - - if (_.isEmpty(value)) { - return ROW_HEIGHT - } - - const lines = Math.ceil(value.length / (this.rowCharLimit * 0.95)) - - return Math.max(lines, 1) * ROW_HEIGHT + 4 - } - - private calculateRowHeight = ({index}: {index: number}): number => { - return this.calculateMessageHeight(index) - } + private calculateRowHeight = ({index}: {index: number}): number => + calculateMessageHeight(index, this.props.data, this.rowCharLimit) private headerRenderer = ({key, style, columnIndex}) => { const column = getColumnFromData(this.props.data, columnIndex) diff --git a/ui/src/logs/components/TimeRangeDropdown.tsx b/ui/src/logs/components/TimeRangeDropdown.tsx index ef48dd5cc..d34ae2a3d 100644 --- a/ui/src/logs/components/TimeRangeDropdown.tsx +++ b/ui/src/logs/components/TimeRangeDropdown.tsx @@ -8,7 +8,7 @@ import timeRanges from 'src/logs/data/timeRanges' import {DROPDOWN_MENU_MAX_HEIGHT} from 'src/shared/constants/index' import {ErrorHandling} from 'src/shared/decorators/errors' import {ClickOutside} from 'src/shared/components/ClickOutside' -import CustomTimeRange from 'src/shared/components/CustomTimeRange' +import CustomSingularTime from 'src/shared/components/CustomSingularTime' import {TimeRange} from 'src/types' @@ -56,7 +56,7 @@ class TimeRangeDropdown extends Component { } public render() { - const {selected, preventCustomTimeRange, page} = this.props + const {selected, preventCustomTimeRange} = this.props const {customTimeRange, isCustomTimeRangeOpen} = this.state return ( @@ -113,13 +113,11 @@ class TimeRangeDropdown extends Component { {isCustomTimeRangeOpen ? (
-
diff --git a/ui/src/logs/containers/LogsPage.tsx b/ui/src/logs/containers/LogsPage.tsx index 9a89bb828..26c2127f9 100644 --- a/ui/src/logs/containers/LogsPage.tsx +++ b/ui/src/logs/containers/LogsPage.tsx @@ -80,6 +80,7 @@ interface State { liveUpdating: boolean isOverlayVisible: boolean histogramColors: HistogramColor[] + hasScrolled: boolean } class LogsPage extends PureComponent { @@ -106,6 +107,7 @@ class LogsPage extends PureComponent { liveUpdating: false, isOverlayVisible: false, histogramColors: [], + hasScrolled: false, } } @@ -163,6 +165,7 @@ class LogsPage extends PureComponent { tableColumns={this.tableColumns} severityFormat={this.severityFormat} severityLevelColors={this.severityLevelColors} + hasScrolled={this.state.hasScrolled} /> @@ -183,7 +186,10 @@ class LogsPage extends PureComponent { const updatedColumns: string[] = _.get(orderedData, '0', []) const updatedValues = _.slice(orderedData, 1) - return {columns: updatedColumns, values: updatedValues} + return { + columns: updatedColumns, + values: updatedValues, + } } private get logConfigLink(): string { @@ -219,8 +225,8 @@ class LogsPage extends PureComponent { private handleVerticalScroll = () => { if (this.state.liveUpdating) { clearInterval(this.interval) - this.setState({liveUpdating: false}) } + this.setState({liveUpdating: false, hasScrolled: true}) } private handleTagSelection = (selection: {tag: string; key: string}) => { @@ -299,8 +305,8 @@ class LogsPage extends PureComponent { const {liveUpdating} = this.state if (liveUpdating) { - clearInterval(this.interval) this.setState({liveUpdating: false}) + clearInterval(this.interval) } else { this.startUpdating() } @@ -351,8 +357,8 @@ class LogsPage extends PureComponent { } private fetchNewDataset() { - this.props.executeQueriesAsync() this.setState({liveUpdating: true}) + this.props.executeQueriesAsync() } private handleToggleOverlay = (): void => { diff --git a/ui/src/logs/utils/index.ts b/ui/src/logs/utils/index.ts index e11546952..989f089c3 100644 --- a/ui/src/logs/utils/index.ts +++ b/ui/src/logs/utils/index.ts @@ -144,6 +144,8 @@ const computeSeconds = (range: TimeRange) => { if (seconds) { return seconds + } else if (upper && upper.match(/now/) && lower) { + return moment().unix() - moment(lower).unix() } else if (upper && lower) { return moment(upper).unix() - moment(lower).unix() } else { diff --git a/ui/src/logs/utils/table.ts b/ui/src/logs/utils/table.ts index fe846140e..786d4a9d6 100644 --- a/ui/src/logs/utils/table.ts +++ b/ui/src/logs/utils/table.ts @@ -4,6 +4,7 @@ import {getDeep} from 'src/utils/wrappers' import {TableData, LogsTableColumn, SeverityFormat} from 'src/types/logs' import {SeverityFormatOptions} from 'src/logs/constants' +export const ROW_HEIGHT = 26 const CHAR_WIDTH = 9 export const getValuesFromData = (data: TableData): string[][] => @@ -69,6 +70,27 @@ export const getColumnWidth = (column: string): number => { ) } +export const calculateRowCharWidth = (currentMessageWidth: number): number => + Math.floor(currentMessageWidth / CHAR_WIDTH) + +export const calculateMessageHeight = ( + index: number, + data: TableData, + rowCharLimit: number +): number => { + const columns = getColumnsFromData(data) + const columnIndex = columns.indexOf('message') + const value = getValueFromData(data, index, columnIndex) + + if (_.isEmpty(value)) { + return ROW_HEIGHT + } + + const lines = Math.ceil(value.length / (rowCharLimit * 0.95)) + + return Math.max(lines, 1) * ROW_HEIGHT + 4 +} + export const getMessageWidth = ( data: TableData, tableColumns: LogsTableColumn[], diff --git a/ui/src/shared/components/CustomSingularTime.tsx b/ui/src/shared/components/CustomSingularTime.tsx new file mode 100644 index 000000000..c6757216a --- /dev/null +++ b/ui/src/shared/components/CustomSingularTime.tsx @@ -0,0 +1,98 @@ +import React, {Component} from 'react' +import rome from 'rome' +import {ErrorHandling} from 'src/shared/decorators/errors' +import {formatTimeRange} from 'src/shared/utils/time' + +import {TimeRange} from 'src/types' + +interface Props { + onSelected: (timeRange: TimeRange) => void + time: string + timeInterval?: number + onClose?: () => void +} + +interface State { + time: string +} + +@ErrorHandling +class CustomSingularTime extends Component { + private calendar?: any + private containerRef: React.RefObject = React.createRef< + HTMLDivElement + >() + private inputRef: React.RefObject = React.createRef< + HTMLInputElement + >() + + constructor(props: Props) { + super(props) + + this.state = { + time: props.time, + } + } + + public componentDidMount() { + const {time, timeInterval} = this.props + + this.calendar = rome(this.inputRef.current, { + appendTo: this.containerRef.current, + initialValue: formatTimeRange(time), + autoClose: false, + autoHideOnBlur: false, + autoHideOnClick: false, + timeInterval, + }) + + this.calendar.show() + } + + public render() { + return ( +
+
+
+
+ +
+
+
+ Apply +
+
+
+ ) + } + + private handleRefreshCalendar = () => { + if (this.calendar) { + this.calendar.refresh() + } + } + + private handleClick = () => { + const date = this.calendar.getDate() + if (date) { + const lower = date.toISOString() + this.props.onSelected({lower, upper: 'now()'}) + } + + if (this.props.onClose) { + this.props.onClose() + } + } +} + +export default CustomSingularTime diff --git a/ui/src/shared/components/CustomTimeRange.js b/ui/src/shared/components/CustomTimeRange.js index 056605c0c..a3976c010 100644 --- a/ui/src/shared/components/CustomTimeRange.js +++ b/ui/src/shared/components/CustomTimeRange.js @@ -3,6 +3,7 @@ import PropTypes from 'prop-types' import rome from 'rome' import moment from 'moment' +import {formatTimeRange} from 'shared/utils/time' import shortcuts from 'shared/data/timeRangeShortcuts' import {ErrorHandling} from 'src/shared/decorators/errors' const dateFormat = 'YYYY-MM-DD HH:mm' @@ -90,21 +91,7 @@ class CustomTimeRange extends Component { * before passing the string to be parsed. */ _formatTimeRange = timeRange => { - if (!timeRange) { - return '' - } - - if (timeRange === 'now()') { - return moment(new Date()).format(dateFormat) - } - - // If the given time range is relative, create a fixed timestamp based on its value - if (timeRange.match(/^now/)) { - const [, duration, unitOfTime] = timeRange.match(/(\d+)(\w+)/) - moment().subtract(duration, unitOfTime) - } - - return moment(timeRange.replace(/\'/g, '')).format(dateFormat) + return formatTimeRange(timeRange) } handleClick = () => { diff --git a/ui/src/shared/utils/time.ts b/ui/src/shared/utils/time.ts new file mode 100644 index 000000000..3cdde0cfc --- /dev/null +++ b/ui/src/shared/utils/time.ts @@ -0,0 +1,22 @@ +import moment from 'moment' + +const dateFormat = 'YYYY-MM-DD HH:mm' + +export const formatTimeRange = (timeRange: string | null): string => { + if (!timeRange) { + return '' + } + + if (timeRange === 'now()') { + return moment(new Date()).format(dateFormat) + } + + if (timeRange.match(/^now/)) { + const [, duration, unitOfTime] = timeRange.match(/(\d+)(\w+)/) + const d = duration as moment.unitOfTime.DurationConstructor + + moment().subtract(d, unitOfTime) + } + + return moment(timeRange.replace(/\'/g, '')).format(dateFormat) +} From 1448fbb93ef16eb19497c76ad29bc919c80f139d Mon Sep 17 00:00:00 2001 From: Brandon Farmer Date: Tue, 10 Jul 2018 15:05:59 -0700 Subject: [PATCH 2/5] Initial duel load of log viewer data --- ui/src/logs/actions/index.ts | 210 ++++++++++++++---- ui/src/logs/components/LogsTable.tsx | 1 + .../logs/components/PointInTimeDropDown.tsx | 158 +++++++++++++ ui/src/logs/components/TimeRangeDropdown.tsx | 54 ++--- ui/src/logs/containers/LogsPage.tsx | 69 ++++-- ui/src/logs/data/serverLogData.ts | 100 +++++++++ ui/src/logs/data/timePoints.ts | 26 +++ ui/src/logs/reducers/index.ts | 41 +++- ui/src/logs/utils/index.ts | 93 ++++++++ ui/src/logs/utils/table.ts | 23 ++ .../shared/components/CustomSingularTime.tsx | 9 +- ui/src/types/logs.ts | 8 + 12 files changed, 698 insertions(+), 94 deletions(-) create mode 100644 ui/src/logs/components/PointInTimeDropDown.tsx create mode 100644 ui/src/logs/data/serverLogData.ts create mode 100644 ui/src/logs/data/timePoints.ts diff --git a/ui/src/logs/actions/index.ts b/ui/src/logs/actions/index.ts index 60bffbf38..e2769d3d5 100644 --- a/ui/src/logs/actions/index.ts +++ b/ui/src/logs/actions/index.ts @@ -9,6 +9,8 @@ import { buildHistogramQueryConfig, buildTableQueryConfig, buildLogQuery, + buildForwardLogQuery, + buildBackwardLogQuery, parseHistogramQueryResponse, } from 'src/logs/utils' import { @@ -21,8 +23,11 @@ import { // getLogConfig as getLogConfigAJAX, // updateLogConfig as updateLogConfigAJAX, } from 'src/logs/api' +import serverLogData from 'src/logs/data/serverLogData' import {LogsState, Filter, TableData, LogConfig} from 'src/types/logs' +const INITIAL_LIMIT = 1000 + const defaultTableData: TableData = { columns: [ 'time', @@ -61,7 +66,40 @@ export enum ActionTypes { DecrementQueryCount = 'LOGS_DECREMENT_QUERY_COUNT', ConcatMoreLogs = 'LOGS_CONCAT_MORE_LOGS', SetConfig = 'SET_CONFIG', + SetTableRelativeTime = 'SET_TABLE_RELATIVE_TIME', + SetTableCustomTime = 'SET_TABLE_CUSTOM_TIME', + SetTableForwardData = 'SET_TABLE_FORWARD_DATA', + SetTableBackwardData = 'SET_TABLE_BACKWARD_DATA', } + +export interface SetTableForwardDataAction { + type: ActionTypes.SetTableForwardData + payload: { + data: TableData + } +} + +export interface SetTableBackwardDataAction { + type: ActionTypes.SetTableBackwardData + payload: { + data: TableData + } +} + +export interface SetTableRelativeTimeAction { + type: ActionTypes.SetTableRelativeTime + payload: { + time: number + } +} + +export interface SetTableCustomTimeAction { + type: ActionTypes.SetTableCustomTime + payload: { + time: string + } +} + export interface ConcatMoreLogsAction { type: ActionTypes.ConcatMoreLogs payload: { @@ -194,6 +232,10 @@ export type Action = | IncrementQueryCountAction | ConcatMoreLogsAction | SetConfigsAction + | SetTableCustomTimeAction + | SetTableRelativeTimeAction + | SetTableForwardDataAction + | SetTableBackwardDataAction const getTimeRange = (state: State): TimeRange | null => getDeep(state, 'logs.timeRange', null) @@ -216,6 +258,131 @@ const getSearchTerm = (state: State): string | null => const getFilters = (state: State): Filter[] => getDeep(state, 'logs.filters', []) +const getTableSelectedTime = (state: State): string => { + const custom = getDeep(state, 'logs.tableTime.custom', '') + + if (!_.isEmpty(custom)) { + return custom + } + + const relative = getDeep(state, 'logs.tableTime.relative', 0) + + return moment() + .subtract(relative, 'seconds') + .toISOString() +} + +export const setTableCustomTime = (time: string): SetTableCustomTimeAction => ({ + type: ActionTypes.SetTableCustomTime, + payload: {time}, +}) + +export const setTableRelativeTime = ( + time: number +): SetTableRelativeTimeAction => ({ + type: ActionTypes.SetTableRelativeTime, + payload: {time}, +}) + +export const setTableForwardData = ( + data: TableData +): SetTableForwardDataAction => ({ + type: ActionTypes.SetTableForwardData, + payload: {data}, +}) + +export const setTableBackwardData = ( + data: TableData +): SetTableBackwardDataAction => ({ + type: ActionTypes.SetTableBackwardData, + payload: {data}, +}) + +export const executeTableForwardQueryAsync = () => async ( + dispatch, + getState: GetState +) => { + const state = getState() + + const time = getTableSelectedTime(state) + const queryConfig = getTableQueryConfig(state) + const namespace = getNamespace(state) + const proxyLink = getProxyLink(state) + const searchTerm = getSearchTerm(state) + const filters = getFilters(state) + + if (!_.every([queryConfig, time, namespace, proxyLink])) { + return + } + + try { + dispatch(incrementQueryCount()) + + const query = buildForwardLogQuery(time, queryConfig, filters, searchTerm) + const response = await executeQueryAsync( + proxyLink, + namespace, + `${query} ORDER BY time ASC LIMIT ${INITIAL_LIMIT}` + ) + + const series = getDeep(response, 'results.0.series.0', defaultTableData) + + const result = { + columns: series.columns, + values: _.reverse(series.values), + } + + dispatch(setTableForwardData(result)) + } finally { + dispatch(decrementQueryCount()) + } +} + +export const executeTableBackwardQueryAsync = () => async ( + dispatch, + getState: GetState +) => { + const state = getState() + + const time = getTableSelectedTime(state) + const queryConfig = getTableQueryConfig(state) + const namespace = getNamespace(state) + const proxyLink = getProxyLink(state) + const searchTerm = getSearchTerm(state) + const filters = getFilters(state) + + if (!_.every([queryConfig, time, namespace, proxyLink])) { + return + } + + try { + dispatch(incrementQueryCount()) + + const query = buildBackwardLogQuery(time, queryConfig, filters, searchTerm) + const response = await executeQueryAsync( + proxyLink, + namespace, + `${query} ORDER BY time DESC LIMIT ${INITIAL_LIMIT}` + ) + + const series = getDeep(response, 'results.0.series.0', defaultTableData) + + dispatch(setTableBackwardData(series)) + } finally { + dispatch(decrementQueryCount()) + } +} + +export const setTableCustomTimeAsync = (time: string) => async dispatch => { + await dispatch(setTableCustomTime(time)) + await dispatch(executeTableQueryAsync()) +} + +export const setTableRelativeTimeAsync = (time: number) => async dispatch => { + await dispatch(setTableRelativeTime(time)) + await dispatch(executeTableQueryAsync()) +} + export const changeFilter = (id: string, operator: string, value: string) => ({ type: ActionTypes.ChangeFilter, payload: {id, operator, value}, @@ -271,44 +438,11 @@ export const executeHistogramQueryAsync = () => async ( } } -const setTableData = (series: TableData): SetTableData => ({ - type: ActionTypes.SetTableData, - payload: {data: {columns: series.columns, values: series.values}}, -}) - -export const executeTableQueryAsync = () => async ( - dispatch, - getState: GetState -): Promise => { - const state = getState() - - const queryConfig = getTableQueryConfig(state) - const timeRange = getTimeRange(state) - const namespace = getNamespace(state) - const proxyLink = getProxyLink(state) - const searchTerm = getSearchTerm(state) - const filters = getFilters(state) - - if (!_.every([queryConfig, timeRange, namespace, proxyLink])) { - return - } - - try { - dispatch(incrementQueryCount()) - - const query = buildLogQuery(timeRange, queryConfig, filters, searchTerm) - const response = await executeQueryAsync( - proxyLink, - namespace, - `${query} ORDER BY time DESC LIMIT 1000` - ) - - const series = getDeep(response, 'results.0.series.0', defaultTableData) - - dispatch(setTableData(series)) - } finally { - dispatch(decrementQueryCount()) - } +export const executeTableQueryAsync = () => async (dispatch): Promise => { + await Promise.all([ + dispatch(executeTableForwardQueryAsync()), + dispatch(executeTableBackwardQueryAsync()), + ]) } export const decrementQueryCount = () => ({ diff --git a/ui/src/logs/components/LogsTable.tsx b/ui/src/logs/components/LogsTable.tsx index db878a03f..200208a81 100644 --- a/ui/src/logs/components/LogsTable.tsx +++ b/ui/src/logs/components/LogsTable.tsx @@ -323,6 +323,7 @@ class LogsTable extends Component { } private loadMoreRows = async () => { + return const data = getValuesFromData(this.props.data) const {timeRange} = this.props const lastTime = getDeep( diff --git a/ui/src/logs/components/PointInTimeDropDown.tsx b/ui/src/logs/components/PointInTimeDropDown.tsx new file mode 100644 index 000000000..15d8a7af2 --- /dev/null +++ b/ui/src/logs/components/PointInTimeDropDown.tsx @@ -0,0 +1,158 @@ +import React, {Component, MouseEvent} from 'react' +import classnames from 'classnames' +import moment from 'moment' + +import FancyScrollbar from 'src/shared/components/FancyScrollbar' +import timePoints from 'src/logs/data/timePoints' +import {DROPDOWN_MENU_MAX_HEIGHT} from 'src/shared/constants/index' +import {ErrorHandling} from 'src/shared/decorators/errors' +import {ClickOutside} from 'src/shared/components/ClickOutside' +import CustomSingularTime from 'src/shared/components/CustomSingularTime' + +interface Props { + customTime?: string + relativeTime?: number + onChooseCustomTime: (time: string) => void + onChooseRelativeTime: (time: number) => void +} + +interface State { + isOpen: boolean + isTimeSelectorOpen: boolean +} + +const dateFormat = 'YYYY-MM-DD HH:mm' +const format = t => moment(t.replace(/\'/g, '')).format(dateFormat) + +@ErrorHandling +class TimeRangeDropdown extends Component { + constructor(props) { + super(props) + + this.state = { + isOpen: false, + isTimeSelectorOpen: false, + } + } + + public render() { + const {isTimeSelectorOpen} = this.state + + return ( + +
+
+
+ + {this.timeInputValue} + +
+ +
+ {isTimeSelectorOpen ? ( + +
+ +
+ + ) : null} +
+ + ) + } + + private get dropdownClassName(): string { + const {isOpen} = this.state + const absoluteTimeRange = !!this.props.customTime + + return classnames('dropdown', { + 'dropdown-290': absoluteTimeRange, + 'dropdown-120': !absoluteTimeRange, + open: isOpen, + }) + } + + private handleCustomSelection = (time: string) => { + this.handleCloseCustomTime() + this.props.onChooseCustomTime(time) + this.setState({isOpen: false}) + } + + private handleSelection = (e: MouseEvent) => { + e.preventDefault() + const {dataset} = e.target as HTMLAnchorElement + this.props.onChooseRelativeTime(+dataset.value) + this.setState({isOpen: false}) + } + + private get timeInputValue(): string { + if (!this.props.customTime) { + const point = timePoints.find(p => p.value === this.props.relativeTime) + if (point) { + return point.text + } + + return 'None' + } + + return format(this.props.customTime) + } + + private handleClickOutside = () => { + this.setState({isOpen: false}) + } + + private toggleMenu = () => { + this.setState({isOpen: !this.state.isOpen}) + } + + private handleCloseCustomTime = () => { + this.setState({isTimeSelectorOpen: false}) + } + + private handleOpenCustomTime = () => { + this.setState({isTimeSelectorOpen: true}) + } +} +export default TimeRangeDropdown diff --git a/ui/src/logs/components/TimeRangeDropdown.tsx b/ui/src/logs/components/TimeRangeDropdown.tsx index d34ae2a3d..9bcb7debe 100644 --- a/ui/src/logs/components/TimeRangeDropdown.tsx +++ b/ui/src/logs/components/TimeRangeDropdown.tsx @@ -8,7 +8,7 @@ import timeRanges from 'src/logs/data/timeRanges' import {DROPDOWN_MENU_MAX_HEIGHT} from 'src/shared/constants/index' import {ErrorHandling} from 'src/shared/decorators/errors' import {ClickOutside} from 'src/shared/components/ClickOutside' -import CustomSingularTime from 'src/shared/components/CustomSingularTime' +import CustomTimeRange from 'src/shared/components/CustomTimeRange' import {TimeRange} from 'src/types' @@ -23,8 +23,6 @@ interface Props { } onChooseTimeRange: (timeRange: TimeRange) => void - preventCustomTimeRange?: boolean - page?: string } interface State { @@ -36,10 +34,6 @@ interface State { @ErrorHandling class TimeRangeDropdown extends Component { - public static defaultProps = { - page: 'default', - } - constructor(props) { super(props) const {lower, upper} = props.selected @@ -56,7 +50,7 @@ class TimeRangeDropdown extends Component { } public render() { - const {selected, preventCustomTimeRange} = this.props + const {selected} = this.props const {customTimeRange, isCustomTimeRangeOpen} = this.state return ( @@ -79,25 +73,21 @@ class TimeRangeDropdown extends Component { autoHeight={true} maxHeight={DROPDOWN_MENU_MAX_HEIGHT} > - {preventCustomTimeRange ? null : ( -
-
  • Absolute Time
    • -
  • - - Date Picker - -
    • -
    - )} -
  • - {preventCustomTimeRange ? '' : 'Relative '}Time -
    • +
    +
  • Absolute Time
    • +
  • + + Date Picker + +
    • +
    +
  • Relative Time
    • {timeRanges.map(item => { return (
  • @@ -113,11 +103,13 @@ class TimeRangeDropdown extends Component { {isCustomTimeRangeOpen ? (
    -
    @@ -129,9 +121,7 @@ class TimeRangeDropdown extends Component { private get dropdownClassName(): string { const {isOpen} = this.state - const {lower, upper} = _.get(this.props, 'selected', {upper: '', lower: ''}) - const absoluteTimeRange = !_.isEmpty(lower) && !_.isEmpty(upper) return classnames('dropdown', { diff --git a/ui/src/logs/containers/LogsPage.tsx b/ui/src/logs/containers/LogsPage.tsx index 26c2127f9..4105d31c5 100644 --- a/ui/src/logs/containers/LogsPage.tsx +++ b/ui/src/logs/containers/LogsPage.tsx @@ -5,6 +5,8 @@ import {connect} from 'react-redux' import {AutoSizer} from 'react-virtualized' import { + setTableCustomTimeAsync, + setTableRelativeTimeAsync, getSourceAndPopulateNamespacesAsync, setTimeRangeAsync, setNamespaceAsync, @@ -26,14 +28,10 @@ import OptionsOverlay from 'src/logs/components/OptionsOverlay' import SearchBar from 'src/logs/components/LogsSearchBar' import FilterBar from 'src/logs/components/LogsFilterBar' import LogsTable from 'src/logs/components/LogsTable' +import PointInTimeDropDown from 'src/logs/components/PointInTimeDropDown' import {getDeep} from 'src/utils/wrappers' import {colorForSeverity} from 'src/logs/utils/colors' import OverlayTechnology from 'src/reusable_ui/components/overlays/OverlayTechnology' -import { - orderTableColumns, - filterTableColumns, -} from 'src/dashboards/utils/tableGraph' - import {SeverityFormatOptions} from 'src/logs/constants' import {Source, Namespace, TimeRange} from 'src/types' @@ -46,6 +44,7 @@ import { LogConfig, TableData, } from 'src/types/logs' +import {applyChangesToTableData} from 'src/logs/utils/table' interface Props { sources: Source[] @@ -59,6 +58,8 @@ interface Props { changeZoomAsync: (timeRange: TimeRange) => void executeQueriesAsync: () => void setSearchTermAsync: (searchTerm: string) => void + setTableRelativeTime: (time: number) => void + setTableCustomTime: (time: string) => void fetchMoreAsync: (queryTimeEnd: string, lastTime: number) => Promise addFilter: (filter: Filter) => void removeFilter: (id: string) => void @@ -73,6 +74,14 @@ interface Props { queryCount: number logConfig: LogConfig logConfigLink: string + tableInfiniteData: { + forward: TableData + backward: TableData + } + tableTime: { + custom: string + relative: number + } } interface State { @@ -134,7 +143,7 @@ class LogsPage extends PureComponent { public render() { const {liveUpdating} = this.state - const {searchTerm, filters, queryCount, timeRange} = this.props + const {searchTerm, filters, queryCount, timeRange, tableTime} = this.props return ( <> @@ -142,6 +151,17 @@ class LogsPage extends PureComponent { {this.header}
    {this.chart} +
    +
    + Go to + +
    +
    { ) } - private get tableData(): TableData { - const {tableData} = this.props - const tableColumns = this.tableColumns - const columns = _.get(tableData, 'columns', []) - const values = _.get(tableData, 'values', []) - const data = [columns, ...values] + private handleChooseCustomTime = (time: string) => { + this.props.setTableCustomTime(time) + } - const filteredData = filterTableColumns(data, tableColumns) - const orderedData = orderTableColumns(filteredData, tableColumns) - const updatedColumns: string[] = _.get(orderedData, '0', []) - const updatedValues = _.slice(orderedData, 1) + private handleChooseRelativeTime = (time: number) => { + this.props.setTableRelativeTime(time) + } + + private get tableData(): TableData { + const forwardData = applyChangesToTableData( + this.props.tableInfiniteData.forward, + this.tableColumns + ) + + const backwardData = applyChangesToTableData( + this.props.tableInfiniteData.backward, + this.tableColumns + ) return { - columns: updatedColumns, - values: updatedValues, + columns: forwardData.columns, + values: [...forwardData.values, ...backwardData.values], } } @@ -436,6 +463,8 @@ const mapStateToProps = ({ filters, queryCount, logConfig, + tableTime, + tableInfiniteData, }, }) => ({ sources, @@ -449,7 +478,9 @@ const mapStateToProps = ({ filters, queryCount, logConfig, + tableTime, logConfigLink: logViewer, + tableInfiniteData, }) const mapDispatchToProps = { @@ -464,6 +495,8 @@ const mapDispatchToProps = { removeFilter, changeFilter, fetchMoreAsync, + setTableCustomTime: setTableCustomTimeAsync, + setTableRelativeTime: setTableRelativeTimeAsync, getConfig: getLogConfigAsync, updateConfig: updateLogConfigAsync, } diff --git a/ui/src/logs/data/serverLogData.ts b/ui/src/logs/data/serverLogData.ts new file mode 100644 index 000000000..ab006ee2e --- /dev/null +++ b/ui/src/logs/data/serverLogData.ts @@ -0,0 +1,100 @@ +export default { + columns: [ + { + name: 'severity', + position: 1, + encodings: [ + { + type: 'visibility', + value: 'visible', + }, + { + type: 'label', + value: 'icon', + }, + { + type: 'label', + value: 'text', + }, + ], + }, + { + name: 'timestamp', + position: 2, + encodings: [ + { + type: 'visibility', + value: 'visible', + }, + ], + }, + { + name: 'message', + position: 3, + encodings: [ + { + type: 'visibility', + value: 'visible', + }, + ], + }, + { + name: 'facility', + position: 4, + encodings: [ + { + type: 'visibility', + value: 'visible', + }, + ], + }, + { + name: 'time', + position: 0, + encodings: [ + { + type: 'visibility', + value: 'hidden', + }, + ], + }, + { + name: 'procid', + position: 5, + encodings: [ + { + type: 'visibility', + value: 'visible', + }, + { + type: 'displayName', + value: 'Proc ID', + }, + ], + }, + { + name: 'host', + position: 7, + encodings: [ + { + type: 'visibility', + value: 'visible', + }, + ], + }, + { + name: 'appname', + position: 6, + encodings: [ + { + type: 'visibility', + value: 'visible', + }, + { + type: 'displayName', + value: 'Application', + }, + ], + }, + ], +} diff --git a/ui/src/logs/data/timePoints.ts b/ui/src/logs/data/timePoints.ts new file mode 100644 index 000000000..fa07be2da --- /dev/null +++ b/ui/src/logs/data/timePoints.ts @@ -0,0 +1,26 @@ +export default [ + { + text: '1 minute ago', + value: 60, + }, + { + text: '5 minute ago', + value: 300, + }, + { + text: '10 minute ago', + value: 600, + }, + { + text: '30 minute ago', + value: 1800, + }, + { + text: '1 hour ago', + value: 3600, + }, + { + text: '3 hour ago', + value: 10800, + }, +] diff --git a/ui/src/logs/reducers/index.ts b/ui/src/logs/reducers/index.ts index e930904d0..fcd7683f2 100644 --- a/ui/src/logs/reducers/index.ts +++ b/ui/src/logs/reducers/index.ts @@ -13,7 +13,21 @@ import { } from 'src/logs/actions' import {SeverityFormatOptions} from 'src/logs/constants' -import {LogsState} from 'src/types/logs' +import {LogsState, TableData} from 'src/types/logs' + +const defaultTableData: TableData = { + columns: [ + 'time', + 'severity', + 'timestamp', + 'facility', + 'procid', + 'application', + 'host', + 'message', + ], + values: [], +} const defaultState: LogsState = { currentSource: null, @@ -32,6 +46,11 @@ const defaultState: LogsState = { severityFormat: SeverityFormatOptions.dotText, severityLevelColors: [], }, + tableTime: {}, + tableInfiniteData: { + forward: defaultTableData, + backward: defaultTableData, + }, } const removeFilter = ( @@ -135,11 +154,31 @@ export default (state: LogsState = defaultState, action: Action) => { return {...state, tableQueryConfig: action.payload.queryConfig} case ActionTypes.SetTableData: return {...state, tableData: action.payload.data} + case ActionTypes.SetTableForwardData: + return { + ...state, + tableInfiniteData: { + ...state.tableInfiniteData, + forward: action.payload.data, + }, + } + case ActionTypes.SetTableBackwardData: + return { + ...state, + tableInfiniteData: { + ...state.tableInfiniteData, + backward: action.payload.data, + }, + } case ActionTypes.ChangeZoom: return {...state, timeRange: action.payload.timeRange} case ActionTypes.SetSearchTerm: const {searchTerm} = action.payload return {...state, searchTerm} + case ActionTypes.SetTableCustomTime: + return {...state, tableTime: {custom: action.payload.time}} + case ActionTypes.SetTableRelativeTime: + return {...state, tableTime: {relative: action.payload.time}} case ActionTypes.AddFilter: return addFilter(state, action) case ActionTypes.RemoveFilter: diff --git a/ui/src/logs/utils/index.ts b/ui/src/logs/utils/index.ts index 989f089c3..5dec529e9 100644 --- a/ui/src/logs/utils/index.ts +++ b/ui/src/logs/utils/index.ts @@ -115,6 +115,99 @@ export const filtersClause = (filters: Filter[]): string => { ).join(' AND ') } +export function buildInfiniteWhereClause({ + lower, + upper, + tags, + areTagsAccepted, +}: QueryConfig): string { + const timeClauses = [] + + if (lower) { + timeClauses.push(`time >= '${lower}'`) + } + + if (upper) { + timeClauses.push(`time < '${upper}'`) + } + + const tagClauses = _.keys(tags).map(k => { + const operator = areTagsAccepted ? '=' : '!=' + + if (tags[k].length > 1) { + const joinedOnOr = tags[k] + .map(v => `"${k}"${operator}'${v}'`) + .join(' OR ') + return `(${joinedOnOr})` + } + + return `"${k}"${operator}'${tags[k]}'` + }) + + const subClauses = timeClauses.concat(tagClauses) + if (!subClauses.length) { + return '' + } + + return ` WHERE ${subClauses.join(' AND ')}` +} + +export function buildGeneralLogQuery( + condition: string, + config: QueryConfig, + filters: Filter[], + searchTerm: string | null = null +) { + const {groupBy, fill = NULL_STRING} = config + const select = buildSelect(config, '') + const dimensions = buildGroupBy(groupBy) + const fillClause = groupBy.time ? buildFill(fill) : '' + + if (!_.isEmpty(searchTerm)) { + condition = `${condition} AND message =~ ${new RegExp(searchTerm)}` + } + + if (!_.isEmpty(filters)) { + condition = `${condition} AND ${filtersClause(filters)}` + } + + return `${select}${condition}${dimensions}${fillClause}` +} + +export function buildBackwardLogQuery( + upper: string, + config: QueryConfig, + filters: Filter[], + searchTerm: string | null = null +) { + const {tags, areTagsAccepted} = config + + const condition = buildInfiniteWhereClause({ + upper, + tags, + areTagsAccepted, + }) + + return buildGeneralLogQuery(condition, config, filters, searchTerm) +} + +export function buildForwardLogQuery( + lower: string, + config: QueryConfig, + filters: Filter[], + searchTerm: string | null = null +) { + const {tags, areTagsAccepted} = config + + const condition = buildInfiniteWhereClause({ + lower, + tags, + areTagsAccepted, + }) + + return buildGeneralLogQuery(condition, config, filters, searchTerm) +} + export function buildLogQuery( timeRange: TimeRange, config: QueryConfig, diff --git a/ui/src/logs/utils/table.ts b/ui/src/logs/utils/table.ts index 786d4a9d6..02464e4f2 100644 --- a/ui/src/logs/utils/table.ts +++ b/ui/src/logs/utils/table.ts @@ -3,6 +3,10 @@ import moment from 'moment' import {getDeep} from 'src/utils/wrappers' import {TableData, LogsTableColumn, SeverityFormat} from 'src/types/logs' import {SeverityFormatOptions} from 'src/logs/constants' +import { + orderTableColumns, + filterTableColumns, +} from 'src/dashboards/utils/tableGraph' export const ROW_HEIGHT = 26 const CHAR_WIDTH = 9 @@ -119,3 +123,22 @@ export const getMessageWidth = ( return calculatedWidth - CHAR_WIDTH } + +export const applyChangesToTableData = ( + tableData: TableData, + tableColumns: LogsTableColumn[] +): TableData => { + const columns = _.get(tableData, 'columns', []) + const values = _.get(tableData, 'values', []) + const data = [columns, ...values] + + const filteredData = filterTableColumns(data, tableColumns) + const orderedData = orderTableColumns(filteredData, tableColumns) + const updatedColumns: string[] = _.get(orderedData, '0', []) + const updatedValues = _.slice(orderedData, 1) + + return { + columns: updatedColumns, + values: updatedValues, + } +} diff --git a/ui/src/shared/components/CustomSingularTime.tsx b/ui/src/shared/components/CustomSingularTime.tsx index c6757216a..b2c119ec1 100644 --- a/ui/src/shared/components/CustomSingularTime.tsx +++ b/ui/src/shared/components/CustomSingularTime.tsx @@ -3,10 +3,8 @@ import rome from 'rome' import {ErrorHandling} from 'src/shared/decorators/errors' import {formatTimeRange} from 'src/shared/utils/time' -import {TimeRange} from 'src/types' - interface Props { - onSelected: (timeRange: TimeRange) => void + onSelected: (time: string) => void time: string timeInterval?: number onClose?: () => void @@ -66,6 +64,7 @@ class CustomSingularTime extends Component {
    @@ -85,8 +84,8 @@ class CustomSingularTime extends Component { private handleClick = () => { const date = this.calendar.getDate() if (date) { - const lower = date.toISOString() - this.props.onSelected({lower, upper: 'now()'}) + const time = date.toISOString() + this.props.onSelected(time) } if (this.props.onClose) { diff --git a/ui/src/types/logs.ts b/ui/src/types/logs.ts index 404032261..c08441cdb 100644 --- a/ui/src/types/logs.ts +++ b/ui/src/types/logs.ts @@ -32,6 +32,14 @@ export interface LogsState { filters: Filter[] queryCount: number logConfig: LogConfig + tableInfiniteData: { + forward: TableData + backward: TableData + } + tableTime: { + custom?: string + relative?: string + } } export interface LogConfig { From a66b3de051bec67f884f3978bfa09a2f8f3b447f Mon Sep 17 00:00:00 2001 From: Brandon Farmer Date: Wed, 11 Jul 2018 14:48:36 -0700 Subject: [PATCH 3/5] Log viewer infinite scrolling initial implementation Co-authored-by: Iris Scholten --- ui/src/logs/actions/index.ts | 232 ++++++++++----------------- ui/src/logs/components/LogsTable.tsx | 114 ++++++++----- ui/src/logs/containers/LogsPage.tsx | 42 ++++- ui/src/logs/data/serverLogData.ts | 42 ++++- ui/src/logs/reducers/index.ts | 48 +++++- ui/src/types/logs.ts | 1 + 6 files changed, 281 insertions(+), 198 deletions(-) diff --git a/ui/src/logs/actions/index.ts b/ui/src/logs/actions/index.ts index e2769d3d5..8b8afbfd5 100644 --- a/ui/src/logs/actions/index.ts +++ b/ui/src/logs/actions/index.ts @@ -23,10 +23,10 @@ import { // getLogConfig as getLogConfigAJAX, // updateLogConfig as updateLogConfigAJAX, } from 'src/logs/api' -import serverLogData from 'src/logs/data/serverLogData' +import {serverLogData} from 'src/logs/data/serverLogData' import {LogsState, Filter, TableData, LogConfig} from 'src/types/logs' -const INITIAL_LIMIT = 1000 +const INITIAL_LIMIT = 100 const defaultTableData: TableData = { columns: [ @@ -65,11 +65,17 @@ export enum ActionTypes { IncrementQueryCount = 'LOGS_INCREMENT_QUERY_COUNT', DecrementQueryCount = 'LOGS_DECREMENT_QUERY_COUNT', ConcatMoreLogs = 'LOGS_CONCAT_MORE_LOGS', + PrependMoreLogs = 'LOGS_PREPEND_MORE_LOGS', SetConfig = 'SET_CONFIG', SetTableRelativeTime = 'SET_TABLE_RELATIVE_TIME', SetTableCustomTime = 'SET_TABLE_CUSTOM_TIME', SetTableForwardData = 'SET_TABLE_FORWARD_DATA', SetTableBackwardData = 'SET_TABLE_BACKWARD_DATA', + ClearRowsAdded = 'CLEAR_ROWS_ADDED', +} + +export interface ClearRowsAddedAction { + type: ActionTypes.ClearRowsAdded } export interface SetTableForwardDataAction { @@ -107,6 +113,13 @@ export interface ConcatMoreLogsAction { } } +export interface PrependMoreLogsAction { + type: ActionTypes.PrependMoreLogs + payload: { + series: TableData + } +} + export interface IncrementQueryCountAction { type: ActionTypes.IncrementQueryCount } @@ -231,11 +244,13 @@ export type Action = | DecrementQueryCountAction | IncrementQueryCountAction | ConcatMoreLogsAction + | PrependMoreLogsAction | SetConfigsAction | SetTableCustomTimeAction | SetTableRelativeTimeAction | SetTableForwardDataAction | SetTableBackwardDataAction + | ClearRowsAddedAction const getTimeRange = (state: State): TimeRange | null => getDeep(state, 'logs.timeRange', null) @@ -272,6 +287,10 @@ const getTableSelectedTime = (state: State): string => { .toISOString() } +export const clearRowsAdded = () => ({ + type: ActionTypes.ClearRowsAdded, +}) + export const setTableCustomTime = (time: string): SetTableCustomTimeAction => ({ type: ActionTypes.SetTableCustomTime, payload: {time}, @@ -442,6 +461,7 @@ export const executeTableQueryAsync = () => async (dispatch): Promise => { await Promise.all([ dispatch(executeTableForwardQueryAsync()), dispatch(executeTableBackwardQueryAsync()), + dispatch(clearRowsAdded()), ]) } @@ -521,14 +541,13 @@ export const setTableQueryConfigAsync = () => async ( } } -export const fetchMoreAsync = ( - queryTimeEnd: string, - lastTime: number -) => async (dispatch, getState): Promise => { +export const fetchMoreAsync = (queryTimeEnd: string) => async ( + dispatch, + getState +): Promise => { const state = getState() const tableQueryConfig = getTableQueryConfig(state) - const time = moment(lastTime).toISOString() - const timeRange = {lower: queryTimeEnd, upper: time} + const timeRange = {lower: queryTimeEnd} const newQueryConfig = { ...tableQueryConfig, range: timeRange, @@ -540,11 +559,17 @@ export const fetchMoreAsync = ( const params = [namespace, proxyLink, tableQueryConfig] if (_.every(params)) { - const query = buildLogQuery(timeRange, newQueryConfig, filters, searchTerm) + const query = buildBackwardLogQuery( + queryTimeEnd, + newQueryConfig, + filters, + searchTerm + ) + const response = await executeQueryAsync( proxyLink, namespace, - `${query} ORDER BY time DESC LIMIT 1000` + `${query} ORDER BY time DESC LIMIT ${INITIAL_LIMIT}` ) const series = getDeep(response, 'results.0.series.0', defaultTableData) @@ -552,11 +577,57 @@ export const fetchMoreAsync = ( } } +export const fetchNewerAsync = (queryTimeStart: string) => async ( + dispatch, + getState +): Promise => { + const state = getState() + const tableQueryConfig = getTableQueryConfig(state) + const timeRange = {lower: queryTimeStart} + const newQueryConfig = { + ...tableQueryConfig, + range: timeRange, + } + const namespace = getNamespace(state) + const proxyLink = getProxyLink(state) + const searchTerm = getSearchTerm(state) + const filters = getFilters(state) + const params = [namespace, proxyLink, tableQueryConfig] + + if (_.every(params)) { + const query = buildForwardLogQuery( + queryTimeStart, + newQueryConfig, + filters, + searchTerm + ) + + const response = await executeQueryAsync( + proxyLink, + namespace, + `${query} ORDER BY time ASC LIMIT ${INITIAL_LIMIT}` + ) + + const series = getDeep(response, 'results.0.series.0', defaultTableData) + await dispatch( + PrependMoreLogs({ + columns: series.columns, + values: _.reverse(series.values), + }) + ) + } +} + export const ConcatMoreLogs = (series: TableData): ConcatMoreLogsAction => ({ type: ActionTypes.ConcatMoreLogs, payload: {series}, }) +export const PrependMoreLogs = (series: TableData): PrependMoreLogsAction => ({ + type: ActionTypes.PrependMoreLogs, + payload: {series}, +}) + export const setNamespaceAsync = (namespace: Namespace) => async ( dispatch ): Promise => { @@ -639,147 +710,6 @@ export const changeZoomAsync = (timeRange: TimeRange) => async ( } } -const serverLogData = { - columns: [ - { - name: 'severity', - position: 1, - encodings: [ - { - type: 'visibility', - value: 'visible', - }, - { - type: 'label', - value: 'icon', - }, - { - type: 'label', - value: 'text', - }, - { - type: 'color', - value: 'emerg', - name: 'ruby', - }, - { - type: 'color', - value: 'alert', - name: 'fire', - }, - { - type: 'color', - value: 'crit', - name: 'curacao', - }, - { - type: 'color', - value: 'err', - name: 'tiger', - }, - { - type: 'color', - value: 'warning', - name: 'pineapple', - }, - { - type: 'color', - value: 'notice', - name: 'rainforest', - }, - { - type: 'color', - value: 'info', - name: 'star', - }, - { - type: 'color', - value: 'debug', - name: 'wolf', - }, - ], - }, - { - name: 'timestamp', - position: 2, - encodings: [ - { - type: 'visibility', - value: 'visible', - }, - ], - }, - { - name: 'message', - position: 3, - encodings: [ - { - type: 'visibility', - value: 'visible', - }, - ], - }, - { - name: 'facility', - position: 4, - encodings: [ - { - type: 'visibility', - value: 'visible', - }, - ], - }, - { - name: 'time', - position: 0, - encodings: [ - { - type: 'visibility', - value: 'hidden', - }, - ], - }, - { - name: 'procid', - position: 5, - encodings: [ - { - type: 'visibility', - value: 'visible', - }, - { - type: 'displayName', - value: 'Proc ID', - }, - ], - }, - { - name: 'host', - position: 7, - encodings: [ - { - type: 'visibility', - value: 'visible', - }, - ], - }, - { - name: 'appname', - position: 6, - encodings: [ - { - type: 'visibility', - value: 'visible', - }, - { - type: 'displayName', - value: 'Application', - }, - ], - }, - ], -} - export const getLogConfigAsync = (url: string) => async ( dispatch: Dispatch ): Promise => { diff --git a/ui/src/logs/components/LogsTable.tsx b/ui/src/logs/components/LogsTable.tsx index 200208a81..f99501e24 100644 --- a/ui/src/logs/components/LogsTable.tsx +++ b/ui/src/logs/components/LogsTable.tsx @@ -24,7 +24,6 @@ import { getColumnsFromData, } from 'src/logs/utils/table' -import timeRanges from 'src/logs/data/timeRanges' import { SeverityFormatOptions, SeverityColorOptions, @@ -45,14 +44,20 @@ interface Props { onScrollVertical: () => void onScrolledToTop: () => void onTagSelection: (selection: {tag: string; key: string}) => void - fetchMore: (queryTimeEnd: string, time: number) => Promise + fetchMore: (time: string) => Promise + fetchNewer: (time: string) => void + hasScrolled: boolean count: number timeRange: TimeRange + queryCount: number tableColumns: LogsTableColumn[] severityFormat: SeverityFormat severityLevelColors: SeverityLevelColor[] scrollToRow?: number - hasScrolled: boolean + tableInfiniteData: { + forward: TableData + backward: TableData + } } interface State { @@ -62,9 +67,22 @@ interface State { currentMessageWidth: number isMessageVisible: boolean lastQueryTime: number + firstQueryTime: number visibleColumnsCount: number } +const calculateScrollTop = (currentMessageWidth, data, scrollToRow) => { + const rowCharLimit = calculateRowCharWidth(currentMessageWidth) + + return _.reduce( + _.range(0, scrollToRow), + (acc, index) => { + return acc + calculateMessageHeight(index, data, rowCharLimit) + }, + 0 + ) +} + class LogsTable extends Component { public static getDerivedStateFromProps(props, state): State { const { @@ -73,7 +91,6 @@ class LogsTable extends Component { data, tableColumns, severityFormat, - hasScrolled, } = props const currentMessageWidth = getMessageWidth( data, @@ -82,20 +99,16 @@ class LogsTable extends Component { ) let lastQueryTime = _.get(state, 'lastQueryTime', null) + let firstQueryTime = _.get(state, 'firstQueryTime', null) let scrollTop = _.get(state, 'scrollTop', 0) if (isScrolledToTop) { lastQueryTime = null + firstQueryTime = null scrollTop = 0 - } else if (scrollToRow && !hasScrolled) { - const rowCharLimit = calculateRowCharWidth(currentMessageWidth) + } - scrollTop = _.reduce( - _.range(0, scrollToRow), - (acc, index) => { - return acc + calculateMessageHeight(index, data, rowCharLimit) - }, - 0 - ) + if (scrollToRow) { + scrollTop = calculateScrollTop(currentMessageWidth, data, scrollToRow) } const scrollLeft = _.get(state, 'scrollLeft', 0) @@ -112,12 +125,14 @@ class LogsTable extends Component { ...state, isQuerying: false, lastQueryTime, + firstQueryTime, scrollTop, scrollLeft, currentRow: -1, currentMessageWidth, isMessageVisible, visibleColumnsCount, + scrollToRow, } } @@ -144,9 +159,12 @@ class LogsTable extends Component { currentRow: -1, currentMessageWidth: 0, lastQueryTime: null, + firstQueryTime: null, isMessageVisible, visibleColumnsCount, } + + this.loadMoreUpRows = _.throttle(this.loadMoreUpRows, 5000) } public componentDidUpdate() { @@ -208,8 +226,8 @@ class LogsTable extends Component { {({registerChild, onRowsRendered}) => ( @@ -253,13 +271,13 @@ class LogsTable extends Component { columnCount: number, registerChild: (g: Grid) => void ) => { - const {hasScrolled, scrollToRow} = this.props + const {scrollToRow} = this.props const {scrollLeft, scrollTop} = this.state - const result: {scrollToRow?: number} & any = { + const result: any = { width, height, rowHeight: this.calculateRowHeight, - rowCount: getValuesFromData(this.props.data).length, + rowCount: this.rowCount(), scrollLeft, scrollTop, cellRenderer: this.cellRenderer, @@ -273,7 +291,7 @@ class LogsTable extends Component { }, } - if (!hasScrolled && scrollToRow) { + if (scrollToRow) { result.scrollToRow = scrollToRow } @@ -302,6 +320,10 @@ class LogsTable extends Component { this.setState({scrollTop}) + if (scrollTop < 200 && scrollTop < previousTop) { + this.loadMoreUpRows() + } + if (scrollTop === 0) { this.props.onScrolledToTop() } else if (scrollTop !== previousTop) { @@ -322,36 +344,56 @@ class LogsTable extends Component { } } - private loadMoreRows = async () => { - return - const data = getValuesFromData(this.props.data) - const {timeRange} = this.props + private loadMoreUpRows = async () => { + // Prevent multiple queries at the same time + const {queryCount} = this.props + if (queryCount > 0) { + return + } + + const data = getValuesFromData(this.props.tableInfiniteData.forward) + const firstTime = getDeep(data, '0.0', new Date().getTime() / 1000) + const {firstQueryTime} = this.state + if (firstQueryTime && firstQueryTime > firstTime) { + return + } + + this.setState({firstQueryTime: firstTime}) + await this.props.fetchNewer(moment(firstTime).toISOString()) + } + + private loadMoreDownRows = async () => { + // Prevent multiple queries at the same time + const {queryCount} = this.props + if (queryCount > 0) { + return + } + + const data = getValuesFromData(this.props.tableInfiniteData.backward) + const lastTime = getDeep( data, `${data.length - 1}.0`, new Date().getTime() / 1000 ) - const upper = getDeep(timeRange, 'upper', null) - const lower = getDeep(timeRange, 'lower', null) - if (this.state.lastQueryTime && this.state.lastQueryTime <= lastTime) { + // Guard against fetching on scrolling back up then down + const {lastQueryTime} = this.state + if (lastQueryTime && lastQueryTime <= lastTime) { return } - const firstQueryTime = getDeep(data, '0.0', null) - let queryTimeEnd = lower - if (!upper) { - const foundTimeRange = timeRanges.find(range => range.lower === lower) - queryTimeEnd = moment(firstQueryTime) - .subtract(foundTimeRange.seconds, 'seconds') - .toISOString() - } this.setState({lastQueryTime: lastTime}) - await this.props.fetchMore(queryTimeEnd, lastTime) + await this.props.fetchMore(moment(lastTime).toISOString()) + } + + private rowCount = (): number => { + const data = this.props.tableInfiniteData + return data.forward.values.length + data.backward.values.length } private isRowLoaded = ({index}) => { - return !!getValuesFromData(this.props.data)[index] + return index < this.rowCount() - 1 } private handleWindowResize = () => { diff --git a/ui/src/logs/containers/LogsPage.tsx b/ui/src/logs/containers/LogsPage.tsx index 4105d31c5..a57983aa7 100644 --- a/ui/src/logs/containers/LogsPage.tsx +++ b/ui/src/logs/containers/LogsPage.tsx @@ -1,4 +1,4 @@ -import React, {PureComponent} from 'react' +import React, {Component} from 'react' import uuid from 'uuid' import _ from 'lodash' import {connect} from 'react-redux' @@ -17,6 +17,7 @@ import { removeFilter, changeFilter, fetchMoreAsync, + fetchNewerAsync, getLogConfigAsync, updateLogConfigAsync, } from 'src/logs/actions' @@ -60,12 +61,14 @@ interface Props { setSearchTermAsync: (searchTerm: string) => void setTableRelativeTime: (time: number) => void setTableCustomTime: (time: string) => void - fetchMoreAsync: (queryTimeEnd: string, lastTime: number) => Promise + fetchMoreAsync: (queryTimeEnd: string) => Promise + fetchNewerAsync: (queryTimeEnd: string) => Promise addFilter: (filter: Filter) => void removeFilter: (id: string) => void changeFilter: (id: string, operator: string, value: string) => void getConfig: (url: string) => Promise updateConfig: (url: string, config: LogConfig) => Promise + newRowsAdded: number timeRange: TimeRange histogramData: HistogramData tableData: TableData @@ -92,7 +95,7 @@ interface State { hasScrolled: boolean } -class LogsPage extends PureComponent { +class LogsPage extends Component { public static getDerivedStateFromProps(props: Props) { const severityLevelColors: SeverityLevelColor[] = _.get( props.logConfig, @@ -107,6 +110,7 @@ class LogsPage extends PureComponent { } private interval: NodeJS.Timer + private loadingNewer: boolean = false constructor(props: Props) { super(props) @@ -142,7 +146,7 @@ class LogsPage extends PureComponent { } public render() { - const {liveUpdating} = this.state + // const {liveUpdating} = this.state const {searchTerm, filters, queryCount, timeRange, tableTime} = this.props return ( @@ -175,17 +179,21 @@ class LogsPage extends PureComponent { />
    @@ -194,6 +202,27 @@ class LogsPage extends PureComponent { ) } + private fetchNewer = (time: string) => { + this.loadingNewer = true + this.props.fetchNewerAsync(time) + } + + private get tableScrollToRow() { + if (this.loadingNewer && this.props.newRowsAdded) { + this.loadingNewer = false + return this.props.newRowsAdded || 0 + } + + if (this.state.hasScrolled) { + return + } + + return Math.max( + _.get(this.props, 'tableInfiniteData.forward.values.length', 0) - 3, + 0 + ) + } + private handleChooseCustomTime = (time: string) => { this.props.setTableCustomTime(time) } @@ -453,6 +482,7 @@ const mapStateToProps = ({ config: {logViewer}, }, logs: { + newRowsAdded, currentSource, currentNamespaces, timeRange, @@ -481,6 +511,7 @@ const mapStateToProps = ({ tableTime, logConfigLink: logViewer, tableInfiniteData, + newRowsAdded, }) const mapDispatchToProps = { @@ -495,6 +526,7 @@ const mapDispatchToProps = { removeFilter, changeFilter, fetchMoreAsync, + fetchNewerAsync, setTableCustomTime: setTableCustomTimeAsync, setTableRelativeTime: setTableRelativeTimeAsync, getConfig: getLogConfigAsync, diff --git a/ui/src/logs/data/serverLogData.ts b/ui/src/logs/data/serverLogData.ts index ab006ee2e..0898602ba 100644 --- a/ui/src/logs/data/serverLogData.ts +++ b/ui/src/logs/data/serverLogData.ts @@ -1,4 +1,4 @@ -export default { +export const serverLogData = { columns: [ { name: 'severity', @@ -16,6 +16,46 @@ export default { type: 'label', value: 'text', }, + { + type: 'color', + value: 'emerg', + name: 'ruby', + }, + { + type: 'color', + value: 'alert', + name: 'fire', + }, + { + type: 'color', + value: 'crit', + name: 'curacao', + }, + { + type: 'color', + value: 'err', + name: 'tiger', + }, + { + type: 'color', + value: 'warning', + name: 'pineapple', + }, + { + type: 'color', + value: 'notice', + name: 'rainforest', + }, + { + type: 'color', + value: 'info', + name: 'star', + }, + { + type: 'color', + value: 'debug', + name: 'wolf', + }, ], }, { diff --git a/ui/src/logs/reducers/index.ts b/ui/src/logs/reducers/index.ts index fcd7683f2..56004b2e5 100644 --- a/ui/src/logs/reducers/index.ts +++ b/ui/src/logs/reducers/index.ts @@ -9,6 +9,7 @@ import { DecrementQueryCountAction, IncrementQueryCountAction, ConcatMoreLogsAction, + PrependMoreLogsAction, SetConfigsAction, } from 'src/logs/actions' @@ -51,6 +52,7 @@ const defaultState: LogsState = { forward: defaultTableData, backward: defaultTableData, }, + newRowsAdded: 0, } const removeFilter = ( @@ -111,13 +113,45 @@ const concatMoreLogs = ( const { series: {values}, } = action.payload - const {tableData} = state - const vals = [...tableData.values, ...values] + const {tableInfiniteData} = state + const {backward} = tableInfiniteData + const vals = [...backward.values, ...values] + return { ...state, - tableData: { - columns: tableData.columns, - values: vals, + tableInfiniteData: { + ...tableInfiniteData, + backward: { + columns: backward.columns, + values: vals, + }, + }, + } +} + +const prependMoreLogs = ( + state: LogsState, + action: PrependMoreLogsAction +): LogsState => { + const { + series: {values}, + } = action.payload + const {tableInfiniteData} = state + const {forward} = tableInfiniteData + const vals = [...values, ...forward.values] + + const uniqueValues = _.uniqBy(vals, '0') + const newRowsAdded = uniqueValues.length - forward.values.length + + return { + ...state, + newRowsAdded, + tableInfiniteData: { + ...tableInfiniteData, + forward: { + columns: forward.columns, + values: uniqueValues, + }, }, } } @@ -154,6 +188,8 @@ export default (state: LogsState = defaultState, action: Action) => { return {...state, tableQueryConfig: action.payload.queryConfig} case ActionTypes.SetTableData: return {...state, tableData: action.payload.data} + case ActionTypes.ClearRowsAdded: + return {...state, newRowsAdded: null} case ActionTypes.SetTableForwardData: return { ...state, @@ -191,6 +227,8 @@ export default (state: LogsState = defaultState, action: Action) => { return decrementQueryCount(state, action) case ActionTypes.ConcatMoreLogs: return concatMoreLogs(state, action) + case ActionTypes.PrependMoreLogs: + return prependMoreLogs(state, action) case ActionTypes.SetConfig: return setConfigs(state, action) default: diff --git a/ui/src/types/logs.ts b/ui/src/types/logs.ts index c08441cdb..f9ba17a0a 100644 --- a/ui/src/types/logs.ts +++ b/ui/src/types/logs.ts @@ -40,6 +40,7 @@ export interface LogsState { custom?: string relative?: string } + newRowsAdded: number } export interface LogConfig { From f64107fc307071536b185e40ecf2c2d2c18837ff Mon Sep 17 00:00:00 2001 From: Brandon Farmer Date: Wed, 11 Jul 2018 15:53:38 -0700 Subject: [PATCH 4/5] Use proper application header value --- ui/src/logs/reducers/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/logs/reducers/index.ts b/ui/src/logs/reducers/index.ts index 56004b2e5..ec9f0ffcb 100644 --- a/ui/src/logs/reducers/index.ts +++ b/ui/src/logs/reducers/index.ts @@ -23,7 +23,7 @@ const defaultTableData: TableData = { 'timestamp', 'facility', 'procid', - 'application', + 'appname', 'host', 'message', ], From 92354eb67e06ee6de8ba063af85ea8558e9437e0 Mon Sep 17 00:00:00 2001 From: Brandon Farmer Date: Wed, 11 Jul 2018 19:36:51 -0700 Subject: [PATCH 5/5] Add types and extract constants for log viewer --- ui/src/logs/actions/index.ts | 2 +- ui/src/logs/components/LogsTable.tsx | 18 +++++++++++------- ui/src/logs/components/PointInTimeDropDown.tsx | 2 +- ui/src/logs/containers/LogsPage.tsx | 1 - 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ui/src/logs/actions/index.ts b/ui/src/logs/actions/index.ts index 8b8afbfd5..97cc47ab5 100644 --- a/ui/src/logs/actions/index.ts +++ b/ui/src/logs/actions/index.ts @@ -26,7 +26,7 @@ import { import {serverLogData} from 'src/logs/data/serverLogData' import {LogsState, Filter, TableData, LogConfig} from 'src/types/logs' -const INITIAL_LIMIT = 100 +export const INITIAL_LIMIT = 100 const defaultTableData: TableData = { columns: [ diff --git a/ui/src/logs/components/LogsTable.tsx b/ui/src/logs/components/LogsTable.tsx index f99501e24..3cd96dd02 100644 --- a/ui/src/logs/components/LogsTable.tsx +++ b/ui/src/logs/components/LogsTable.tsx @@ -37,6 +37,7 @@ import { SeverityFormat, SeverityLevelColor, } from 'src/types/logs' +import {INITIAL_LIMIT} from 'src/logs/actions' interface Props { data: TableData @@ -164,7 +165,7 @@ class LogsTable extends Component { visibleColumnsCount, } - this.loadMoreUpRows = _.throttle(this.loadMoreUpRows, 5000) + this.loadMoreAboveRows = _.throttle(this.loadMoreAboveRows, 5000) } public componentDidUpdate() { @@ -226,8 +227,8 @@ class LogsTable extends Component { {({registerChild, onRowsRendered}) => ( @@ -321,7 +322,7 @@ class LogsTable extends Component { this.setState({scrollTop}) if (scrollTop < 200 && scrollTop < previousTop) { - this.loadMoreUpRows() + this.loadMoreAboveRows() } if (scrollTop === 0) { @@ -344,7 +345,7 @@ class LogsTable extends Component { } } - private loadMoreUpRows = async () => { + private loadMoreAboveRows = async () => { // Prevent multiple queries at the same time const {queryCount} = this.props if (queryCount > 0) { @@ -362,7 +363,7 @@ class LogsTable extends Component { await this.props.fetchNewer(moment(firstTime).toISOString()) } - private loadMoreDownRows = async () => { + private loadMoreBelowRows = async () => { // Prevent multiple queries at the same time const {queryCount} = this.props if (queryCount > 0) { @@ -389,7 +390,10 @@ class LogsTable extends Component { private rowCount = (): number => { const data = this.props.tableInfiniteData - return data.forward.values.length + data.backward.values.length + return ( + getDeep(data, 'forward.values.length', 0) + + getDeep(data, 'backward.values.length', 0) + ) } private isRowLoaded = ({index}) => { diff --git a/ui/src/logs/components/PointInTimeDropDown.tsx b/ui/src/logs/components/PointInTimeDropDown.tsx index 15d8a7af2..7cfeeab9c 100644 --- a/ui/src/logs/components/PointInTimeDropDown.tsx +++ b/ui/src/logs/components/PointInTimeDropDown.tsx @@ -22,7 +22,7 @@ interface State { } const dateFormat = 'YYYY-MM-DD HH:mm' -const format = t => moment(t.replace(/\'/g, '')).format(dateFormat) +const format = (t: string) => moment(t.replace(/\'/g, '')).format(dateFormat) @ErrorHandling class TimeRangeDropdown extends Component { diff --git a/ui/src/logs/containers/LogsPage.tsx b/ui/src/logs/containers/LogsPage.tsx index a57983aa7..75cb946ab 100644 --- a/ui/src/logs/containers/LogsPage.tsx +++ b/ui/src/logs/containers/LogsPage.tsx @@ -146,7 +146,6 @@ class LogsPage extends Component { } public render() { - // const {liveUpdating} = this.state const {searchTerm, filters, queryCount, timeRange, tableTime} = this.props return (