From 1c19faa1f5c0c3f8a94e892b296ed6c0a067ad35 Mon Sep 17 00:00:00 2001 From: Brandon Farmer Date: Tue, 12 Jun 2018 10:50:49 -0700 Subject: [PATCH] Implement load on scroll Co-authored-by: Deniz Kusefoglu --- ui/src/logs/actions/index.ts | 63 ++++++++++-- ui/src/logs/components/LogsFilter.tsx | 8 +- ui/src/logs/components/LogsTable.tsx | 138 +++++++++++++++++++------- ui/src/logs/containers/LogsPage.tsx | 28 ++++-- ui/src/logs/reducers/index.ts | 25 ++++- ui/src/types/logs.ts | 7 +- 6 files changed, 212 insertions(+), 57 deletions(-) diff --git a/ui/src/logs/actions/index.ts b/ui/src/logs/actions/index.ts index 2bd233a5fa..aecc89780f 100644 --- a/ui/src/logs/actions/index.ts +++ b/ui/src/logs/actions/index.ts @@ -1,3 +1,4 @@ +import moment from 'moment' import _ from 'lodash' import {Source, Namespace, TimeRange, QueryConfig} from 'src/types' import {getSource} from 'src/shared/apis' @@ -10,14 +11,9 @@ import { import {getDeep} from 'src/utils/wrappers' import buildQuery from 'src/utils/influxql' import {executeQueryAsync} from 'src/logs/api' -import {LogsState, Filter} from 'src/types/logs' +import {LogsState, Filter, TableData} from 'src/types/logs' -interface TableData { - columns: string[] - values: string[] -} - -const defaultTableData = { +const defaultTableData: TableData = { columns: [ 'time', 'severity', @@ -54,6 +50,14 @@ export enum ActionTypes { ChangeFilter = 'LOGS_CHANGE_FILTER', IncrementQueryCount = 'LOGS_INCREMENT_QUERY_COUNT', DecrementQueryCount = 'LOGS_DECREMENT_QUERY_COUNT', + ConcatMoreLogs = 'LOGS_CONCAT_MORE_LOGS', +} + +export interface ConcatMoreLogsAction { + type: ActionTypes.ConcatMoreLogs + payload: { + series: TableData + } } export interface IncrementQueryCountAction { @@ -173,6 +177,7 @@ export type Action = | ChangeFilterAction | DecrementQueryCountAction | IncrementQueryCountAction + | ConcatMoreLogsAction const getTimeRange = (state: State): TimeRange | null => getDeep(state, 'logs.timeRange', null) @@ -243,7 +248,7 @@ export const executeHistogramQueryAsync = () => async ( const setTableData = (series: TableData): SetTableData => ({ type: ActionTypes.SetTableData, - payload: {data: {columns: series.columns, values: _.reverse(series.values)}}, + payload: {data: {columns: series.columns, values: series.values}}, }) export const executeTableQueryAsync = () => async ( @@ -261,7 +266,11 @@ export const executeTableQueryAsync = () => async ( if (_.every([queryConfig, timeRange, namespace, proxyLink])) { const query = buildLogQuery(timeRange, queryConfig, filters, searchTerm) - const response = await executeQueryAsync(proxyLink, namespace, query) + const response = await executeQueryAsync( + proxyLink, + namespace, + `${query} ORDER BY time DESC LIMIT 1000` + ) const series = getDeep(response, 'results.0.series.0', defaultTableData) @@ -348,6 +357,42 @@ export const setTableQueryConfigAsync = () => async ( } } +export const fetchMoreAsync = ( + queryTimeEnd: string, + lastTime: number +) => async (dispatch, getState): Promise => { + const state = getState() + const tableQueryConfig = getTableQueryConfig(state) + const time = moment(lastTime).toISOString() + const timeRange = {lower: queryTimeEnd, upper: time} + const newQueryConfig = { + ...tableQueryConfig, + range: timeRange, + } + const namespace = getNamespace(state) + const proxyLink = getProxyLink(state) + const searchTerm = getSearchTerm(state) + const filters = getFilters(state) + const params = [namespace, proxyLink, tableQueryConfig] + + if (_.every(params)) { + const query = buildLogQuery(timeRange, newQueryConfig, filters, searchTerm) + const response = await executeQueryAsync( + proxyLink, + namespace, + `${query} ORDER BY time DESC LIMIT 1000` + ) + + const series = getDeep(response, 'results.0.series.0', defaultTableData) + await dispatch(ConcatMoreLogs(series)) + } +} + +export const ConcatMoreLogs = (series: TableData): ConcatMoreLogsAction => ({ + type: ActionTypes.ConcatMoreLogs, + payload: {series}, +}) + export const setNamespaceAsync = (namespace: Namespace) => async ( dispatch ): Promise => { diff --git a/ui/src/logs/components/LogsFilter.tsx b/ui/src/logs/components/LogsFilter.tsx index c17b8814e0..3f8190aa26 100644 --- a/ui/src/logs/components/LogsFilter.tsx +++ b/ui/src/logs/components/LogsFilter.tsx @@ -125,13 +125,17 @@ class LogsFilter extends PureComponent { private stopEditing(): void { const id = getDeep(this.props, 'filter.id', '') - const {operator, value} = this.state + const {operator, value, editing} = this.state + const {filter} = this.props + + if (!editing || (filter.operator === operator && filter.value === value)) { + return + } let state = {} if (['!=', '==', '=~'].includes(operator) && value !== '') { this.props.onChangeFilter(id, operator, value) } else { - const {filter} = this.props state = {operator: filter.operator, value: filter.value} } diff --git a/ui/src/logs/components/LogsTable.tsx b/ui/src/logs/components/LogsTable.tsx index 8005b3a084..152ac2daf2 100644 --- a/ui/src/logs/components/LogsTable.tsx +++ b/ui/src/logs/components/LogsTable.tsx @@ -1,8 +1,10 @@ import _ from 'lodash' +import moment from 'moment' import classnames from 'classnames' import React, {Component, MouseEvent} from 'react' -import {Grid, AutoSizer} from 'react-virtualized' +import {Grid, AutoSizer, InfiniteLoader} from 'react-virtualized' import FancyScrollbar from 'src/shared/components/FancyScrollbar' +import {getDeep} from 'src/utils/wrappers' import { getColumnFromData, @@ -16,6 +18,10 @@ import { getColumnsFromData, } from 'src/logs/utils/table' +import timeRanges from 'src/logs/data/timeRanges' + +import {TimeRange} from 'src/types' + const ROW_HEIGHT = 26 const CHAR_WIDTH = 9 interface Props { @@ -27,7 +33,9 @@ interface Props { onScrollVertical: () => void onScrolledToTop: () => void onTagSelection: (selection: {tag: string; key: string}) => void + fetchMore: (queryTimeEnd: string, time: number) => Promise count: number + timeRange: TimeRange } interface State { @@ -35,20 +43,26 @@ interface State { scrollTop: number currentRow: number currentMessageWidth: number + lastQueryTime: number } class LogsTable extends Component { public static getDerivedStateFromProps(props, state): State { const {isScrolledToTop} = props + let lastQueryTime = _.get(state, 'lastQueryTime', null) let scrollTop = _.get(state, 'scrollTop', 0) if (isScrolledToTop) { + lastQueryTime = null scrollTop = 0 } const scrollLeft = _.get(state, 'scrollLeft', 0) return { + ...state, + isQuerying: false, + lastQueryTime, scrollTop, scrollLeft, currentRow: -1, @@ -56,13 +70,13 @@ class LogsTable extends Component { } } - private grid: React.RefObject + private grid: Grid | null private headerGrid: React.RefObject constructor(props: Props) { super(props) - this.grid = React.createRef() + this.grid = null this.headerGrid = React.createRef() this.state = { @@ -70,17 +84,22 @@ class LogsTable extends Component { scrollLeft: 0, currentRow: -1, currentMessageWidth: 0, + lastQueryTime: null, } } public componentDidUpdate() { - this.grid.current.recomputeGridSize() + if (this.grid) { + this.grid.recomputeGridSize() + } this.headerGrid.current.recomputeGridSize() } public componentDidMount() { window.addEventListener('resize', this.handleWindowResize) - this.grid.current.recomputeGridSize() + if (this.grid) { + this.grid.recomputeGridSize() + } this.headerGrid.current.recomputeGridSize() } @@ -115,39 +134,89 @@ class LogsTable extends Component { /> )} - - {({width, height}) => ( - - - + + {({registerChild, onRowsRendered}) => ( + + {({width, height}) => ( + + { + registerChild(ref) + this.grid = ref + }} + style={{height: this.calculateTotalHeight()}} + /> + + )} + )} - + ) } + private handleRowRender = onRowsRendered => ({ + rowStartIndex, + rowStopIndex, + }) => { + onRowsRendered({startIndex: rowStartIndex, stopIndex: rowStopIndex}) + } + + private loadMoreRows = async () => { + const data = getValuesFromData(this.props.data) + const {timeRange} = this.props + const lastTime = getDeep( + data, + `${data.length - 1}.0`, + new Date().getTime() / 1000 + ) + const upper = getDeep(timeRange, 'upper', null) + const lower = getDeep(timeRange, 'lower', null) + + if (this.state.lastQueryTime && this.state.lastQueryTime <= lastTime) { + return + } + const firstQueryTime = getDeep(data, '0.0', null) + let queryTimeEnd = lower + if (!upper) { + const foundTimeRange = timeRanges.find(range => range.lower === lower) + queryTimeEnd = moment(firstQueryTime) + .subtract(foundTimeRange.seconds, 'seconds') + .toISOString() + } + + await this.setState({lastQueryTime: lastTime}) + await this.props.fetchMore(queryTimeEnd, lastTime) + } + + private isRowLoaded = ({index}) => { + return !!getValuesFromData(this.props.data)[index] + } + private handleWindowResize = () => { this.setState({currentMessageWidth: getMessageWidth(this.props.data)}) } @@ -178,8 +247,9 @@ class LogsTable extends Component { } private calculateTotalHeight = (): number => { + const data = getValuesFromData(this.props.data) return _.reduce( - getValuesFromData(this.props.data), + data, (acc, __, index) => { return acc + this.calculateMessageHeight(index) }, diff --git a/ui/src/logs/containers/LogsPage.tsx b/ui/src/logs/containers/LogsPage.tsx index b996e12283..a427fa78c0 100644 --- a/ui/src/logs/containers/LogsPage.tsx +++ b/ui/src/logs/containers/LogsPage.tsx @@ -11,6 +11,7 @@ import { addFilter, removeFilter, changeFilter, + fetchMoreAsync, } from 'src/logs/actions' import {getSourcesAsync} from 'src/shared/actions/sources' import LogViewerHeader from 'src/logs/components/LogViewerHeader' @@ -36,6 +37,7 @@ interface Props { changeZoomAsync: (timeRange: TimeRange) => void executeQueriesAsync: () => void setSearchTermAsync: (searchTerm: string) => void + fetchMoreAsync: (queryTimeEnd: string, lastTime: number) => Promise addFilter: (filter: Filter) => void removeFilter: (id: string) => void changeFilter: (id: string, operator: string, value: string) => void @@ -77,7 +79,7 @@ class LogsPage extends PureComponent { this.props.getSources() if (this.props.currentNamespace) { - this.props.executeQueriesAsync() + this.fetchNewDataset() } this.startUpdating() @@ -89,9 +91,7 @@ class LogsPage extends PureComponent { public render() { const {liveUpdating} = this.state - const {searchTerm, filters, queryCount} = this.props - - const count = getDeep(this.props, 'tableData.values.length', 0) + const {searchTerm, filters, queryCount, timeRange} = this.props return (
@@ -103,7 +103,7 @@ class LogsPage extends PureComponent { onSearch={this.handleSubmitSearch} /> { onScrolledToTop={this.handleScrollToTop} isScrolledToTop={liveUpdating} onTagSelection={this.handleTagSelection} + fetchMore={this.props.fetchMoreAsync} + timeRange={timeRange} />
@@ -158,11 +160,11 @@ class LogsPage extends PureComponent { value: selection.tag, operator: '==', }) - this.props.executeQueriesAsync() + this.fetchNewDataset() } private handleInterval = () => { - this.props.executeQueriesAsync() + this.fetchNewDataset() } private get histogramTotal(): number { @@ -233,7 +235,7 @@ class LogsPage extends PureComponent { private handleFilterDelete = (id: string): void => { this.props.removeFilter(id) - this.props.executeQueriesAsync() + this.fetchNewDataset() } private handleFilterChange = ( @@ -242,12 +244,13 @@ class LogsPage extends PureComponent { value: string ) => { this.props.changeFilter(id, operator, value) + this.fetchNewDataset() this.props.executeQueriesAsync() } private handleChooseTimerange = (timeRange: TimeRange) => { this.props.setTimeRangeAsync(timeRange) - this.props.executeQueriesAsync() + this.fetchNewDataset() } private handleChooseSource = (sourceID: string) => { @@ -261,8 +264,14 @@ class LogsPage extends PureComponent { private handleChartZoom = (lower, upper) => { if (lower) { this.props.changeZoomAsync({lower, upper}) + this.setState({liveUpdating: true}) } } + + private fetchNewDataset() { + this.props.executeQueriesAsync() + this.setState({liveUpdating: true}) + } } const mapStateToProps = ({ @@ -302,6 +311,7 @@ const mapDispatchToProps = { addFilter, removeFilter, changeFilter, + fetchMoreAsync, } export default connect(mapStateToProps, mapDispatchToProps)(LogsPage) diff --git a/ui/src/logs/reducers/index.ts b/ui/src/logs/reducers/index.ts index 74e88e8eec..f36052aa3d 100644 --- a/ui/src/logs/reducers/index.ts +++ b/ui/src/logs/reducers/index.ts @@ -7,6 +7,7 @@ import { ChangeFilterAction, DecrementQueryCountAction, IncrementQueryCountAction, + ConcatMoreLogsAction, } from 'src/logs/actions' import {LogsState} from 'src/types/logs' @@ -17,9 +18,9 @@ const defaultState: LogsState = { currentNamespace: null, histogramQueryConfig: null, tableQueryConfig: null, - tableData: [], + tableData: {columns: [], values: []}, histogramData: [], - searchTerm: null, + searchTerm: '', filters: [], queryCount: 0, } @@ -75,6 +76,24 @@ const incrementQueryCount = ( return {...state, queryCount: queryCount + 1} } +const concatMoreLogs = ( + state: LogsState, + action: ConcatMoreLogsAction +): LogsState => { + const { + series: {values}, + } = action.payload + const {tableData} = state + const vals = [...tableData.values, ...values] + return { + ...state, + tableData: { + columns: tableData.columns, + values: vals, + }, + } +} + export default (state: LogsState = defaultState, action: Action) => { switch (action.type) { case ActionTypes.SetSource: @@ -109,6 +128,8 @@ export default (state: LogsState = defaultState, action: Action) => { return incrementQueryCount(state, action) case ActionTypes.DecrementQueryCount: return decrementQueryCount(state, action) + case ActionTypes.ConcatMoreLogs: + return concatMoreLogs(state, action) default: return state } diff --git a/ui/src/types/logs.ts b/ui/src/types/logs.ts index 96a1cc66eb..58899d93bc 100644 --- a/ui/src/types/logs.ts +++ b/ui/src/types/logs.ts @@ -7,6 +7,11 @@ export interface Filter { operator: string } +export interface TableData { + columns: string[] + values: object[] +} + export interface LogsState { currentSource: Source | null currentNamespaces: Namespace[] @@ -15,7 +20,7 @@ export interface LogsState { histogramQueryConfig: QueryConfig | null histogramData: object[] tableQueryConfig: QueryConfig | null - tableData: object[] + tableData: TableData searchTerm: string | null filters: Filter[] queryCount: number