Implement load on scroll

Co-authored-by: Deniz Kusefoglu <deniz@influxdata.com>

ui/src/logs/actions/index.ts

@@ -1,3 +1,4 @@
+import moment from 'moment'
 import _ from 'lodash'
 import {Source, Namespace, TimeRange, QueryConfig} from 'src/types'
 import {getSource} from 'src/shared/apis'
@@ -10,14 +11,9 @@ import {
 import {getDeep} from 'src/utils/wrappers'
 import buildQuery from 'src/utils/influxql'
 import {executeQueryAsync} from 'src/logs/api'
-import {LogsState, Filter} from 'src/types/logs'
+import {LogsState, Filter, TableData} from 'src/types/logs'
 
-interface TableData {
+const defaultTableData: TableData = {
-  columns: string[]
-  values: string[]
-}
-
-const defaultTableData = {
   columns: [
     'time',
     'severity',
@@ -54,6 +50,14 @@ export enum ActionTypes {
   ChangeFilter = 'LOGS_CHANGE_FILTER',
   IncrementQueryCount = 'LOGS_INCREMENT_QUERY_COUNT',
   DecrementQueryCount = 'LOGS_DECREMENT_QUERY_COUNT',
+  ConcatMoreLogs = 'LOGS_CONCAT_MORE_LOGS',
+}
+
+export interface ConcatMoreLogsAction {
+  type: ActionTypes.ConcatMoreLogs
+  payload: {
+    series: TableData
+  }
 }
 
 export interface IncrementQueryCountAction {
@@ -173,6 +177,7 @@ export type Action =
   | ChangeFilterAction
   | DecrementQueryCountAction
   | IncrementQueryCountAction
+  | ConcatMoreLogsAction
 
 const getTimeRange = (state: State): TimeRange | null =>
   getDeep<TimeRange | null>(state, 'logs.timeRange', null)
@@ -243,7 +248,7 @@ export const executeHistogramQueryAsync = () => async (
 
 const setTableData = (series: TableData): SetTableData => ({
   type: ActionTypes.SetTableData,
-  payload: {data: {columns: series.columns, values: _.reverse(series.values)}},
+  payload: {data: {columns: series.columns, values: series.values}},
 })
 
 export const executeTableQueryAsync = () => async (
@@ -261,7 +266,11 @@ export const executeTableQueryAsync = () => async (
 
   if (_.every([queryConfig, timeRange, namespace, proxyLink])) {
     const query = buildLogQuery(timeRange, queryConfig, filters, searchTerm)
-    const response = await executeQueryAsync(proxyLink, namespace, query)
+    const response = await executeQueryAsync(
+      proxyLink,
+      namespace,
+      `${query} ORDER BY time DESC LIMIT 1000`
+    )
 
     const series = getDeep(response, 'results.0.series.0', defaultTableData)
 
@@ -348,6 +357,42 @@ export const setTableQueryConfigAsync = () => async (
   }
 }
 
+export const fetchMoreAsync = (
+  queryTimeEnd: string,
+  lastTime: number
+) => async (dispatch, getState): Promise<void> => {
+  const state = getState()
+  const tableQueryConfig = getTableQueryConfig(state)
+  const time = moment(lastTime).toISOString()
+  const timeRange = {lower: queryTimeEnd, upper: time}
+  const newQueryConfig = {
+    ...tableQueryConfig,
+    range: timeRange,
+  }
+  const namespace = getNamespace(state)
+  const proxyLink = getProxyLink(state)
+  const searchTerm = getSearchTerm(state)
+  const filters = getFilters(state)
+  const params = [namespace, proxyLink, tableQueryConfig]
+
+  if (_.every(params)) {
+    const query = buildLogQuery(timeRange, newQueryConfig, filters, searchTerm)
+    const response = await executeQueryAsync(
+      proxyLink,
+      namespace,
+      `${query} ORDER BY time DESC LIMIT 1000`
+    )
+
+    const series = getDeep(response, 'results.0.series.0', defaultTableData)
+    await dispatch(ConcatMoreLogs(series))
+  }
+}
+
+export const ConcatMoreLogs = (series: TableData): ConcatMoreLogsAction => ({
+  type: ActionTypes.ConcatMoreLogs,
+  payload: {series},
+})
+
 export const setNamespaceAsync = (namespace: Namespace) => async (
   dispatch
 ): Promise<void> => {

ui/src/logs/components/LogsFilter.tsx

@@ -125,13 +125,17 @@ class LogsFilter extends PureComponent<Props, State> {
 
   private stopEditing(): void {
     const id = getDeep(this.props, 'filter.id', '')
-    const {operator, value} = this.state
+    const {operator, value, editing} = this.state
+    const {filter} = this.props
+
+    if (!editing || (filter.operator === operator && filter.value === value)) {
+      return
+    }
 
     let state = {}
     if (['!=', '==', '=~'].includes(operator) && value !== '') {
       this.props.onChangeFilter(id, operator, value)
     } else {
-      const {filter} = this.props
       state = {operator: filter.operator, value: filter.value}
     }
 

ui/src/logs/components/LogsTable.tsx

@@ -1,8 +1,10 @@
 import _ from 'lodash'
+import moment from 'moment'
 import classnames from 'classnames'
 import React, {Component, MouseEvent} from 'react'
-import {Grid, AutoSizer} from 'react-virtualized'
+import {Grid, AutoSizer, InfiniteLoader} from 'react-virtualized'
 import FancyScrollbar from 'src/shared/components/FancyScrollbar'
+import {getDeep} from 'src/utils/wrappers'
 
 import {
   getColumnFromData,
@@ -16,6 +18,10 @@ import {
   getColumnsFromData,
 } from 'src/logs/utils/table'
 
+import timeRanges from 'src/logs/data/timeRanges'
+
+import {TimeRange} from 'src/types'
+
 const ROW_HEIGHT = 26
 const CHAR_WIDTH = 9
 interface Props {
@@ -27,7 +33,9 @@ interface Props {
   onScrollVertical: () => void
   onScrolledToTop: () => void
   onTagSelection: (selection: {tag: string; key: string}) => void
+  fetchMore: (queryTimeEnd: string, time: number) => Promise<void>
   count: number
+  timeRange: TimeRange
 }
 
 interface State {
@@ -35,20 +43,26 @@ interface State {
   scrollTop: number
   currentRow: number
   currentMessageWidth: number
+  lastQueryTime: number
 }
 
 class LogsTable extends Component<Props, State> {
   public static getDerivedStateFromProps(props, state): State {
     const {isScrolledToTop} = props
 
+    let lastQueryTime = _.get(state, 'lastQueryTime', null)
     let scrollTop = _.get(state, 'scrollTop', 0)
     if (isScrolledToTop) {
+      lastQueryTime = null
       scrollTop = 0
     }
 
     const scrollLeft = _.get(state, 'scrollLeft', 0)
 
     return {
+      ...state,
+      isQuerying: false,
+      lastQueryTime,
       scrollTop,
       scrollLeft,
       currentRow: -1,
@@ -56,13 +70,13 @@ class LogsTable extends Component<Props, State> {
     }
   }
 
-  private grid: React.RefObject<Grid>
+  private grid: Grid | null
   private headerGrid: React.RefObject<Grid>
 
   constructor(props: Props) {
     super(props)
 
-    this.grid = React.createRef()
+    this.grid = null
     this.headerGrid = React.createRef()
 
     this.state = {
@@ -70,17 +84,22 @@ class LogsTable extends Component<Props, State> {
       scrollLeft: 0,
       currentRow: -1,
       currentMessageWidth: 0,
+      lastQueryTime: null,
     }
   }
 
   public componentDidUpdate() {
-    this.grid.current.recomputeGridSize()
+    if (this.grid) {
+      this.grid.recomputeGridSize()
+    }
     this.headerGrid.current.recomputeGridSize()
   }
 
   public componentDidMount() {
     window.addEventListener('resize', this.handleWindowResize)
-    this.grid.current.recomputeGridSize()
+    if (this.grid) {
+      this.grid.recomputeGridSize()
+    }
     this.headerGrid.current.recomputeGridSize()
   }
 
@@ -115,39 +134,89 @@ class LogsTable extends Component<Props, State> {
             />
           )}
         </AutoSizer>
-        <AutoSizer>
+        <InfiniteLoader
-          {({width, height}) => (
+          isRowLoaded={this.isRowLoaded}
-            <FancyScrollbar
+          loadMoreRows={this.loadMoreRows}
-              style={{
+          rowCount={this.props.count}
-                width,
+        >
-                height,
+          {({registerChild, onRowsRendered}) => (
-                marginTop: `${ROW_HEIGHT}px`,
+            <AutoSizer>
-              }}
+              {({width, height}) => (
-              setScrollTop={this.handleScrollbarScroll}
+                <FancyScrollbar
-              scrollTop={this.state.scrollTop}
+                  style={{
-              autoHide={false}
+                    width,
-            >
+                    height,
-              <Grid
+                    marginTop: `${ROW_HEIGHT}px`,
-                height={height}
+                  }}
-                rowHeight={this.calculateRowHeight}
+                  setScrollTop={this.handleScrollbarScroll}
-                rowCount={getValuesFromData(this.props.data).length}
+                  scrollTop={this.state.scrollTop}
-                width={width}
+                  autoHide={false}
-                scrollLeft={this.state.scrollLeft}
+                >
-                scrollTop={this.state.scrollTop}
+                  <Grid
-                onScroll={this.handleScroll}
+                    height={height}
-                cellRenderer={this.cellRenderer}
+                    rowHeight={this.calculateRowHeight}
-                columnCount={columnCount}
+                    rowCount={getValuesFromData(this.props.data).length}
-                columnWidth={this.getColumnWidth}
+                    width={width}
-                ref={this.grid}
+                    scrollLeft={this.state.scrollLeft}
-                style={{height: this.calculateTotalHeight()}}
+                    scrollTop={this.state.scrollTop}
-              />
+                    onScroll={this.handleScroll}
-            </FancyScrollbar>
+                    cellRenderer={this.cellRenderer}
+                    onSectionRendered={this.handleRowRender(onRowsRendered)}
+                    columnCount={columnCount}
+                    columnWidth={this.getColumnWidth}
+                    ref={(ref: Grid) => {
+                      registerChild(ref)
+                      this.grid = ref
+                    }}
+                    style={{height: this.calculateTotalHeight()}}
+                  />
+                </FancyScrollbar>
+              )}
+            </AutoSizer>
           )}
-        </AutoSizer>
+        </InfiniteLoader>
       </div>
     )
   }
 
+  private handleRowRender = onRowsRendered => ({
+    rowStartIndex,
+    rowStopIndex,
+  }) => {
+    onRowsRendered({startIndex: rowStartIndex, stopIndex: rowStopIndex})
+  }
+
+  private loadMoreRows = async () => {
+    const data = getValuesFromData(this.props.data)
+    const {timeRange} = this.props
+    const lastTime = getDeep(
+      data,
+      `${data.length - 1}.0`,
+      new Date().getTime() / 1000
+    )
+    const upper = getDeep<string>(timeRange, 'upper', null)
+    const lower = getDeep<string>(timeRange, 'lower', null)
+
+    if (this.state.lastQueryTime && this.state.lastQueryTime <= lastTime) {
+      return
+    }
+    const firstQueryTime = getDeep<number>(data, '0.0', null)
+    let queryTimeEnd = lower
+    if (!upper) {
+      const foundTimeRange = timeRanges.find(range => range.lower === lower)
+      queryTimeEnd = moment(firstQueryTime)
+        .subtract(foundTimeRange.seconds, 'seconds')
+        .toISOString()
+    }
+
+    await this.setState({lastQueryTime: lastTime})
+    await this.props.fetchMore(queryTimeEnd, lastTime)
+  }
+
+  private isRowLoaded = ({index}) => {
+    return !!getValuesFromData(this.props.data)[index]
+  }
+
   private handleWindowResize = () => {
     this.setState({currentMessageWidth: getMessageWidth(this.props.data)})
   }
@@ -178,8 +247,9 @@ class LogsTable extends Component<Props, State> {
   }
 
   private calculateTotalHeight = (): number => {
+    const data = getValuesFromData(this.props.data)
     return _.reduce(
-      getValuesFromData(this.props.data),
+      data,
       (acc, __, index) => {
         return acc + this.calculateMessageHeight(index)
       },

ui/src/logs/containers/LogsPage.tsx

@@ -11,6 +11,7 @@ import {
   addFilter,
   removeFilter,
   changeFilter,
+  fetchMoreAsync,
 } from 'src/logs/actions'
 import {getSourcesAsync} from 'src/shared/actions/sources'
 import LogViewerHeader from 'src/logs/components/LogViewerHeader'
@@ -36,6 +37,7 @@ interface Props {
   changeZoomAsync: (timeRange: TimeRange) => void
   executeQueriesAsync: () => void
   setSearchTermAsync: (searchTerm: string) => void
+  fetchMoreAsync: (queryTimeEnd: string, lastTime: number) => Promise<void>
   addFilter: (filter: Filter) => void
   removeFilter: (id: string) => void
   changeFilter: (id: string, operator: string, value: string) => void
@@ -77,7 +79,7 @@ class LogsPage extends PureComponent<Props, State> {
     this.props.getSources()
 
     if (this.props.currentNamespace) {
-      this.props.executeQueriesAsync()
+      this.fetchNewDataset()
     }
 
     this.startUpdating()
@@ -89,9 +91,7 @@ class LogsPage extends PureComponent<Props, State> {
 
   public render() {
     const {liveUpdating} = this.state
-    const {searchTerm, filters, queryCount} = this.props
+    const {searchTerm, filters, queryCount, timeRange} = this.props
-
-    const count = getDeep(this.props, 'tableData.values.length', 0)
 
     return (
       <div className="page">
@@ -103,7 +103,7 @@ class LogsPage extends PureComponent<Props, State> {
             onSearch={this.handleSubmitSearch}
           />
           <FilterBar
-            numResults={count}
+            numResults={this.histogramTotal}
             filters={filters || []}
             onDelete={this.handleFilterDelete}
             onFilterChange={this.handleFilterChange}
@@ -116,6 +116,8 @@ class LogsPage extends PureComponent<Props, State> {
             onScrolledToTop={this.handleScrollToTop}
             isScrolledToTop={liveUpdating}
             onTagSelection={this.handleTagSelection}
+            fetchMore={this.props.fetchMoreAsync}
+            timeRange={timeRange}
           />
         </div>
       </div>
@@ -158,11 +160,11 @@ class LogsPage extends PureComponent<Props, State> {
       value: selection.tag,
       operator: '==',
     })
-    this.props.executeQueriesAsync()
+    this.fetchNewDataset()
   }
 
   private handleInterval = () => {
-    this.props.executeQueriesAsync()
+    this.fetchNewDataset()
   }
 
   private get histogramTotal(): number {
@@ -233,7 +235,7 @@ class LogsPage extends PureComponent<Props, State> {
 
   private handleFilterDelete = (id: string): void => {
     this.props.removeFilter(id)
-    this.props.executeQueriesAsync()
+    this.fetchNewDataset()
   }
 
   private handleFilterChange = (
@@ -242,12 +244,13 @@ class LogsPage extends PureComponent<Props, State> {
     value: string
   ) => {
     this.props.changeFilter(id, operator, value)
+    this.fetchNewDataset()
     this.props.executeQueriesAsync()
   }
 
   private handleChooseTimerange = (timeRange: TimeRange) => {
     this.props.setTimeRangeAsync(timeRange)
-    this.props.executeQueriesAsync()
+    this.fetchNewDataset()
   }
 
   private handleChooseSource = (sourceID: string) => {
@@ -261,8 +264,14 @@ class LogsPage extends PureComponent<Props, State> {
   private handleChartZoom = (lower, upper) => {
     if (lower) {
       this.props.changeZoomAsync({lower, upper})
+      this.setState({liveUpdating: true})
     }
   }
+
+  private fetchNewDataset() {
+    this.props.executeQueriesAsync()
+    this.setState({liveUpdating: true})
+  }
 }
 
 const mapStateToProps = ({
@@ -302,6 +311,7 @@ const mapDispatchToProps = {
   addFilter,
   removeFilter,
   changeFilter,
+  fetchMoreAsync,
 }
 
 export default connect(mapStateToProps, mapDispatchToProps)(LogsPage)

ui/src/logs/reducers/index.ts

@@ -7,6 +7,7 @@ import {
   ChangeFilterAction,
   DecrementQueryCountAction,
   IncrementQueryCountAction,
+  ConcatMoreLogsAction,
 } from 'src/logs/actions'
 import {LogsState} from 'src/types/logs'
 
@@ -17,9 +18,9 @@ const defaultState: LogsState = {
   currentNamespace: null,
   histogramQueryConfig: null,
   tableQueryConfig: null,
-  tableData: [],
+  tableData: {columns: [], values: []},
   histogramData: [],
-  searchTerm: null,
+  searchTerm: '',
   filters: [],
   queryCount: 0,
 }
@@ -75,6 +76,24 @@ const incrementQueryCount = (
   return {...state, queryCount: queryCount + 1}
 }
 
+const concatMoreLogs = (
+  state: LogsState,
+  action: ConcatMoreLogsAction
+): LogsState => {
+  const {
+    series: {values},
+  } = action.payload
+  const {tableData} = state
+  const vals = [...tableData.values, ...values]
+  return {
+    ...state,
+    tableData: {
+      columns: tableData.columns,
+      values: vals,
+    },
+  }
+}
+
 export default (state: LogsState = defaultState, action: Action) => {
   switch (action.type) {
     case ActionTypes.SetSource:
@@ -109,6 +128,8 @@ export default (state: LogsState = defaultState, action: Action) => {
       return incrementQueryCount(state, action)
     case ActionTypes.DecrementQueryCount:
       return decrementQueryCount(state, action)
+    case ActionTypes.ConcatMoreLogs:
+      return concatMoreLogs(state, action)
     default:
       return state
   }

ui/src/types/logs.ts

@@ -7,6 +7,11 @@ export interface Filter {
   operator: string
 }
 
+export interface TableData {
+  columns: string[]
+  values: object[]
+}
+
 export interface LogsState {
   currentSource: Source | null
   currentNamespaces: Namespace[]
@@ -15,7 +20,7 @@ export interface LogsState {
   histogramQueryConfig: QueryConfig | null
   histogramData: object[]
   tableQueryConfig: QueryConfig | null
-  tableData: object[]
+  tableData: TableData
   searchTerm: string | null
   filters: Filter[]
   queryCount: number