Exponential backoff for finding timeframe to query logs from

ui/src/logs/actions/index.ts

@@ -9,9 +9,10 @@ import {
   buildHistogramQueryConfig,
   buildTableQueryConfig,
   buildLogQuery,
-  buildForwardLogQuery,
+  buildInfiniteLogQuery,
-  buildBackwardLogQuery,
   parseHistogramQueryResponse,
+  findBackwardLower,
+  findForwardUpper,
 } from 'src/logs/utils'
 import {logConfigServerToUI, logConfigUIToServer} from 'src/logs/utils/config'
 import {getDeep} from 'src/utils/wrappers'
@@ -367,7 +368,23 @@ export const executeTableForwardQueryAsync = () => async (
   try {
     dispatch(incrementQueryCount())
 
-    const query = buildForwardLogQuery(time, queryConfig, filters, searchTerm)
+    const upper = await findForwardUpper(
+      time,
+      queryConfig,
+      filters,
+      searchTerm,
+      proxyLink,
+      namespace
+    )
+
+    const query: string = await buildInfiniteLogQuery(
+      time,
+      upper,
+      queryConfig,
+      filters,
+      searchTerm
+    )
+
     const response = await executeQueryAsync(
       proxyLink,
       namespace,
@@ -407,7 +424,23 @@ export const executeTableBackwardQueryAsync = () => async (
   try {
     dispatch(incrementQueryCount())
 
-    const query = buildBackwardLogQuery(time, queryConfig, filters, searchTerm)
+    const lower: string = await findBackwardLower(
+      time,
+      queryConfig,
+      filters,
+      searchTerm,
+      proxyLink,
+      namespace
+    )
+
+    const query: string = await buildInfiniteLogQuery(
+      lower,
+      time,
+      queryConfig,
+      filters,
+      searchTerm
+    )
+
     const response = await executeQueryAsync(
       proxyLink,
       namespace,
@@ -577,7 +610,7 @@ export const setTableQueryConfigAsync = () => async (
   }
 }
 
-export const fetchMoreAsync = (queryTimeEnd: string) => async (
+export const fetchOlderLogsAsync = (queryTimeEnd: string) => async (
   dispatch,
   getState
 ): Promise<void> => {
@@ -595,7 +628,17 @@ export const fetchMoreAsync = (queryTimeEnd: string) => async (
   const params = [namespace, proxyLink, tableQueryConfig]
 
   if (_.every(params)) {
-    const query = buildBackwardLogQuery(
+    const lower = await findBackwardLower(
+      queryTimeEnd,
+      newQueryConfig,
+      filters,
+      searchTerm,
+      proxyLink,
+      namespace
+    )
+
+    const query = await buildInfiniteLogQuery(
+      lower,
       queryTimeEnd,
       newQueryConfig,
       filters,
@@ -613,7 +656,7 @@ export const fetchMoreAsync = (queryTimeEnd: string) => async (
   }
 }
 
-export const fetchNewerAsync = (queryTimeStart: string) => async (
+export const fetchNewerLogsAsync = (queryTimeStart: string) => async (
   dispatch,
   getState
 ): Promise<void> => {
@@ -631,10 +674,20 @@ export const fetchNewerAsync = (queryTimeStart: string) => async (
   const params = [namespace, proxyLink, tableQueryConfig]
 
   if (_.every(params)) {
-    const query = buildForwardLogQuery(
+    const upper = await findForwardUpper(
       queryTimeStart,
       newQueryConfig,
       filters,
+      searchTerm,
+      proxyLink,
+      namespace
+    )
+
+    const query: string = await buildInfiniteLogQuery(
+      queryTimeStart,
+      upper,
+      newQueryConfig,
+      filters,
       searchTerm
     )
 

ui/src/logs/containers/LogsPage.tsx

@@ -22,8 +22,8 @@ import {
   addFilter,
   removeFilter,
   changeFilter,
-  fetchMoreAsync,
+  fetchOlderLogsAsync,
-  fetchNewerAsync,
+  fetchNewerLogsAsync,
   getLogConfigAsync,
   updateLogConfigAsync,
 } from 'src/logs/actions'
@@ -81,8 +81,8 @@ interface Props {
   setSearchTermAsync: (searchTerm: string) => void
   setTableRelativeTime: (time: number) => void
   setTableCustomTime: (time: string) => void
-  fetchMoreAsync: (queryTimeEnd: string) => Promise<void>
+  fetchOlderLogsAsync: (queryTimeEnd: string) => Promise<void>
-  fetchNewerAsync: (queryTimeEnd: string) => Promise<void>
+  fetchNewerLogsAsync: (queryTimeEnd: string) => Promise<void>
   addFilter: (filter: Filter) => void
   removeFilter: (id: string) => void
   changeFilter: (id: string, operator: string, value: string) => void
@@ -195,7 +195,7 @@ class LogsPage extends Component<Props, State> {
               onScrolledToTop={this.handleScrollToTop}
               isScrolledToTop={false}
               onTagSelection={this.handleTagSelection}
-              fetchMore={this.props.fetchMoreAsync}
+              fetchMore={this.props.fetchOlderLogsAsync}
               fetchNewer={this.fetchNewer}
               timeRange={timeRange}
               scrollToRow={this.tableScrollToRow}
@@ -215,7 +215,7 @@ class LogsPage extends Component<Props, State> {
 
   private fetchNewer = (time: string) => {
     this.loadingNewer = true
-    this.props.fetchNewerAsync(time)
+    this.props.fetchNewerLogsAsync(time)
   }
 
   private get tableScrollToRow() {
@@ -506,7 +506,6 @@ class LogsPage extends Component<Props, State> {
 
   private handleSubmitSearch = (value: string): void => {
     this.props.setSearchTermAsync(value)
-    this.setState({liveUpdating: LiveUpdating.Play})
   }
 
   private handleFilterDelete = (id: string): void => {
@@ -689,8 +688,8 @@ const mapDispatchToProps = {
   addFilter,
   removeFilter,
   changeFilter,
-  fetchMoreAsync,
+  fetchOlderLogsAsync,
-  fetchNewerAsync,
+  fetchNewerLogsAsync,
   setTableCustomTime: setTableCustomTimeAsync,
   setTableRelativeTime: setTableRelativeTimeAsync,
   getConfig: getLogConfigAsync,

ui/src/logs/utils/index.ts

@@ -14,8 +14,10 @@ import {
 } from 'src/utils/influxql'
 
 import {HistogramData} from 'src/types/histogram'
+import {executeQueryAsync} from 'src/logs/api'
 
 const BIN_COUNT = 30
+const LOOK_BACK_LIMIT = 2592000
 
 const histogramFields = [
   {
@@ -174,7 +176,116 @@ export function buildGeneralLogQuery(
   return `${select}${condition}${dimensions}${fillClause}`
 }
 
-export function buildBackwardLogQuery(
+export async function findCount(
+  lower,
+  upper,
+  config,
+  filters,
+  searchTerm,
+  proxyLink,
+  namespace
+): Promise<number> {
+  const {database, retentionPolicy, measurement} = config
+  const rpSegment = retentionPolicy ? `"${retentionPolicy}"` : ''
+  const fullyQualifiedMeasurement = `"${database}".${rpSegment}."${measurement}"`
+  const select = `SELECT count(message) FROM ${fullyQualifiedMeasurement}`
+  let condition = `WHERE time >= '${lower}' AND time <='${upper}'`
+
+  if (!_.isEmpty(searchTerm)) {
+    condition = `${condition} AND message =~ ${new RegExp(searchTerm)}`
+  }
+
+  if (!_.isEmpty(filters)) {
+    condition = `${condition} AND ${filtersClause(filters)}`
+  }
+
+  const query = `${select} ${condition} FILL(0)`
+  const result = await executeQueryAsync(proxyLink, namespace, query)
+  return getDeep<number>(result, 'results.0.series.0.values.0.1', 0)
+}
+
+export async function findBackwardLower(
+  upper: string,
+  config: QueryConfig,
+  filters: Filter[],
+  searchTerm: string | null = null,
+  proxyLink: string,
+  namespace: Namespace
+): Promise<string> {
+  const parsedUpper = moment(upper)
+
+  let secondsBack = 30
+  let currentLower = parsedUpper.subtract(secondsBack, 'seconds')
+
+  while (true) {
+    if (secondsBack > LOOK_BACK_LIMIT) {
+      // One day
+      break
+    }
+
+    const count = await findCount(
+      currentLower.toISOString(),
+      upper,
+      config,
+      filters,
+      searchTerm,
+      proxyLink,
+      namespace
+    )
+
+    if (count >= 400) {
+      break
+    }
+
+    secondsBack *= secondsBack
+    currentLower = parsedUpper.subtract(secondsBack, 'seconds')
+  }
+
+  return currentLower.toISOString()
+}
+
+export async function findForwardUpper(
+  lower: string,
+  config: QueryConfig,
+  filters: Filter[],
+  searchTerm: string | null = null,
+  proxyLink: string,
+  namespace: Namespace
+): Promise<string> {
+  const parsedLower = moment(lower)
+
+  let secondsBack = 30
+  let currentUpper = parsedLower.add(secondsBack, 'seconds')
+
+  while (true) {
+    if (secondsBack > LOOK_BACK_LIMIT) {
+      // One day
+      break
+    }
+
+    const count = await findCount(
+      lower,
+      currentUpper.toISOString(),
+      config,
+      filters,
+      searchTerm,
+      proxyLink,
+      namespace
+    )
+
+    if (count >= 400) {
+      break
+    }
+
+    secondsBack *= secondsBack
+    currentUpper = parsedLower.add(secondsBack, 'seconds')
+  }
+
+  return currentUpper.toISOString()
+}
+
+export async function buildInfiniteLogQuery(
+  lower: string,
   upper: string,
   config: QueryConfig,
   filters: Filter[],
@@ -182,25 +293,9 @@ export function buildBackwardLogQuery(
 ) {
   const {tags, areTagsAccepted} = config
 
-  const condition = buildInfiniteWhereClause({
-    upper,
-    tags,
-    areTagsAccepted,
-  })
-
-  return buildGeneralLogQuery(condition, config, filters, searchTerm)
-}
-
-export function buildForwardLogQuery(
-  lower: string,
-  config: QueryConfig,
-  filters: Filter[],
-  searchTerm: string | null = null
-) {
-  const {tags, areTagsAccepted} = config
-
   const condition = buildInfiniteWhereClause({
     lower,
+    upper,
     tags,
     areTagsAccepted,
   })