From ca9e4b797b08ddcae9cdc6bc76fad284d0659fb5 Mon Sep 17 00:00:00 2001 From: ebb-tide Date: Fri, 4 May 2018 14:35:08 -0700 Subject: [PATCH 1/6] Flatten groupbys when parsing data for csv download in dashboard --- ui/src/shared/components/AutoRefresh.tsx | 4 ++- ui/src/shared/components/Layout.js | 6 ++-- ui/src/shared/components/LayoutCell.js | 8 ++--- ui/src/shared/parsing/resultsToCSV.js | 37 ++++++++++++------------ 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/ui/src/shared/components/AutoRefresh.tsx b/ui/src/shared/components/AutoRefresh.tsx index cfbb2f5a5..53ee8381a 100644 --- a/ui/src/shared/components/AutoRefresh.tsx +++ b/ui/src/shared/components/AutoRefresh.tsx @@ -4,6 +4,7 @@ import _ from 'lodash' import {fetchTimeSeries} from 'src/shared/apis/query' import {DEFAULT_TIME_SERIES} from 'src/shared/constants/series' import {TimeSeriesServerResponse, TimeSeriesResponse} from 'src/types/series' +import {timeSeriesToTableGraph} from 'src/utils/timeSeriesTransformers' interface Axes { bounds: { @@ -129,8 +130,9 @@ const AutoRefresh = ( isFetching: false, }) + const {data} = timeSeriesToTableGraph(newSeries) if (grabDataForDownload) { - grabDataForDownload(newSeries) + grabDataForDownload(data) } } catch (err) { console.error(err) diff --git a/ui/src/shared/components/Layout.js b/ui/src/shared/components/Layout.js index b38adbdef..aa441602b 100644 --- a/ui/src/shared/components/Layout.js +++ b/ui/src/shared/components/Layout.js @@ -23,7 +23,7 @@ const getSource = (cell, source, sources, defaultSource) => { @ErrorHandling class LayoutState extends Component { state = { - celldata: [], + celldata: [[]], } grabDataForDownload = celldata => { @@ -122,7 +122,7 @@ const Layout = ( ) -const {arrayOf, bool, func, number, shape, string} = PropTypes +const {array, arrayOf, bool, func, number, shape, string} = PropTypes Layout.contextTypes = { source: shape(), @@ -200,7 +200,7 @@ LayoutState.propTypes = {...propTypes} Layout.propTypes = { ...propTypes, grabDataForDownload: func, - celldata: arrayOf(shape()), + celldata: arrayOf(array), } export default LayoutState diff --git a/ui/src/shared/components/LayoutCell.js b/ui/src/shared/components/LayoutCell.js index 7bbb05cba..6f66844b6 100644 --- a/ui/src/shared/components/LayoutCell.js +++ b/ui/src/shared/components/LayoutCell.js @@ -8,9 +8,9 @@ import LayoutCellMenu from 'shared/components/LayoutCellMenu' import LayoutCellHeader from 'shared/components/LayoutCellHeader' import {notify} from 'src/shared/actions/notifications' import {notifyCSVDownloadFailed} from 'src/shared/copy/notifications' -import {dashboardtoCSV} from 'shared/parsing/resultsToCSV' import download from 'src/external/download.js' import {ErrorHandling} from 'src/shared/decorators/errors' +import {dataToCSV} from 'src/shared/parsing/resultsToCSV' @ErrorHandling class LayoutCell extends Component { @@ -26,7 +26,7 @@ class LayoutCell extends Component { const joinedName = cell.name.split(' ').join('_') const {celldata} = this.props try { - download(dashboardtoCSV(celldata), `${joinedName}.csv`, 'text/plain') + download(dataToCSV(celldata), `${joinedName}.csv`, 'text/plain') } catch (error) { notify(notifyCSVDownloadFailed()) console.error(error) @@ -79,7 +79,7 @@ class LayoutCell extends Component { } } -const {arrayOf, bool, func, node, number, shape, string} = PropTypes +const {array, arrayOf, bool, func, node, number, shape, string} = PropTypes LayoutCell.propTypes = { cell: shape({ @@ -96,7 +96,7 @@ LayoutCell.propTypes = { onSummonOverlayTechnologies: func, isEditable: bool, onCancelEditCell: func, - celldata: arrayOf(shape()), + celldata: arrayOf(array), } export default LayoutCell diff --git a/ui/src/shared/parsing/resultsToCSV.js b/ui/src/shared/parsing/resultsToCSV.js index c3d3fdd02..c73698fd1 100644 --- a/ui/src/shared/parsing/resultsToCSV.js +++ b/ui/src/shared/parsing/resultsToCSV.js @@ -1,5 +1,6 @@ import _ from 'lodash' import moment from 'moment' +import {map} from 'fast.js' export const formatDate = timestamp => moment(timestamp).format('M/D/YYYY h:mm:ss.SSSSSSSSS A') @@ -30,24 +31,22 @@ export const resultsToCSV = results => { return {flag: 'ok', name, CSVString} } -export const dashboardtoCSV = data => { - const columnNames = _.flatten( - data.map(r => _.get(r, 'results[0].series[0].columns', [])) - ) - const timeIndices = columnNames - .map((e, i) => (e === 'time' ? i : -1)) - .filter(e => e >= 0) - - let values = data.map(r => _.get(r, 'results[0].series[0].values', [])) - values = _.unzip(values).map(v => _.flatten(v)) - if (timeIndices) { - values.map(v => { - timeIndices.forEach(i => (v[i] = formatDate(v[i]))) - return v - }) +export const dataToCSV = ([titleRow, ...valueRows]) => { + if (_.isEmpty(titleRow)) { + return '' } - const CSVString = [columnNames.join(',')] - .concat(values.map(v => v.join(','))) - .join('\n') - return CSVString + if (_.isEmpty(valueRows)) { + return ['date', titleRow.slice(1)].join(',') + } + if (titleRow[0] === 'time') { + const titlesString = ['date', titleRow.slice(1)].join(',') + + const valuesString = map(valueRows, ([timestamp, ...values]) => [ + [formatDate(timestamp), ...values].join(','), + ]).join('\n') + return `${titlesString}\n${valuesString}` + } + const allRows = [titleRow, ...valueRows] + const allRowsStringArray = map(allRows, r => r.join(',')) + return allRowsStringArray.join('\n') } From 980b8e6c3e2dad8eee09be979c4e979f6b42f00c Mon Sep 17 00:00:00 2001 From: ebb-tide Date: Fri, 4 May 2018 14:35:29 -0700 Subject: [PATCH 2/6] Bestow csv download ability on table graphs --- ui/src/shared/components/RefreshingGraph.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/src/shared/components/RefreshingGraph.js b/ui/src/shared/components/RefreshingGraph.js index 2e4d1586a..c8369e960 100644 --- a/ui/src/shared/components/RefreshingGraph.js +++ b/ui/src/shared/components/RefreshingGraph.js @@ -118,6 +118,7 @@ const RefreshingGraph = ({ decimalPlaces={decimalPlaces} editQueryStatus={editQueryStatus} resizerTopHeight={resizerTopHeight} + grabDataForDownload={grabDataForDownload} handleSetHoverTime={handleSetHoverTime} isInCEO={isInCEO} /> From 1f03956fc1b733b9d1121c9426abea3875b2d5b7 Mon Sep 17 00:00:00 2001 From: ebb-tide Date: Fri, 4 May 2018 15:54:57 -0700 Subject: [PATCH 3/6] Prevent data transformation if no csv download option --- ui/src/shared/components/AutoRefresh.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/shared/components/AutoRefresh.tsx b/ui/src/shared/components/AutoRefresh.tsx index 53ee8381a..32c09ccb5 100644 --- a/ui/src/shared/components/AutoRefresh.tsx +++ b/ui/src/shared/components/AutoRefresh.tsx @@ -130,8 +130,8 @@ const AutoRefresh = ( isFetching: false, }) - const {data} = timeSeriesToTableGraph(newSeries) if (grabDataForDownload) { + const {data} = timeSeriesToTableGraph(newSeries) grabDataForDownload(data) } } catch (err) { From 50cf43bafbf7d34bcd47f67a8a2fc2f512ba419f Mon Sep 17 00:00:00 2001 From: ebb-tide Date: Fri, 4 May 2018 16:32:28 -0700 Subject: [PATCH 4/6] Use Flatten groupbys when parsing data for csv download in data explorer --- ui/src/data_explorer/components/VisHeader.js | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/ui/src/data_explorer/components/VisHeader.js b/ui/src/data_explorer/components/VisHeader.js index 79638222a..13f0a3956 100644 --- a/ui/src/data_explorer/components/VisHeader.js +++ b/ui/src/data_explorer/components/VisHeader.js @@ -4,23 +4,21 @@ import classnames from 'classnames' import _ from 'lodash' import {fetchTimeSeriesAsync} from 'shared/actions/timeSeries' -import {resultsToCSV} from 'src/shared/parsing/resultsToCSV.js' +import {timeSeriesToTableGraph} from 'src/utils/timeSeriesTransformers' +import {dataToCSV} from 'src/shared/parsing/resultsToCSV' import download from 'src/external/download.js' import {TEMPLATES} from 'src/shared/constants' -const getCSV = (query, errorThrown) => async () => { +const getDataForCSV = (query, errorThrown) => async () => { try { - const {results} = await fetchTimeSeriesAsync({ + const response = await fetchTimeSeriesAsync({ source: query.host, query, tempVars: TEMPLATES, }) - const {flag, name, CSVString} = resultsToCSV(results) - if (flag === 'no_data') { - errorThrown('no data', 'There are no data to download.') - return - } - download(CSVString, `${name}.csv`, 'text/plain') + const {data} = timeSeriesToTableGraph([{response}]) + + download(dataToCSV(data), `${''}.csv`, 'text/plain') } catch (error) { errorThrown(error, 'Unable to download .csv file') console.error(error) @@ -46,7 +44,7 @@ const VisHeader = ({views, view, onToggleView, query, errorThrown}) => ( {query ? (
.csv From 1feb86dfa95e9fc1f71667bdf1d503863d772c02 Mon Sep 17 00:00:00 2001 From: ebb-tide Date: Fri, 4 May 2018 16:50:34 -0700 Subject: [PATCH 5/6] Removed unused functions, rename files, and add tests --- ui/src/data_explorer/components/VisHeader.js | 2 +- ui/src/shared/components/LayoutCell.js | 2 +- .../parsing/{resultsToCSV.js => dataToCSV.js} | 26 ----- ui/test/shared/parsing/dataToCSV.test.js | 46 ++++++++ ui/test/shared/parsing/resultsToCSV.test.js | 105 ------------------ 5 files changed, 48 insertions(+), 133 deletions(-) rename ui/src/shared/parsing/{resultsToCSV.js => dataToCSV.js} (52%) create mode 100644 ui/test/shared/parsing/dataToCSV.test.js delete mode 100644 ui/test/shared/parsing/resultsToCSV.test.js diff --git a/ui/src/data_explorer/components/VisHeader.js b/ui/src/data_explorer/components/VisHeader.js index 13f0a3956..7c05417e8 100644 --- a/ui/src/data_explorer/components/VisHeader.js +++ b/ui/src/data_explorer/components/VisHeader.js @@ -5,7 +5,7 @@ import _ from 'lodash' import {fetchTimeSeriesAsync} from 'shared/actions/timeSeries' import {timeSeriesToTableGraph} from 'src/utils/timeSeriesTransformers' -import {dataToCSV} from 'src/shared/parsing/resultsToCSV' +import {dataToCSV} from 'src/shared/parsing/dataToCSV' import download from 'src/external/download.js' import {TEMPLATES} from 'src/shared/constants' diff --git a/ui/src/shared/components/LayoutCell.js b/ui/src/shared/components/LayoutCell.js index 6f66844b6..f1658e8ac 100644 --- a/ui/src/shared/components/LayoutCell.js +++ b/ui/src/shared/components/LayoutCell.js @@ -10,7 +10,7 @@ import {notify} from 'src/shared/actions/notifications' import {notifyCSVDownloadFailed} from 'src/shared/copy/notifications' import download from 'src/external/download.js' import {ErrorHandling} from 'src/shared/decorators/errors' -import {dataToCSV} from 'src/shared/parsing/resultsToCSV' +import {dataToCSV} from 'src/shared/parsing/dataToCSV' @ErrorHandling class LayoutCell extends Component { diff --git a/ui/src/shared/parsing/resultsToCSV.js b/ui/src/shared/parsing/dataToCSV.js similarity index 52% rename from ui/src/shared/parsing/resultsToCSV.js rename to ui/src/shared/parsing/dataToCSV.js index c73698fd1..8edbd4afc 100644 --- a/ui/src/shared/parsing/resultsToCSV.js +++ b/ui/src/shared/parsing/dataToCSV.js @@ -5,32 +5,6 @@ import {map} from 'fast.js' export const formatDate = timestamp => moment(timestamp).format('M/D/YYYY h:mm:ss.SSSSSSSSS A') -export const resultsToCSV = results => { - if (!_.get(results, ['0', 'series', '0'])) { - return {flag: 'no_data', name: '', CSVString: ''} - } - - const {name, columns, values} = _.get(results, ['0', 'series', '0']) - - if (columns[0] === 'time') { - const [, ...cols] = columns - const CSVString = [['date', ...cols].join(',')] - .concat( - values.map(([timestamp, ...measurements]) => - // MS Excel format - [formatDate(timestamp), ...measurements].join(',') - ) - ) - .join('\n') - return {flag: 'ok', name, CSVString} - } - - const CSVString = [columns.join(',')] - .concat(values.map(row => row.join(','))) - .join('\n') - return {flag: 'ok', name, CSVString} -} - export const dataToCSV = ([titleRow, ...valueRows]) => { if (_.isEmpty(titleRow)) { return '' diff --git a/ui/test/shared/parsing/dataToCSV.test.js b/ui/test/shared/parsing/dataToCSV.test.js new file mode 100644 index 000000000..d46e620cf --- /dev/null +++ b/ui/test/shared/parsing/dataToCSV.test.js @@ -0,0 +1,46 @@ +import {dataToCSV, formatDate} from 'shared/parsing/dataToCSV' +import moment from 'moment' + +describe('formatDate', () => { + it('converts timestamp to an excel compatible date string', () => { + const timestamp = 1000000000000 + const result = formatDate(timestamp) + expect(moment(result, 'M/D/YYYY h:mm:ss.SSSSSSSSS A').valueOf()).toBe( + timestamp + ) + }) +}) + +describe('dataToCSV', () => { + it('parses data, an array of arrays, to a csv string', () => { + const data = [[1, 2], [3, 4], [5, 6], [7, 8]] + const returned = dataToCSV(data) + const expected = `1,2\n3,4\n5,6\n7,8` + + expect(returned).toEqual(expected) + }) + + it('converts values to dates if title of first column is time.', () => { + const data = [ + ['time', 'something'], + [1505262600000, 0.06163066773148772], + [1505264400000, 2.616484718180463], + [1505266200000, 1.6174323943535571], + ] + const returned = dataToCSV(data) + const expected = `date,something\n${formatDate( + 1505262600000 + )},0.06163066773148772\n${formatDate( + 1505264400000 + )},2.616484718180463\n${formatDate(1505266200000)},1.6174323943535571` + + expect(returned).toEqual(expected) + }) + + it('returns an empty string if data is empty', () => { + const data = [[]] + const returned = dataToCSV(data) + const expected = '' + expect(returned).toEqual(expected) + }) +}) diff --git a/ui/test/shared/parsing/resultsToCSV.test.js b/ui/test/shared/parsing/resultsToCSV.test.js deleted file mode 100644 index fdc84ace2..000000000 --- a/ui/test/shared/parsing/resultsToCSV.test.js +++ /dev/null @@ -1,105 +0,0 @@ -import { - resultsToCSV, - formatDate, - dashboardtoCSV, -} from 'shared/parsing/resultsToCSV' -import moment from 'moment' - -describe('formatDate', () => { - it('converts timestamp to an excel compatible date string', () => { - const timestamp = 1000000000000 - const result = formatDate(timestamp) - expect(moment(result, 'M/D/YYYY h:mm:ss.SSSSSSSSS A').valueOf()).toBe( - timestamp - ) - }) -}) - -describe('resultsToCSV', () => { - it('parses results, a time series data structure, to an object with name and CSVString keys', () => { - const results = [ - { - statement_id: 0, - series: [ - { - name: 'procstat', - columns: ['time', 'mean_cpu_usage'], - values: [ - [1505262600000, 0.06163066773148772], - [1505264400000, 2.616484718180463], - [1505266200000, 1.6174323943535571], - ], - }, - ], - }, - ] - const response = resultsToCSV(results) - const expected = { - flag: 'ok', - name: 'procstat', - CSVString: `date,mean_cpu_usage\n${formatDate( - 1505262600000 - )},0.06163066773148772\n${formatDate( - 1505264400000 - )},2.616484718180463\n${formatDate(1505266200000)},1.6174323943535571`, - } - expect(Object.keys(response).sort()).toEqual( - ['flag', 'name', 'CSVString'].sort() - ) - expect(response.flag).toBe(expected.flag) - expect(response.name).toBe(expected.name) - expect(response.CSVString).toBe(expected.CSVString) - }) -}) - -describe('dashboardtoCSV', () => { - it('parses the array of timeseries data displayed by the dashboard cell to a CSVstring for download', () => { - const data = [ - { - results: [ - { - statement_id: 0, - series: [ - { - name: 'procstat', - columns: ['time', 'mean_cpu_usage'], - values: [ - [1505262600000, 0.06163066773148772], - [1505264400000, 2.616484718180463], - [1505266200000, 1.6174323943535571], - ], - }, - ], - }, - ], - }, - { - results: [ - { - statement_id: 0, - series: [ - { - name: 'procstat', - columns: ['not-time', 'mean_cpu_usage'], - values: [ - [1505262600000, 0.06163066773148772], - [1505264400000, 2.616484718180463], - [1505266200000, 1.6174323943535571], - ], - }, - ], - }, - ], - }, - ] - const result = dashboardtoCSV(data) - const expected = `time,mean_cpu_usage,not-time,mean_cpu_usage\n${formatDate( - 1505262600000 - )},0.06163066773148772,1505262600000,0.06163066773148772\n${formatDate( - 1505264400000 - )},2.616484718180463,1505264400000,2.616484718180463\n${formatDate( - 1505266200000 - )},1.6174323943535571,1505266200000,1.6174323943535571` - expect(result).toBe(expected) - }) -}) From 313f3a8aebb90de385aed2618aa5008aeb70b1cb Mon Sep 17 00:00:00 2001 From: ebb-tide Date: Fri, 4 May 2018 17:26:02 -0700 Subject: [PATCH 6/6] Add a filename with db.rp.measurement... to csvs from data explorer --- ui/src/data_explorer/components/VisHeader.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ui/src/data_explorer/components/VisHeader.js b/ui/src/data_explorer/components/VisHeader.js index 7c05417e8..615608d6a 100644 --- a/ui/src/data_explorer/components/VisHeader.js +++ b/ui/src/data_explorer/components/VisHeader.js @@ -2,6 +2,7 @@ import React from 'react' import PropTypes from 'prop-types' import classnames from 'classnames' import _ from 'lodash' +import moment from 'moment' import {fetchTimeSeriesAsync} from 'shared/actions/timeSeries' import {timeSeriesToTableGraph} from 'src/utils/timeSeriesTransformers' @@ -17,8 +18,13 @@ const getDataForCSV = (query, errorThrown) => async () => { tempVars: TEMPLATES, }) const {data} = timeSeriesToTableGraph([{response}]) + const db = _.get(query, ['queryConfig', 'database'], '') + const rp = _.get(query, ['queryConfig', 'retentionPolicy'], '') + const measurement = _.get(query, ['queryConfig', 'measurement'], '') - download(dataToCSV(data), `${''}.csv`, 'text/plain') + const timestring = moment().format('YYYY-MM-DD-HH-mm') + const name = `${db}.${rp}.${measurement}.${timestring}` + download(dataToCSV(data), `${name}.csv`, 'text/plain') } catch (error) { errorThrown(error, 'Unable to download .csv file') console.error(error)