From b139eb4a18fbfc36793f8bd0b4f088f6642fe7fa Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Thu, 12 Dec 2024 16:42:11 +0000 Subject: [PATCH] update CSV to adhere to strict rfc compliance on msg.columns --- .../@node-red/nodes/core/parsers/70-CSV.js | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js index 38cda9a6f..26fdb636f 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js +++ b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js @@ -367,20 +367,21 @@ module.exports = function(RED) { const sendHeadersAlways = node.hdrout === "all" const sendHeaders = !dontSendHeaders && (sendHeadersOnce || sendHeadersAlways) const quoteables = [node.sep, node.quo, "\n", "\r"] - const templateQuoteables = [node.sep, '"', "\n", "\r"] + const templateQuoteables = [node.sep, node.quo, "\n", "\r"] + const templateQuoteablesStrict = [',', '"', "\n", "\r"] let badTemplateWarnOnce = true const columnStringToTemplateArray = function (col, sep) { // NOTE: enforce strict column template parsing in RFC4180 mode const parsed = csv.parse(col, { separator: sep, quote: node.quo, outputStyle: 'array', strict: true }) - if (parsed.headers.length > 0) { node.goodtmpl = true } else { node.goodtmpl = false } - return parsed.headers.length ? parsed.headers : null + if (parsed.data?.length === 1) { node.goodtmpl = true } else { node.goodtmpl = false } + return node.goodtmpl ? parsed.data[0] : null } - const templateArrayToColumnString = function (template, keepEmptyColumns) { - // NOTE: enforce strict column template parsing in RFC4180 mode - const parsed = csv.parse('', {headers: template, headersOnly:true, separator: node.sep, quote: node.quo, outputStyle: 'array', strict: true }) + const templateArrayToColumnString = function (template, keepEmptyColumns, separator = ',', quotables = templateQuoteablesStrict) { + // NOTE: defaults to strict column template parsing (commas and double quotes) + const parsed = csv.parse('', {headers: template, headersOnly:true, separator, quote: node.quo, outputStyle: 'array', strict: true }) return keepEmptyColumns - ? parsed.headers.map(e => addQuotes(e || '', { separator: node.sep, quoteables: templateQuoteables})).join(node.sep) + ? parsed.headers.map(e => addQuotes(e || '', { separator, quoteables: quotables })).join(separator) : parsed.header // exclues empty columns // TODO: resolve inconsistency between CSV->JSON and JSON->CSV // CSV->JSON: empty columns are excluded @@ -441,13 +442,13 @@ module.exports = function(RED) { if (sendHeaders && node.hdrSent === false) { if (hasTemplate(template) === false) { if (msg.hasOwnProperty("columns")) { - template = columnStringToTemplateArray(msg.columns || "", node.sep) || [''] + template = columnStringToTemplateArray(msg.columns || "", ",") || [''] } else { template = Object.keys(inputData[0]) || [''] } } - stringBuilder.push(templateArrayToColumnString(template, true)) + stringBuilder.push(templateArrayToColumnString(template, true, node.sep, templateQuoteables)) // use user set separator for output data. if (sendHeadersOnce) { node.hdrSent = true } } @@ -475,7 +476,7 @@ module.exports = function(RED) { } else { /*** row is an object ***/ if (hasTemplate(template) === false && (msg.hasOwnProperty("columns"))) { - template = columnStringToTemplateArray(msg.columns || "", node.sep) + template = columnStringToTemplateArray(msg.columns || "", ",") } if (hasTemplate(template) === false) { /*** row is an object but we still don't have a template ***/ @@ -483,6 +484,7 @@ module.exports = function(RED) { node.warn(RED._("csv.errors.obj_csv")) badTemplateWarnOnce = false } + template = Object.keys(row) || [''] const rowData = [] for (let header in inputData[0]) { if (row.hasOwnProperty(header)) { @@ -518,7 +520,7 @@ module.exports = function(RED) { // join lines, don't forget to add the last new line msg.payload = stringBuilder.join(node.ret) + node.ret - msg.columns = templateArrayToColumnString(template) + msg.columns = templateArrayToColumnString(template) // always strict commas + double quotes for if (msg.payload !== '') { send(msg) } done() } @@ -615,16 +617,15 @@ module.exports = function(RED) { } if (msg.parts.index + 1 === msg.parts.count) { msg.payload = node.store - msg.columns = csvParseResult.header - // msg._mode = 'RFC4180 mode' + // msg.columns = csvParseResult.header + msg.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns delete msg.parts send(msg) node.store = [] } } else { - msg.columns = csvParseResult.header - // msg._mode = 'RFC4180 mode' + msg.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns msg.payload = data send(msg); // finally send the array } @@ -633,7 +634,8 @@ module.exports = function(RED) { const len = data.length for (let row = 0; row < len; row++) { const newMessage = RED.util.cloneMessage(msg) - newMessage.columns = csvParseResult.header + // newMessage.columns = csvParseResult.header + newMessage.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns newMessage.payload = data[row] if (!has_parts) { newMessage.parts = {