From e4fdf24545034c0cf1019fa35fc465bb32e42741 Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Thu, 5 Dec 2024 16:00:54 +0000 Subject: [PATCH 1/6] Ensure node.sep is honoured when generating CSV --- .../@node-red/nodes/core/parsers/70-CSV.js | 18 +++++++++--------- .../nodes/core/parsers/lib/csv/index.js | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js index c6a91d61a..92d12d0df 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js +++ b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js @@ -171,7 +171,7 @@ module.exports = function(RED) { } // join lines, don't forget to add the last new line msg.payload = ou.join(node.ret) + node.ret; - msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(','); + msg.columns = template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep); if (msg.payload !== '') { send(msg); } @@ -289,14 +289,14 @@ module.exports = function(RED) { } if (msg.parts.index + 1 === msg.parts.count) { msg.payload = node.store; - msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); + msg.columns = template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).filter(v => v).join(node.sep); delete msg.parts; send(msg); node.store = []; } } else { - msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); + msg.columns = template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).filter(v => v).join(node.sep); send(msg); // finally send the array } } @@ -304,7 +304,7 @@ module.exports = function(RED) { var len = a.length; for (var i = 0; i < len; i++) { var newMessage = RED.util.cloneMessage(msg); - newMessage.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); + newMessage.columns = template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).filter(v => v).join(node.sep); newMessage.payload = a[i]; if (!has_parts) { newMessage.parts = { @@ -367,7 +367,7 @@ module.exports = function(RED) { const sendHeadersAlways = node.hdrout === "all" const sendHeaders = !dontSendHeaders && (sendHeadersOnce || sendHeadersAlways) const quoteables = [node.sep, node.quo, "\n", "\r"] - const templateQuoteables = [',', '"', "\n", "\r"] + const templateQuoteables = [node.sep, '"', "\n", "\r"] let badTemplateWarnOnce = true const columnStringToTemplateArray = function (col, sep) { @@ -378,9 +378,9 @@ module.exports = function(RED) { } const templateArrayToColumnString = function (template, keepEmptyColumns) { // NOTE: enforce strict column template parsing in RFC4180 mode - const parsed = csv.parse('', {headers: template, headersOnly:true, separator: ',', quote: node.quo, outputStyle: 'array', strict: true }) + const parsed = csv.parse('', {headers: template, headersOnly:true, separator: node.sep, quote: node.quo, outputStyle: 'array', strict: true }) return keepEmptyColumns - ? parsed.headers.map(e => addQuotes(e || '', { separator: ',', quoteables: templateQuoteables})) + ? parsed.headers.map(e => addQuotes(e || '', { separator: node.sep, quoteables: templateQuoteables})).join(node.sep) : parsed.header // exclues empty columns // TODO: resolve inconsistency between CSV->JSON and JSON->CSV // CSV->JSON: empty columns are excluded @@ -441,7 +441,7 @@ module.exports = function(RED) { if (sendHeaders && node.hdrSent === false) { if (hasTemplate(template) === false) { if (msg.hasOwnProperty("columns")) { - template = columnStringToTemplateArray(msg.columns || "", ",") || [''] + template = columnStringToTemplateArray(msg.columns || "", node.sep) || [''] } else { template = Object.keys(inputData[0]) || [''] @@ -475,7 +475,7 @@ module.exports = function(RED) { } else { /*** row is an object ***/ if (hasTemplate(template) === false && (msg.hasOwnProperty("columns"))) { - template = columnStringToTemplateArray(msg.columns || "", ",") + template = columnStringToTemplateArray(msg.columns || "", node.sep) } if (hasTemplate(template) === false) { /*** row is an object but we still don't have a template ***/ diff --git a/packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js b/packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js index 73cf4b292..dd2fb2db9 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js +++ b/packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js @@ -200,9 +200,9 @@ function parse(csvIn, parseOptions) { if (!headers[i]) { continue } - quotedHeaders.push(quoteCell(headers[i], { quote, separator: ',' })) + quotedHeaders.push(quoteCell(headers[i], { quote, separator })) } - finalResult.header = quotedHeaders.join(',') // always quote headers and join with comma + finalResult.header = quotedHeaders.join(separator) // always quote headers and join with comma // output is an array of arrays [[],[],[]] if (ouputArrays || headersOnly) { From 2c3fbb14679534cc0a399316619cd32a30a1b976 Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Thu, 12 Dec 2024 16:40:43 +0000 Subject: [PATCH 2/6] revert changes to legacy mode --- .../node_modules/@node-red/nodes/core/parsers/70-CSV.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js index 92d12d0df..38cda9a6f 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js +++ b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js @@ -171,7 +171,7 @@ module.exports = function(RED) { } // join lines, don't forget to add the last new line msg.payload = ou.join(node.ret) + node.ret; - msg.columns = template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep); + msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(','); if (msg.payload !== '') { send(msg); } @@ -289,14 +289,14 @@ module.exports = function(RED) { } if (msg.parts.index + 1 === msg.parts.count) { msg.payload = node.store; - msg.columns = template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).filter(v => v).join(node.sep); + msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); delete msg.parts; send(msg); node.store = []; } } else { - msg.columns = template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).filter(v => v).join(node.sep); + msg.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); send(msg); // finally send the array } } @@ -304,7 +304,7 @@ module.exports = function(RED) { var len = a.length; for (var i = 0; i < len; i++) { var newMessage = RED.util.cloneMessage(msg); - newMessage.columns = template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).filter(v => v).join(node.sep); + newMessage.columns = template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); newMessage.payload = a[i]; if (!has_parts) { newMessage.parts = { From 6af3c8c2a9422e22129b62ee5f211b9ca5964a83 Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Thu, 12 Dec 2024 16:41:26 +0000 Subject: [PATCH 3/6] revert changes to csv parser --- .../@node-red/nodes/core/parsers/lib/csv/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js b/packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js index dd2fb2db9..73cf4b292 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js +++ b/packages/node_modules/@node-red/nodes/core/parsers/lib/csv/index.js @@ -200,9 +200,9 @@ function parse(csvIn, parseOptions) { if (!headers[i]) { continue } - quotedHeaders.push(quoteCell(headers[i], { quote, separator })) + quotedHeaders.push(quoteCell(headers[i], { quote, separator: ',' })) } - finalResult.header = quotedHeaders.join(separator) // always quote headers and join with comma + finalResult.header = quotedHeaders.join(',') // always quote headers and join with comma // output is an array of arrays [[],[],[]] if (ouputArrays || headersOnly) { From b139eb4a18fbfc36793f8bd0b4f088f6642fe7fa Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Thu, 12 Dec 2024 16:42:11 +0000 Subject: [PATCH 4/6] update CSV to adhere to strict rfc compliance on msg.columns --- .../@node-red/nodes/core/parsers/70-CSV.js | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js index 38cda9a6f..26fdb636f 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js +++ b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js @@ -367,20 +367,21 @@ module.exports = function(RED) { const sendHeadersAlways = node.hdrout === "all" const sendHeaders = !dontSendHeaders && (sendHeadersOnce || sendHeadersAlways) const quoteables = [node.sep, node.quo, "\n", "\r"] - const templateQuoteables = [node.sep, '"', "\n", "\r"] + const templateQuoteables = [node.sep, node.quo, "\n", "\r"] + const templateQuoteablesStrict = [',', '"', "\n", "\r"] let badTemplateWarnOnce = true const columnStringToTemplateArray = function (col, sep) { // NOTE: enforce strict column template parsing in RFC4180 mode const parsed = csv.parse(col, { separator: sep, quote: node.quo, outputStyle: 'array', strict: true }) - if (parsed.headers.length > 0) { node.goodtmpl = true } else { node.goodtmpl = false } - return parsed.headers.length ? parsed.headers : null + if (parsed.data?.length === 1) { node.goodtmpl = true } else { node.goodtmpl = false } + return node.goodtmpl ? parsed.data[0] : null } - const templateArrayToColumnString = function (template, keepEmptyColumns) { - // NOTE: enforce strict column template parsing in RFC4180 mode - const parsed = csv.parse('', {headers: template, headersOnly:true, separator: node.sep, quote: node.quo, outputStyle: 'array', strict: true }) + const templateArrayToColumnString = function (template, keepEmptyColumns, separator = ',', quotables = templateQuoteablesStrict) { + // NOTE: defaults to strict column template parsing (commas and double quotes) + const parsed = csv.parse('', {headers: template, headersOnly:true, separator, quote: node.quo, outputStyle: 'array', strict: true }) return keepEmptyColumns - ? parsed.headers.map(e => addQuotes(e || '', { separator: node.sep, quoteables: templateQuoteables})).join(node.sep) + ? parsed.headers.map(e => addQuotes(e || '', { separator, quoteables: quotables })).join(separator) : parsed.header // exclues empty columns // TODO: resolve inconsistency between CSV->JSON and JSON->CSV // CSV->JSON: empty columns are excluded @@ -441,13 +442,13 @@ module.exports = function(RED) { if (sendHeaders && node.hdrSent === false) { if (hasTemplate(template) === false) { if (msg.hasOwnProperty("columns")) { - template = columnStringToTemplateArray(msg.columns || "", node.sep) || [''] + template = columnStringToTemplateArray(msg.columns || "", ",") || [''] } else { template = Object.keys(inputData[0]) || [''] } } - stringBuilder.push(templateArrayToColumnString(template, true)) + stringBuilder.push(templateArrayToColumnString(template, true, node.sep, templateQuoteables)) // use user set separator for output data. if (sendHeadersOnce) { node.hdrSent = true } } @@ -475,7 +476,7 @@ module.exports = function(RED) { } else { /*** row is an object ***/ if (hasTemplate(template) === false && (msg.hasOwnProperty("columns"))) { - template = columnStringToTemplateArray(msg.columns || "", node.sep) + template = columnStringToTemplateArray(msg.columns || "", ",") } if (hasTemplate(template) === false) { /*** row is an object but we still don't have a template ***/ @@ -483,6 +484,7 @@ module.exports = function(RED) { node.warn(RED._("csv.errors.obj_csv")) badTemplateWarnOnce = false } + template = Object.keys(row) || [''] const rowData = [] for (let header in inputData[0]) { if (row.hasOwnProperty(header)) { @@ -518,7 +520,7 @@ module.exports = function(RED) { // join lines, don't forget to add the last new line msg.payload = stringBuilder.join(node.ret) + node.ret - msg.columns = templateArrayToColumnString(template) + msg.columns = templateArrayToColumnString(template) // always strict commas + double quotes for if (msg.payload !== '') { send(msg) } done() } @@ -615,16 +617,15 @@ module.exports = function(RED) { } if (msg.parts.index + 1 === msg.parts.count) { msg.payload = node.store - msg.columns = csvParseResult.header - // msg._mode = 'RFC4180 mode' + // msg.columns = csvParseResult.header + msg.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns delete msg.parts send(msg) node.store = [] } } else { - msg.columns = csvParseResult.header - // msg._mode = 'RFC4180 mode' + msg.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns msg.payload = data send(msg); // finally send the array } @@ -633,7 +634,8 @@ module.exports = function(RED) { const len = data.length for (let row = 0; row < len; row++) { const newMessage = RED.util.cloneMessage(msg) - newMessage.columns = csvParseResult.header + // newMessage.columns = csvParseResult.header + newMessage.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns newMessage.payload = data[row] if (!has_parts) { newMessage.parts = { From 82c756b091cd2683e9f4b95f76292051937439e4 Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Thu, 12 Dec 2024 16:43:38 +0000 Subject: [PATCH 5/6] add test for correct CSV output when separator in non-comma --- test/nodes/core/parsers/70-CSV_spec.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/nodes/core/parsers/70-CSV_spec.js b/test/nodes/core/parsers/70-CSV_spec.js index f362ecdbf..b7ba60f58 100644 --- a/test/nodes/core/parsers/70-CSV_spec.js +++ b/test/nodes/core/parsers/70-CSV_spec.js @@ -2077,6 +2077,26 @@ describe('CSV node (RFC Mode)', function () { }); }); + it('should convert a simple object back to a tsv with headers using a tab as a separator', function (done) { + const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", sep: "\t", ret: '\n', hdrout: "all", wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test + { id: "n2", type: "helper" }]; + helper.load(csvNode, flow, function () { + const n1 = helper.getNode("n1"); + const n2 = helper.getNode("n2"); + n2.on("input", function (msg) { + try { + msg.should.have.property('payload', 'd\tb\tc\ta\n1\tfoo\t"ba""r"\tdi,ng\n'); + msg.should.have.property('columns', 'd,b,c,a'); // Strict RFC columns + done(); + } catch (e) { + done(e); + } + }); + const testJson = { d: 1, b: "foo", c: "ba\"r", a: "di,ng" }; + n1.emit("input", { payload: testJson }); + }); + }); + it('should handle a template with spaces in the property names', function (done) { const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "a,b o,c p,,e", ret: '\n', wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test { id: "n2", type: "helper" }]; From 16005a462d4bd32aeba770d663f7be5ecf0bba2d Mon Sep 17 00:00:00 2001 From: Steve-Mcl Date: Thu, 12 Dec 2024 16:44:16 +0000 Subject: [PATCH 6/6] update tests to check msg.columns is strictly RFC compliant --- test/nodes/core/parsers/70-CSV_spec.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/nodes/core/parsers/70-CSV_spec.js b/test/nodes/core/parsers/70-CSV_spec.js index b7ba60f58..9f6749b9f 100644 --- a/test/nodes/core/parsers/70-CSV_spec.js +++ b/test/nodes/core/parsers/70-CSV_spec.js @@ -2067,6 +2067,7 @@ describe('CSV node (RFC Mode)', function () { n2.on("input", function (msg) { try { msg.should.have.property('payload', '1\tfoo\t"ba""r"\tdi,ng\n'); + msg.should.have.property('columns', 'd,b,c,a'); // Strict RFC columns done(); } catch (e) { done(e); @@ -2106,6 +2107,7 @@ describe('CSV node (RFC Mode)', function () { n2.on("input", function (msg) { try { msg.should.have.property('payload', '4,foo,true,,0\n'); + msg.should.have.property('columns', 'a,b o,c p,e'); // Strict RFC columns done(); } catch (e) { done(e); @@ -2126,6 +2128,7 @@ describe('CSV node (RFC Mode)', function () { try { // 'payload', 'a"a,b\'b\nA1,B1\nA2,B2\n'); // Legacy msg.should.have.property('payload', '"a""a",b\'b\nA1,B1\nA2,B2\n'); // RFC-vs-Legacy difference - RFC4180 Section 2.6, 2.7 quote handling + msg.should.have.property('columns', '"a""a",b\'b'); // RCF compliant column names done(); } catch (e) { done(e); @@ -2191,6 +2194,7 @@ describe('CSV node (RFC Mode)', function () { n2.on("input", function (msg) { try { msg.should.have.property('payload', '1,3,2,4\n4,2,3,1\n'); + msg.should.have.property('columns', 'd,b,c,a'); // Strict RFC columns done(); } catch (e) { done(e); } @@ -2209,6 +2213,7 @@ describe('CSV node (RFC Mode)', function () { n2.on("input", function (msg) { try { msg.should.have.property('payload', 'd,b,c,a\n1,3,2,4\n4,"f\ng",3,1\n'); + msg.should.have.property('columns', 'd,b,c,a'); // Strict RFC columns done(); } catch (e) { done(e); } @@ -2228,6 +2233,7 @@ describe('CSV node (RFC Mode)', function () { try { // 'payload', ',0,1,foo,"ba""r","di,ng","fa\nba"\n'); msg.should.have.property('payload', ',0,1,foo\n'); // RFC-vs-Legacy difference - respect that user has specified a template with 4 columns + msg.should.have.property('columns', 'a,b,c,d'); done(); } catch (e) { done(e); } @@ -2347,6 +2353,7 @@ describe('CSV node (RFC Mode)', function () { n2.on("input", function (msg) { try { msg.should.have.property('payload', '{},"text,with,commas","This ""is"" a banana","{""sub"":""object""}"\n'); + msg.should.have.property('columns', 'a,b,c,d'); done(); } catch (e) { done(e); }