mirror of https://github.com/node-red/node-red.git
Merge pull request #4982 from node-red/4977-fix-csv-sep
Ensure node.sep is honoured when generating CSVpull/4990/head
Nick O'Leary
GitHub
commit
aa79aa5479
|
|
@ -367,20 +367,21 @@ module.exports = function(RED) {
|
|||
const sendHeadersAlways = node.hdrout === "all"
|
||||
const sendHeaders = !dontSendHeaders && (sendHeadersOnce || sendHeadersAlways)
|
||||
const quoteables = [node.sep, node.quo, "\n", "\r"]
|
||||
const templateQuoteables = [',', '"', "\n", "\r"]
|
||||
const templateQuoteables = [node.sep, node.quo, "\n", "\r"]
|
||||
const templateQuoteablesStrict = [',', '"', "\n", "\r"]
|
||||
let badTemplateWarnOnce = true
|
||||
|
||||
const columnStringToTemplateArray = function (col, sep) {
|
||||
// NOTE: enforce strict column template parsing in RFC4180 mode
|
||||
const parsed = csv.parse(col, { separator: sep, quote: node.quo, outputStyle: 'array', strict: true })
|
||||
if (parsed.headers.length > 0) { node.goodtmpl = true } else { node.goodtmpl = false }
|
||||
return parsed.headers.length ? parsed.headers : null
|
||||
if (parsed.data?.length === 1) { node.goodtmpl = true } else { node.goodtmpl = false }
|
||||
return node.goodtmpl ? parsed.data[0] : null
|
||||
}
|
||||
const templateArrayToColumnString = function (template, keepEmptyColumns) {
|
||||
// NOTE: enforce strict column template parsing in RFC4180 mode
|
||||
const parsed = csv.parse('', {headers: template, headersOnly:true, separator: ',', quote: node.quo, outputStyle: 'array', strict: true })
|
||||
const templateArrayToColumnString = function (template, keepEmptyColumns, separator = ',', quotables = templateQuoteablesStrict) {
|
||||
// NOTE: defaults to strict column template parsing (commas and double quotes)
|
||||
const parsed = csv.parse('', {headers: template, headersOnly:true, separator, quote: node.quo, outputStyle: 'array', strict: true })
|
||||
return keepEmptyColumns
|
||||
? parsed.headers.map(e => addQuotes(e || '', { separator: ',', quoteables: templateQuoteables}))
|
||||
? parsed.headers.map(e => addQuotes(e || '', { separator, quoteables: quotables })).join(separator)
|
||||
: parsed.header // exclues empty columns
|
||||
// TODO: resolve inconsistency between CSV->JSON and JSON->CSV
|
||||
// CSV->JSON: empty columns are excluded
|
||||
|
|
@ -447,7 +448,7 @@ module.exports = function(RED) {
|
|||
template = Object.keys(inputData[0]) || ['']
|
||||
}
|
||||
}
|
||||
stringBuilder.push(templateArrayToColumnString(template, true))
|
||||
stringBuilder.push(templateArrayToColumnString(template, true, node.sep, templateQuoteables)) // use user set separator for output data.
|
||||
if (sendHeadersOnce) { node.hdrSent = true }
|
||||
}
|
||||
|
||||
|
|
@ -483,6 +484,7 @@ module.exports = function(RED) {
|
|||
node.warn(RED._("csv.errors.obj_csv"))
|
||||
badTemplateWarnOnce = false
|
||||
}
|
||||
template = Object.keys(row) || ['']
|
||||
const rowData = []
|
||||
for (let header in inputData[0]) {
|
||||
if (row.hasOwnProperty(header)) {
|
||||
|
|
@ -518,7 +520,7 @@ module.exports = function(RED) {
|
|||
|
||||
// join lines, don't forget to add the last new line
|
||||
msg.payload = stringBuilder.join(node.ret) + node.ret
|
||||
msg.columns = templateArrayToColumnString(template)
|
||||
msg.columns = templateArrayToColumnString(template) // always strict commas + double quotes for
|
||||
if (msg.payload !== '') { send(msg) }
|
||||
done()
|
||||
}
|
||||
|
|
@ -615,16 +617,15 @@ module.exports = function(RED) {
|
|||
}
|
||||
if (msg.parts.index + 1 === msg.parts.count) {
|
||||
msg.payload = node.store
|
||||
msg.columns = csvParseResult.header
|
||||
// msg._mode = 'RFC4180 mode'
|
||||
// msg.columns = csvParseResult.header
|
||||
msg.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns
|
||||
delete msg.parts
|
||||
send(msg)
|
||||
node.store = []
|
||||
}
|
||||
}
|
||||
else {
|
||||
msg.columns = csvParseResult.header
|
||||
// msg._mode = 'RFC4180 mode'
|
||||
msg.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns
|
||||
msg.payload = data
|
||||
send(msg); // finally send the array
|
||||
}
|
||||
|
|
@ -633,7 +634,8 @@ module.exports = function(RED) {
|
|||
const len = data.length
|
||||
for (let row = 0; row < len; row++) {
|
||||
const newMessage = RED.util.cloneMessage(msg)
|
||||
newMessage.columns = csvParseResult.header
|
||||
// newMessage.columns = csvParseResult.header
|
||||
newMessage.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns
|
||||
newMessage.payload = data[row]
|
||||
if (!has_parts) {
|
||||
newMessage.parts = {
|
||||
|
|
|
|||
|
|
@ -2067,6 +2067,27 @@ describe('CSV node (RFC Mode)', function () {
|
|||
n2.on("input", function (msg) {
|
||||
try {
|
||||
msg.should.have.property('payload', '1\tfoo\t"ba""r"\tdi,ng\n');
|
||||
msg.should.have.property('columns', 'd,b,c,a'); // Strict RFC columns
|
||||
done();
|
||||
} catch (e) {
|
||||
done(e);
|
||||
}
|
||||
});
|
||||
const testJson = { d: 1, b: "foo", c: "ba\"r", a: "di,ng" };
|
||||
n1.emit("input", { payload: testJson });
|
||||
});
|
||||
});
|
||||
|
||||
it('should convert a simple object back to a tsv with headers using a tab as a separator', function (done) {
|
||||
const flow = [{ id: "n1", type: "csv", spec: "rfc", temp: "", sep: "\t", ret: '\n', hdrout: "all", wires: [["n2"]] }, // RFC-vs-Legacy difference - use line separator \n to satisfy original test
|
||||
{ id: "n2", type: "helper" }];
|
||||
helper.load(csvNode, flow, function () {
|
||||
const n1 = helper.getNode("n1");
|
||||
const n2 = helper.getNode("n2");
|
||||
n2.on("input", function (msg) {
|
||||
try {
|
||||
msg.should.have.property('payload', 'd\tb\tc\ta\n1\tfoo\t"ba""r"\tdi,ng\n');
|
||||
msg.should.have.property('columns', 'd,b,c,a'); // Strict RFC columns
|
||||
done();
|
||||
} catch (e) {
|
||||
done(e);
|
||||
|
|
@ -2086,6 +2107,7 @@ describe('CSV node (RFC Mode)', function () {
|
|||
n2.on("input", function (msg) {
|
||||
try {
|
||||
msg.should.have.property('payload', '4,foo,true,,0\n');
|
||||
msg.should.have.property('columns', 'a,b o,c p,e'); // Strict RFC columns
|
||||
done();
|
||||
} catch (e) {
|
||||
done(e);
|
||||
|
|
@ -2106,6 +2128,7 @@ describe('CSV node (RFC Mode)', function () {
|
|||
try {
|
||||
// 'payload', 'a"a,b\'b\nA1,B1\nA2,B2\n'); // Legacy
|
||||
msg.should.have.property('payload', '"a""a",b\'b\nA1,B1\nA2,B2\n'); // RFC-vs-Legacy difference - RFC4180 Section 2.6, 2.7 quote handling
|
||||
msg.should.have.property('columns', '"a""a",b\'b'); // RCF compliant column names
|
||||
done();
|
||||
} catch (e) {
|
||||
done(e);
|
||||
|
|
@ -2171,6 +2194,7 @@ describe('CSV node (RFC Mode)', function () {
|
|||
n2.on("input", function (msg) {
|
||||
try {
|
||||
msg.should.have.property('payload', '1,3,2,4\n4,2,3,1\n');
|
||||
msg.should.have.property('columns', 'd,b,c,a'); // Strict RFC columns
|
||||
done();
|
||||
}
|
||||
catch (e) { done(e); }
|
||||
|
|
@ -2189,6 +2213,7 @@ describe('CSV node (RFC Mode)', function () {
|
|||
n2.on("input", function (msg) {
|
||||
try {
|
||||
msg.should.have.property('payload', 'd,b,c,a\n1,3,2,4\n4,"f\ng",3,1\n');
|
||||
msg.should.have.property('columns', 'd,b,c,a'); // Strict RFC columns
|
||||
done();
|
||||
}
|
||||
catch (e) { done(e); }
|
||||
|
|
@ -2208,6 +2233,7 @@ describe('CSV node (RFC Mode)', function () {
|
|||
try {
|
||||
// 'payload', ',0,1,foo,"ba""r","di,ng","fa\nba"\n');
|
||||
msg.should.have.property('payload', ',0,1,foo\n'); // RFC-vs-Legacy difference - respect that user has specified a template with 4 columns
|
||||
msg.should.have.property('columns', 'a,b,c,d');
|
||||
done();
|
||||
}
|
||||
catch (e) { done(e); }
|
||||
|
|
@ -2327,6 +2353,7 @@ describe('CSV node (RFC Mode)', function () {
|
|||
n2.on("input", function (msg) {
|
||||
try {
|
||||
msg.should.have.property('payload', '{},"text,with,commas","This ""is"" a banana","{""sub"":""object""}"\n');
|
||||
msg.should.have.property('columns', 'a,b,c,d');
|
||||
done();
|
||||
}
|
||||
catch (e) { done(e); }
|
||||
|
|
|
|||
Loading…
Reference in New Issue