Node-Red
/
node-red
mirror of https://github.com/node-red/node-red.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update CSV to adhere to strict rfc compliance on msg.columns

pull/4982/head
Steve-Mcl 2024-12-12 16:42:11 +00:00
parent 6af3c8c2a9
commit b139eb4a18
1 changed files with 18 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js vendored
View File

@ -367,20 +367,21 @@ module.exports = function(RED) {
const sendHeadersAlways = node.hdrout === "all"
const sendHeaders = !dontSendHeaders && (sendHeadersOnce || sendHeadersAlways)
const quoteables = [node.sep, node.quo, "\n", "\r"]
const templateQuoteables = [node.sep, '"', "\n", "\r"]
const templateQuoteables = [node.sep, node.quo, "\n", "\r"]
const templateQuoteablesStrict = [',', '"', "\n", "\r"]
let badTemplateWarnOnce = true
const columnStringToTemplateArray = function (col, sep) {
// NOTE: enforce strict column template parsing in RFC4180 mode
const parsed = csv.parse(col, { separator: sep, quote: node.quo, outputStyle: 'array', strict: true })
if (parsed.headers.length > 0) { node.goodtmpl = true } else { node.goodtmpl = false }
return parsed.headers.length ? parsed.headers : null
if (parsed.data?.length === 1) { node.goodtmpl = true } else { node.goodtmpl = false }
return node.goodtmpl ? parsed.data[0] : null
}
const templateArrayToColumnString = function (template, keepEmptyColumns) {
// NOTE: enforce strict column template parsing in RFC4180 mode
const parsed = csv.parse('', {headers: template, headersOnly:true, separator: node.sep, quote: node.quo, outputStyle: 'array', strict: true })
const templateArrayToColumnString = function (template, keepEmptyColumns, separator = ',', quotables = templateQuoteablesStrict) {
// NOTE: defaults to strict column template parsing (commas and double quotes)
const parsed = csv.parse('', {headers: template, headersOnly:true, separator, quote: node.quo, outputStyle: 'array', strict: true })
return keepEmptyColumns
? parsed.headers.map(e => addQuotes(e || '', { separator: node.sep, quoteables: templateQuoteables})).join(node.sep)
? parsed.headers.map(e => addQuotes(e || '', { separator, quoteables: quotables })).join(separator)
: parsed.header // exclues empty columns
// TODO: resolve inconsistency between CSV->JSON and JSON->CSV
// CSV->JSON: empty columns are excluded
@ -441,13 +442,13 @@ module.exports = function(RED) {
if (sendHeaders && node.hdrSent === false) {
if (hasTemplate(template) === false) {
if (msg.hasOwnProperty("columns")) {
template = columnStringToTemplateArray(msg.columns || "", node.sep) || ['']
template = columnStringToTemplateArray(msg.columns || "", ",") || ['']
}
else {
template = Object.keys(inputData[0]) || ['']
}
}
stringBuilder.push(templateArrayToColumnString(template, true))
stringBuilder.push(templateArrayToColumnString(template, true, node.sep, templateQuoteables)) // use user set separator for output data.
if (sendHeadersOnce) { node.hdrSent = true }
}
@ -475,7 +476,7 @@ module.exports = function(RED) {
} else {
/*** row is an object ***/
if (hasTemplate(template) === false && (msg.hasOwnProperty("columns"))) {
template = columnStringToTemplateArray(msg.columns || "", node.sep)
template = columnStringToTemplateArray(msg.columns || "", ",")
}
if (hasTemplate(template) === false) {
/*** row is an object but we still don't have a template ***/
@ -483,6 +484,7 @@ module.exports = function(RED) {
node.warn(RED._("csv.errors.obj_csv"))
badTemplateWarnOnce = false
}
template = Object.keys(row) || ['']
const rowData = []
for (let header in inputData[0]) {
if (row.hasOwnProperty(header)) {
@ -518,7 +520,7 @@ module.exports = function(RED) {
// join lines, don't forget to add the last new line
msg.payload = stringBuilder.join(node.ret) + node.ret
msg.columns = templateArrayToColumnString(template)
msg.columns = templateArrayToColumnString(template) // always strict commas + double quotes for
if (msg.payload !== '') { send(msg) }
done()
}
@ -615,16 +617,15 @@ module.exports = function(RED) {
}
if (msg.parts.index + 1 === msg.parts.count) {
msg.payload = node.store
msg.columns = csvParseResult.header
// msg._mode = 'RFC4180 mode'
// msg.columns = csvParseResult.header
msg.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns
delete msg.parts
send(msg)
node.store = []
}
}
else {
msg.columns = csvParseResult.header
// msg._mode = 'RFC4180 mode'
msg.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns
msg.payload = data
send(msg); // finally send the array
}
@ -633,7 +634,8 @@ module.exports = function(RED) {
const len = data.length
for (let row = 0; row < len; row++) {
const newMessage = RED.util.cloneMessage(msg)
newMessage.columns = csvParseResult.header
// newMessage.columns = csvParseResult.header
newMessage.columns = templateArrayToColumnString(csvParseResult.headers) // always strict commas + double quotes for msg.columns
newMessage.payload = data[row]
if (!has_parts) {
newMessage.parts = {