diff --git a/CliClient/package-lock.json b/CliClient/package-lock.json index 61d3a79dc9..c3ab8c3115 100644 --- a/CliClient/package-lock.json +++ b/CliClient/package-lock.json @@ -70,6 +70,21 @@ "readable-stream": "^2.0.6" } }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + }, + "dependencies": { + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" + } + } + }, "array-equal": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", @@ -493,6 +508,11 @@ "iconv-lite": "~0.4.13" } }, + "entities": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz", + "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=" + }, "es6-promise-pool": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/es6-promise-pool/-/es6-promise-pool-2.5.0.tgz", @@ -1117,6 +1137,14 @@ "type-check": "~0.3.2" } }, + "linkify-it": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.0.3.tgz", + "integrity": "sha1-2UpGSPmxwXnWT6lykSaL22zpQ08=", + "requires": { + "uc.micro": "^1.0.1" + } + }, "lodash": { "version": "4.17.4", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", @@ -1159,6 +1187,18 @@ "highlight.js": "~9.12.0" } }, + "markdown-it": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-8.4.2.tgz", + "integrity": "sha512-GcRz3AWTqSUphY3vsUqQSFMbgR38a4Lh3GWlHRh/7MRwz8mcu9n2IO7HOh+bXHrR9kOPDl5RNCaEsrneb+xhHQ==", + "requires": { + "argparse": "^1.0.7", + "entities": "~1.1.1", + "linkify-it": "^2.0.0", + "mdurl": "^1.0.1", + "uc.micro": "^1.0.5" + } + }, "md5": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/md5/-/md5-2.2.1.tgz", @@ -1169,6 +1209,11 @@ "is-buffer": "~1.1.1" } }, + "mdurl": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", + "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=" + }, "mime": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/mime/-/mime-2.0.3.tgz", @@ -2039,6 +2084,11 @@ "prelude-ls": "~1.1.2" } }, + "uc.micro": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.5.tgz", + "integrity": "sha512-JoLI4g5zv5qNyT09f4YAvEZIIV1oOjqnewYg5D38dkQljIzpPT296dbIGvKro3digYI1bkb7W6EP1y4uDlmzLg==" + }, "uglify-js": { "version": "3.3.25", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.25.tgz", diff --git a/CliClient/package.json b/CliClient/package.json index 9f840d2ca7..20466d1a80 100644 --- a/CliClient/package.json +++ b/CliClient/package.json @@ -43,6 +43,7 @@ "jssha": "^2.3.0", "levenshtein": "^1.0.5", "lodash": "^4.17.4", + "markdown-it": "^8.4.2", "md5": "^2.2.1", "mime": "^2.0.3", "moment": "^2.18.1", diff --git a/CliClient/tests/markdownUtils.js b/CliClient/tests/markdownUtils.js index 5a68ed6e26..9203b19ed8 100644 --- a/CliClient/tests/markdownUtils.js +++ b/CliClient/tests/markdownUtils.js @@ -39,6 +39,7 @@ describe('markdownUtils', function() { ['![something](http://test.com/img.png)', ['http://test.com/img.png']], ['![something](http://test.com/img.png) ![something2](http://test.com/img2.png)', ['http://test.com/img.png', 'http://test.com/img2.png']], ['![something](http://test.com/img.png "Some description")', ['http://test.com/img.png']], + ['![something](https://test.com/ohoh_(123).png)', ['https://test.com/ohoh_(123).png']], ]; for (let i = 0; i < testCases.length; i++) { diff --git a/ReactNativeClient/lib/ClipperServer.js b/ReactNativeClient/lib/ClipperServer.js index 557150d49b..f0119d73a9 100644 --- a/ReactNativeClient/lib/ClipperServer.js +++ b/ReactNativeClient/lib/ClipperServer.js @@ -115,14 +115,21 @@ class ClipperServer { async downloadImage_(url) { const tempDir = Setting.value('tempDir'); - const name = filename(url); - let fileExt = safeFileExtension(fileExtension(url).toLowerCase()); + + const isDataUrl = url && url.toLowerCase().indexOf('data:') === 0; + + const name = isDataUrl ? md5(Math.random() + '_' + Date.now()) : filename(url); + let fileExt = isDataUrl ? mimeUtils.toFileExtension(mimeUtils.fromDataUrl(url)) : safeFileExtension(fileExtension(url).toLowerCase()); if (fileExt) fileExt = '.' + fileExt; let imagePath = tempDir + '/' + safeFilename(name) + fileExt; if (await shim.fsDriver().exists(imagePath)) imagePath = tempDir + '/' + safeFilename(name) + '_' + md5(Math.random() + '_' + Date.now()).substr(0,10) + fileExt; try { - const result = await shim.fetchBlob(url, { path: imagePath }); + if (isDataUrl) { + await shim.imageFromDataUrl(url, imagePath); + } else { + await shim.fetchBlob(url, { path: imagePath }); + } return imagePath; } catch (error) { this.logger().warn('Cannot download image at ' + url, error); @@ -276,11 +283,19 @@ class ClipperServer { let note = await this.requestNoteToNote(requestNote); const imageUrls = markdownUtils.extractImageUrls(note.body); + + this.logger().info('Request (' + requestId + '): Downloading images: ' + imageUrls.length); + let result = await this.downloadImages_(imageUrls); + + this.logger().info('Request (' + requestId + '): Creating resources from paths: ' + Object.getOwnPropertyNames(result).length); + result = await this.createResourcesFromPaths_(result); await this.removeTempFiles_(result); note.body = this.replaceImageUrlsByResources_(note.body, result); + this.logger().info('Request (' + requestId + '): Saving note...'); + note = await Note.save(note); if (requestNote.tags) { diff --git a/ReactNativeClient/lib/markdownUtils.js b/ReactNativeClient/lib/markdownUtils.js index 985e9fc9f2..ff61db5047 100644 --- a/ReactNativeClient/lib/markdownUtils.js +++ b/ReactNativeClient/lib/markdownUtils.js @@ -1,4 +1,5 @@ const urlUtils = require('lib/urlUtils'); +const MarkdownIt = require('markdown-it'); const markdownUtils = { @@ -20,15 +21,32 @@ const markdownUtils = { }, extractImageUrls(md) { - // ![some text](http://path/to/image) - const regex = new RegExp(/!\[.*?\]\(([^\s\)]+).*?\)/, 'g') - let match = regex.exec(md); + const markdownIt = new MarkdownIt(); + const env = {}; + const tokens = markdownIt.parse(md, env); const output = []; - while (match) { - const url = match[1]; - if (output.indexOf(url) < 0) output.push(url); - match = regex.exec(md); + + const searchUrls = (tokens) => { + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i]; + + if (token.type === 'image') { + for (let j = 0; j < token.attrs.length; j++) { + const a = token.attrs[j]; + if (a[0] === 'src' && a.length >= 2 && a[1]) { + output.push(a[1]); + } + } + } + + if (token.children && token.children.length) { + searchUrls(token.children); + } + } } + + searchUrls(tokens); + return output; }, diff --git a/ReactNativeClient/lib/shim-init-node.js b/ReactNativeClient/lib/shim-init-node.js index 12edfe87e6..8d4836d0ee 100644 --- a/ReactNativeClient/lib/shim-init-node.js +++ b/ReactNativeClient/lib/shim-init-node.js @@ -173,7 +173,7 @@ function shimInit() { if (shim.isElectron()) { const nativeImage = require('electron').nativeImage; let image = nativeImage.createFromDataURL(imageDataUrl); - if (image.isEmpty()) throw new Error('Could not convert data URL to image'); + if (image.isEmpty()) throw new Error('Could not convert data URL to image'); // Would throw for example if the image format is no supported (eg. image/gif) if (options.cropRect) { // Crop rectangle values need to be rounded or the crop() call will fail const c = options.cropRect;