Joplin
/
joplin
mirror of https://github.com/laurent22/joplin.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

All: Fixes #992: Allow non-ASCII chars when exporting MD and handle duplicate filenames

pull/998/head
Laurent Cozic 2018-11-20 00:42:21 +00:00
parent 7bfc3e1256
commit 07b724d65b
10 changed files with 135 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CliClient/package-lock.json generated
View File

@ -2890,11 +2890,6 @@
"resolved": "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz",
"integrity": "sha1-5z3T17DXxe2G+6xrCufYxqadUPo="
},
"unidecode": {
"version": "0.1.8",
"resolved": "https://registry.npmjs.org/unidecode/-/unidecode-0.1.8.tgz",
"integrity": "sha1-77swFTi8RSRqmsjFWdcvAVMFBT4="
},
"uniq": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz",

1
CliClient/package.json
View File

@ -68,7 +68,6 @@
"tar": "^4.4.0",
"tcp-port-used": "^0.1.2",
"tkwidgets": "^0.5.26",
"unidecode": "^0.1.8",
"url-parse": "^1.2.0",
"uuid": "^3.0.1",
"valid-url": "^1.0.9",

1
CliClient/run_test.sh
View File

@ -30,6 +30,7 @@ npm test tests-build/models_Folder.js
npm test tests-build/models_Note.js
npm test tests-build/models_Tag.js
npm test tests-build/models_Setting.js
npm test tests-build/pathUtils.js
npm test tests-build/services_InteropService.js
npm test tests-build/services_ResourceService.js
npm test tests-build/urlUtils.js

38
CliClient/tests/pathUtils.js Normal file
View File

@ -0,0 +1,38 @@
require('app-module-path').addPath(__dirname);
const { friendlySafeFilename } = require('lib/path-utils.js');
const { fileContentEqual, setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi, sleep, clearDatabase, switchClient, syncTargetId, objectsEqual, checkThrowAsync } = require('test-utils.js');
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});
describe('pathUtils', function() {
beforeEach(async (done) => {
done();
});
it('should create friendly safe filename', async (done) => {
const testCases = [
['生活', '生活'],
['not/good', 'not_good'],
['really/not/good', 'really_not_good'],
['con', '___'],
['no space at the end ', 'no space at the end'],
['nor dots...', 'nor dots'],
['thatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylong', 'thatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylongthatsreallylong'],
];
for (let i = 0; i < testCases.length; i++) {
const t = testCases[i];
expect(friendlySafeFilename(t[0])).toBe(t[1]);
}
expect(!!friendlySafeFilename('')).toBe(true);
expect(!!friendlySafeFilename('...')).toBe(true);
done();
});
});

26
CliClient/tests/services_InteropService.js
View File

@ -331,4 +331,30 @@ describe('services_InteropService', function() {
expect(obj.body).toBe(items[i].body);
}
}));
it('should export MD with unicode filenames', asyncTest(async () => {
const service = new InteropService();
let folder1 = await Folder.save({ title: 'folder1' });
let folder2 = await Folder.save({ title: 'ジョプリン' });
let note1 = await Note.save({ title: '生活', parent_id: folder1.id });
let note2 = await Note.save({ title: '生活', parent_id: folder1.id });
let note2b = await Note.save({ title: '生活', parent_id: folder1.id });
let note3 = await Note.save({ title: '', parent_id: folder1.id });
let note4 = await Note.save({ title: '', parent_id: folder1.id });
let note5 = await Note.save({ title: 'salut, ça roule ?', parent_id: folder1.id });
let note6 = await Note.save({ title: 'ジョプリン', parent_id: folder2.id });
const outDir = exportDir();
await service.export({ path: outDir, format: 'md' });
expect(await shim.fsDriver().exists(outDir + '/folder1/生活.md')).toBe(true);
expect(await shim.fsDriver().exists(outDir + '/folder1/生活 (1).md')).toBe(true);
expect(await shim.fsDriver().exists(outDir + '/folder1/生活 (2).md')).toBe(true);
expect(await shim.fsDriver().exists(outDir + '/folder1/Untitled.md')).toBe(true);
expect(await shim.fsDriver().exists(outDir + '/folder1/Untitled (1).md')).toBe(true);
expect(await shim.fsDriver().exists(outDir + '/folder1/salut, ça roule _.md')).toBe(true);
expect(await shim.fsDriver().exists(outDir + '/ジョプリン/ジョプリン.md')).toBe(true);
}));
});

5
ElectronClient/app/package-lock.json generated
View File

@ -6981,11 +6981,6 @@
"random-bytes": "~1.0.0"
}
},
"unidecode": {
"version": "0.1.8",
"resolved": "https://registry.npmjs.org/unidecode/-/unidecode-0.1.8.tgz",
"integrity": "sha1-77swFTi8RSRqmsjFWdcvAVMFBT4="
},
"union-value": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",

1
ElectronClient/app/package.json
View File

@ -127,7 +127,6 @@
"syswide-cas": "^5.1.0",
"tar": "^4.4.4",
"tcp-port-used": "^0.1.2",
"unidecode": "^0.1.8",
"url-parse": "^1.4.1",
"uuid": "^3.2.1",
"valid-url": "^1.0.9",

19
ReactNativeClient/lib/fs-driver-base.js
View File

@ -1,3 +1,5 @@
const { filename, fileExtension } = require('lib/path-utils');
class FsDriverBase {
async isDirectory(path) {
@ -19,6 +21,23 @@ class FsDriverBase {
return output;
}
async findUniqueFilename(name) {
let counter = 1;
let nameNoExt = filename(name, true);
let extension = fileExtension(name);
if (extension) extension = '.' + extension;
let nameToTry = nameNoExt + extension;
while (true) {
const exists = await this.exists(nameToTry);
if (!exists) return nameToTry;
nameToTry = nameNoExt + ' (' + counter + ')' + extension;
counter++;
if (counter >= 1000) nameToTry = nameNoExt + ' (' + ((new Date()).getTime()) + ')' + extension;
if (counter >= 10000) throw new Error('Cannot find unique title');
}
}
}
module.exports = FsDriverBase;

49
ReactNativeClient/lib/path-utils.js
View File

@ -1,3 +1,5 @@
const { _ } = require('lib/locale');
function dirname(path) {
if (!path) throw new Error('Path is empty');
let s = path.split(/\/|\\/);
@ -11,9 +13,9 @@ function basename(path) {
return s[s.length - 1];
}
function filename(path) {
function filename(path, includeDir = false) {
if (!path) throw new Error('Path is empty');
let output = basename(path);
let output = includeDir ? path : basename(path);
if (output.indexOf('.') < 0) return output;
output = output.split('.');
@ -48,6 +50,47 @@ function safeFilename(e, maxLength = null, allowSpaces = false) {
return output.substr(0, maxLength);
}
let friendlySafeFilename_blackListChars = '/<>:\'"\\|?*';
for (let i = 0; i < 32; i++) {
friendlySafeFilename_blackListChars += String.fromCharCode(i);
}
const friendlySafeFilename_blackListNames = [".", "..", "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9"];
function friendlySafeFilename(e, maxLength = null) {
if (maxLength === null) maxLength = 255;
if (!e || !e.replace) return _('Untitled');
let output = '';
for (let i = 0; i < e.length; i++) {
const c = e[i];
if (friendlySafeFilename_blackListChars.indexOf(c) >= 0) {
output += '_';
} else {
output += c;
}
}
if (output.length <= 4) {
if (friendlySafeFilename_blackListNames.indexOf(output.toUpperCase()) >= 0) {
output = '___';
}
}
while (output.length) {
const c = output[output.length - 1];
if (c === ' ' || c === '.') {
output = output.substr(0, output.length - 1);
} else {
break;
}
}
if (!output) return _('Untitled');
return output.substr(0, maxLength);
}
function toSystemSlashes(path, os = null) {
if (os === null) os = process.platform;
if (os === 'win32') return path.replace(/\//g, "\\");
@ -62,4 +105,4 @@ function ltrimSlashes(path) {
return path.replace(/^\/+/, '');
}
module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFilename, safeFileExtension, toSystemSlashes, rtrimSlashes, ltrimSlashes };
module.exports = { basename, dirname, filename, isHidden, fileExtension, safeFilename, friendlySafeFilename, safeFileExtension, toSystemSlashes, rtrimSlashes, ltrimSlashes };

10
ReactNativeClient/lib/services/InteropService_Exporter_Md.js
View File

@ -1,10 +1,9 @@
const InteropService_Exporter_Base = require('lib/services/InteropService_Exporter_Base');
const { basename, filename, safeFilename } = require('lib/path-utils.js');
const { basename, filename, friendlySafeFilename } = require('lib/path-utils.js');
const BaseModel = require('lib/BaseModel');
const Folder = require('lib/models/Folder');
const Note = require('lib/models/Note');
const { shim } = require('lib/shim');
const unidecode = require('unidecode');
class InteropService_Exporter_Md extends InteropService_Exporter_Base {
@ -21,7 +20,8 @@ class InteropService_Exporter_Md extends InteropService_Exporter_Base {
let output = '';
while (true) {
if (item.type_ === BaseModel.TYPE_FOLDER) {
output = safeFilename(item.title, null, true) + '/' + output;
output = friendlySafeFilename(item.title, null, true) + '/' + output;
output = await shim.fsDriver().findUniqueFilename(output);
}
if (!item.parent_id) return output;
item = await Folder.load(item.parent_id);
@ -32,7 +32,6 @@ class InteropService_Exporter_Md extends InteropService_Exporter_Base {
async processItem(ItemClass, item) {
if ([BaseModel.TYPE_NOTE, BaseModel.TYPE_FOLDER].indexOf(item.type_) < 0) return;
const filename = safeFilename(item.title, null, true);
const dirPath = this.destDir_ + '/' + (await this.makeDirPath_(item));
if (this.createdDirs_.indexOf(dirPath) < 0) {
@ -41,7 +40,8 @@ class InteropService_Exporter_Md extends InteropService_Exporter_Base {
}
if (item.type_ === BaseModel.TYPE_NOTE) {
const noteFilePath = dirPath + '/' + safeFilename(unidecode(item.title), null, true) + '.md';
let noteFilePath = dirPath + '/' + friendlySafeFilename(item.title, null, true) + '.md';
noteFilePath = await shim.fsDriver().findUniqueFilename(noteFilePath);
const noteContent = await Note.serializeForEdit(item);
await shim.fsDriver().writeFile(noteFilePath, noteContent, 'utf-8');
}