Joplin
/
joplin
mirror of https://github.com/laurent22/joplin.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improved enex import

pull/41/head
Laurent Cozic 2017-06-27 00:20:01 +01:00
parent 6aa7ea0c6c
commit 4178d1f1de
8 changed files with 232 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
CliClient/app/import-enex.js
View File

@ -12,25 +12,6 @@ import jsSHA from "jssha";
const Promise = require('promise');
const fs = require('fs-extra');
const stringToStream = require('string-to-stream')
let existingTimestamps = [];
function uniqueCreatedTimestamp(timestamp) {
if (existingTimestamps.indexOf(timestamp) < 0) {
existingTimestamps.push(timestamp);
return timestamp;
}
for (let i = 1; i <= 999; i++) {
let t = timestamp + i;
if (existingTimestamps.indexOf(t) < 0) {
existingTimestamps.push(t);
return t;
}
}
return timestamp;
}
function dateToTimestamp(s, zeroIfInvalid = false) {
let m = moment(s, 'YYYYMMDDTHHmmssZ');
@ -69,7 +50,7 @@ function createNoteId(note) {
}
async function fuzzyMatch(note) {
let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', note.created_time);
let notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ?', [note.created_time]);
if (!notes.length) return null;
if (notes.length === 1) return notes[0];
@ -92,6 +73,7 @@ async function saveNoteToStorage(note, fuzzyMatching = false) {
let result = {
noteCreated: false,
noteUpdated: false,
noteSkipped: false,
resourcesCreated: 0,
};
@ -103,14 +85,16 @@ async function saveNoteToStorage(note, fuzzyMatching = false) {
// TODO: also save resources
if (!Object.getOwnPropertyNames(diff).length) return;
if (!Object.getOwnPropertyNames(diff).length) {
result.noteSkipped = true;
// TODO: also save resources
return result;
}
diff.id = existingNote.id;
diff.type_ = existingNote.type_;
return Note.save(diff, { autoTimestamp: false }).then(() => {
result.noteUpdated = true;
return result;
});
await Note.save(diff, { autoTimestamp: false })
result.noteUpdated = true;
} else {
for (let i = 0; i < note.resources.length; i++) {
let resource = note.resources[i];
@ -128,14 +112,14 @@ async function saveNoteToStorage(note, fuzzyMatching = false) {
result.resourcesCreated++;
}
return Note.save(note, {
await Note.save(note, {
isNew: true,
autoTimestamp: false,
}).then(() => {
result.noteCreated = true;
return result;
});
result.noteCreated = true;
}
return result;
}
function importEnex(parentFolderId, filePath, importOptions = null) {
@ -144,11 +128,35 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
if (!('onProgress' in importOptions)) importOptions.onProgress = function(state) {};
if (!('onError' in importOptions)) importOptions.onError = function(error) {};
// Some notes were created with the exact same timestamp, for example when they were
// batch imported. In order to make fuzzy matching easier, this function ensures
// that each timestamp is unique.
let existingTimestamps = [];
function uniqueCreatedTimestamp(timestamp) {
return timestamp;
if (existingTimestamps.indexOf(timestamp) < 0) {
existingTimestamps.push(timestamp);
return timestamp;
}
for (let i = 1; i <= 999; i++) {
let t = timestamp + i;
if (existingTimestamps.indexOf(t) < 0) {
existingTimestamps.push(t);
return t;
}
}
return timestamp;
}
return new Promise((resolve, reject) => {
let progressState = {
loaded: 0,
created: 0,
updated: 0,
skipped: 0,
resourcesCreated: 0,
};
@ -182,7 +190,8 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
}
async function processNotes() {
if (processingNotes) return;
if (processingNotes) return false;
processingNotes = true;
stream.pause();
@ -204,6 +213,8 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
progressState.updated++;
} else if (result.noteCreated) {
progressState.created++;
} else if (result.noteSkipped) {
progressState.skipped++;
}
progressState.resourcesCreated += result.resourcesCreated;
importOptions.onProgress(progressState);
@ -214,6 +225,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
return promiseChain(chain).then(() => {
stream.resume();
processingNotes = false;
return true;
});
}
@ -354,12 +366,12 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
saxStream.on('end', function() {
// Wait till there is no more notes to process.
let iid = setInterval(() => {
if (notes.length) {
processNotes();
} else {
clearInterval(iid);
resolve();
}
processNotes().then((allDone) => {
if (allDone) {
clearTimeout(iid);
resolve();
}
});
}, 500);
});

246
CliClient/app/main.js
View File

@ -63,10 +63,11 @@ commands.push({
aliases: ['mkdir'],
description: 'Creates a new notebook',
action: function(args, end) {
Folder.save({ title: args['notebook'] }).catch((error) => {
this.log(error);
}).then((folder) => {
Folder.save({ title: args['notebook'] }).then((folder) => {
switchCurrentFolder(folder);
}).catch((error) => {
this.log(error);
}).then(() => {
end();
});
},
@ -187,34 +188,38 @@ commands.push({
usage: 'rm <pattern>',
description: 'Deletes the given item. For a notebook, all the notes within that notebook will be deleted. Use `rm ../<notebook>` to delete a notebook.',
action: async function(args, end) {
let pattern = args['pattern'];
let itemType = null;
try {
let pattern = args['pattern'];
let itemType = null;
if (pattern.indexOf('*') < 0) { // Handle it as a simple title
if (pattern.substr(0, 3) == '../') {
itemType = BaseModel.MODEL_TYPE_FOLDER;
pattern = pattern.substr(3);
} else {
itemType = BaseModel.MODEL_TYPE_NOTE;
}
if (pattern.indexOf('*') < 0) { // Handle it as a simple title
if (pattern.substr(0, 3) == '../') {
itemType = BaseModel.MODEL_TYPE_FOLDER;
pattern = pattern.substr(3);
} else {
itemType = BaseModel.MODEL_TYPE_NOTE;
}
let item = await BaseItem.loadItemByField(itemType, 'title', pattern);
if (!item) return cmdError(this, _('No item with title "%s" found.', pattern), end);
await BaseItem.deleteItem(itemType, item.id);
let item = await BaseItem.loadItemByField(itemType, 'title', pattern);
if (!item) throw new Error(_('No item with title "%s" found.', pattern));
await BaseItem.deleteItem(itemType, item.id);
if (currentFolder && currentFolder.id == item.id) {
let f = await Folder.defaultFolder();
switchCurrentFolder(f);
}
} else { // Handle it as a glob pattern
let notes = await Note.previews(currentFolder.id, { titlePattern: pattern });
if (!notes.length) return cmdError(this, _('No note matches this pattern: "%s"', pattern), end);
let ok = await cmdPromptConfirm(this, _('%d notes match this pattern. Delete them?', notes.length));
if (ok) {
for (let i = 0; i < notes.length; i++) {
await Note.delete(notes[i].id);
if (currentFolder && currentFolder.id == item.id) {
let f = await Folder.defaultFolder();
switchCurrentFolder(f);
}
} else { // Handle it as a glob pattern
let notes = await Note.previews(currentFolder.id, { titlePattern: pattern });
if (!notes.length) throw new Error(_('No note matches this pattern: "%s"', pattern));
let ok = await cmdPromptConfirm(this, _('%d notes match this pattern. Delete them?', notes.length));
if (ok) {
for (let i = 0; i < notes.length; i++) {
await Note.delete(notes[i].id);
}
}
}
} catch (error) {
this.log(error);
}
end();
@ -226,15 +231,19 @@ commands.push({
usage: 'mv <pattern> <notebook>',
description: 'Moves the notes matching <pattern> to <notebook>.',
action: async function(args, end) {
let pattern = args['pattern'];
try {
let pattern = args['pattern'];
let folder = await Folder.loadByField('title', args['notebook']);
if (!folder) return cmdError(this, _('No folder with title "%s"', args['notebook']), end);
let notes = await Note.previews(currentFolder.id, { titlePattern: pattern });
if (!notes.length) return cmdError(this, _('No note matches this pattern: "%s"', pattern), end);
let folder = await Folder.loadByField('title', args['notebook']);
if (!folder) throw new Error(_('No folder with title "%s"', args['notebook']));
let notes = await Note.previews(currentFolder.id, { titlePattern: pattern });
if (!notes.length) throw new Error(_('No note matches this pattern: "%s"', pattern));
for (let i = 0; i < notes.length; i++) {
await Note.save({ id: notes[i].id, parent_id: folder.id });
for (let i = 0; i < notes.length; i++) {
await Note.save({ id: notes[i].id, parent_id: folder.id });
}
} catch (error) {
this.log(error);
}
end();
@ -252,41 +261,45 @@ commands.push({
['-t, --type <type>', 'Displays only the items of the specific type(s). Can be `n` for notes, `t` for todos, or `nt` for notes and todos (eg. `-tt` would display only the todos, while `-ttd` would display notes and todos.'],
],
action: async function(args, end) {
let pattern = args['pattern'];
let suffix = '';
let items = [];
let options = args.options;
try {
let pattern = args['pattern'];
let suffix = '';
let items = [];
let options = args.options;
let queryOptions = {};
if (options.lines) queryOptions.limit = options.lines;
if (options.sort) {
queryOptions.orderBy = options.sort;
queryOptions.orderByDir = 'ASC';
}
if (options.reverse === true) queryOptions.orderByDir = queryOptions.orderByDir == 'ASC' ? 'DESC' : 'ASC';
queryOptions.caseInsensitive = true;
if (options.type) {
queryOptions.itemTypes = [];
if (options.type.indexOf('n') >= 0) queryOptions.itemTypes.push('note');
if (options.type.indexOf('t') >= 0) queryOptions.itemTypes.push('todo');
}
if (pattern) queryOptions.titlePattern = pattern;
if (pattern == '..') {
items = await Folder.all(queryOptions);
suffix = '/';
} else {
items = await Note.previews(currentFolder.id, queryOptions);
}
for (let i = 0; i < items.length; i++) {
let item = items[i];
let line = '';
if (!!item.is_todo) {
line += sprintf('[%s] ', !!item.todo_completed ? 'X' : ' ');
let queryOptions = {};
if (options.lines) queryOptions.limit = options.lines;
if (options.sort) {
queryOptions.orderBy = options.sort;
queryOptions.orderByDir = 'ASC';
}
line += item.title + suffix;
this.log(line);
if (options.reverse === true) queryOptions.orderByDir = queryOptions.orderByDir == 'ASC' ? 'DESC' : 'ASC';
queryOptions.caseInsensitive = true;
if (options.type) {
queryOptions.itemTypes = [];
if (options.type.indexOf('n') >= 0) queryOptions.itemTypes.push('note');
if (options.type.indexOf('t') >= 0) queryOptions.itemTypes.push('todo');
}
if (pattern) queryOptions.titlePattern = pattern;
if (pattern == '..') {
items = await Folder.all(queryOptions);
suffix = '/';
} else {
items = await Note.previews(currentFolder.id, queryOptions);
}
for (let i = 0; i < items.length; i++) {
let item = items[i];
let line = '';
if (!!item.is_todo) {
line += sprintf('[%s] ', !!item.todo_completed ? 'X' : ' ');
}
line += item.title + suffix;
this.log(line);
}
} catch (Error) {
this.log(error);
}
end();
@ -315,59 +328,64 @@ commands.push({
['--fuzzy-matching', 'For debugging purposes. Do not use.'],
],
action: async function(args, end) {
let filePath = args.file;
let folder = null;
let folderTitle = args['notebook'];
try {
let filePath = args.file;
let folder = null;
let folderTitle = args['notebook'];
if (folderTitle) {
folder = await Folder.loadByField('title', folderTitle);
if (!folder) return cmdError(this, _('Folder does not exists: "%s"', folderTitle), end);
} else {
folderTitle = filename(filePath);
folderTitle = _('Imported - %s', folderTitle);
let inc = 0;
while (true) {
let t = folderTitle + (inc ? ' (' + inc + ')' : '');
let f = await Folder.loadByField('title', t);
if (!f) {
folderTitle = t;
break;
if (folderTitle) {
folder = await Folder.loadByField('title', folderTitle);
if (!folder) return cmdError(this, _('Folder does not exists: "%s"', folderTitle), end);
} else {
folderTitle = filename(filePath);
folderTitle = _('Imported - %s', folderTitle);
let inc = 0;
while (true) {
let t = folderTitle + (inc ? ' (' + inc + ')' : '');
let f = await Folder.loadByField('title', t);
if (!f) {
folderTitle = t;
break;
}
inc++;
}
inc++;
}
let ok = await cmdPromptConfirm(this, _('File "%s" will be imported into notebook "%s". Continue?', basename(filePath), folderTitle))
if (!ok) {
end();
return;
}
let redrawnCalled = false;
let options = {
fuzzyMatching: args.options['fuzzy-matching'] === true,
onProgress: (progressState) => {
let line = [];
line.push(_('Found: %d.', progressState.loaded));
line.push(_('Created: %d.', progressState.created));
if (progressState.updated) line.push(_('Updated: %d.', progressState.updated));
if (progressState.skipped) line.push(_('Skipped: %d.', progressState.skipped));
if (progressState.resourcesCreated) line.push(_('Resources: %d.', progressState.resourcesCreated));
redrawnCalled = true;
vorpal.ui.redraw(line.join(' '));
},
onError: (error) => {
let s = error.trace ? error.trace : error.toString();
this.log(s);
},
}
folder = !folder ? await Folder.save({ title: folderTitle }) : folder;
this.log(_('Importing notes...'));
await importEnex(folder.id, filePath, options);
if (redrawnCalled) vorpal.ui.redraw.done();
this.log(_('Done.'));
} catch (error) {
this.log(error);
}
let ok = await cmdPromptConfirm(this, _('File "%s" will be imported into notebook "%s". Continue?', basename(filePath), folderTitle))
if (!ok) {
end();
return;
}
let redrawnCalled = false;
let options = {
fuzzyMatching: args.options['fuzzy-matching'] === true,
onProgress: (progressState) => {
let line = [];
line.push(_('Found: %d.', progressState.loaded));
line.push(_('Created: %d.', progressState.created));
if (progressState.updated) line.push(_('Updated: %d.', progressState.updated));
if (progressState.resourcesCreated) line.push(_('Resources: %d.', progressState.resourcesCreated));
redrawnCalled = true;
vorpal.ui.redraw(line.join(' '));
},
onError: (error) => {
let s = error.trace ? error.trace : error.toString();
this.log(s);
},
}
folder = !folder ? await Folder.save({ title: folderTitle }) : folder;
this.log(_('Importing notes...'));
await importEnex(folder.id, filePath, options);
if (redrawnCalled) vorpal.ui.redraw.done();
this.log(_('Done.'));
end();
},
});

2
CliClient/package.json
View File

@ -1,6 +1,6 @@
{
"name": "joplin-cli",
"version": "0.8.9",
"version": "0.8.11",
"bin": {
"joplin": "./main.sh"
},

12
lib/base-model.js
View File

@ -271,7 +271,17 @@ class BaseModel {
}
static filter(model) {
return model;
if (!model) return model;
let output = Object.assign({}, model);
for (let n in output) {
if (!output.hasOwnProperty(n)) continue;
// The SQLite database doesn't have booleans so cast everything to int
if (output[n] === true) output[n] = 1;
if (output[n] === false) output[n] = 0;
}
return output;
}
static delete(id, options = null) {

30
lib/database.js
View File

@ -1,7 +1,9 @@
import { uuid } from 'lib/uuid.js';
import { promiseChain } from 'lib/promise-utils.js';
import { Logger } from 'lib/logger.js'
import { time } from 'lib/time-utils.js'
import { _ } from 'lib/locale.js'
import { sprintf } from 'sprintf-js';
const structureSql = `
CREATE TABLE folders (
@ -193,11 +195,29 @@ class Database {
});
}
exec(sql, params = null) {
this.logQuery(sql, params);
return this.driver().exec(sql, params).catch((error) => {
throw this.sqliteErrorToJsError(error, sql, params);
});
async exec(sql, params = null) {
let result = null;
let waitTime = 50;
let totalWaitTime = 0;
while (true) {
try {
this.logQuery(sql, params);
let result = await this.driver().exec(sql, params);
return result;; // No exception was thrown
} catch (error) {
throw error;
if (error && error.code == 'SQLITE_IOERR') {
if (totalWaitTime >= 20000) throw error;
this.logger().warn(sprintf('SQLITE_IOERR: will retry in %s milliseconds', waitTime));
this.logger().warn('Error was: ' + error.toString());
await time.msleep(waitTime);
totalWaitTime += waitTime;
waitTime *= 1.5;
} else {
throw this.sqliteErrorToJsError(error, sql, params);
}
}
}
}
transactionExecBatch(queries) {

2
lib/models/note.js
View File

@ -105,7 +105,7 @@ class Note extends BaseItem {
static filter(note) {
if (!note) return note;
let output = Object.assign({}, note);
let output = super.filter(note);
if ('longitude' in output) output.longitude = Number(!output.longitude ? 0 : output.longitude).toFixed(8);
if ('latitude' in output) output.latitude = Number(!output.latitude ? 0 : output.latitude).toFixed(8);
if ('altitude' in output) output.altitude = Number(!output.altitude ? 0 : output.altitude).toFixed(4);

3
lib/onedrive-api.js
View File

@ -224,7 +224,8 @@ class OneDriveApi {
enableServerDestroy(server);
console.info('Please open this URL in your browser to authentify the application: ' + authCodeUrl);
console.info('Please open this URL in your browser to authentify the application:');
console.info(authCodeUrl);
});
}

12
lib/time-utils.js
View File

@ -18,6 +18,18 @@ let time = {
return moment.unix(ms / 1000).utc().format('YYYY-MM-DDTHH:mm:ss.SSS') + 'Z';
},
msleep(ms) {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve();
}, ms);
});
},
sleep(seconds) {
return this.msleep(seconds * 1000);
},
}
export { time };