Joplin
/
joplin
mirror of https://github.com/laurent22/joplin.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Sync fixes

pull/41/head
Laurent Cozic 2017-07-19 20:15:55 +01:00
parent df3e5ac40c
commit 5ca8647d35
18 changed files with 264 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CliClient/app/command-sync.js
View File

@ -7,6 +7,7 @@ import { vorpalUtils } from './vorpal-utils.js';
import { Synchronizer } from 'lib/synchronizer.js'; import { Synchronizer } from 'lib/synchronizer.js';
const locker = require('proper-lockfile'); const locker = require('proper-lockfile');
const fs = require('fs-extra'); const fs = require('fs-extra');
const osTmpdir = require('os-tmpdir');
class Command extends BaseCommand { class Command extends BaseCommand {
@ -34,7 +35,7 @@ class Command extends BaseCommand {
static lockFile(filePath) { static lockFile(filePath) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
locker.lock(filePath, (error, release) => { locker.lock(filePath, { stale: 1000 * 60 * 5 }, (error, release) => {
if (error) { if (error) {
reject(error); reject(error);
return; return;
@ -61,7 +62,7 @@ class Command extends BaseCommand {
async action(args) { async action(args) {
this.releaseLockFn_ = null; this.releaseLockFn_ = null;
const lockFilePath = Setting.value('tempDir') + '/synclock'; const lockFilePath = osTmpdir() + '/synclock';
if (!await fs.pathExists(lockFilePath)) await fs.writeFile(lockFilePath, 'synclock'); if (!await fs.pathExists(lockFilePath)) await fs.writeFile(lockFilePath, 'synclock');
if (await Command.isLocked(lockFilePath)) throw new Error(_('Synchronisation is already in progress.')); if (await Command.isLocked(lockFilePath)) throw new Error(_('Synchronisation is already in progress.'));
@ -71,7 +72,7 @@ class Command extends BaseCommand {
try { try {
this.syncTarget_ = Setting.value('sync.target'); this.syncTarget_ = Setting.value('sync.target');
if (args.options.target) this.syncTarget_ = args.options.target; if (args.options.target) this.syncTarget_ = args.options.target;
let syncInitOptions = {}; let syncInitOptions = {};
if (args.options['filesystem-path']) syncInitOptions['sync.filesystem.path'] = args.options['filesystem-path']; if (args.options['filesystem-path']) syncInitOptions['sync.filesystem.path'] = args.options['filesystem-path'];
@ -100,6 +101,7 @@ class Command extends BaseCommand {
options.context = context; options.context = context;
let newContext = await sync.start(options); let newContext = await sync.start(options);
Setting.setValue('sync.context', JSON.stringify(newContext)); Setting.setValue('sync.context', JSON.stringify(newContext));
vorpalUtils.redrawDone(); vorpalUtils.redrawDone();
await app().refreshCurrentFolder(); await app().refreshCurrentFolder();

2
CliClient/app/import-enex.js
View File

@ -97,8 +97,8 @@ async function saveNoteResources(note) {
let existingResource = await Resource.load(toSave.id); let existingResource = await Resource.load(toSave.id);
if (existingResource) continue; if (existingResource) continue;
await Resource.save(toSave, { isNew: true });
await filePutContents(Resource.fullPath(toSave), resource.data) await filePutContents(Resource.fullPath(toSave), resource.data)
await Resource.save(toSave, { isNew: true });
resourcesCreated++; resourcesCreated++;
} }
return resourcesCreated; return resourcesCreated;

20
CliClient/locales/en_GB.po
View File

@ -8,7 +8,7 @@ msgid ""
msgstr "" msgstr ""
"Project-Id-Version: Joplin-CLI 1.0.0\n" "Project-Id-Version: Joplin-CLI 1.0.0\n"
"Report-Msgid-Bugs-To: \n" "Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2017-07-18 23:34+0100\n" "POT-Creation-Date: 2017-07-19 20:00+0100\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n" "Language-Team: LANGUAGE <LL@li.org>\n"
@ -287,40 +287,40 @@ msgstr ""
msgid "Displays summary about the notes and notebooks." msgid "Displays summary about the notes and notebooks."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:24 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:26
msgid "Synchronizes with remote storage." msgid "Synchronizes with remote storage."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:29 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:31
msgid "Sync to provided target (defaults to sync.target config value)" msgid "Sync to provided target (defaults to sync.target config value)"
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:30 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:32
msgid "For \"filesystem\" target only: Path to sync to." msgid "For \"filesystem\" target only: Path to sync to."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:67 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:69
msgid "Synchronisation is already in progress." msgid "Synchronisation is already in progress."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96
#, javascript-format #, javascript-format
msgid "Synchronization target: %s" msgid "Synchronization target: %s"
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:94 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:98
msgid "Cannot initialize synchronizer." msgid "Cannot initialize synchronizer."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:100
msgid "Starting synchronization..." msgid "Starting synchronization..."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:107 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:112
msgid "Done." msgid "Done."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:122 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:127
#: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60 #: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60
msgid "Cancelling..." msgid "Cancelling..."
msgstr "" msgstr ""

20
CliClient/locales/fr_FR.po
View File

@ -7,7 +7,7 @@ msgid ""
msgstr "" msgstr ""
"Project-Id-Version: Joplin-CLI 1.0.0\n" "Project-Id-Version: Joplin-CLI 1.0.0\n"
"Report-Msgid-Bugs-To: \n" "Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2017-07-18 23:18+0100\n" "POT-Creation-Date: 2017-07-19 19:53+0100\n"
"PO-Revision-Date: 2017-07-18 13:27+0100\n" "PO-Revision-Date: 2017-07-18 13:27+0100\n"
"Last-Translator: \n" "Last-Translator: \n"
"Language-Team: \n" "Language-Team: \n"
@ -310,42 +310,42 @@ msgstr "Assigner la valeur [value] à la propriété <name> de la <note> donnée
msgid "Displays summary about the notes and notebooks." msgid "Displays summary about the notes and notebooks."
msgstr "Afficher un résumé des notes et carnets." msgstr "Afficher un résumé des notes et carnets."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:24 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:26
msgid "Synchronizes with remote storage." msgid "Synchronizes with remote storage."
msgstr "Synchroniser les notes et carnets." msgstr "Synchroniser les notes et carnets."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:29 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:31
msgid "Sync to provided target (defaults to sync.target config value)" msgid "Sync to provided target (defaults to sync.target config value)"
msgstr "" msgstr ""
"Synchroniser avec la cible donnée (par défaut, la valeur de configuration " "Synchroniser avec la cible donnée (par défaut, la valeur de configuration "
"`sync.target`)." "`sync.target`)."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:30 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:32
msgid "For \"filesystem\" target only: Path to sync to." msgid "For \"filesystem\" target only: Path to sync to."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:67 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:69
msgid "Synchronisation is already in progress." msgid "Synchronisation is already in progress."
msgstr "Synchronisation est déjà en cours." msgstr "Synchronisation est déjà en cours."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96
#, javascript-format #, javascript-format
msgid "Synchronization target: %s" msgid "Synchronization target: %s"
msgstr "Cible de la synchronisation : %s" msgstr "Cible de la synchronisation : %s"
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:94 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:98
msgid "Cannot initialize synchronizer." msgid "Cannot initialize synchronizer."
msgstr "Impossible d'initialiser le synchroniseur." msgstr "Impossible d'initialiser le synchroniseur."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:100
msgid "Starting synchronization..." msgid "Starting synchronization..."
msgstr "Commencement de la synchronisation..." msgstr "Commencement de la synchronisation..."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:107 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:112
msgid "Done." msgid "Done."
msgstr "Terminé." msgstr "Terminé."
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:122 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:127
#: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60 #: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60
msgid "Cancelling..." msgid "Cancelling..."
msgstr "Annulation..." msgstr "Annulation..."

20
CliClient/locales/joplin.pot
View File

@ -8,7 +8,7 @@ msgid ""
msgstr "" msgstr ""
"Project-Id-Version: Joplin-CLI 1.0.0\n" "Project-Id-Version: Joplin-CLI 1.0.0\n"
"Report-Msgid-Bugs-To: \n" "Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2017-07-18 23:34+0100\n" "POT-Creation-Date: 2017-07-19 20:00+0100\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n" "Language-Team: LANGUAGE <LL@li.org>\n"
@ -287,40 +287,40 @@ msgstr ""
msgid "Displays summary about the notes and notebooks." msgid "Displays summary about the notes and notebooks."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:24 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:26
msgid "Synchronizes with remote storage." msgid "Synchronizes with remote storage."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:29 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:31
msgid "Sync to provided target (defaults to sync.target config value)" msgid "Sync to provided target (defaults to sync.target config value)"
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:30 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:32
msgid "For \"filesystem\" target only: Path to sync to." msgid "For \"filesystem\" target only: Path to sync to."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:67 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:69
msgid "Synchronisation is already in progress." msgid "Synchronisation is already in progress."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:92 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96
#, javascript-format #, javascript-format
msgid "Synchronization target: %s" msgid "Synchronization target: %s"
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:94 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:98
msgid "Cannot initialize synchronizer." msgid "Cannot initialize synchronizer."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:96 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:100
msgid "Starting synchronization..." msgid "Starting synchronization..."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:107 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:112
msgid "Done." msgid "Done."
msgstr "" msgstr ""
#: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:122 #: /mnt/d/Web/www/joplin/CliClient/app/command-sync.js:127
#: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60 #: /mnt/d/Web/www/joplin/ReactNativeClient/lib/synchronizer.js:60
msgid "Cancelling..." msgid "Cancelling..."
msgstr "" msgstr ""

3
CliClient/package.json
View File

@ -7,7 +7,7 @@
"url": "https://github.com/laurent22/joplin" "url": "https://github.com/laurent22/joplin"
}, },
"url": "git://github.com/laurent22/joplin.git", "url": "git://github.com/laurent22/joplin.git",
"version": "0.8.58", "version": "0.8.62",
"bin": { "bin": {
"joplin": "./main_launcher.js" "joplin": "./main_launcher.js"
}, },
@ -24,6 +24,7 @@
"moment": "^2.18.1", "moment": "^2.18.1",
"moment-timezone": "^0.5.13", "moment-timezone": "^0.5.13",
"node-fetch": "^1.7.1", "node-fetch": "^1.7.1",
"os-tmpdir": "^1.0.2",
"promise": "^7.1.1", "promise": "^7.1.1",
"proper-lockfile": "^2.0.1", "proper-lockfile": "^2.0.1",
"query-string": "4.3.4", "query-string": "4.3.4",

9
CliClient/tests/synchronizer.js
View File

@ -6,6 +6,7 @@ import { setupDatabase, setupDatabaseAndSynchronizer, db, synchronizer, fileApi,
import { Folder } from 'lib/models/folder.js'; import { Folder } from 'lib/models/folder.js';
import { Note } from 'lib/models/note.js'; import { Note } from 'lib/models/note.js';
import { Tag } from 'lib/models/tag.js'; import { Tag } from 'lib/models/tag.js';
import { Database } from 'lib/database.js';
import { Setting } from 'lib/models/setting.js'; import { Setting } from 'lib/models/setting.js';
import { BaseItem } from 'lib/models/base-item.js'; import { BaseItem } from 'lib/models/base-item.js';
import { BaseModel } from 'lib/base-model.js'; import { BaseModel } from 'lib/base-model.js';
@ -16,6 +17,8 @@ process.on('unhandledRejection', (reason, p) => {
jasmine.DEFAULT_TIMEOUT_INTERVAL = 9000; // The first test is slow because the database needs to be built jasmine.DEFAULT_TIMEOUT_INTERVAL = 9000; // The first test is slow because the database needs to be built
const syncTargetId = Database.enumId('syncTarget', 'memory');
async function allItems() { async function allItems() {
let folders = await Folder.all(); let folders = await Folder.all();
let notes = await Note.all(); let notes = await Note.all();
@ -237,7 +240,7 @@ describe('Synchronizer', function() {
expect(files.length).toBe(1); expect(files.length).toBe(1);
expect(files[0].path).toBe(Folder.systemPath(folder1)); expect(files[0].path).toBe(Folder.systemPath(folder1));
let deletedItems = await BaseItem.deletedItems(); let deletedItems = await BaseItem.deletedItems(syncTargetId);
expect(deletedItems.length).toBe(0); expect(deletedItems.length).toBe(0);
done(); done();
@ -259,7 +262,7 @@ describe('Synchronizer', function() {
await synchronizer().start(); await synchronizer().start();
let items = await allItems(); let items = await allItems();
expect(items.length).toBe(1); expect(items.length).toBe(1);
let deletedItems = await BaseItem.deletedItems(); let deletedItems = await BaseItem.deletedItems(syncTargetId);
expect(deletedItems.length).toBe(0); expect(deletedItems.length).toBe(0);
done(); done();
@ -565,7 +568,7 @@ describe('Synchronizer', function() {
await synchronizer().start(); await synchronizer().start();
await Note.save({ id: n1.id, is_conflict: 1 }); await Note.save({ id: n1.id, is_conflict: 1 });
await Note.delete(n1.id); await Note.delete(n1.id);
const deletedItems = await BaseItem.deletedItems(); const deletedItems = await BaseItem.deletedItems(syncTargetId);
expect(deletedItems.length).toBe(0); expect(deletedItems.length).toBe(0);

3
CliClient/tests/test-utils.js
View File

@ -72,6 +72,9 @@ function clearDatabase(id = null) {
'DELETE FROM resources', 'DELETE FROM resources',
'DELETE FROM tags', 'DELETE FROM tags',
'DELETE FROM note_tags', 'DELETE FROM note_tags',
'DELETE FROM deleted_items',
'DELETE FROM sync_items',
]; ];
return databases_[id].transactionExecBatch(queries); return databases_[id].transactionExecBatch(queries);

4
CliClient/yarn.lock
View File

@ -666,7 +666,7 @@ babel-register@^6.24.1:
mkdirp "^0.5.1" mkdirp "^0.5.1"
source-map-support "^0.4.2" source-map-support "^0.4.2"
babel-runtime@^6.18.0, babel-runtime@^6.22.0, babel-runtime@^6.23.0: babel-runtime@^6.18.0, babel-runtime@^6.22.0:
version "6.23.0" version "6.23.0"
resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.23.0.tgz#0a9489f144de70efb3ce4300accdb329e2fc543b" resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.23.0.tgz#0a9489f144de70efb3ce4300accdb329e2fc543b"
dependencies: dependencies:
@ -1641,7 +1641,7 @@ os-homedir@^1.0.0:
version "1.0.2" version "1.0.2"
resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3"
os-tmpdir@^1.0.0, os-tmpdir@^1.0.1: os-tmpdir@^1.0.0, os-tmpdir@^1.0.1, os-tmpdir@^1.0.2:
version "1.0.2" version "1.0.2"
resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"

4
ReactNativeClient/android/app/build.gradle
View File

@ -90,8 +90,8 @@ android {
applicationId "net.cozic.joplin" applicationId "net.cozic.joplin"
minSdkVersion 16 minSdkVersion 16
targetSdkVersion 22 targetSdkVersion 22
versionCode 21 versionCode 23
versionName "0.9.8" versionName "0.9.10"
ndk { ndk {
abiFilters "armeabi-v7a", "x86" abiFilters "armeabi-v7a", "x86"
} }

5
ReactNativeClient/lib/base-model.js
View File

@ -52,10 +52,13 @@ class BaseModel {
static fieldNames(withPrefix = false) { static fieldNames(withPrefix = false) {
let output = this.db().tableFieldNames(this.tableName()); let output = this.db().tableFieldNames(this.tableName());
if (!withPrefix) return output; if (!withPrefix) return output;
let p = withPrefix === true ? this.tableName() : withPrefix;
let temp = []; let temp = [];
for (let i = 0; i < output.length; i++) { for (let i = 0; i < output.length; i++) {
temp.push(this.tableName() + '.' + output[i]); temp.push(p + '.' + output[i]);
} }
return temp; return temp;
} }

2
ReactNativeClient/lib/components/side-menu-content.js
View File

@ -73,7 +73,7 @@ class SideMenuContentComponent extends Component {
sync.cancel(); sync.cancel();
} else { } else {
if (reg.oneDriveApi().auth()) { if (reg.oneDriveApi().auth()) {
sync.start(); reg.scheduleSync(1);
} else { } else {
this.props.dispatch({ type: 'SIDE_MENU_CLOSE' }); this.props.dispatch({ type: 'SIDE_MENU_CLOSE' });

8
ReactNativeClient/lib/database.js
View File

@ -166,6 +166,11 @@ class Database {
throw new Error('Unknown enum type or id: ' + type + ', ' + id); throw new Error('Unknown enum type or id: ' + type + ', ' + id);
} }
static enumIds(type) {
if (type == 'syncTarget') return [1,2,3];
throw new Error('Unknown enum type: ' + type);
}
static formatValue(type, value) { static formatValue(type, value) {
if (value === null || value === undefined) return null; if (value === null || value === undefined) return null;
if (type == this.TYPE_INT) return Number(value); if (type == this.TYPE_INT) return Number(value);
@ -182,7 +187,8 @@ class Database {
var line = lines[i]; var line = lines[i];
if (line == '') continue; if (line == '') continue;
if (line.substr(0, 2) == "--") continue; if (line.substr(0, 2) == "--") continue;
statement += line; statement += line.trim();
if (line[line.length - 1] == ',') statement += ' ';
if (line[line.length - 1] == ';') { if (line[line.length - 1] == ';') {
output.push(statement); output.push(statement);
statement = ''; statement = '';

83
ReactNativeClient/lib/joplin-database.js
View File

@ -42,13 +42,6 @@ CREATE INDEX notes_is_conflict ON notes (is_conflict);
CREATE INDEX notes_is_todo ON notes (is_todo); CREATE INDEX notes_is_todo ON notes (is_todo);
CREATE INDEX notes_order ON notes (\`order\`); CREATE INDEX notes_order ON notes (\`order\`);
CREATE TABLE deleted_items (
id INTEGER PRIMARY KEY,
item_type INT NOT NULL,
item_id TEXT NOT NULL,
deleted_time INT NOT NULL
);
CREATE TABLE tags ( CREATE TABLE tags (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
title TEXT NOT NULL DEFAULT "", title TEXT NOT NULL DEFAULT "",
@ -97,10 +90,6 @@ CREATE TABLE table_fields (
field_default TEXT field_default TEXT
); );
CREATE TABLE version (
version INT NOT NULL
);
CREATE TABLE sync_items ( CREATE TABLE sync_items (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
sync_target INT NOT NULL, sync_target INT NOT NULL,
@ -114,6 +103,17 @@ CREATE INDEX sync_items_sync_target ON sync_items (sync_target);
CREATE INDEX sync_items_item_type ON sync_items (item_type); CREATE INDEX sync_items_item_type ON sync_items (item_type);
CREATE INDEX sync_items_item_id ON sync_items (item_id); CREATE INDEX sync_items_item_id ON sync_items (item_id);
CREATE TABLE deleted_items (
id INTEGER PRIMARY KEY,
item_type INT NOT NULL,
item_id TEXT NOT NULL,
deleted_time INT NOT NULL
);
CREATE TABLE version (
version INT NOT NULL
);
INSERT INTO version (version) VALUES (1); INSERT INTO version (version) VALUES (1);
`; `;
@ -187,28 +187,59 @@ class JoplinDatabase extends Database {
}); });
} }
async upgradeDatabase(fromVersion) {
// INSTRUCTIONS TO UPGRADE THE DATABASE:
//
// 1. Add the new version number to the existingDatabaseVersions array
// 2. Add the upgrade logic to the "switch (targetVersion)" statement below
const existingDatabaseVersions = [1, 2];
let currentVersionIndex = existingDatabaseVersions.indexOf(fromVersion);
if (currentVersionIndex == existingDatabaseVersions.length - 1) return false;
while (currentVersionIndex < existingDatabaseVersions.length - 1) {
const targetVersion = existingDatabaseVersions[currentVersionIndex + 1];
this.logger().info("Converting database to version " + targetVersion);
let queries = [];
if (targetVersion == 2) {
const newTableSql = `
CREATE TABLE deleted_items (
id INTEGER PRIMARY KEY,
item_type INT NOT NULL,
item_id TEXT NOT NULL,
deleted_time INT NOT NULL,
sync_target INT NOT NULL
);
`;
queries.push({ sql: 'DROP TABLE deleted_items' });
queries.push({ sql: this.sqlStringToLines(newTableSql)[0] });
queries.push({ sql: "CREATE INDEX deleted_items_sync_target ON deleted_items (sync_target)" });
}
queries.push({ sql: 'UPDATE version SET version = ?', params: [targetVersion] });
await this.transactionExecBatch(queries);
currentVersionIndex++;
}
return true;
}
async initialize() { async initialize() {
this.logger().info('Checking for database schema update...'); this.logger().info('Checking for database schema update...');
for (let initLoopCount = 1; initLoopCount <= 2; initLoopCount++) { for (let initLoopCount = 1; initLoopCount <= 2; initLoopCount++) {
try { try {
// await this.exec('DROP TABLE folders');
// await this.exec('DROP TABLE notes');
// await this.exec('DROP TABLE deleted_items');
// await this.exec('DROP TABLE tags');
// await this.exec('DROP TABLE note_tags');
// await this.exec('DROP TABLE resources');
// await this.exec('DROP TABLE settings');
// await this.exec('DROP TABLE table_fields');
// await this.exec('DROP TABLE version');
// await this.exec('DROP TABLE sync_items');
let row = await this.selectOne('SELECT * FROM version LIMIT 1'); let row = await this.selectOne('SELECT * FROM version LIMIT 1');
this.logger().info('Current database version', row); let currentVersion = row.version;
this.logger().info('Current database version', currentVersion);
// TODO: version update logic const upgraded = await this.upgradeDatabase(currentVersion);
// TODO: only do this if db has been updated: if (upgraded) await this.refreshTableFields();
// return this.refreshTableFields();
} catch (error) { } catch (error) {
if (error && error.code != 0 && error.code != 'SQLITE_ERROR') throw this.sqliteErrorToJsError(error); if (error && error.code != 0 && error.code != 'SQLITE_ERROR') throw this.sqliteErrorToJsError(error);

143
ReactNativeClient/lib/models/base-item.js
View File

@ -131,31 +131,36 @@ class BaseItem extends BaseModel {
await super.batchDelete(ids, options); await super.batchDelete(ids, options);
if (trackDeleted) { if (trackDeleted) {
const syncTargetIds = Database.enumIds('syncTarget');
let queries = []; let queries = [];
let now = time.unixMs(); let now = time.unixMs();
for (let i = 0; i < ids.length; i++) { for (let i = 0; i < ids.length; i++) {
if (conflictNoteIds.indexOf(ids[i]) >= 0) continue; if (conflictNoteIds.indexOf(ids[i]) >= 0) continue;
queries.push({ // For each deleted item, for each sync target, we need to add an entry in deleted_items.
sql: 'INSERT INTO deleted_items (item_type, item_id, deleted_time) VALUES (?, ?, ?)', // That way, each target can later delete the remote item.
params: [this.modelType(), ids[i], now], for (let j = 0; j < syncTargetIds.length; j++) {
}); queries.push({
sql: 'INSERT INTO deleted_items (item_type, item_id, deleted_time, sync_target) VALUES (?, ?, ?, ?)',
params: [this.modelType(), ids[i], now, syncTargetIds[j]],
});
}
} }
await this.db().transactionExecBatch(queries); await this.db().transactionExecBatch(queries);
} }
} }
static deletedItems() { static deletedItems(syncTarget) {
return this.db().selectAll('SELECT * FROM deleted_items'); return this.db().selectAll('SELECT * FROM deleted_items WHERE sync_target = ?', [syncTarget]);
} }
static async deletedItemCount() { static async deletedItemCount(syncTarget) {
let r = await this.db().selectOne('SELECT count(*) as total FROM deleted_items'); let r = await this.db().selectOne('SELECT count(*) as total FROM deleted_items WHERE sync_target = ?', [syncTarget]);
return r['total']; return r['total'];
} }
static remoteDeletedItem(itemId) { static remoteDeletedItem(syncTarget, itemId) {
return this.db().exec('DELETE FROM deleted_items WHERE item_id = ?', [itemId]); return this.db().exec('DELETE FROM deleted_items WHERE item_id = ? AND sync_target = ?', [itemId, syncTarget]);
} }
static serialize_format(propName, propValue) { static serialize_format(propName, propValue) {
@ -275,33 +280,131 @@ class BaseItem extends BaseModel {
for (let i = 0; i < classNames.length; i++) { for (let i = 0; i < classNames.length; i++) {
const className = classNames[i]; const className = classNames[i];
const ItemClass = this.getClass(className); const ItemClass = this.getClass(className);
const fieldNames = ItemClass.fieldNames(true); let fieldNames = ItemClass.fieldNames('items');
fieldNames.push('sync_time');
// // NEVER SYNCED:
// 'SELECT * FROM [ITEMS] WHERE id NOT INT (SELECT item_id FROM sync_items WHERE sync_target = ?)'
// // CHANGED:
// 'SELECT * FROM [ITEMS] items JOIN sync_items s ON s.item_id = items.id WHERE sync_target = ? AND'
let extraWhere = className == 'Note' ? 'AND is_conflict = 0' : ''; let extraWhere = className == 'Note' ? 'AND is_conflict = 0' : '';
// First get all the items that have never been synced under this sync target
let sql = sprintf(` let sql = sprintf(`
SELECT %s FROM %s SELECT %s
LEFT JOIN sync_items t ON t.item_id = %s.id FROM %s items
WHERE WHERE id NOT IN (
(t.id IS NULL OR t.sync_time < %s.updated_time) SELECT item_id FROM sync_items WHERE sync_target = %d
%s )
%s
LIMIT %d LIMIT %d
`, `,
this.db().escapeFields(fieldNames), this.db().escapeFields(fieldNames),
this.db().escapeField(ItemClass.tableName()), this.db().escapeField(ItemClass.tableName()),
this.db().escapeField(ItemClass.tableName()), Number(syncTarget),
this.db().escapeField(ItemClass.tableName()),
extraWhere, extraWhere,
limit); limit);
const items = await ItemClass.modelSelectAll(sql); let neverSyncedItem = await ItemClass.modelSelectAll(sql);
//for (let i = 0; i < neverSyncedItem.length; i++) neverSyncedItem[i].sync_time = 0;
// console.info(sql);
// console.info('NEVER', neverSyncedItem);
// Secondly get the items that have been synced under this sync target but that have been changed since then
const newLimit = limit - neverSyncedItem.length;
let changedItems = [];
if (newLimit > 0) {
fieldNames.push('sync_time');
let sql = sprintf(`
SELECT %s FROM %s items
JOIN sync_items s ON s.item_id = items.id
WHERE sync_target = %d
AND s.sync_time < items.updated_time
%s
LIMIT %d
`,
this.db().escapeFields(fieldNames),
this.db().escapeField(ItemClass.tableName()),
Number(syncTarget),
extraWhere,
newLimit);
changedItems = await ItemClass.modelSelectAll(sql);
}
// console.info('CHANGED', changedItems);
const items = neverSyncedItem.concat(changedItems);
if (i >= classNames.length - 1) { if (i >= classNames.length - 1) {
return { hasMore: items.length >= limit, items: items }; return { hasMore: items.length >= limit, items: items };
} else { } else {
if (items.length) return { hasMore: true, items: items }; if (items.length) return { hasMore: true, items: items };
} }
//let extraWhere = className == 'Note' ? 'AND is_conflict = 0' : '';
// First get all the items that have never been synced under this sync target
// let sql = sprintf(`
// SELECT %s FROM %s items
// LEFT JOIN sync_items t ON t.item_id = items.id
// WHERE (t.id IS NULL OR t.sync_target != %d) %s
// LIMIT %d
// `,
// this.db().escapeFields(fieldNames),
// this.db().escapeField(ItemClass.tableName()),
// Number(syncTarget),
// extraWhere,
// limit);
// let neverSyncedItem = await ItemClass.modelSelectAll(sql);
// for (let i = 0; i < neverSyncedItem.length; i++) neverSyncedItem[i].sync_time = 0;
// console.info(sql);
// console.info('NEVER', neverSyncedItem);
// // Secondly get the items that have been synced under this sync target but that have been changed since then
// const newLimit = limit - neverSyncedItem.length;
// let changedItems = [];
// if (newLimit > 0) {
// let sql = sprintf(`
// SELECT %s FROM %s items
// LEFT JOIN sync_items t ON t.item_id = items.id
// WHERE (t.sync_time < items.updated_time AND t.sync_target = %d) %s
// LIMIT %d
// `,
// this.db().escapeFields(fieldNames),
// this.db().escapeField(ItemClass.tableName()),
// Number(syncTarget),
// extraWhere,
// newLimit);
// changedItems = await ItemClass.modelSelectAll(sql);
// }
// console.info('CHANGED', changedItems);
// const items = neverSyncedItem.concat(changedItems);
// if (i >= classNames.length - 1) {
// return { hasMore: items.length >= limit, items: items };
// } else {
// if (items.length) return { hasMore: true, items: items };
// }
} }
throw new Error('Unreachable'); throw new Error('Unreachable');

12
ReactNativeClient/lib/registry.js
View File

@ -74,7 +74,9 @@ reg.synchronizer = async () => {
return reg.synchronizer_; return reg.synchronizer_;
} }
reg.scheduleSync = async () => { reg.scheduleSync = async (delay = null) => {
if (delay === null) delay = 1000 * 10;
if (reg.scheduleSyncId_) { if (reg.scheduleSyncId_) {
clearTimeout(reg.scheduleSyncId_); clearTimeout(reg.scheduleSyncId_);
reg.scheduleSyncId_ = null; reg.scheduleSyncId_ = null;
@ -92,8 +94,12 @@ reg.scheduleSync = async () => {
} }
const sync = await reg.synchronizer(); const sync = await reg.synchronizer();
sync.start();
}, 1000 * 10); let context = Setting.value('sync.context');
context = context ? JSON.parse(context) : {};
let newContext = await sync.start({ context: context });
Setting.setValue('sync.context', JSON.stringify(newContext));
}, delay);
} }
reg.setDb = (v) => { reg.setDb = (v) => {

2
ReactNativeClient/lib/services/report.js
View File

@ -34,7 +34,7 @@ class ReportService {
}; };
output.toDelete = { output.toDelete = {
total: await BaseItem.deletedItemCount(), total: await BaseItem.deletedItemCount(syncTarget),
}; };
output.conflicted = { output.conflicted = {

15
ReactNativeClient/lib/synchronizer.js
View File

@ -202,7 +202,7 @@ class Synchronizer {
let content = await ItemClass.serialize(local); let content = await ItemClass.serialize(local);
let action = null; let action = null;
let updateSyncTimeOnly = true; let updateSyncTimeOnly = true;
let reason = ''; let reason = '';
if (!remote) { if (!remote) {
if (!local.sync_time) { if (!local.sync_time) {
@ -230,7 +230,14 @@ class Synchronizer {
if (local.type_ == BaseModel.TYPE_RESOURCE && (action == 'createRemote' || (action == 'itemConflict' && remote))) { if (local.type_ == BaseModel.TYPE_RESOURCE && (action == 'createRemote' || (action == 'itemConflict' && remote))) {
let remoteContentPath = this.resourceDirName_ + '/' + local.id; let remoteContentPath = this.resourceDirName_ + '/' + local.id;
let resourceContent = await Resource.content(local); let resourceContent = '';
try {
resourceContent = await Resource.content(local);
} catch (error) {
error.message = 'Cannot read resource content: ' + local.id + ': ' + error.message;
this.logger().error(error);
this.progressReport_.errors.push(error);
}
await this.api().put(remoteContentPath, resourceContent); await this.api().put(remoteContentPath, resourceContent);
} }
@ -302,7 +309,7 @@ class Synchronizer {
// Delete the remote items that have been deleted locally. // Delete the remote items that have been deleted locally.
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
let deletedItems = await BaseItem.deletedItems(); let deletedItems = await BaseItem.deletedItems(syncTargetId);
for (let i = 0; i < deletedItems.length; i++) { for (let i = 0; i < deletedItems.length; i++) {
if (this.cancelling()) break; if (this.cancelling()) break;
@ -311,7 +318,7 @@ class Synchronizer {
this.logSyncOperation('deleteRemote', null, { id: item.item_id }, 'local has been deleted'); this.logSyncOperation('deleteRemote', null, { id: item.item_id }, 'local has been deleted');
await this.api().delete(path); await this.api().delete(path);
if (this.randomFailure(options, 3)) return; if (this.randomFailure(options, 3)) return;
await BaseItem.remoteDeletedItem(item.item_id); await BaseItem.remoteDeletedItem(syncTargetId, item.item_id);
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------