From 857864230778b97776562443cd6122e064220644 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Mon, 12 Jun 2017 22:56:27 +0100 Subject: [PATCH] various changes --- CliClient/app/cmd.js | 8 +- CliClient/package.json | 3 +- ReactNativeClient/src/base-model.js | 3 +- ReactNativeClient/src/database.js | 1 + .../src/file-api-driver-local.js | 18 +- ReactNativeClient/src/file-api.js | 4 + ReactNativeClient/src/models/change.js | 2 - ReactNativeClient/src/models/folder.js | 35 +++ ReactNativeClient/src/models/setting.js | 3 +- .../src/services/note-folder-service.js | 16 +- ReactNativeClient/src/string-utils.js | 7 +- ReactNativeClient/src/synchronizer.js | 237 +++++++++++++----- structure.sql | 2 +- 13 files changed, 255 insertions(+), 84 deletions(-) diff --git a/CliClient/app/cmd.js b/CliClient/app/cmd.js index c8ddecd149..63e6e10402 100644 --- a/CliClient/app/cmd.js +++ b/CliClient/app/cmd.js @@ -1,3 +1,5 @@ +require('source-map-support').install(); + import { FileApi } from 'src/file-api.js'; import { FileApiDriverLocal } from 'src/file-api-driver-local.js'; import { Database } from 'src/database.js'; @@ -5,6 +7,7 @@ import { DatabaseDriverNode } from 'src/database-driver-node.js'; import { BaseModel } from 'src/base-model.js'; import { Folder } from 'src/models/folder.js'; import { Note } from 'src/models/note.js'; +import { Setting } from 'src/models/setting.js'; import { Synchronizer } from 'src/synchronizer.js'; import { uuid } from 'src/uuid.js'; import { sprintf } from 'sprintf-js'; @@ -53,7 +56,9 @@ let synchronizer = new Synchronizer(db, fileApi); db.open({ name: '/home/laurent/Temp/test.sqlite3' }).then(() => { BaseModel.db_ = db; - +}).then(() => { + return Setting.load(); +}).then(() => { let commands = []; let currentFolder = null; @@ -291,6 +296,7 @@ db.open({ name: '/home/laurent/Temp/test.sqlite3' }).then(() => { commands.push({ usage: 'ls [list-title]', + alias: 'll', description: 'Lists items in [list-title].', action: function (args, end) { let folderTitle = args['list-title']; diff --git a/CliClient/package.json b/CliClient/package.json index c40b8a802e..f4f7d5fe5e 100644 --- a/CliClient/package.json +++ b/CliClient/package.json @@ -14,6 +14,7 @@ "promise": "^7.1.1", "react": "16.0.0-alpha.6", "sax": "^1.2.2", + "source-map-support": "^0.4.15", "sprintf-js": "^1.1.1", "sqlite3": "^3.1.8", "string-to-stream": "^1.1.0", @@ -29,7 +30,7 @@ }, "scripts": { "babelbuild": "babel app -d build", - "build": "babel-changed app -d build && babel-changed app/src/models -d build/src/models && babel-changed app/src/services -d build/src/services", + "build": "babel-changed app -d build --source-maps && babel-changed app/src/models -d build/src/models --source-maps && babel-changed app/src/services -d build/src/services --source-maps", "clean": "babel-changed --reset" } } diff --git a/ReactNativeClient/src/base-model.js b/ReactNativeClient/src/base-model.js index abb12042c6..aed87df8eb 100644 --- a/ReactNativeClient/src/base-model.js +++ b/ReactNativeClient/src/base-model.js @@ -73,7 +73,6 @@ class BaseModel { static load(id) { return this.loadByField('id', id); - //return this.db().selectOne('SELECT * FROM ' + this.tableName() + ' WHERE id = ?', [id]); } static loadByField(fieldName, fieldValue) { @@ -139,7 +138,7 @@ class BaseModel { query.id = itemId; - Log.info('Saving', o); + Log.info('Saving', JSON.stringify(o)); return query; } diff --git a/ReactNativeClient/src/database.js b/ReactNativeClient/src/database.js index 328c0dd0ac..cab8ee5d9a 100644 --- a/ReactNativeClient/src/database.js +++ b/ReactNativeClient/src/database.js @@ -283,6 +283,7 @@ class Database { } refreshTableFields() { + Log.info('Initializing tables...'); let queries = []; queries.push(this.wrapQuery('DELETE FROM table_fields')); diff --git a/ReactNativeClient/src/file-api-driver-local.js b/ReactNativeClient/src/file-api-driver-local.js index 627229861c..80cac7d974 100644 --- a/ReactNativeClient/src/file-api-driver-local.js +++ b/ReactNativeClient/src/file-api-driver-local.js @@ -25,15 +25,29 @@ class FileApiDriverLocal { return Math.round(m.toDate().getTime() / 1000); } - metadataFromStats_(name, stats) { + metadataFromStats_(path, stats) { return { - name: name, + path: path, createdTime: this.statTimeToUnixTimestamp_(stats.birthtime), updatedTime: this.statTimeToUnixTimestamp_(stats.mtime), + createdTimeOrig: stats.birthtime, + updatedTimeOrig: stats.mtime, isDir: stats.isDirectory(), }; } + setFileTimestamp(path, timestamp) { + return new Promise((resolve, reject) => { + fs.utimes(path, timestamp, timestamp, (error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); + } + list(path) { return new Promise((resolve, reject) => { fs.readdir(path, (error, items) => { diff --git a/ReactNativeClient/src/file-api.js b/ReactNativeClient/src/file-api.js index bd4557e2fc..8c2ed589e4 100644 --- a/ReactNativeClient/src/file-api.js +++ b/ReactNativeClient/src/file-api.js @@ -35,6 +35,10 @@ class FileApi { }); } + setFileTimestamp(path, timestamp) { + return this.driver_.setFileTimestamp(this.baseDir_ + '/' + path, timestamp); + } + mkdir(path) { return this.driver_.mkdir(this.baseDir_ + '/' + path); } diff --git a/ReactNativeClient/src/models/change.js b/ReactNativeClient/src/models/change.js index 7cf7ced119..e52b930195 100644 --- a/ReactNativeClient/src/models/change.js +++ b/ReactNativeClient/src/models/change.js @@ -24,8 +24,6 @@ class Change extends BaseModel { static deleteMultiple(ids) { if (ids.length == 0) return Promise.resolve(); - console.warn('TODO: deleteMultiple: CHECK THAT IT WORKS'); - let queries = []; for (let i = 0; i < ids.length; i++) { queries.push(['DELETE FROM changes WHERE id = ?', [ids[i]]]); diff --git a/ReactNativeClient/src/models/folder.js b/ReactNativeClient/src/models/folder.js index bd3e8760ee..7de67ee335 100644 --- a/ReactNativeClient/src/models/folder.js +++ b/ReactNativeClient/src/models/folder.js @@ -4,6 +4,7 @@ import { promiseChain } from 'src/promise-chain.js'; import { Note } from 'src/models/note.js'; import { folderItemFilename } from 'src/string-utils.js' import { _ } from 'src/locale.js'; +import moment from 'moment'; class Folder extends BaseModel { @@ -19,6 +20,40 @@ class Folder extends BaseModel { return this.filename(folder); } + static systemMetadataPath(parent, folder) { + return this.systemPath(parent, folder) + '/.folder.md'; + } + + // TODO: share with Note class + static toFriendlyString_format(propName, propValue) { + if (['created_time', 'updated_time'].indexOf(propName) >= 0) { + if (!propValue) return ''; + propValue = moment.unix(propValue).format('YYYY-MM-DD hh:mm:ss'); + } else if (propValue === null || propValue === undefined) { + propValue = ''; + } + + return propValue; + } + + // TODO: share with Note class + static toFriendlyString(folder) { + let shownKeys = ['created_time', 'updated_time']; + let output = []; + + output.push(folder.title); + output.push(''); + output.push(''); // For consistency with the notes, leave an empty line where the body should be + output.push(''); + for (let i = 0; i < shownKeys.length; i++) { + let v = folder[shownKeys[i]]; + v = this.toFriendlyString_format(shownKeys[i], v); + output.push(shownKeys[i] + ': ' + v); + } + + return output.join("\n"); + } + static useUuid() { return true; } diff --git a/ReactNativeClient/src/models/setting.js b/ReactNativeClient/src/models/setting.js index 33d84e5151..e3a523f395 100644 --- a/ReactNativeClient/src/models/setting.js +++ b/ReactNativeClient/src/models/setting.js @@ -116,7 +116,8 @@ Setting.defaults_ = { 'sessionId': { value: '', type: 'string' }, 'user.email': { value: '', type: 'string' }, 'user.session': { value: '', type: 'string' }, - 'sync.lastRevId': { value: 0, type: 'int' }, + 'sync.lastRevId': { value: 0, type: 'int' }, // DEPRECATED + 'sync.lastUpdateTime': { value: 0, type: 'int' }, }; export { Setting }; \ No newline at end of file diff --git a/ReactNativeClient/src/services/note-folder-service.js b/ReactNativeClient/src/services/note-folder-service.js index a9099c1024..9a08cffd8f 100644 --- a/ReactNativeClient/src/services/note-folder-service.js +++ b/ReactNativeClient/src/services/note-folder-service.js @@ -10,8 +10,9 @@ import { Registry } from 'src/registry.js'; class NoteFolderService extends BaseService { static save(type, item, oldItem) { + let diff = null; if (oldItem) { - let diff = BaseModel.diffObjects(oldItem, item); + diff = BaseModel.diffObjects(oldItem, item); if (!Object.getOwnPropertyNames(diff).length) { Log.info('Item not changed - not saved'); return Promise.resolve(item); @@ -27,9 +28,16 @@ class NoteFolderService extends BaseService { let isNew = !item.id; let output = null; - return ItemClass.save(item).then((item) => { - output = item; - if (isNew && type == 'note') return Note.updateGeolocation(item.id); + + let toSave = item; + if (diff !== null) { + toSave = diff; + toSave.id = item.id; + } + + return ItemClass.save(toSave).then((savedItem) => { + output = Object.assign(item, savedItem); + if (isNew && type == 'note') return Note.updateGeolocation(output.id); }).then(() => { // Registry.synchronizer().start(); return output; diff --git a/ReactNativeClient/src/string-utils.js b/ReactNativeClient/src/string-utils.js index abd4d08f20..0fe1131297 100644 --- a/ReactNativeClient/src/string-utils.js +++ b/ReactNativeClient/src/string-utils.js @@ -114,9 +114,10 @@ function escapeFilename(s, maxLength = 32) { } function folderItemFilename(item) { - let output = escapeFilename(item.title).trim(); - if (!output.length) output = '_'; - return output + '.' + item.id.substr(0, 7); + return item.id; + // let output = escapeFilename(item.title).trim(); + // if (!output.length) output = '_'; + // return output + '.' + item.id.substr(0, 7); } export { removeDiacritics, escapeFilename, folderItemFilename }; \ No newline at end of file diff --git a/ReactNativeClient/src/synchronizer.js b/ReactNativeClient/src/synchronizer.js index 6ad2f5a915..62868377ef 100644 --- a/ReactNativeClient/src/synchronizer.js +++ b/ReactNativeClient/src/synchronizer.js @@ -46,15 +46,15 @@ class Synchronizer { } } - remoteFileByName(remoteFiles, name) { + remoteFileByPath(remoteFiles, path) { for (let i = 0; i < remoteFiles.length; i++) { - if (remoteFiles[i].name == name) return remoteFiles[i]; + if (remoteFiles[i].path == path) return remoteFiles[i]; } return null; } conflictDir(remoteFiles) { - let d = this.remoteFileByName('Conflicts'); + let d = this.remoteFileByPath('Conflicts'); if (!d) { return this.api().mkdir('Conflicts').then(() => { return 'Conflicts'; @@ -69,11 +69,11 @@ class Synchronizer { if (item.isDir) return Promise.resolve(); return this.conflictDir().then((conflictDirPath) => { - let p = path.basename(item.name).split('.'); + let p = path.basename(item.path).split('.'); let pos = item.isDir ? p.length - 1 : p.length - 2; p.splice(pos, 0, moment().format('YYYYMMDDThhmmss')); - let newName = p.join('.'); - return this.api().move(item.name, conflictDirPath + '/' + newName); + let newPath = p.join('.'); + return this.api().move(item.path, conflictDirPath + '/' + newPath); }); } @@ -86,6 +86,7 @@ class Synchronizer { }).then((changes) => { let mergedChanges = Change.mergeChanges(changes); let chain = []; + const lastSyncTime = Setting.value('sync.lastUpdateTime'); for (let i = 0; i < mergedChanges.length; i++) { let c = mergedChanges[i]; chain.push(() => { @@ -102,11 +103,13 @@ class Synchronizer { p = Promise.resolve(); } else if (c.type == Change.TYPE_CREATE) { p = this.loadParentAndItem(c).then((result) => { - if (!result.item) return; // Change refers to an object that doesn't exist (has probably been deleted directly in the database) + let item = result.item; + let parent = result.parent; + if (!item) return; // Change refers to an object that doesn't exist (has probably been deleted directly in the database) - let path = ItemClass.systemPath(result.parent, result.item); + let path = ItemClass.systemPath(parent, item); - let remoteFile = this.remoteFileByName(remoteFiles, path); + let remoteFile = this.remoteFileByPath(remoteFiles, path); let p = null; if (remoteFile) { p = this.moveConflict(remoteFile); @@ -116,7 +119,39 @@ class Synchronizer { return p.then(() => { if (c.item_type == BaseModel.ITEM_TYPE_FOLDER) { - return this.api().mkdir(path); + return this.api().mkdir(path).then(() => { + return this.api().put(Folder.systemMetadataPath(parent, item), Folder.toFriendlyString(item)); + }).then(() => { + return this.api().setFileTimestamp(Folder.systemMetadataPath(parent, item), item.updated_time); + }); + } else { + return this.api().put(path, Note.toFriendlyString(item)).then(() => { + return this.api().setFileTimestamp(path, item.updated_time); + }); + } + }); + }); + } else if (c.type == Change.TYPE_UPDATE) { + p = this.loadParentAndItem(c).then((result) => { + if (!result.item) return; // Change refers to an object that doesn't exist (has probably been deleted directly in the database) + + let path = ItemClass.systemPath(result.parent, result.item); + + let remoteFile = this.remoteFileByPath(remoteFiles, path); + let p = null; + if (remoteFile && remoteFile.updatedTime > lastSyncTime) { + console.info('CONFLICT:', lastSyncTime, remoteFile); + //console.info(moment.unix(remoteFile.updatedTime), moment.unix(result.item.updated_time)); + p = this.moveConflict(remoteFile); + } else { + p = Promise.resolve(); + } + + console.info('Uploading change:', JSON.stringify(result.item)); + + return p.then(() => { + if (c.item_type == BaseModel.ITEM_TYPE_FOLDER) { + return this.api().put(Folder.systemMetadataPath(result.parent, result.item), Folder.toFriendlyString(result.item)); } else { return this.api().put(path, Note.toFriendlyString(result.item)); } @@ -124,7 +159,6 @@ class Synchronizer { }); } - // TODO: handle UPDATE // TODO: handle DELETE return p.then(() => { @@ -142,11 +176,17 @@ class Synchronizer { } return promiseChain(chain); + // }).then(() => { + // console.info(remoteFiles); + // for (let i = 0; i < remoteFiles.length; i++) { + // const remoteFile = remoteFiles[i]; + + // } }).catch((error) => { - Log.warn('Synchronization was interrupted due to an error:', error); + Log.error('Synchronization was interrupted due to an error:', error); }).then(() => { - Log.info('IDs to delete: ', processedChangeIds); - // Change.deleteMultiple(processedChangeIds); + //Log.info('IDs to delete: ', processedChangeIds); + //return Change.deleteMultiple(processedChangeIds); }).then(() => { this.processState('downloadChanges'); }); @@ -240,63 +280,126 @@ class Synchronizer { } processState_downloadChanges() { - let maxRevId = null; - let hasMore = false; - this.api().get('synchronizer', { rev_id: Setting.value('sync.lastRevId') }).then((syncOperations) => { - hasMore = syncOperations.has_more; - let chain = []; - for (let i = 0; i < syncOperations.items.length; i++) { - let syncOp = syncOperations.items[i]; - if (syncOp.id > maxRevId) maxRevId = syncOp.id; + // return this.api().list('', true).then((items) => { + // remoteFiles = items; + // return Change.all(); - let ItemClass = null; - if (syncOp.item_type == 'folder') { - ItemClass = Folder; - } else if (syncOp.item_type == 'note') { - ItemClass = Note; - } - if (syncOp.type == 'create') { - chain.push(() => { - let item = ItemClass.fromApiResult(syncOp.item); - // TODO: automatically handle NULL fields by checking type and default value of field - if ('parent_id' in item && !item.parent_id) item.parent_id = ''; - return ItemClass.save(item, { isNew: true, trackChanges: false }); - }); - } + // let maxRevId = null; + // let hasMore = false; + // this.api().get('synchronizer', { rev_id: Setting.value('sync.lastRevId') }).then((syncOperations) => { + // hasMore = syncOperations.has_more; + // let chain = []; + // for (let i = 0; i < syncOperations.items.length; i++) { + // let syncOp = syncOperations.items[i]; + // if (syncOp.id > maxRevId) maxRevId = syncOp.id; - if (syncOp.type == 'update') { - chain.push(() => { - return ItemClass.load(syncOp.item_id).then((item) => { - if (!item) return; - item = ItemClass.applyPatch(item, syncOp.item); - return ItemClass.save(item, { trackChanges: false }); - }); - }); - } + // let ItemClass = null; + // if (syncOp.item_type == 'folder') { + // ItemClass = Folder; + // } else if (syncOp.item_type == 'note') { + // ItemClass = Note; + // } - if (syncOp.type == 'delete') { - chain.push(() => { - return ItemClass.delete(syncOp.item_id, { trackChanges: false }); - }); - } - } - return promiseChain(chain); - }).then(() => { - Log.info('All items synced. has_more = ', hasMore); - if (maxRevId) { - Setting.setValue('sync.lastRevId', maxRevId); - return Setting.saveAll(); - } - }).then(() => { - if (hasMore) { - this.processState('downloadChanges'); - } else { - this.processState('idle'); - } - }).catch((error) => { - Log.warn('Sync error', error); - }); + // if (syncOp.type == 'create') { + // chain.push(() => { + // let item = ItemClass.fromApiResult(syncOp.item); + // // TODO: automatically handle NULL fields by checking type and default value of field + // if ('parent_id' in item && !item.parent_id) item.parent_id = ''; + // return ItemClass.save(item, { isNew: true, trackChanges: false }); + // }); + // } + + // if (syncOp.type == 'update') { + // chain.push(() => { + // return ItemClass.load(syncOp.item_id).then((item) => { + // if (!item) return; + // item = ItemClass.applyPatch(item, syncOp.item); + // return ItemClass.save(item, { trackChanges: false }); + // }); + // }); + // } + + // if (syncOp.type == 'delete') { + // chain.push(() => { + // return ItemClass.delete(syncOp.item_id, { trackChanges: false }); + // }); + // } + // } + // return promiseChain(chain); + // }).then(() => { + // Log.info('All items synced. has_more = ', hasMore); + // if (maxRevId) { + // Setting.setValue('sync.lastRevId', maxRevId); + // return Setting.saveAll(); + // } + // }).then(() => { + // if (hasMore) { + // this.processState('downloadChanges'); + // } else { + // this.processState('idle'); + // } + // }).catch((error) => { + // Log.warn('Sync error', error); + // }); + + // let maxRevId = null; + // let hasMore = false; + // this.api().get('synchronizer', { rev_id: Setting.value('sync.lastRevId') }).then((syncOperations) => { + // hasMore = syncOperations.has_more; + // let chain = []; + // for (let i = 0; i < syncOperations.items.length; i++) { + // let syncOp = syncOperations.items[i]; + // if (syncOp.id > maxRevId) maxRevId = syncOp.id; + + // let ItemClass = null; + // if (syncOp.item_type == 'folder') { + // ItemClass = Folder; + // } else if (syncOp.item_type == 'note') { + // ItemClass = Note; + // } + + // if (syncOp.type == 'create') { + // chain.push(() => { + // let item = ItemClass.fromApiResult(syncOp.item); + // // TODO: automatically handle NULL fields by checking type and default value of field + // if ('parent_id' in item && !item.parent_id) item.parent_id = ''; + // return ItemClass.save(item, { isNew: true, trackChanges: false }); + // }); + // } + + // if (syncOp.type == 'update') { + // chain.push(() => { + // return ItemClass.load(syncOp.item_id).then((item) => { + // if (!item) return; + // item = ItemClass.applyPatch(item, syncOp.item); + // return ItemClass.save(item, { trackChanges: false }); + // }); + // }); + // } + + // if (syncOp.type == 'delete') { + // chain.push(() => { + // return ItemClass.delete(syncOp.item_id, { trackChanges: false }); + // }); + // } + // } + // return promiseChain(chain); + // }).then(() => { + // Log.info('All items synced. has_more = ', hasMore); + // if (maxRevId) { + // Setting.setValue('sync.lastRevId', maxRevId); + // return Setting.saveAll(); + // } + // }).then(() => { + // if (hasMore) { + // this.processState('downloadChanges'); + // } else { + // this.processState('idle'); + // } + // }).catch((error) => { + // Log.warn('Sync error', error); + // }); } processState(state) { diff --git a/structure.sql b/structure.sql index 7196a9a440..1d65724198 100755 --- a/structure.sql +++ b/structure.sql @@ -99,4 +99,4 @@ CREATE TABLE `files` ( `is_encrypted` tinyint(1) NOT NULL default '0', `encryption_method` int(11) NOT NULL default '0', PRIMARY KEY (`id`) -) CHARACTER SET=utf8; +) CHARACTER SET=utf8; \ No newline at end of file