From 1a1e264fa450b02a3e657f8fae6e8060bb51dc19 Mon Sep 17 00:00:00 2001 From: Laurent Cozic Date: Sun, 21 Jan 2018 19:45:32 +0000 Subject: [PATCH] All: Refactored so that memory and file sync target use same delta logic --- .../lib/file-api-driver-local.js | 112 ++---------------- .../lib/file-api-driver-memory.js | 50 ++------ ReactNativeClient/lib/file-api.js | 112 +++++++++++++++++- ReactNativeClient/lib/synchronizer.js | 2 +- 4 files changed, 129 insertions(+), 147 deletions(-) diff --git a/ReactNativeClient/lib/file-api-driver-local.js b/ReactNativeClient/lib/file-api-driver-local.js index 3f6ea8430a..104d35b852 100644 --- a/ReactNativeClient/lib/file-api-driver-local.js +++ b/ReactNativeClient/lib/file-api-driver-local.js @@ -1,5 +1,5 @@ -const BaseItem = require('lib/models/BaseItem.js'); const { time } = require('lib/time-utils.js'); +const { basicDelta } = require('lib/file-api'); // NOTE: when synchronising with the file system the time resolution is the second (unlike milliseconds for OneDrive for instance). // What it means is that if, for example, client 1 changes a note at time t, and client 2 changes the same note within the same second, @@ -67,113 +67,15 @@ class FileApiDriverLocal { } } - contextFromOptions_(options) { - let output = { - timestamp: 0, - filesAtTimestamp: [], - statsCache: null, + async delta(path, options) { + const getStatFn = async (path) => { + const stats = await this.fsDriver().readDirStats(path); + return this.metadataFromStats_(stats); }; - if (!options || !options.context) return output; - const d = new Date(options.context.timestamp); - - output.timestamp = isNaN(d.getTime()) ? 0 : options.context.timestamp; - output.filesAtTimestamp = Array.isArray(options.context.filesAtTimestamp) ? options.context.filesAtTimestamp.slice() : []; - output.statsCache = options.context && options.context.statsCache ? options.context.statsCache : null; - - return output; - } - - async delta(path, options) { - const outputLimit = 1000; - const itemIds = await options.allItemIdsHandler(); - try { - const context = this.contextFromOptions_(options); - - let newContext = { - timestamp: context.timestamp, - filesAtTimestamp: context.filesAtTimestamp.slice(), - statsCache: context.statsCache, - }; - - // Stats are cached until all items have been processed (until hasMore is false) - if (newContext.statsCache === null) { - const stats = await this.fsDriver().readDirStats(path); - newContext.statsCache = this.metadataFromStats_(stats); - newContext.statsCache.sort(function(a, b) { - return a.updated_time - b.updated_time; - }); - } - - let output = []; - - // Find out which files have been changed since the last time. Note that we keep - // both the timestamp of the most recent change, *and* the items that exactly match - // this timestamp. This to handle cases where an item is modified while this delta - // function is running. For example: - // t0: Item 1 is changed - // t0: Sync items - run delta function - // t0: While delta() is running, modify Item 2 - // Since item 2 was modified within the same millisecond, it would be skipped in the - // next sync if we relied exclusively on a timestamp. - for (let i = 0; i < newContext.statsCache.length; i++) { - const stat = newContext.statsCache[i]; - - if (stat.isDir) continue; - - if (stat.updated_time < context.timestamp) continue; - - // Special case for items that exactly match the timestamp - if (stat.updated_time === context.timestamp) { - if (context.filesAtTimestamp.indexOf(stat.path) >= 0) continue; - } - - if (stat.updated_time > newContext.timestamp) { - newContext.timestamp = stat.updated_time; - newContext.filesAtTimestamp = []; - } - - newContext.filesAtTimestamp.push(stat.path); - output.push(stat); - - if (output.length >= outputLimit) break; - } - - if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided'); - - let deletedItems = []; - for (let i = 0; i < itemIds.length; i++) { - if (output.length + deletedItems.length >= outputLimit) break; - - const itemId = itemIds[i]; - let found = false; - for (let j = 0; j < newContext.statsCache.length; j++) { - const item = newContext.statsCache[j]; - if (BaseItem.pathToId(item.path) == itemId) { - found = true; - break; - } - } - - if (!found) { - deletedItems.push({ - path: BaseItem.systemPath(itemId), - isDeleted: true, - }); - } - } - - output = output.concat(deletedItems); - - const hasMore = output.length >= outputLimit; - if (!hasMore) newContext.statsCache = null; - - return { - hasMore: hasMore, - context: newContext, - items: output, - }; + const output = await basicDelta(path, getStatFn, options); + return output; } catch(error) { throw this.fsErrorToJsError_(error, path); } diff --git a/ReactNativeClient/lib/file-api-driver-memory.js b/ReactNativeClient/lib/file-api-driver-memory.js index 317796ec2c..29ac3fdb39 100644 --- a/ReactNativeClient/lib/file-api-driver-memory.js +++ b/ReactNativeClient/lib/file-api-driver-memory.js @@ -1,5 +1,6 @@ const { time } = require('lib/time-utils.js'); const fs = require('fs-extra'); +const { basicDelta } = require('lib/file-api'); class FileApiDriverMemory { @@ -144,48 +145,17 @@ class FileApiDriverMemory { } async delta(path, options = null) { - let limit = 3; - - let output = { - hasMore: false, - context: {}, - items: [], + const getStatFn = async (path) => { + let output = this.items_.slice(); + for (let i = 0; i < output.length; i++) { + const item = Object.assign({}, output[i]); + item.path = item.path.substr(path.length + 1); + output[i] = item; + } + return output; }; - let context = options ? options.context : null; - let fromTime = 0; - - if (context) fromTime = context.fromTime; - - let sortedItems = this.items_.slice().concat(this.deletedItems_); - sortedItems.sort((a, b) => { - if (a.updated_time < b.updated_time) return -1; - if (a.updated_time > b.updated_time) return +1; - return 0; - }); - - let hasMore = false; - let items = []; - let maxTime = 0; - for (let i = 0; i < sortedItems.length; i++) { - let item = sortedItems[i]; - if (item.updated_time >= fromTime) { - item = Object.assign({}, item); - item.path = item.path.substr(path.length + 1); - items.push(item); - if (item.updated_time > maxTime) maxTime = item.updated_time; - } - - if (items.length >= limit) { - hasMore = true; - break; - } - } - - output.items = items; - output.hasMore = hasMore; - output.context = { fromTime: maxTime }; - + const output = await basicDelta(path, getStatFn, options); return output; } diff --git a/ReactNativeClient/lib/file-api.js b/ReactNativeClient/lib/file-api.js index ebb3876861..4c25d33aff 100644 --- a/ReactNativeClient/lib/file-api.js +++ b/ReactNativeClient/lib/file-api.js @@ -1,6 +1,7 @@ const { isHidden } = require('lib/path-utils.js'); const { Logger } = require('lib/logger.js'); const { shim } = require('lib/shim'); +const BaseItem = require('lib/models/BaseItem.js'); const JoplinError = require('lib/JoplinError'); class FileApi { @@ -120,4 +121,113 @@ class FileApi { } -module.exports = { FileApi }; \ No newline at end of file +function basicDeltaContextFromOptions_(options) { + let output = { + timestamp: 0, + filesAtTimestamp: [], + statsCache: null, + }; + + if (!options || !options.context) return output; + const d = new Date(options.context.timestamp); + + output.timestamp = isNaN(d.getTime()) ? 0 : options.context.timestamp; + output.filesAtTimestamp = Array.isArray(options.context.filesAtTimestamp) ? options.context.filesAtTimestamp.slice() : []; + output.statsCache = options.context && options.context.statsCache ? options.context.statsCache : null; + + return output; +} + +// This is the basic delta algorithm, which can be used in case the cloud service does not have +// a built-on delta API. OneDrive and Dropbox have one for example, but Nextcloud and obviously +// the file system do not. +async function basicDelta(path, getStatFn, options) { + const outputLimit = 1000; + const itemIds = await options.allItemIdsHandler(); + if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided'); + + const context = basicDeltaContextFromOptions_(options); + + let newContext = { + timestamp: context.timestamp, + filesAtTimestamp: context.filesAtTimestamp.slice(), + statsCache: context.statsCache, + }; + + // Stats are cached until all items have been processed (until hasMore is false) + if (newContext.statsCache === null) { + newContext.statsCache = await getStatFn(path); + newContext.statsCache.sort(function(a, b) { + return a.updated_time - b.updated_time; + }); + } + + let output = []; + + // Find out which files have been changed since the last time. Note that we keep + // both the timestamp of the most recent change, *and* the items that exactly match + // this timestamp. This to handle cases where an item is modified while this delta + // function is running. For example: + // t0: Item 1 is changed + // t0: Sync items - run delta function + // t0: While delta() is running, modify Item 2 + // Since item 2 was modified within the same millisecond, it would be skipped in the + // next sync if we relied exclusively on a timestamp. + for (let i = 0; i < newContext.statsCache.length; i++) { + const stat = newContext.statsCache[i]; + + if (stat.isDir) continue; + + if (stat.updated_time < context.timestamp) continue; + + // Special case for items that exactly match the timestamp + if (stat.updated_time === context.timestamp) { + if (context.filesAtTimestamp.indexOf(stat.path) >= 0) continue; + } + + if (stat.updated_time > newContext.timestamp) { + newContext.timestamp = stat.updated_time; + newContext.filesAtTimestamp = []; + } + + newContext.filesAtTimestamp.push(stat.path); + output.push(stat); + + if (output.length >= outputLimit) break; + } + + let deletedItems = []; + for (let i = 0; i < itemIds.length; i++) { + if (output.length + deletedItems.length >= outputLimit) break; + + const itemId = itemIds[i]; + let found = false; + for (let j = 0; j < newContext.statsCache.length; j++) { + const item = newContext.statsCache[j]; + if (BaseItem.pathToId(item.path) == itemId) { + found = true; + break; + } + } + + if (!found) { + deletedItems.push({ + path: BaseItem.systemPath(itemId), + isDeleted: true, + }); + } + } + + output = output.concat(deletedItems); + + const hasMore = output.length >= outputLimit; + if (!hasMore) newContext.statsCache = null; + + return { + hasMore: hasMore, + context: newContext, + items: output, + }; +} + +module.exports = { FileApi, basicDelta }; \ No newline at end of file diff --git a/ReactNativeClient/lib/synchronizer.js b/ReactNativeClient/lib/synchronizer.js index deedd19bf8..2ea5538c1a 100644 --- a/ReactNativeClient/lib/synchronizer.js +++ b/ReactNativeClient/lib/synchronizer.js @@ -501,7 +501,7 @@ class Synchronizer { if (action == 'createLocal' || action == 'updateLocal') { if (content === null) { - this.logger().warn('Remote has been deleted between now and the list() call? In that case it will be handled during the next sync: ' + path); + this.logger().warn('Remote has been deleted between now and the delta() call? In that case it will be handled during the next sync: ' + path); continue; } content = ItemClass.filter(content);