All: Improved workflow of downloading and decrypting data during sync

ElectronClient/app/gui/MainScreen.jsx

@@ -448,7 +448,7 @@ class MainScreenComponent extends React.Component {
 			if (this.props.hasDisabledSyncItems) {
 				msg = <span>{_('Some items cannot be synchronised.')} <a href="#" onClick={() => { onViewDisabledItemsClick() }}>{_('View them now')}</a></span>
 			} else if (this.props.showMissingMasterKeyMessage) {
-				msg = <span>{_('Some items cannot be decrypted.')} <a href="#" onClick={() => { onViewMasterKeysClick() }}>{_('Set the password')}</a></span>
+				msg = <span>{_('One or more master keys need a password.')} <a href="#" onClick={() => { onViewMasterKeysClick() }}>{_('Set the password')}</a></span>
 			}
 
 			messageComp = (

ReactNativeClient/lib/BaseApplication.js

@@ -59,6 +59,8 @@ class BaseApplication {
 		// be derived from the state and not set directly since that would make the
 		// state and UI out of sync.
 		this.currentFolder_ = null;
+
+		this.decryptionWorker_resourceMetadataButNotBlobDecrypted = this.decryptionWorker_resourceMetadataButNotBlobDecrypted.bind(this);
 	}
 
 	logger() {
@@ -285,6 +287,19 @@ class BaseApplication {
 		}
 	}
 
+	async decryptionWorker_resourceMetadataButNotBlobDecrypted(event) {
+		this.scheduleAutoAddResources();
+	}
+
+	scheduleAutoAddResources() {
+		if (this.scheduleAutoAddResourcesIID_) return;
+
+		this.scheduleAutoAddResourcesIID_ = setTimeout(() => {
+			this.scheduleAutoAddResourcesIID_ = null;
+			ResourceFetcher.instance().autoAddResources();
+		}, 1000);
+	}
+
 	reducerActionToString(action) {
 		let o = [action.type];
 		if ('id' in action) o.push(action.id);
@@ -314,7 +329,7 @@ class BaseApplication {
 	}
 
 	async generalMiddleware(store, next, action) {
-		this.logger().debug('Reducer action', this.reducerActionToString(action));
+		// this.logger().debug('Reducer action', this.reducerActionToString(action));
 
 		const result = next(action);
 		const newState = store.getState();
@@ -608,6 +623,7 @@ class BaseApplication {
 		DecryptionWorker.instance().setLogger(this.logger_);
 		DecryptionWorker.instance().setEncryptionService(EncryptionService.instance());
 		await EncryptionService.instance().loadMasterKeysFromSettings();
+		DecryptionWorker.instance().on('resourceMetadataButNotBlobDecrypted', this.decryptionWorker_resourceMetadataButNotBlobDecrypted);
 
 		ResourceFetcher.instance().setFileApi(() => { return reg.syncTarget().fileApi() });
 		ResourceFetcher.instance().setLogger(this.logger_);

ReactNativeClient/lib/reducer.js

@@ -535,6 +535,12 @@ const reducer = (state = defaultState, action) => {
 				newState.masterKeys = action.items;
 				break;
 
+			case 'MASTERKEY_SET_NOT_LOADED':
+
+				newState = Object.assign({}, state);
+				newState.notLoadedMasterKeys = action.ids;
+				break;
+
 			case 'MASTERKEY_ADD_NOT_LOADED':
 
 				if (state.notLoadedMasterKeys.indexOf(action.id) < 0) {

ReactNativeClient/lib/services/DecryptionWorker.js

@@ -1,4 +1,5 @@
 const BaseItem = require('lib/models/BaseItem');
+const MasterKey = require('lib/models/MasterKey');
 const Resource = require('lib/models/Resource');
 const ResourceService = require('lib/services/ResourceService');
 const { Logger } = require('lib/logger.js');
@@ -75,6 +76,20 @@ class DecryptionWorker {
 			return;
 		}
 
+		const loadedMasterKeyCount = await this.encryptionService().loadedMasterKeysCount();
+		if (!loadedMasterKeyCount) {
+			this.logger().info('DecryptionWorker: cannot start because no master key is currently loaded.');
+			const ids = await MasterKey.allIds();
+
+			if (ids.length) {
+				this.dispatch({
+					type: 'MASTERKEY_SET_NOT_LOADED',
+					ids: ids,
+				});
+			}
+			return;
+		}
+
 		this.logger().info('DecryptionWorker: starting decryption...');
 
 		this.state_ = 'started';
@@ -101,7 +116,7 @@ class DecryptionWorker {
 					});
 					
 					// Don't log in production as it results in many messages when importing many items
-					// this.logger().info('DecryptionWorker: decrypting: ' + item.id + ' (' + ItemClass.tableName() + ')');
+					// this.logger().debug('DecryptionWorker: decrypting: ' + item.id + ' (' + ItemClass.tableName() + ')');
 					try {
 						const decryptedItem = await ItemClass.decrypt(item);
 
@@ -115,6 +130,10 @@ class DecryptionWorker {
 						if (decryptedItem.type_ === Resource.modelType() && !decryptedItem.encryption_blob_encrypted) {
 							this.eventEmitter_.emit('resourceDecrypted', { id: decryptedItem.id });
 						}
+
+						if (decryptedItem.type_ === Resource.modelType() && !decryptedItem.encryption_applied && !!decryptedItem.encryption_blob_encrypted) {
+							this.eventEmitter_.emit('resourceMetadataButNotBlobDecrypted', { id: decryptedItem.id });
+						}
 					} catch (error) {
 						excludedIds.push(item.id);
 
@@ -154,10 +173,10 @@ class DecryptionWorker {
 
 		const downloadedButEncryptedBlobCount = await Resource.downloadedButEncryptedBlobCount();
 
-		this.dispatchReport({ state: 'idle' });
-
 		this.state_ = 'idle';
 
+		this.dispatchReport({ state: 'idle' });
+
 		if (downloadedButEncryptedBlobCount) {
 			this.logger().info('DecryptionWorker: Some resources have been downloaded but are not decrypted yet. Scheduling another decryption. Resource count: ' + downloadedButEncryptedBlobCount);
 			this.scheduleStart();

ReactNativeClient/lib/services/ResourceFetcher.js

@@ -120,6 +120,7 @@ class ResourceFetcher extends BaseService {
 			// sync. The other ones have been done using migrations/20.js. This code can be removed
 			// after a few months.
 			if (resource && resource.size < 0 && localResourceContentPath && !resource.encryption_blob_encrypted) {
+				await shim.fsDriver().waitTillExists(localResourceContentPath);
 				await ResourceService.autoSetFileSizes();
 			}
 
@@ -194,7 +195,9 @@ class ResourceFetcher extends BaseService {
 		});
 	}
 
-	async autoAddResources(limit) {
+	async autoAddResources(limit = null) {
+		if (limit === null) limit = 10;
+
 		if (this.addingResources_) return;
 		this.addingResources_ = true;
 

ReactNativeClient/lib/services/ResourceService.js

@@ -111,8 +111,8 @@ class ResourceService extends BaseService {
 		}
 	}
 
-	static async autoSetFileSize(resourceId, filePath) {
+	static async autoSetFileSize(resourceId, filePath, waitTillExists = true) {
-		const itDoes = await shim.fsDriver().waitTillExists(filePath);
+		const itDoes = await shim.fsDriver().waitTillExists(filePath, waitTillExists ? 10000 : 0);
 		if (!itDoes) {
 			// this.logger().warn('Trying to set file size on non-existent resource:', resourceId, filePath);
 			return;
@@ -125,7 +125,7 @@ class ResourceService extends BaseService {
 		const resources = await Resource.needFileSizeSet();
 
 		for (const r of resources) {
-			await this.autoSetFileSize(r.id, Resource.fullPath(r));
+			await this.autoSetFileSize(r.id, Resource.fullPath(r), false);
 		}
 	}
 

ReactNativeClient/root.js

@@ -86,7 +86,7 @@ const logReducerAction = function(action) {
 	let msg = [action.type];
 	if (action.routeName) msg.push(action.routeName);
 
-	reg.logger().info('Reducer action', msg.join(', '));
+	// reg.logger().debug('Reducer action', msg.join(', '));
 }
 
 const generalMiddleware = store => next => async (action) => {