Tools: Improve and simplify how to build the apps (#2538)

* Improving CLI build

* Improving CLI build

* Remove requirement to build the tools

* Moved Electron app one level down

* Clean up Electron build

* Moved tools to sub-dir

* Updated root script

* update root

* update root

* update root

* update root

* update root

* update root

* Updated build

* Added doc

* Update CI config

* Should not lint index.js

* Fixing jetify

* Fixed linter errors

* Fixed pod build

* Fixed Windows build

.eslintignore

@@ -6,6 +6,7 @@ _releases/
 Assets/
 CliClient/build
 CliClient/locales
+CliClient/locales-build
 CliClient/node_modules
 CliClient/tests-build
 CliClient/tests/enex_to_md
@@ -22,12 +23,12 @@ Clipper/joplin-webclipper/icons
 Clipper/joplin-webclipper/popup/build
 Clipper/joplin-webclipper/popup/node_modules
 docs/
-ElectronClient/app/dist
-ElectronClient/app/lib
-ElectronClient/app/lib/vendor/sjcl-rn.js
-ElectronClient/app/lib/vendor/sjcl.js
-ElectronClient/app/locales
-ElectronClient/app/node_modules
+ElectronClient/dist
+ElectronClient/lib
+ElectronClient/lib/vendor/sjcl-rn.js
+ElectronClient/lib/vendor/sjcl.js
+ElectronClient/locales
+ElectronClient/node_modules
 highlight.pack.js
 node_modules/
 ReactNativeClient/android
@@ -45,17 +46,22 @@ Server/docs/
 Server/dist/
 Server/bin/
 Server/node_modules/
-ElectronClient/app/packageInfo.js
+ElectronClient/packageInfo.js
 ReactNativeClient/pluginAssets/
 ReactNativeClient/lib/joplin-renderer/vendor/fountain.min.js
 ReactNativeClient/lib/joplin-renderer/assets/
+ReactNativeClient/lib/rnInjectedJs/
 
 # AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
-ElectronClient/app/gui/ShareNoteDialog.js
-ReactNativeClient/PluginAssetsLoader.js
-ReactNativeClient/lib/JoplinServerApi.js
+CliClient/tests-build/lib/JoplinServerApi.js
+ElectronClient/app/lib/joplin-renderer/MdToHtml/rules/mermaid.js
+ElectronClient/app/lib/joplin-renderer/MdToHtml/rules/sanitize_html.js
+ElectronClient/app/lib/JoplinServerApi.js
+ElectronClient/gui/ResourceScreen.js
+ElectronClient/gui/ShareNoteDialog.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/mermaid.js
-ElectronClient/app/gui/ResourceScreen.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/sanitize_html.js
+ReactNativeClient/lib/JoplinServerApi.js
+ReactNativeClient/PluginAssetsLoader.js
 ReactNativeClient/setUpQuickActions.js
 # AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD

.gitignore

@@ -40,18 +40,22 @@ Tools/github_oauth_token.txt
 _releases
 ReactNativeClient/lib/csstojs/
 ReactNativeClient/lib/rnInjectedJs/
-ElectronClient/app/gui/note-viewer/fonts/
-ElectronClient/app/gui/note-viewer/lib.js
+ElectronClient/gui/note-viewer/fonts/
+ElectronClient/gui/note-viewer/lib.js
 Tools/commit_hook.txt
 .vscode/*
 *.map
 
 # AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
-ElectronClient/app/gui/ShareNoteDialog.js
-ReactNativeClient/PluginAssetsLoader.js
-ReactNativeClient/lib/JoplinServerApi.js
+CliClient/tests-build/lib/JoplinServerApi.js
+ElectronClient/app/lib/joplin-renderer/MdToHtml/rules/mermaid.js
+ElectronClient/app/lib/joplin-renderer/MdToHtml/rules/sanitize_html.js
+ElectronClient/app/lib/JoplinServerApi.js
+ElectronClient/gui/ResourceScreen.js
+ElectronClient/gui/ShareNoteDialog.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/mermaid.js
-ElectronClient/app/gui/ResourceScreen.js
 ReactNativeClient/lib/joplin-renderer/MdToHtml/rules/sanitize_html.js
+ReactNativeClient/lib/JoplinServerApi.js
+ReactNativeClient/PluginAssetsLoader.js
 ReactNativeClient/setUpQuickActions.js
 # AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD

.travis.yml

@@ -55,12 +55,8 @@ before_install:
 
 script:
   - |
-    # Copy lib
-    rsync -aP --delete ReactNativeClient/lib/ ElectronClient/app/lib/
-
     # Install tools
     npm install
-    npm run tsc
     cd Tools
     npm install
     cd ..
@@ -70,7 +66,6 @@ script:
     # and that would break the desktop release.
     if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
       cd CliClient
-      npm install
       ./run_test.sh
       testResult=$?
       if [ $testResult -ne 0 ]; then
@@ -113,5 +108,5 @@ script:
     fi
 
     # Prepare the Electron app and build it
-    cd ElectronClient/app
-    npm install && USE_HARD_LINKS=false yarn dist
+    cd ElectronClient
+    USE_HARD_LINKS=false yarn dist

BUILD.md

@@ -2,70 +2,71 @@
 
 # General information
 
-- All the applications share the same library, which, for historical reasons, is in ReactNativeClient/lib. This library is copied to the relevant directories when building each app.
+- All the applications share the same library, which, for historical reasons, is in `ReactNativeClient/lib`. This library is copied to the relevant directories when building each app.
 - In general, most of the backend (anything to do with the database, synchronisation, data import or export, etc.) is shared across all the apps, so when making a change please consider how it will affect all the apps.
 
-# TypeScript
+## TypeScript
 
-Most of the application is written in JavaScript, however new classes and files should generally be written in [TypeScript](https://www.typescriptlang.org/). Even if you don't write TypeScript code, you will need to build the existing .ts and .tsx files. This is done from the root of the project, by running `npm run tsc`.
+Most of the application is written in JavaScript, however new classes and files should generally be written in [TypeScript](https://www.typescriptlang.org/). Even if you don't write TypeScript code, you will need to build the existing .ts and .tsx files. This is done from the root of the project, by running `npm run build`.
 
-If you are modifying TypeScript code, the best is to have the compiler watch for changes from a terminal. To do so, run `npm run tsc-watch`.
+If you are modifying TypeScript code, the best is to have the compiler watch for changes from a terminal. To do so, run `npm run watch`.
 
 All TypeScript files are generated next to the .ts or .tsx file. So for example, if there's a file "lib/MyClass.ts", there will be a generated "lib/MyClass.js" next to it. If you create a new TypeScript file, make sure you add the generated .js file to .gitignore. It is implemented that way as it requires minimal changes to integrate TypeScript in the existing JavaScript code base.
 
-## macOS dependencies
-
-	brew install yarn node
-	echo 'export PATH="/usr/local/opt/gettext/bin:$PATH"' >> ~/.bash_profile
-	source ~/.bash_profile
-
-## Linux and Windows (WSL) dependencies
+## Required dependencies
 
 - Install yarn - https://yarnpkg.com/lang/en/docs/install/
 - Install node v10.x (check with `node --version`) - https://nodejs.org/en/
-- If you get a node-gyp related error you might need to manually install it: `npm install -g node-gyp`
 
-# Building the tools
-
-Before building any of the applications, you need to build the tools and pre-commit hooks:
+# Building the desktop application
 
 ```
-npm install && cd Tools && npm install && cd ..
-```
-
-# Building the Electron application
-
-## Linux and macOS
-
-```
-npm run copyLib
-npm run tsc
-cd ElectronClient/app
 npm install
-yarn dist
+cd ElectronClient
+npm run start
 ```
 
+# Building the Terminal application
+
+```
+npm install
+cd CliClient
+npm run start
+```
+
+# Building the Mobile application
+
+First you need to setup React Native to build projects with native code. For this, follow the instructions on the [Get Started](https://facebook.github.io/react-native/docs/getting-started.html) tutorial, in the "React Native CLI Quickstart" tab.
+
+Then:
+
+```
+npm install
+cd ReactNativeClient
+react-native run-ios
+# Or: react-native run-android
+```
+
+To run the iOS application, it might be easier to open the file `ios/Joplin.xcworkspace` on XCode and the app from there.
+
+On iOS you might also need to install the Podfiles:
+
+```
+cd ReactNativeClient/ios
+pod install
+```
+
+# Troubleshooting desktop application
+
+## On Linux and macOS
+
 If there's an error `while loading shared libraries: libgconf-2.so.4: cannot open shared object file: No such file or directory`, run `sudo apt-get install libgconf-2-4`
 
 If you get a node-gyp related error you might need to manually install it: `npm install -g node-gyp`.
 
 If you get the error `libtool: unrecognized option '-static'`, follow the instructions [in this post](https://stackoverflow.com/a/38552393/561309) to use the correct libtool version.
 
-That will create the executable file in the `dist` directory.
-
-From `/ElectronClient` you can also run `run.sh` to run the app for testing.
-
-## Windows
-
-Run the following commands on Windows Command prompt running as Administrator:
-
-```
-xcopy /C /I /H /R /Y /S ReactNativeClient\lib ElectronClient\app\lib
-npm run tsc
-cd ElectronClient\app
-npm install
-yarn dist
-```
+## On Windows
 
 If node-gyp does not works (MSBUILD: error MSB3428: Could not load the Visual C++ component "VCBuild.exe"), you might need to install the `windows-build-tools` using `npm install --global windows-build-tools`.
 
@@ -75,7 +76,7 @@ If you get an `error MSB8020: The build tools for v140 cannot be found.` try to
 
 The [building\_win32\_tips on this page](./readme/building_win32_tips.md) might be helpful.
 
-## Troubleshooting desktop application
+## Other issues
 
 > The application window doesn't open or is white
 
@@ -91,27 +92,3 @@ This is an indication that there's an early initialisation error. Try this:
 You should not use WSL at all because this is a GUI app that lives outside of WSL, and the WSL layer can cause all kind of very hard to debug issues. It can also lock files in node_modules that cannot be unlocked when the app crashes (you need to restart your computer). Likewise, don't run the TypeScript watch command from WSL.
 
 So everything should be done from a Windows Command prompt running as Administrator. You can use `run.bat` to run the app in dev mode.
-
-# Building the Mobile application
-
-First you need to setup React Native to build projects with native code. For this, follow the instructions on the [Get Started](https://facebook.github.io/react-native/docs/getting-started.html) tutorial, in the "React Native CLI Quickstart" tab.
-
-Then:
-
-```
-npm run tsc
-cd ReactNativeClient
-npm install
-react-native run-ios
-# Or: react-native run-android
-```
-
-# Building the Terminal application
-
-```
-cd CliClient
-npm install
-./build.sh
-```
-
-Run `run.sh` to start the application for testing.

CliClient/.gitignore

@@ -1,5 +1,3 @@
-build/*
-!build/locales
 node_modules/
 app/src
 tests-build/
@@ -21,4 +19,5 @@ tests/sync
 out.txt
 linkToLocal.sh
 yarn-error.log
-tests/support/dropbox-auth.txt
+tests/support/dropbox-auth.txt
+build/

CliClient/build.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-BUILD_DIR="$ROOT_DIR/build"
-
-rsync -a --exclude "node_modules/" "$ROOT_DIR/app/" "$BUILD_DIR/"
-rsync -a --delete "$ROOT_DIR/../ReactNativeClient/lib/" "$BUILD_DIR/lib/"
-cp "$ROOT_DIR/package.json" "$BUILD_DIR"
-
-# Don't add TypeScript here or make it silent as output of Cli app must be clean
-# cd $ROOT_DIR/..
-# npm run tsc
-# cd $ROOT_DIR
-
-chmod 755 "$BUILD_DIR/main.js"

CliClient/gulpfile.js

@@ -0,0 +1,22 @@
+const gulp = require('gulp');
+const fs = require('fs-extra');
+const utils = require('../Tools/gulp/utils');
+const tasks = {
+	copyLib: require('../Tools/gulp/tasks/copyLib'),
+};
+
+const buildDir = `${__dirname}/build`;
+
+tasks.build = {
+	fn: async () => {
+		await utils.copyDir(`${__dirname}/app`, buildDir, {
+			excluded: ['node_modules'],
+		});
+		await utils.copyDir(`${__dirname}/locales-build`, `${buildDir}/locales`);
+		await tasks.copyLib.fn();
+		await utils.copyFile(`${__dirname}/package.json`, `${buildDir}/package.json`);
+		fs.chmodSync(`${buildDir}/main.js`, 0o755);
+	},
+};
+
+gulp.task('build', tasks.build.fn);

CliClient/locales-build/index.js

@@ -35,39 +35,39 @@ locales['sv'] = require('./sv.json');
 locales['tr_TR'] = require('./tr_TR.json');
 locales['zh_CN'] = require('./zh_CN.json');
 locales['zh_TW'] = require('./zh_TW.json');
-stats['ar'] = {"percentDone":94};
-stats['eu'] = {"percentDone":40};
-stats['bs_BA'] = {"percentDone":88};
-stats['bg_BG'] = {"percentDone":79};
-stats['ca'] = {"percentDone":62};
-stats['hr_HR'] = {"percentDone":33};
-stats['cs_CZ'] = {"percentDone":97};
-stats['da_DK'] = {"percentDone":87};
-stats['de_DE'] = {"percentDone":98};
-stats['et_EE'] = {"percentDone":78};
-stats['en_GB'] = {"percentDone":100};
-stats['en_US'] = {"percentDone":100};
-stats['es_ES'] = {"percentDone":97};
-stats['eo'] = {"percentDone":45};
-stats['fr_FR'] = {"percentDone":98};
-stats['gl_ES'] = {"percentDone":51};
-stats['it_IT'] = {"percentDone":93};
-stats['nl_NL'] = {"percentDone":89};
-stats['nl_BE'] = {"percentDone":40};
-stats['nb_NO'] = {"percentDone":91};
-stats['fa'] = {"percentDone":39};
-stats['pl_PL'] = {"percentDone":77};
-stats['pt_PT'] = {"percentDone":93};
-stats['pt_BR'] = {"percentDone":90};
-stats['ro'] = {"percentDone":40};
-stats['sl_SI'] = {"percentDone":51};
-stats['sv'] = {"percentDone":69};
-stats['tr_TR'] = {"percentDone":94};
-stats['el_GR'] = {"percentDone":96};
-stats['ru_RU'] = {"percentDone":98};
-stats['sr_RS'] = {"percentDone":77};
-stats['zh_CN'] = {"percentDone":97};
-stats['zh_TW'] = {"percentDone":94};
-stats['ja_JP'] = {"percentDone":98};
-stats['ko'] = {"percentDone":92};
-module.exports = { locales: locales, stats: stats };
+stats['ar'] = { 'percentDone': 94 };
+stats['eu'] = { 'percentDone': 40 };
+stats['bs_BA'] = { 'percentDone': 88 };
+stats['bg_BG'] = { 'percentDone': 79 };
+stats['ca'] = { 'percentDone': 62 };
+stats['hr_HR'] = { 'percentDone': 33 };
+stats['cs_CZ'] = { 'percentDone': 97 };
+stats['da_DK'] = { 'percentDone': 87 };
+stats['de_DE'] = { 'percentDone': 98 };
+stats['et_EE'] = { 'percentDone': 78 };
+stats['en_GB'] = { 'percentDone': 100 };
+stats['en_US'] = { 'percentDone': 100 };
+stats['es_ES'] = { 'percentDone': 97 };
+stats['eo'] = { 'percentDone': 45 };
+stats['fr_FR'] = { 'percentDone': 98 };
+stats['gl_ES'] = { 'percentDone': 51 };
+stats['it_IT'] = { 'percentDone': 93 };
+stats['nl_NL'] = { 'percentDone': 89 };
+stats['nl_BE'] = { 'percentDone': 40 };
+stats['nb_NO'] = { 'percentDone': 91 };
+stats['fa'] = { 'percentDone': 39 };
+stats['pl_PL'] = { 'percentDone': 77 };
+stats['pt_PT'] = { 'percentDone': 93 };
+stats['pt_BR'] = { 'percentDone': 90 };
+stats['ro'] = { 'percentDone': 40 };
+stats['sl_SI'] = { 'percentDone': 51 };
+stats['sv'] = { 'percentDone': 69 };
+stats['tr_TR'] = { 'percentDone': 94 };
+stats['el_GR'] = { 'percentDone': 96 };
+stats['ru_RU'] = { 'percentDone': 98 };
+stats['sr_RS'] = { 'percentDone': 77 };
+stats['zh_CN'] = { 'percentDone': 97 };
+stats['zh_TW'] = { 'percentDone': 94 };
+stats['ja_JP'] = { 'percentDone': 98 };
+stats['ko'] = { 'percentDone': 92 };
+module.exports = { locales: locales, stats: stats };

CliClient/package.json

@@ -4,7 +4,9 @@
   "license": "MIT",
   "author": "Laurent Cozic",
   "scripts": {
-    "postinstall": "patch-package"
+    "postinstall": "patch-package && npm run build",
+    "build": "gulp build",
+    "start": "gulp build -L && node 'build/main.js' --profile ~/Temp/TestNotes2 --stack-trace-enabled --log-level debug --env dev"
   },
   "bugs": {
     "url": "https://github.com/laurent22/joplin/issues"
@@ -109,6 +111,8 @@
     "node-html-parser": "^1.2.4"
   },
   "devDependencies": {
-    "jasmine": "^3.5.0"
+    "gulp": "^4.0.2",
+    "jasmine": "^3.5.0",
+    "temp": "^0.9.1"
   }
 }

CliClient/publish.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-echo "Deprecated! Use `node Tools/release-cli.js`"
-exit 1
-
-set -e
-
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd "$SCRIPT_DIR"
-npm version patch
-touch "$SCRIPT_DIR/app/main.js"
-bash $SCRIPT_DIR/build.sh
-cp "$SCRIPT_DIR/package.json" build/
-cp "$SCRIPT_DIR/../README.md" build/
-cd "$SCRIPT_DIR/build"
-npm publish
-
-NEW_VERSION=$(cat package.json | jq -r .version)
-git add -A
-git commit -m "CLI v$NEW_VERSION"
-git tag "cli-v$NEW_VERSION"
-git push && git push --tags

CliClient/run.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-set -e
-CLIENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-bash "$CLIENT_DIR/build.sh" && node "$CLIENT_DIR/build/main.js" --profile ~/Temp/TestNotes2 --stack-trace-enabled --log-level debug --env dev "$@"
-# bash "$CLIENT_DIR/build.sh" && node "$CLIENT_DIR/build/main.js" --profile ~/.config/joplin --stack-trace-enabled --log-level debug --env dev "$@"

ElectronClient/.gitignore

@@ -1,8 +1,8 @@
-app/node_modules/
-app/packageInfo.js
+node_modules/
+packageInfo.js
 dist/
-app/lib/
-app/gui/*.min.js
-app/plugins/*.min.js
+lib/
+gui/*.min.js
+plugins/*.min.js
 .DS_Store
-app/gui/note-viewer/pluginAssets/
+gui/note-viewer/pluginAssets/

ElectronClient/app/compile-package-info.js

@@ -1,42 +0,0 @@
-const fs = require('fs-extra');
-const execSync = require('child_process').execSync;
-
-// Electron Builder strip off certain important keys from package.json, which we need, in particular build.appId
-// so this script is used to preserve the keys that we need.
-
-const packageInfo = require(`${__dirname}/package.json`);
-
-let removeKeys = ['scripts', 'devDependencies', 'optionalDependencies', 'dependencies'];
-
-for (let i = 0; i < removeKeys.length; i++) {
-	delete packageInfo[removeKeys[i]];
-}
-
-const appId = packageInfo.build.appId;
-
-delete packageInfo.build;
-packageInfo.build = { appId: appId };
-
-let branch;
-let hash;
-try {
-	// Use stdio: 'pipe' so that execSync doesn't print error directly to stdout
-	branch = execSync('git rev-parse --abbrev-ref HEAD', { stdio: 'pipe' }).toString().trim();
-	hash   = execSync('git log --pretty="%h" -1', { stdio: 'pipe' }).toString().trim();
-	// The builds in CI are done from a 'detached HEAD' state
-	if (branch === 'HEAD') branch = 'master';
-} catch (err) {
-	// Don't display error object as it's a "fatal" error, but
-	// not for us, since is it not critical information
-	// https://github.com/laurent22/joplin/issues/2256
-	console.info('Warning: Could not get git info (it will not be displayed in About dialog box)');
-}
-if (typeof branch !== 'undefined' && typeof hash !== 'undefined') {
-	packageInfo.git = { branch: branch, hash: hash };
-}
-
-let fileContent = `// Auto-generated by compile-package-info.js\n// Do not change directly\nconst packageInfo = ${JSON.stringify(packageInfo, null, 4)};`;
-fileContent += '\n';
-fileContent += 'module.exports = packageInfo;';
-
-fs.writeFileSync(`${__dirname}/packageInfo.js`, fileContent);

ElectronClient/build.bat

@@ -1,16 +0,0 @@
-@echo off
-setlocal
-SET mypath=%~dp0
-set script_dir=%mypath:~0,-1%
-
-xcopy /C /I /H /R /Y /S /Q %script_dir%\..\ReactNativeClient\lib %script_dir%\app\lib
-REM if %errorlevel% neq 0 exit /b %errorlevel%
-
-rem Note that TypeScript must be installed globally for this to work
-REM cd %script_dir%\..
-REM call tsc
-REM if %errorlevel% neq 0 exit /b %errorlevel%
-
-cd %script_dir%\app
-call npm run compile
-REM if %errorlevel% neq 0 exit /b %errorlevel%

ElectronClient/build.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-BUILD_DIR="$ROOT_DIR/app"
-
-rsync -a --delete "$ROOT_DIR/../ReactNativeClient/lib/" "$BUILD_DIR/lib/"
-
-# cd "$ROOT_DIR/.."
-# npm run tsc
-
-cd "$BUILD_DIR"
-npm run compile