Joplin
/
joplin
mirror of https://github.com/laurent22/joplin.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into release-2.4

pull/5488/head
Laurent Cozic 2021-09-19 13:04:36 +01:00
parent 2e04656b54 e447acc076
commit 4244f712e1
63 changed files with 1096 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.eslintignore
View File

@ -76,6 +76,9 @@ packages/app-cli/app/command-e2ee.js.map
packages/app-cli/app/command-settingschema.d.ts
packages/app-cli/app/command-settingschema.js
packages/app-cli/app/command-settingschema.js.map
packages/app-cli/app/command-testing.d.ts
packages/app-cli/app/command-testing.js
packages/app-cli/app/command-testing.js.map
packages/app-cli/app/services/plugins/PluginRunner.d.ts
packages/app-cli/app/services/plugins/PluginRunner.js
packages/app-cli/app/services/plugins/PluginRunner.js.map
@ -109,6 +112,9 @@ packages/app-cli/tests/services/plugins/sandboxProxy.js.map
packages/app-cli/tests/testUtils.d.ts
packages/app-cli/tests/testUtils.js
packages/app-cli/tests/testUtils.js.map
packages/app-cli/tools/populateDatabase.d.ts
packages/app-cli/tools/populateDatabase.js
packages/app-cli/tools/populateDatabase.js.map
packages/app-desktop/ElectronAppWrapper.d.ts
packages/app-desktop/ElectronAppWrapper.js
packages/app-desktop/ElectronAppWrapper.js.map

6
.gitignore vendored
View File

@ -61,6 +61,9 @@ packages/app-cli/app/command-e2ee.js.map
packages/app-cli/app/command-settingschema.d.ts
packages/app-cli/app/command-settingschema.js
packages/app-cli/app/command-settingschema.js.map
packages/app-cli/app/command-testing.d.ts
packages/app-cli/app/command-testing.js
packages/app-cli/app/command-testing.js.map
packages/app-cli/app/services/plugins/PluginRunner.d.ts
packages/app-cli/app/services/plugins/PluginRunner.js
packages/app-cli/app/services/plugins/PluginRunner.js.map
@ -94,6 +97,9 @@ packages/app-cli/tests/services/plugins/sandboxProxy.js.map
packages/app-cli/tests/testUtils.d.ts
packages/app-cli/tests/testUtils.js
packages/app-cli/tests/testUtils.js.map
packages/app-cli/tools/populateDatabase.d.ts
packages/app-cli/tools/populateDatabase.js
packages/app-cli/tools/populateDatabase.js.map
packages/app-desktop/ElectronAppWrapper.d.ts
packages/app-desktop/ElectronAppWrapper.js
packages/app-desktop/ElectronAppWrapper.js.map

12
Joplin_install_and_update.sh
View File

@ -137,8 +137,8 @@ fi
#-----------------------------------------------------
print 'Downloading Joplin...'
TEMP_DIR=$(mktemp -d)
wget -O ${TEMP_DIR}/Joplin.AppImage https://github.com/laurent22/joplin/releases/download/v${RELEASE_VERSION}/Joplin-${RELEASE_VERSION}.AppImage
wget -O ${TEMP_DIR}/joplin.png https://joplinapp.org/images/Icon512.png
wget -O "${TEMP_DIR}/Joplin.AppImage" "https://github.com/laurent22/joplin/releases/download/v${RELEASE_VERSION}/Joplin-${RELEASE_VERSION}.AppImage"
wget -O "${TEMP_DIR}/joplin.png" https://joplinapp.org/images/Icon512.png
#-----------------------------------------------------
print 'Installing Joplin...'
@ -149,7 +149,7 @@ rm -f ~/.joplin/*.AppImage ~/.local/share/applications/joplin.desktop ~/.joplin/
mkdir -p ~/.joplin/
# Download the latest version
mv ${TEMP_DIR}/Joplin.AppImage ~/.joplin/Joplin.AppImage
mv "${TEMP_DIR}/Joplin.AppImage" ~/.joplin/Joplin.AppImage
# Gives execution privileges
chmod +x ~/.joplin/Joplin.AppImage
@ -159,7 +159,7 @@ print "${COLOR_GREEN}OK${COLOR_RESET}"
#-----------------------------------------------------
print 'Installing icon...'
mkdir -p ~/.local/share/icons/hicolor/512x512/apps
mv ${TEMP_DIR}/joplin.png ~/.local/share/icons/hicolor/512x512/apps/joplin.png
mv "${TEMP_DIR}/joplin.png" ~/.local/share/icons/hicolor/512x512/apps/joplin.png
print "${COLOR_GREEN}OK${COLOR_RESET}"
# Detect desktop environment
@ -222,7 +222,7 @@ fi
print "${COLOR_GREEN}Joplin version${COLOR_RESET} ${RELEASE_VERSION} ${COLOR_GREEN}installed.${COLOR_RESET}"
# Record version
echo $RELEASE_VERSION > ~/.joplin/VERSION
echo "$RELEASE_VERSION" > ~/.joplin/VERSION
#-----------------------------------------------------
if [[ "$SHOW_CHANGELOG" == true ]]; then
@ -232,5 +232,5 @@ fi
#-----------------------------------------------------
print "Cleaning up..."
rm -rf $TEMP_DIR
rm -rf "$TEMP_DIR"
print "${COLOR_GREEN}OK${COLOR_RESET}"

4
packages/app-cli/.gitignore vendored
View File

@ -23,4 +23,6 @@ tests/support/dropbox-auth.txt
tests/support/nextcloud-auth.json
tests/support/onedrive-auth.txt
build/
patches/
patches/
createUsers-*.txt
tools/temp/

2
packages/app-cli/app/cli-utils.js
View File

@ -89,7 +89,7 @@ cliUtils.makeCommandArgs = function(cmd, argv) {
flags = cliUtils.parseFlags(flags);
if (!flags.arg) {
booleanFlags.push(flags.short);
if (flags.short) booleanFlags.push(flags.short);
if (flags.long) booleanFlags.push(flags.long);
}

95
packages/app-cli/app/command-testing.ts Normal file
View File

@ -0,0 +1,95 @@
const { BaseCommand } = require('./base-command.js');
import { reg } from '@joplin/lib/registry';
import Note from '@joplin/lib/models/Note';
import uuid from '@joplin/lib/uuid';
import populateDatabase from '@joplin/lib/services/debug/populateDatabase';
function randomElement(array: any[]): any {
if (!array.length) return null;
return array[Math.floor(Math.random() * array.length)];
}
function itemCount(args: any) {
const count = Number(args.arg0);
if (!count || isNaN(count)) throw new Error('Note count must be specified');
return count;
}
class Command extends BaseCommand {
usage() {
return 'testing <command> [arg0]';
}
description() {
return 'testing';
}
enabled() {
return false;
}
options(): any[] {
return [
['--folder-count <count>', 'Folders to create'],
['--note-count <count>', 'Notes to create'],
['--tag-count <count>', 'Tags to create'],
['--tags-per-note <count>', 'Tags per note'],
['--silent', 'Silent'],
];
}
async action(args: any) {
const { command, options } = args;
if (command === 'populate') {
await populateDatabase(reg.db(), {
folderCount: options['folder-count'],
noteCount: options['note-count'],
tagCount: options['tag-count'],
tagsPerNote: options['tags-per-note'],
silent: options['silent'],
});
}
const promises: any[] = [];
if (command === 'createRandomNotes') {
const noteCount = itemCount(args);
for (let i = 0; i < noteCount; i++) {
promises.push(Note.save({
title: `Note ${uuid.createNano()}`,
}));
}
}
if (command === 'updateRandomNotes') {
const noteCount = itemCount(args);
const noteIds = await Note.allIds();
for (let i = 0; i < noteCount; i++) {
const noteId = randomElement(noteIds);
promises.push(Note.save({
id: noteId,
title: `Note ${uuid.createNano()}`,
}));
}
}
if (command === 'deleteRandomNotes') {
const noteCount = itemCount(args);
const noteIds = await Note.allIds();
for (let i = 0; i < noteCount; i++) {
const noteId = randomElement(noteIds);
promises.push(Note.delete(noteId));
}
}
await Promise.all(promises);
}
}
module.exports = Command;

52
packages/app-cli/createUsers.sh Executable file
View File

@ -0,0 +1,52 @@
#!/bin/bash
# Start the server with:
#
# JOPLIN_IS_TESTING=1 npm run start-dev
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
# curl --data '{"action": "clearDatabase"}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
# SMALL
# curl --data '{"action": "createTestUsers", "count": 400, "fromNum": 1}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
NUM=398
while [ "$NUM" -lt 400 ]; do
NUM=$(( NUM + 1 ))
echo "User $NUM"
CMD_FILE="$SCRIPT_DIR/createUsers-$NUM.txt"
PROFILE_DIR=~/.config/joplindev-testing-$NUM
USER_EMAIL="user$NUM@example.com"
rm -rf "$CMD_FILE" "$PROFILE_DIR"
touch "$CMD_FILE"
FLAG_FOLDER_COUNT=100
FLAG_NOTE_COUNT=1000
FLAG_TAG_COUNT=20
if [ "$NUM" -gt 300 ]; then
FLAG_FOLDER_COUNT=2000
FLAG_NOTE_COUNT=10000
FLAG_TAG_COUNT=200
fi
if [ "$NUM" -gt 399 ]; then
FLAG_FOLDER_COUNT=10000
FLAG_NOTE_COUNT=150000
FLAG_TAG_COUNT=2000
fi
echo "testing populate --silent --folder-count $FLAG_FOLDER_COUNT --note-count $FLAG_NOTE_COUNT --tag-count $FLAG_TAG_COUNT" >> "$CMD_FILE"
echo "config keychain.supported 0" >> "$CMD_FILE"
echo "config sync.target 10" >> "$CMD_FILE"
echo "config sync.10.username $USER_EMAIL" >> "$CMD_FILE"
echo "config sync.10.password hunter1hunter2hunter3" >> "$CMD_FILE"
echo "sync" >> "$CMD_FILE"
npm start -- --profile "$PROFILE_DIR" batch "$CMD_FILE"
done

1
packages/app-cli/package.json
View File

@ -10,6 +10,7 @@
"test-ci": "jest --config=jest.config.js --forceExit",
"build": "gulp build",
"start": "gulp build -L && node \"build/main.js\" --stack-trace-enabled --log-level debug --env dev",
"start-no-build": "node \"build/main.js\" --stack-trace-enabled --log-level debug --env dev",
"tsc": "node node_modules/typescript/bin/tsc --project tsconfig.json",
"watch": "node node_modules/typescript/bin/tsc --watch --project tsconfig.json"
},

6
packages/app-cli/tests/HtmlToMd.ts
View File

@ -84,4 +84,10 @@ describe('HtmlToMd', function() {
}
}));
it('should allow disabling escape', async () => {
const htmlToMd = new HtmlToMd();
expect(htmlToMd.parse('https://test.com/1_2_3.pdf', { disableEscapeContent: true })).toBe('https://test.com/1_2_3.pdf');
expect(htmlToMd.parse('https://test.com/1_2_3.pdf', { disableEscapeContent: false })).toBe('https://test.com/1\\_2\\_3.pdf');
});
});

2
packages/app-cli/tests/html_to_md/sub_sup_insert_strikethrough.html
View File

@ -1 +1 @@
X<sub>1</sub> X<sup>1</sup> <ins>Insert</ins> <s>Strike</s>
X<sub>1</sub> X<sup>1</sup> <ins>Insert</ins> <span style="text-decoration: underline;">Insert alt</span> <s>Strike</s>

2
packages/app-cli/tests/html_to_md/sub_sup_insert_strikethrough.md
View File

@ -1 +1 @@
X<sub>1</sub> X<sup>1</sup> <ins>Insert</ins> ~~Strike~~
X<sub>1</sub> X<sup>1</sup> <ins>Insert</ins> <ins>Insert alt</ins> ~~Strike~~

108
packages/app-cli/tools/populateDatabase.ts Normal file
View File

@ -0,0 +1,108 @@
// This script can be used to simulate a running production environment, by
// having multiple users in parallel changing notes and synchronising.
//
// To get it working:
//
// - Run the Postgres database -- `sudo docker-compose --file docker-compose.db-dev.yml up`
// - Update the DB parameters in ~/joplin-credentials/server.env to use the dev
// database
// - Run the server - `JOPLIN_IS_TESTING=1 npm run start-dev`
// - Then run this script - `node populateDatabase.js`
//
// Currently it doesn't actually create the users, so that should be done using:
//
// curl --data '{"action": "createTestUsers", "count": 400, "fromNum": 1}' -H 'Content-Type: application/json' http://api.joplincloud.local:22300/api/debug
//
// That will create n users with email `user<n>@example.com`
import * as fs from 'fs-extra';
import { homedir } from 'os';
import { execCommand2 } from '@joplin/tools/tool-utils';
import { chdir } from 'process';
const minUserNum = 1;
const maxUserNum = 400;
const cliDir = `${__dirname}/..`;
const tempDir = `${__dirname}/temp`;
function randomInt(min: number, max: number) {
min = Math.ceil(min);
max = Math.floor(max);
return Math.floor(Math.random() * (max - min + 1)) + min;
}
const processing_: Record<number, boolean> = {};
const processUser = async (userNum: number) => {
if (processing_[userNum]) {
console.info(`User already being processed: ${userNum} - skipping`);
return;
}
processing_[userNum] = true;
try {
const userEmail = `user${userNum}@example.com`;
const userPassword = 'hunter1hunter2hunter3';
const commandFile = `${tempDir}/populateDatabase-${userNum}.txt`;
const profileDir = `${homedir()}/.config/joplindev-populate/joplindev-testing-${userNum}`;
const commands: string[] = [];
const jackpot = Math.random() >= 0.95 ? 100 : 1;
commands.push(`testing createRandomNotes ${randomInt(1, 500 * jackpot)}`);
commands.push(`testing updateRandomNotes ${randomInt(1, 1500 * jackpot)}`);
commands.push(`testing deleteRandomNotes ${randomInt(1, 200 * jackpot)}`);
commands.push('config keychain.supported 0');
commands.push('config sync.target 10');
commands.push(`config sync.10.username ${userEmail}`);
commands.push(`config sync.10.password ${userPassword}`);
commands.push('sync');
await fs.writeFile(commandFile, commands.join('\n'), 'utf8');
await chdir(cliDir);
await execCommand2(['npm', 'run', 'start-no-build', '--', '--profile', profileDir, 'batch', commandFile]);
} catch (error) {
console.error(`Could not process user ${userNum}:`, error);
} finally {
delete processing_[userNum];
}
};
const waitForProcessing = (count: number) => {
return new Promise((resolve) => {
const iid = setInterval(() => {
if (Object.keys(processing_).length <= count) {
clearInterval(iid);
resolve(null);
}
}, 100);
});
};
const main = async () => {
await fs.mkdirp(tempDir);
// Build the app once before starting, because we'll use start-no-build to
// run the scripts (faster)
await execCommand2(['npm', 'run', 'build']);
const focusUserNum = 400;
while (true) {
let userNum = randomInt(minUserNum, maxUserNum);
if (Math.random() >= .7) userNum = focusUserNum;
void processUser(userNum);
await waitForProcessing(10);
}
};
main().catch((error) => {
console.error('Fatal error', error);
process.exit(1);
});

5
packages/app-desktop/gui/MainScreen/commands/toggleNoteList.ts
View File

@ -19,6 +19,11 @@ export const runtime = (): CommandRuntime => {
visible: !layoutItemProp(layout, 'noteList', 'visible'),
});
// Toggling the sidebar will affect the size of most other on-screen components.
// Dispatching a window resize event is a bit of a hack, but it ensures that any
// component that watches for resizes will be accurately notified
window.dispatchEvent(new Event('resize'));
context.dispatch({
type: 'MAIN_LAYOUT_SET',
value: newLayout,

5
packages/app-desktop/gui/MainScreen/commands/toggleSideBar.ts
View File

@ -19,6 +19,11 @@ export const runtime = (): CommandRuntime => {
visible: !layoutItemProp(layout, 'sideBar', 'visible'),
});
// Toggling the sidebar will affect the size of most other on-screen components.
// Dispatching a window resize event is a bit of a hack, but it ensures that any
// component that watches for resizes will be accurately notified
window.dispatchEvent(new Event('resize'));
context.dispatch({
type: 'MAIN_LAYOUT_SET',
value: newLayout,

5
packages/app-desktop/gui/NoteEditor/NoteBody/CodeMirror/CodeMirror.tsx
View File

@ -6,7 +6,8 @@ import { EditorCommand, NoteBodyEditorProps } from '../../utils/types';
import { commandAttachFileToBody, handlePasteEvent } from '../../utils/resourceHandling';
import { ScrollOptions, ScrollOptionTypes } from '../../utils/types';
import { CommandValue } from '../../utils/types';
import { useScrollHandler, usePrevious, cursorPositionToTextOffset, useRootSize } from './utils';
import { useScrollHandler, usePrevious, cursorPositionToTextOffset } from './utils';
import useElementSize from '@joplin/lib/hooks/useElementSize';
import Toolbar from './Toolbar';
import styles_ from './styles';
import { RenderedBody, defaultRenderedBody } from './utils/types';
@ -59,7 +60,7 @@ function CodeMirror(props: NoteBodyEditorProps, ref: any) {
const props_onChangeRef = useRef<Function>(null);
props_onChangeRef.current = props.onChange;
const rootSize = useRootSize({ rootRef });
const rootSize = useElementSize(rootRef);
usePluginServiceRegistration(ref);

20
packages/app-desktop/gui/NoteEditor/NoteBody/CodeMirror/utils/index.ts
View File

@ -1,4 +1,4 @@
import { useEffect, useCallback, useRef, useState } from 'react';
import { useEffect, useCallback, useRef } from 'react';
import shim from '@joplin/lib/shim';
export function cursorPositionToTextOffset(cursorPos: any, body: string) {
@ -89,21 +89,3 @@ export function useScrollHandler(editorRef: any, webviewRef: any, onScroll: Func
return { resetScroll, setEditorPercentScroll, setViewerPercentScroll, editor_scroll };
}
export function useRootSize(dependencies: any) {
const { rootRef } = dependencies;
const [rootSize, setRootSize] = useState({ width: 0, height: 0 });
useEffect(() => {
if (!rootRef.current) return;
const { width, height } = rootRef.current.getBoundingClientRect();
if (rootSize.width !== width || rootSize.height !== height) {
setRootSize({ width: width, height: height });
}
});
return rootSize;
}

1
packages/app-desktop/gui/NoteEditor/NoteBody/CodeMirror/utils/useKeymap.ts
View File

@ -171,6 +171,7 @@ export default function useKeymap(CodeMirror: any) {
'Cmd-Right': 'goLineRightSmart',
'Alt-Backspace': 'delGroupBefore',
'Alt-Delete': 'delGroupAfter',
'Cmd-Backspace': 'delWrappedLineLeft',
'fallthrough': 'basic',
};

2
packages/app-desktop/gui/NoteEditor/NoteBody/TinyMCE/TinyMCE.tsx
View File

@ -793,7 +793,7 @@ const TinyMCE = (props: NoteBodyEditorProps, ref: any) => {
};
}
await loadDocumentAssets(editor, await props.allAssets(props.contentMarkupLanguage));
await loadDocumentAssets(editor, await props.allAssets(props.contentMarkupLanguage, { contentMaxWidthTarget: '.mce-content-body' }));
dispatchDidUpdate(editor);
};

9
packages/app-desktop/gui/NoteEditor/NoteEditor.tsx
View File

@ -14,7 +14,7 @@ import useMarkupToHtml from './utils/useMarkupToHtml';
import useFormNote, { OnLoadEvent } from './utils/useFormNote';
import useFolder from './utils/useFolder';
import styles_ from './styles';
import { NoteEditorProps, FormNote, ScrollOptions, ScrollOptionTypes, OnChangeEvent, NoteBodyEditorProps } from './utils/types';
import { NoteEditorProps, FormNote, ScrollOptions, ScrollOptionTypes, OnChangeEvent, NoteBodyEditorProps, AllAssetsOptions } from './utils/types';
import ResourceEditWatcher from '@joplin/lib/services/ResourceEditWatcher/index';
import CommandService from '@joplin/lib/services/CommandService';
import ToolbarButton from '../ToolbarButton/ToolbarButton';
@ -151,7 +151,7 @@ function NoteEditor(props: NoteEditorProps) {
plugins: props.plugins,
});
const allAssets = useCallback(async (markupLanguage: number): Promise<any[]> => {
const allAssets = useCallback(async (markupLanguage: number, options: AllAssetsOptions = null): Promise<any[]> => {
const theme = themeStyle(props.themeId);
const markupToHtml = markupLanguageUtils.newMarkupToHtml({}, {
@ -159,7 +159,10 @@ function NoteEditor(props: NoteEditorProps) {
customCss: props.customCss,
});
return markupToHtml.allAssets(markupLanguage, theme, { contentMaxWidth: props.contentMaxWidth });
return markupToHtml.allAssets(markupLanguage, theme, {
contentMaxWidth: props.contentMaxWidth,
contentMaxWidthTarget: options.contentMaxWidthTarget,
});
}, [props.themeId, props.customCss, props.contentMaxWidth]);
const handleProvisionalFlag = useCallback(() => {

11
packages/app-desktop/gui/NoteEditor/utils/clipboardUtils.ts
View File

@ -69,8 +69,17 @@ export function htmlToClipboardData(html: string): ClipboardData {
// In that case we need to set both HTML and Text context, otherwise it
// won't be possible to paste the text in, for example, a text editor.
// https://github.com/laurent22/joplin/issues/4788
//
// Also we don't escape the content produced in HTML to MD conversion
// because it's not what would be expected. For example, if the content is
// `* something`, strictly speaking it would be correct to escape to `\*
// something`, however this is not what the user would expect when copying
// text. Likewise for URLs that contain "_". So the resulting Markdown might
// not be perfectly valid but would be closer to what a user would expect.
// If they want accurate MArkdown they can switch to the MD editor.
// https://github.com/laurent22/joplin/issues/5440
return {
text: htmlToMd().parse(copyableContent),
text: htmlToMd().parse(copyableContent, { disableEscapeContent: true }),
html: cleanUpCodeBlocks(copyableContent),
};
}

6
packages/app-desktop/gui/NoteEditor/utils/types.ts
View File

@ -6,6 +6,10 @@ import { MarkupLanguage } from '@joplin/renderer';
import { RenderResult, RenderResultPluginAsset } from '@joplin/renderer/MarkupToHtml';
import { MarkupToHtmlOptions } from './useMarkupToHtml';
export interface AllAssetsOptions {
contentMaxWidthTarget?: string;
}
export interface ToolbarButtonInfos {
[key: string]: ToolbarButtonInfo;
}
@ -55,7 +59,7 @@ export interface NoteBodyEditorProps {
onScroll(event: any): void;
markupToHtml: (markupLanguage: MarkupLanguage, markup: string, options: MarkupToHtmlOptions)=> Promise<RenderResult>;
htmlToMarkdown: Function;
allAssets: (markupLanguage: MarkupLanguage)=> Promise<RenderResultPluginAsset[]>;
allAssets: (markupLanguage: MarkupLanguage, options: AllAssetsOptions)=> Promise<RenderResultPluginAsset[]>;
disabled: boolean;
dispatch: Function;
noteToolbar: any;

5
packages/app-desktop/gui/NoteSearchBar.jsx
View File

@ -139,9 +139,10 @@ class NoteSearchBarComponent extends React.Component {
color: theme.colorFaded,
backgroundColor: theme.backgroundColor,
});
const matchesFoundString = (query.length > 0 && this.props.resultCount > 0) ? (
const matchesFoundString = (query.length > 0) ? (
<div style={textStyle}>
{`${this.props.selectedIndex + 1} / ${this.props.resultCount}`}
{`${this.props.resultCount === 0 ? 0 : this.props.selectedIndex + 1} / ${this.props.resultCount}`}
</div>
) : null;

6
packages/lib/BaseApplication.ts
View File

@ -251,6 +251,12 @@ export default class BaseApplication {
continue;
}
if (arg.indexOf('--user-data-dir=') === 0) {
// Electron-specific flag. Allows users to run the app with chromedriver.
argv.splice(0, 1);
continue;
}
if (arg.length && arg[0] == '-') {
throw new JoplinError(_('Unknown flag: %s', arg), 'flagError');
} else {

2
packages/lib/HtmlToMd.ts
View File

@ -6,6 +6,7 @@ export interface ParseOptions {
anchorNames?: string[];
preserveImageTagsWithSize?: boolean;
baseUrl?: string;
disableEscapeContent?: boolean;
}
export default class HtmlToMd {
@ -20,6 +21,7 @@ export default class HtmlToMd {
emDelimiter: '*',
strongDelimiter: '**',
br: '',
disableEscapeContent: 'disableEscapeContent' in options ? options.disableEscapeContent : false,
});
turndown.use(turndownPluginGfm);
turndown.remove('script');

50
packages/lib/Synchronizer.ts
View File

@ -479,6 +479,31 @@ export default class Synchronizer {
void this.cancel();
});
// ========================================================================
// 2. DELETE_REMOTE
// ------------------------------------------------------------------------
// Delete the remote items that have been deleted locally.
// ========================================================================
if (syncSteps.indexOf('delete_remote') >= 0) {
const deletedItems = await BaseItem.deletedItems(syncTargetId);
for (let i = 0; i < deletedItems.length; i++) {
if (this.cancelling()) break;
const item = deletedItems[i];
const path = BaseItem.systemPath(item.item_id);
this.logSyncOperation('deleteRemote', null, { id: item.item_id }, 'local has been deleted');
await this.apiCall('delete', path);
if (item.item_type === BaseModel.TYPE_RESOURCE) {
const remoteContentPath = resourceRemotePath(item.item_id);
await this.apiCall('delete', remoteContentPath);
}
await BaseItem.remoteDeletedItem(syncTargetId, item.item_id);
}
} // DELETE_REMOTE STEP
// ========================================================================
// 1. UPLOAD
// ------------------------------------------------------------------------
@ -763,31 +788,6 @@ export default class Synchronizer {
}
} // UPLOAD STEP
// ========================================================================
// 2. DELETE_REMOTE
// ------------------------------------------------------------------------
// Delete the remote items that have been deleted locally.
// ========================================================================
if (syncSteps.indexOf('delete_remote') >= 0) {
const deletedItems = await BaseItem.deletedItems(syncTargetId);
for (let i = 0; i < deletedItems.length; i++) {
if (this.cancelling()) break;
const item = deletedItems[i];
const path = BaseItem.systemPath(item.item_id);
this.logSyncOperation('deleteRemote', null, { id: item.item_id }, 'local has been deleted');
await this.apiCall('delete', path);
if (item.item_type === BaseModel.TYPE_RESOURCE) {
const remoteContentPath = resourceRemotePath(item.item_id);
await this.apiCall('delete', remoteContentPath);
}
await BaseItem.remoteDeletedItem(syncTargetId, item.item_id);
}
} // DELETE_REMOTE STEP
// ------------------------------------------------------------------------
// 3. DELTA
// ------------------------------------------------------------------------

59
packages/lib/services/debug/populateDatabase.ts
View File

File diff suppressed because one or more lines are too long

28
packages/lib/services/searchengine/SearchFilter.test.js
View File

@ -386,6 +386,34 @@ describe('services_SearchFilter', function() {
expect(ids(rows).sort()).toEqual(ids(notes0).concat(ids(notes00).concat(ids(notes1))).sort());
}));
it('should support filtering and search term', (async () => {
const notebook1 = await Folder.save({ title: 'notebook1' });
const notebook2 = await Folder.save({ title: 'notebook2' });
const note1 = await Note.save({ title: 'note1', body: 'abcdefg', parent_id: notebook1.id });
await Note.save({ title: 'note2', body: 'body', parent_id: notebook1.id });
await Note.save({ title: 'note3', body: 'abcdefg', parent_id: notebook2.id });
await Note.save({ title: 'note4', body: 'body', parent_id: notebook2.id });
await engine.syncTables();
const testCases = [
{ searchQuery: 'notebook:notebook1 abcdefg' },
{ searchQuery: 'notebook:notebook1 "abcdefg"' },
{ searchQuery: 'notebook:"notebook1" abcdefg' },
{ searchQuery: 'notebook:"notebook1" "abcdefg"' },
{ searchQuery: 'notebook:"notebook1" -tag:* "abcdefg"' },
{ searchQuery: 'notebook:"notebook1" -tag:* abcdefg' },
{ searchQuery: 'notebook:"notebook1" -tag:"*" abcdefg' },
{ searchQuery: 'notebook:"notebook1" -tag:"*" "abcdefg"' },
];
for (const testCase of testCases) {
const rows = await engine.search(testCase.searchQuery, { searchType });
expect(rows.length).toBe(1);
expect(ids(rows)).toContain(note1.id);
}
}));
it('should support filtering by created date', (async () => {
let rows;
const n1 = await Note.save({ title: 'I made this on', body: 'May 20 2020', user_created_time: Date.parse('2020-05-20') });

1
packages/lib/services/searchengine/filterParser.ts
View File

@ -36,6 +36,7 @@ const getTerms = (query: string, validFilters: Set<string>): Term[] => {
if (inQuote) {
terms.push(makeTerm(currentCol, currentTerm));
currentTerm = '';
currentCol = '_';
inQuote = false;
} else {
inQuote = true;

4
packages/renderer/noteStyle.ts
View File

@ -9,6 +9,7 @@ function formatCssSize(v: any): string {
export interface Options {
contentMaxWidth?: number;
contentMaxWidthTarget?: string;
}
export default function(theme: any, options: Options = null) {
@ -21,8 +22,9 @@ export default function(theme: any, options: Options = null) {
const fontFamily = '\'Avenir\', \'Arial\', sans-serif';
const maxWidthTarget = options.contentMaxWidthTarget ? options.contentMaxWidthTarget : '#rendered-md';
const maxWidthCss = options.contentMaxWidth ? `
#rendered-md {
${maxWidthTarget} {
max-width: ${options.contentMaxWidth}px;
margin-left: auto;
margin-right: auto;

10
packages/server/src/db.ts
View File

@ -122,8 +122,16 @@ export function setupSlowQueryLog(connection: DbConnection, slowQueryLogMinDurat
const queryInfos: Record<any, QueryInfo> = {};
// These queries do not return a response, so "query-response" is not
// called.
const ignoredQueries = /^BEGIN|SAVEPOINT|RELEASE SAVEPOINT|COMMIT|ROLLBACK/gi;
connection.on('query', (data) => {
const timeoutId = makeSlowQueryHandler(slowQueryLogMinDuration, connection, data.sql, data.bindings);
const sql: string = data.sql;
if (!sql || sql.match(ignoredQueries)) return;
const timeoutId = makeSlowQueryHandler(slowQueryLogMinDuration, connection, sql, data.bindings);
queryInfos[data.__knexQueryUid] = {
timeoutId,

9
packages/server/src/middleware/notificationHandler.ts
View File

@ -42,6 +42,13 @@ async function handleSqliteInProdNotification(ctx: AppContext) {
}
}
function levelClassName(level: NotificationLevel): string {
if (level === NotificationLevel.Important) return 'is-warning';
if (level === NotificationLevel.Normal) return 'is-info';
if (level === NotificationLevel.Error) return 'is-danger';
throw new Error(`Unknown level: ${level}`);
}
async function makeNotificationViews(ctx: AppContext): Promise<NotificationView[]> {
const markdownIt = new MarkdownIt();
@ -52,7 +59,7 @@ async function makeNotificationViews(ctx: AppContext): Promise<NotificationView[
views.push({
id: n.id,
messageHtml: markdownIt.render(n.message),
level: n.level === NotificationLevel.Important ? 'warning' : 'info',
levelClassName: levelClassName(n.level),
closeUrl: notificationModel.closeUrl(n.id),
});
}

28
packages/server/src/models/ItemModel.test.ts
View File

@ -196,6 +196,34 @@ describe('ItemModel', function() {
expect((await models().user().load(user3.id)).total_item_size).toBe(totalSize3);
});
test('should update total size when an item is deleted', async function() {
const { user: user1 } = await createUserAndSession(1);
await createItemTree3(user1.id, '', '', [
{
id: '000000000000000000000000000000F1',
children: [
{
id: '00000000000000000000000000000001',
},
],
},
]);
const folder1 = await models().item().loadByJopId(user1.id, '000000000000000000000000000000F1');
const note1 = await models().item().loadByJopId(user1.id, '00000000000000000000000000000001');
await models().item().updateTotalSizes();
expect((await models().user().load(user1.id)).total_item_size).toBe(folder1.content_size + note1.content_size);
await models().item().delete(note1.id);
await models().item().updateTotalSizes();
expect((await models().user().load(user1.id)).total_item_size).toBe(folder1.content_size);
});
test('should include shared items in total size calculation', async function() {
const { user: user1, session: session1 } = await createUserAndSession(1);
const { user: user2, session: session2 } = await createUserAndSession(2);

6
packages/server/src/models/ItemModel.ts
View File

@ -623,7 +623,11 @@ export default class ItemModel extends BaseModel<Item> {
} else {
const itemIds: Uuid[] = unique(changes.map(c => c.item_id));
const userItems: UserItem[] = await this.db('user_items').select('user_id').whereIn('item_id', itemIds);
const userIds: Uuid[] = unique(userItems.map(u => u.user_id));
const userIds: Uuid[] = unique(
userItems
.map(u => u.user_id)
.concat(changes.map(c => c.user_id))
);
const totalSizes: TotalSizeRow[] = [];
for (const userId of userIds) {

10
packages/server/src/models/NotificationModel.ts
View File

@ -1,8 +1,10 @@
import { Notification, NotificationLevel, Uuid } from '../services/database/types';
import { ErrorUnprocessableEntity } from '../utils/errors';
import uuidgen from '../utils/uuidgen';
import BaseModel, { ValidateOptions } from './BaseModel';
export enum NotificationKey {
Any = 'any',
ConfirmEmail = 'confirmEmail',
PasswordSet = 'passwordSet',
EmailConfirmed = 'emailConfirmed',
@ -52,6 +54,10 @@ export default class NotificationModel extends BaseModel<Notification> {
level: NotificationLevel.Normal,
message: 'Thank you! Your account has been successfully upgraded to Pro.',
},
[NotificationKey.Any]: {
level: NotificationLevel.Normal,
message: '',
},
};
const type = notificationTypes[key];
@ -72,7 +78,9 @@ export default class NotificationModel extends BaseModel<Notification> {
}
}
return this.save({ key, message, level, owner_id: userId });
const actualKey = key === NotificationKey.Any ? `any_${uuidgen()}` : key;
return this.save({ key: actualKey, message, level, owner_id: userId });
}
public async markAsRead(userId: Uuid, key: NotificationKey): Promise<void> {

2
packages/server/src/models/SubscriptionModel.ts
View File

@ -98,7 +98,7 @@ export default class SubscriptionModel extends BaseModel<Subscription> {
// failed.
//
// We don't update the user can_upload and enabled properties here
// because it's done after a few days from CronService.
// because it's done after a few days from TaskService.
if (!sub.last_payment_failed_time) {
const user = await this.models().user().load(sub.user_id, { fields: ['email', 'id', 'full_name'] });

32
packages/server/src/models/UserModel.ts
View File

@ -180,7 +180,7 @@ export default class UserModel extends BaseModel<User> {
}
public async checkMaxItemSizeLimit(user: User, buffer: Buffer, item: Item, joplinItem: any) {
// If the item is encrypted, we apply a multipler because encrypted
// If the item is encrypted, we apply a multiplier because encrypted
// items can be much larger (seems to be up to twice the size but for
// safety let's go with 2.2).
@ -198,14 +198,20 @@ export default class UserModel extends BaseModel<User> {
));
}
// Also apply a multiplier to take into account E2EE overhead
const maxTotalItemSize = getMaxTotalItemSize(user) * 1.5;
if (maxTotalItemSize && user.total_item_size + itemSize >= maxTotalItemSize) {
throw new ErrorPayloadTooLarge(_('Cannot save %s "%s" because it would go over the total allowed size (%s) for this account',
isNote ? _('note') : _('attachment'),
itemTitle ? itemTitle : item.name,
formatBytes(maxTotalItemSize)
));
// We allow lock files to go through so that sync can happen, which in
// turns allow user to fix oversized account by deleting items.
const isWhiteListed = itemSize < 200 && item.name.startsWith('locks/');
if (!isWhiteListed) {
// Also apply a multiplier to take into account E2EE overhead
const maxTotalItemSize = getMaxTotalItemSize(user) * 1.5;
if (maxTotalItemSize && user.total_item_size + itemSize >= maxTotalItemSize) {
throw new ErrorPayloadTooLarge(_('Cannot save %s "%s" because it would go over the total allowed size (%s) for this account',
isNote ? _('note') : _('attachment'),
itemTitle ? itemTitle : item.name,
formatBytes(maxTotalItemSize)
));
}
}
}
@ -469,4 +475,12 @@ export default class UserModel extends BaseModel<User> {
}, 'UserModel::save');
}
public async saveMulti(users: User[], options: SaveOptions = {}): Promise<void> {
await this.withTransaction(async () => {
for (const user of users) {
await this.save(user, options);
}
}, 'UserModel::saveMulti');
}
}

15
packages/server/src/routes/api/debug.ts
View File

@ -1,5 +1,5 @@
import config from '../../config';
import { createTestUsers } from '../../tools/debugTools';
import { clearDatabase, createTestUsers, CreateTestUsersOptions } from '../../tools/debugTools';
import { bodyFields } from '../../utils/requestUtils';
import Router from '../../utils/Router';
import { RouteType } from '../../utils/types';
@ -12,6 +12,8 @@ router.public = true;
interface Query {
action: string;
count?: number;
fromNum?: number;
}
router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
@ -20,7 +22,16 @@ router.post('api/debug', async (_path: SubPath, ctx: AppContext) => {
console.info(`Action: ${query.action}`);
if (query.action === 'createTestUsers') {
await createTestUsers(ctx.joplin.db, config());
const options: CreateTestUsersOptions = {};
if ('count' in query) options.count = query.count;
if ('fromNum' in query) options.fromNum = query.fromNum;
await createTestUsers(ctx.joplin.db, config(), options);
}
if (query.action === 'clearDatabase') {
await clearDatabase(ctx.joplin.db);
}
});

4
packages/server/src/routes/api/items.ts
View File

@ -5,7 +5,7 @@ import Router from '../../utils/Router';
import { RouteType } from '../../utils/types';
import { AppContext } from '../../utils/types';
import * as fs from 'fs-extra';
import { ErrorForbidden, ErrorMethodNotAllowed, ErrorNotFound, ErrorPayloadTooLarge } from '../../utils/errors';
import { ErrorForbidden, ErrorMethodNotAllowed, ErrorNotFound, ErrorPayloadTooLarge, errorToPlainObject } from '../../utils/errors';
import ItemModel, { ItemSaveOption, SaveFromRawContentItem } from '../../models/ItemModel';
import { requestDeltaPagination, requestPagination } from '../../models/utils/pagination';
import { AclAction } from '../../models/BaseModel';
@ -66,7 +66,9 @@ export async function putItemContents(path: SubPath, ctx: AppContext, isBatch: b
const output = await ctx.joplin.models.item().saveFromRawContent(ctx.joplin.owner, items, saveOptions);
for (const [name] of Object.entries(output)) {
if (output[name].item) output[name].item = ctx.joplin.models.item().toApiOutput(output[name].item) as Item;
if (output[name].error) output[name].error = errorToPlainObject(output[name].error);
}
return output;
}

135
packages/server/src/routes/index/tasks.ts Normal file
View File

@ -0,0 +1,135 @@
import { makeUrl, redirect, SubPath, UrlType } from '../../utils/routeUtils';
import Router from '../../utils/Router';
import { RouteType } from '../../utils/types';
import { AppContext } from '../../utils/types';
import { bodyFields } from '../../utils/requestUtils';
import { ErrorBadRequest, ErrorForbidden } from '../../utils/errors';
import defaultView from '../../utils/defaultView';
import { makeTableView, Row, Table } from '../../utils/views/table';
import { yesOrNo } from '../../utils/strings';
import { formatDateTime } from '../../utils/time';
import { createCsrfTag } from '../../utils/csrf';
import { RunType } from '../../services/TaskService';
import { NotificationKey } from '../../models/NotificationModel';
import { NotificationLevel } from '../../services/database/types';
const router: Router = new Router(RouteType.Web);
router.post('tasks', async (_path: SubPath, ctx: AppContext) => {
const user = ctx.joplin.owner;
if (!user.is_admin) throw new ErrorForbidden();
const taskService = ctx.joplin.services.tasks;
const fields: any = await bodyFields(ctx.req);
if (fields.startTaskButton) {
const errors: Error[] = [];
for (const k of Object.keys(fields)) {
if (k.startsWith('checkbox_')) {
const taskId = k.substr(9);
try {
void taskService.runTask(taskId, RunType.Manual);
} catch (error) {
errors.push(error);
}
}
}
if (errors.length) {
await ctx.joplin.models.notification().add(
user.id,
NotificationKey.Any,
NotificationLevel.Error,
`Some tasks could not be started: ${errors.join('. ')}`
);
}
} else {
throw new ErrorBadRequest('Invalid action');
}
return redirect(ctx, makeUrl(UrlType.Tasks));
});
router.get('tasks', async (_path: SubPath, ctx: AppContext) => {
const user = ctx.joplin.owner;
if (!user.is_admin) throw new ErrorForbidden();
const taskService = ctx.joplin.services.tasks;
const taskRows: Row[] = [];
for (const [taskId, task] of Object.entries(taskService.tasks)) {
const state = taskService.taskState(taskId);
taskRows.push([
{
value: `checkbox_${taskId}`,
checkbox: true,
},
{
value: taskId,
},
{
value: task.description,
},
{
value: task.schedule,
},
{
value: yesOrNo(state.running),
},
{
value: state.lastRunTime ? formatDateTime(state.lastRunTime) : '-',
},
{
value: state.lastCompletionTime ? formatDateTime(state.lastCompletionTime) : '-',
},
]);
}
const table: Table = {
headers: [
{
name: 'select',
label: '',
},
{
name: 'id',
label: 'ID',
},
{
name: 'description',
label: 'Description',
},
{
name: 'schedule',
label: 'Schedule',
},
{
name: 'running',
label: 'Running',
},
{
name: 'lastRunTime',
label: 'Last Run',
},
{
name: 'lastCompletionTime',
label: 'Last Completion',
},
],
rows: taskRows,
};
return {
...defaultView('tasks', 'Tasks'),
content: {
itemTable: makeTableView(table),
postUrl: makeUrl(UrlType.Tasks),
csrfTag: await createCsrfTag(ctx),
},
cssFiles: ['index/tasks'],
};
});
export default router;

18
packages/server/src/routes/routes.ts
View File

@ -1,31 +1,32 @@
import { Routers } from '../utils/routeUtils';
import apiBatch from './api/batch';
import apiBatchItems from './api/batch_items';
import apiDebug from './api/debug';
import apiEvents from './api/events';
import apiBatchItems from './api/batch_items';
import apiItems from './api/items';
import apiPing from './api/ping';
import apiSessions from './api/sessions';
import apiUsers from './api/users';
import apiShares from './api/shares';
import apiShareUsers from './api/share_users';
import apiUsers from './api/users';
import indexChanges from './index/changes';
import indexHelp from './index/help';
import indexHome from './index/home';
import indexItems from './index/items';
import indexLogin from './index/login';
import indexLogout from './index/logout';
import indexNotifications from './index/notifications';
import indexPassword from './index/password';
import indexSignup from './index/signup';
import indexShares from './index/shares';
import indexUsers from './index/users';
import indexStripe from './index/stripe';
import indexTerms from './index/terms';
import indexPrivacy from './index/privacy';
import indexShares from './index/shares';
import indexSignup from './index/signup';
import indexStripe from './index/stripe';
import indexTasks from './index/tasks';
import indexTerms from './index/terms';
import indexUpgrade from './index/upgrade';
import indexHelp from './index/help';
import indexUsers from './index/users';
import defaultRoute from './default';
@ -56,6 +57,7 @@ const routes: Routers = {
'privacy': indexPrivacy,
'upgrade': indexUpgrade,
'help': indexHelp,
'tasks': indexTasks,
'': defaultRoute,
};

42
packages/server/src/services/CronService.ts
View File

@ -1,42 +0,0 @@
import Logger from '@joplin/lib/Logger';
import BaseService from './BaseService';
const cron = require('node-cron');
const logger = Logger.create('cron');
async function runCronTask(name: string, callback: Function) {
const startTime = Date.now();
logger.info(`Running task "${name}"`);
try {
await callback();
} catch (error) {
logger.error(`On task "${name}"`, error);
}
logger.info(`Completed task "${name}" in ${Date.now() - startTime}ms`);
}
export default class CronService extends BaseService {
public async runInBackground() {
cron.schedule('0 */6 * * *', async () => {
await runCronTask('deleteExpiredTokens', async () => this.models.token().deleteExpiredTokens());
});
cron.schedule('0 * * * *', async () => {
await runCronTask('updateTotalSizes', async () => this.models.item().updateTotalSizes());
});
cron.schedule('0 12 * * *', async () => {
await runCronTask('handleBetaUserEmails', async () => this.models.user().handleBetaUserEmails());
});
cron.schedule('0 13 * * *', async () => {
await runCronTask('handleFailedPaymentSubscriptions', async () => this.models.user().handleFailedPaymentSubscriptions());
});
cron.schedule('0 14 * * *', async () => {
await runCronTask('handleOversizedAccounts', async () => this.models.user().handleOversizedAccounts());
});
}
}

82
packages/server/src/services/TaskService.test.ts Normal file
View File

@ -0,0 +1,82 @@
import config from '../config';
import { Models } from '../models/factory';
import { afterAllTests, beforeAllDb, beforeEachDb, expectThrow, models, msleep } from '../utils/testing/testUtils';
import { Env } from '../utils/types';
import TaskService, { RunType, Task } from './TaskService';
const newService = () => {
return new TaskService(Env.Dev, models(), config());
};
describe('TaskService', function() {
beforeAll(async () => {
await beforeAllDb('TaskService');
});
afterAll(async () => {
await afterAllTests();
});
beforeEach(async () => {
await beforeEachDb();
});
test('should register a task', async function() {
const service = newService();
const task: Task = {
id: 'test',
description: '',
run: (_models: Models) => {},
schedule: '',
};
service.registerTask(task);
expect(service.tasks['test']).toBeTruthy();
await expectThrow(async () => service.registerTask(task));
});
test('should run a task', async function() {
const service = newService();
let finishTask = false;
let taskHasRan = false;
const task: Task = {
id: 'test',
description: '',
run: async (_models: Models) => {
const iid = setInterval(() => {
if (finishTask) {
clearInterval(iid);
taskHasRan = true;
}
}, 1);
},
schedule: '',
};
service.registerTask(task);
expect(service.taskState('test').running).toBe(false);
const startTime = new Date();
void service.runTask('test', RunType.Manual);
expect(service.taskState('test').running).toBe(true);
expect(service.taskState('test').lastCompletionTime).toBeFalsy();
expect(service.taskState('test').lastRunTime.getTime()).toBeGreaterThanOrEqual(startTime.getTime());
await msleep(1);
finishTask = true;
await msleep(3);
expect(taskHasRan).toBe(true);
expect(service.taskState('test').running).toBe(false);
expect(service.taskState('test').lastCompletionTime.getTime()).toBeGreaterThan(startTime.getTime());
});
});

108
packages/server/src/services/TaskService.ts Normal file
View File

@ -0,0 +1,108 @@
import Logger from '@joplin/lib/Logger';
import { Models } from '../models/factory';
import BaseService from './BaseService';
const cron = require('node-cron');
const logger = Logger.create('TaskService');
type TaskId = string;
export enum RunType {
Scheduled = 1,
Manual = 2,
}
const runTypeToString = (runType: RunType) => {
if (runType === RunType.Scheduled) return 'scheduled';
if (runType === RunType.Manual) return 'manual';
throw new Error(`Unknown run type: ${runType}`);
};
export interface Task {
id: TaskId;
description: string;
schedule: string;
run(models: Models): void;
}
export type Tasks = Record<TaskId, Task>;
interface TaskState {
running: boolean;
lastRunTime: Date;
lastCompletionTime: Date;
}
const defaultTaskState: TaskState = {
running: false,
lastRunTime: null,
lastCompletionTime: null,
};
export default class TaskService extends BaseService {
private tasks_: Tasks = {};
private taskStates_: Record<TaskId, TaskState> = {};
public registerTask(task: Task) {
if (this.tasks_[task.id]) throw new Error(`Already a task with this ID: ${task.id}`);
this.tasks_[task.id] = task;
this.taskStates_[task.id] = { ...defaultTaskState };
}
public registerTasks(tasks: Task[]) {
for (const task of tasks) this.registerTask(task);
}
public get tasks(): Tasks {
return this.tasks_;
}
public taskState(id: TaskId): TaskState {
if (!this.taskStates_[id]) throw new Error(`No such task: ${id}`);
return this.taskStates_[id];
}
// TODO: add tests
public async runTask(id: TaskId, runType: RunType) {
const state = this.taskState(id);
if (state.running) throw new Error(`Task is already running: ${id}`);
const startTime = Date.now();
this.taskStates_[id] = {
...this.taskStates_[id],
running: true,
lastRunTime: new Date(),
};
try {
logger.info(`Running "${id}" (${runTypeToString(runType)})...`);
await this.tasks_[id].run(this.models);
} catch (error) {
logger.error(`On task "${id}"`, error);
}
this.taskStates_[id] = {
...this.taskStates_[id],
running: false,
lastCompletionTime: new Date(),
};
logger.info(`Completed "${id}" in ${Date.now() - startTime}ms`);
}
public async runInBackground() {
for (const [taskId, task] of Object.entries(this.tasks_)) {
if (!task.schedule) continue;
logger.info(`Scheduling task "${taskId}": ${task.schedule}`);
cron.schedule(task.schedule, async () => {
await this.runTask(taskId, RunType.Scheduled);
});
}
}
}

1
packages/server/src/services/database/types.ts
View File

@ -6,6 +6,7 @@ export enum ItemAddressingType {
}
export enum NotificationLevel {
Error = 5,
Important = 10,
Normal = 20,
}

4
packages/server/src/services/types.ts
View File

@ -1,11 +1,11 @@
import CronService from './CronService';
import EmailService from './EmailService';
import MustacheService from './MustacheService';
import ShareService from './ShareService';
import TaskService from './TaskService';
export interface Services {
share: ShareService;
email: EmailService;
cron: CronService;
mustache: MustacheService;
tasks: TaskService;
}

109
packages/server/src/tools/debugTools.ts
View File

@ -1,9 +1,14 @@
import { DbConnection, dropTables, migrateLatest } from '../db';
import newModelFactory from '../models/factory';
import { AccountType } from '../models/UserModel';
import { UserFlagType } from '../services/database/types';
import { User, UserFlagType } from '../services/database/types';
import { Config } from '../utils/types';
export interface CreateTestUsersOptions {
count?: number;
fromNum?: number;
}
export async function handleDebugCommands(argv: any, db: DbConnection, config: Config): Promise<boolean> {
if (argv.debugCreateTestUsers) {
await createTestUsers(db, config);
@ -14,51 +19,79 @@ export async function handleDebugCommands(argv: any, db: DbConnection, config: C
return true;
}
export async function createTestUsers(db: DbConnection, config: Config) {
export async function clearDatabase(db: DbConnection) {
await dropTables(db);
await migrateLatest(db);
}
export async function createTestUsers(db: DbConnection, config: Config, options: CreateTestUsersOptions = null) {
options = {
count: 0,
fromNum: 1,
...options,
};
const password = 'hunter1hunter2hunter3';
const models = newModelFactory(db, config);
for (let userNum = 1; userNum <= 2; userNum++) {
await models.user().save({
email: `user${userNum}@example.com`,
password,
full_name: `User ${userNum}`,
});
}
if (options.count) {
const models = newModelFactory(db, config);
{
const { user } = await models.subscription().saveUserAndSubscription(
'usersub@example.com',
'With Sub',
AccountType.Basic,
'usr_111',
'sub_111'
);
await models.user().save({ id: user.id, password });
}
const users: User[] = [];
{
const { user, subscription } = await models.subscription().saveUserAndSubscription(
'userfailedpayment@example.com',
'Failed Payment',
AccountType.Basic,
'usr_222',
'sub_222'
);
await models.user().save({ id: user.id, password });
await models.subscription().handlePayment(subscription.stripe_subscription_id, false);
}
for (let i = 0; i < options.count; i++) {
const userNum = i + options.fromNum;
users.push({
email: `user${userNum}@example.com`,
password,
full_name: `User ${userNum}`,
});
}
{
const user = await models.user().save({
email: 'userwithflags@example.com',
password,
full_name: 'User Withflags',
});
await models.user().saveMulti(users);
} else {
await dropTables(db);
await migrateLatest(db);
const models = newModelFactory(db, config);
await models.userFlag().add(user.id, UserFlagType.AccountOverLimit);
for (let userNum = 1; userNum <= 2; userNum++) {
await models.user().save({
email: `user${userNum}@example.com`,
password,
full_name: `User ${userNum}`,
});
}
{
const { user } = await models.subscription().saveUserAndSubscription(
'usersub@example.com',
'With Sub',
AccountType.Basic,
'usr_111',
'sub_111'
);
await models.user().save({ id: user.id, password });
}
{
const { user, subscription } = await models.subscription().saveUserAndSubscription(
'userfailedpayment@example.com',
'Failed Payment',
AccountType.Basic,
'usr_222',
'sub_222'
);
await models.user().save({ id: user.id, password });
await models.subscription().handlePayment(subscription.stripe_subscription_id, false);
}
{
const user = await models.user().save({
email: 'userwithflags@example.com',
password,
full_name: 'User Withflags',
});
await models.userFlag().add(user.id, UserFlagType.AccountOverLimit);
}
}
}

14
packages/server/src/utils/errors.ts
View File

@ -114,3 +114,17 @@ export function errorToString(error: Error): string {
if (error.stack) msg.push(error.stack);
return msg.join(': ');
}
interface PlainObjectError {
httpCode?: number;
message?: string;
code?: string;
}
export function errorToPlainObject(error: any): PlainObjectError {
const output: PlainObjectError = {};
if ('httpCode' in error) output.httpCode = error.httpCode;
if ('code' in error) output.code = error.code;
if ('message' in error) output.message = error.message;
return output;
}

1
packages/server/src/utils/routeUtils.ts
View File

@ -271,6 +271,7 @@ export enum UrlType {
Login = 'login',
Terms = 'terms',
Privacy = 'privacy',
Tasks = 'tasks',
}
export function makeUrl(urlType: UrlType): string {

4
packages/server/src/utils/setupAppContext.ts
View File

@ -7,15 +7,15 @@ import routes from '../routes/routes';
import ShareService from '../services/ShareService';
import { Services } from '../services/types';
import EmailService from '../services/EmailService';
import CronService from '../services/CronService';
import MustacheService from '../services/MustacheService';
import setupTaskService from './setupTaskService';
async function setupServices(env: Env, models: Models, config: Config): Promise<Services> {
const output: Services = {
share: new ShareService(env, models, config),
email: new EmailService(env, models, config),
cron: new CronService(env, models, config),
mustache: new MustacheService(config.viewDir, config.baseUrl),
tasks: setupTaskService(env, models, config),
};
await output.mustache.loadPartials();

49
packages/server/src/utils/setupTaskService.ts Normal file
View File

@ -0,0 +1,49 @@
import { Models } from '../models/factory';
import TaskService, { Task } from '../services/TaskService';
import { Config, Env } from './types';
export default function(env: Env, models: Models, config: Config): TaskService {
const taskService = new TaskService(env, models, config);
let tasks: Task[] = [
{
id: 'deleteExpiredTokens',
description: 'Delete expired tokens',
schedule: '0 */6 * * *',
run: (models: Models) => models.token().deleteExpiredTokens(),
},
{
id: 'updateTotalSizes',
description: 'Update total sizes',
schedule: '0 * * * *',
run: (models: Models) => models.item().updateTotalSizes(),
},
{
id: 'handleOversizedAccounts',
description: 'Process oversized accounts',
schedule: '0 14 * * *',
run: (models: Models) => models.user().handleOversizedAccounts(),
},
];
if (config.isJoplinCloud) {
tasks = tasks.concat([
{
id: 'handleBetaUserEmails',
description: 'Process beta user emails',
schedule: '0 12 * * *',
run: (models: Models) => models.user().handleBetaUserEmails(),
},
{
id: 'handleFailedPaymentSubscriptions',
description: 'Process failed payment subscriptions',
schedule: '0 13 * * *',
run: (models: Models) => models.user().handleFailedPaymentSubscriptions(),
},
]);
}
taskService.registerTasks(tasks);
return taskService;
}

2
packages/server/src/utils/startServices.ts
View File

@ -3,5 +3,5 @@ import { Services } from '../services/types';
export default async function startServices(services: Services) {
void services.share.runInBackground();
void services.email.runInBackground();
void services.cron.runInBackground();
void services.tasks.runInBackground();
}

3
packages/server/src/utils/time.ts
View File

@ -29,7 +29,8 @@ export function msleep(ms: number) {
});
}
export function formatDateTime(ms: number): string {
export function formatDateTime(ms: number | Date): string {
ms = ms instanceof Date ? ms.getTime() : ms;
return `${dayjs(ms).format('D MMM YY HH:mm:ss')} (${defaultTimezone()})`;
}

2
packages/server/src/utils/types.ts
View File

@ -17,7 +17,7 @@ export enum Env {
export interface NotificationView {
id: Uuid;
messageHtml: string;
level: string;
levelClassName: string;
closeUrl: string;
}

27
packages/server/src/utils/views/table.ts
View File

@ -4,11 +4,13 @@ import { setQueryParameters } from '../urlUtils';
const defaultSortOrder = PaginationOrderDir.ASC;
function headerIsSelectedClass(name: string, pagination: Pagination): string {
if (!pagination) return '';
const orderBy = pagination.order[0].by;
return name === orderBy ? 'is-selected' : '';
}
function headerSortIconDir(name: string, pagination: Pagination): string {
if (!pagination) return '';
const orderBy = pagination.order[0].by;
const orderDir = orderBy === name ? pagination.order[0].dir : defaultSortOrder;
return orderDir === PaginationOrderDir.ASC ? 'up' : 'down';
@ -35,6 +37,7 @@ interface HeaderView {
interface RowItem {
value: string;
checkbox?: boolean;
url?: string;
stretch?: boolean;
}
@ -45,6 +48,7 @@ interface RowItemView {
value: string;
classNames: string[];
url: string;
checkbox: boolean;
}
type RowView = RowItemView[];
@ -52,10 +56,10 @@ type RowView = RowItemView[];
export interface Table {
headers: Header[];
rows: Row[];
baseUrl: string;
requestQuery: any;
pageCount: number;
pagination: Pagination;
baseUrl?: string;
requestQuery?: any;
pageCount?: number;
pagination?: Pagination;
}
export interface TableView {
@ -77,7 +81,7 @@ export function makeTablePagination(query: any, defaultOrderField: string, defau
function makeHeaderView(header: Header, parentBaseUrl: string, baseUrlQuery: PaginationQueryParams, pagination: Pagination): HeaderView {
return {
label: header.label,
sortLink: setQueryParameters(parentBaseUrl, { ...baseUrlQuery, 'order_by': header.name, 'order_dir': headerNextOrder(header.name, pagination) }),
sortLink: !pagination ? null : setQueryParameters(parentBaseUrl, { ...baseUrlQuery, 'order_by': header.name, 'order_dir': headerNextOrder(header.name, pagination) }),
classNames: [header.stretch ? 'stretch' : 'nowrap', headerIsSelectedClass(header.name, pagination)],
iconDir: headerSortIconDir(header.name, pagination),
};
@ -89,14 +93,21 @@ function makeRowView(row: Row): RowView {
value: rowItem.value,
classNames: [rowItem.stretch ? 'stretch' : 'nowrap'],
url: rowItem.url,
checkbox: rowItem.checkbox,
};
});
}
export function makeTableView(table: Table): TableView {
const baseUrlQuery = filterPaginationQueryParams(table.requestQuery);
const pagination = table.pagination;
const paginationLinks = createPaginationLinks(pagination.page, table.pageCount, setQueryParameters(table.baseUrl, { ...baseUrlQuery, 'page': 'PAGE_NUMBER' }));
let paginationLinks: PageLink[] = [];
let baseUrlQuery: PaginationQueryParams = null;
let pagination: Pagination = null;
if (table.pageCount) {
baseUrlQuery = filterPaginationQueryParams(table.requestQuery);
pagination = table.pagination;
paginationLinks = createPaginationLinks(pagination.page, table.pageCount, setQueryParameters(table.baseUrl, { ...baseUrlQuery, 'page': 'PAGE_NUMBER' }));
}
return {
headers: table.headers.map(h => makeHeaderView(h, table.baseUrl, baseUrlQuery, pagination)),

11
packages/server/src/views/index/tasks.mustache Normal file
View File

@ -0,0 +1,11 @@
<form method='POST' action="{{postUrl}}">
{{{csrfTag}}}
{{#itemTable}}
{{>table}}
{{/itemTable}}
<div class="block">
<input class="button is-link" type="submit" value="Start selected tasks" name="startTaskButton"/>
</div>
</form>

3
packages/server/src/views/partials/navbar.mustache
View File

@ -16,6 +16,9 @@
{{/global.owner.is_admin}}
<a class="navbar-item" href="{{{global.baseUrl}}}/items">Items</a>
<a class="navbar-item" href="{{{global.baseUrl}}}/changes">Log</a>
{{#global.owner.is_admin}}
<a class="navbar-item" href="{{{global.baseUrl}}}/tasks">Tasks</a>
{{/global.owner.is_admin}}
</div>
<div class="navbar-end">
{{#global.isJoplinCloud}}

2
packages/server/src/views/partials/notifications.mustache
View File

@ -1,6 +1,6 @@
{{#global.hasNotifications}}
{{#global.notifications}}
<div class="notification is-{{level}}" id="notification-{{id}}">
<div class="notification {{levelClassName}}" id="notification-{{id}}">
<button data-close-url="{{closeUrl}}" data-id="{{id}}" class="delete close-notification-button"></button>
{{{messageHtml}}}
</div>

34
packages/server/src/views/partials/table.mustache
View File

@ -1,20 +1,22 @@
<table class="table is-fullwidth is-hoverable">
<thead>
<tr>
{{#headers}}
{{>tableHeader}}
{{/headers}}
</tr>
</thead>
<tbody>
{{#rows}}
<div class="table-container">
<table class="table is-fullwidth is-hoverable ">
<thead>
<tr>
{{#.}}
{{>tableRowItem}}
{{/.}}
{{#headers}}
{{>tableHeader}}
{{/headers}}
</tr>
{{/rows}}
</tbody>
</table>
</thead>
<tbody>
{{#rows}}
<tr>
{{#.}}
{{>tableRowItem}}
{{/.}}
</tr>
{{/rows}}
</tbody>
</table>
</div>
{{>pagination}}

7
packages/server/src/views/partials/tableHeader.mustache
View File

@ -1,3 +1,8 @@
<th class="{{#classNames}}{{.}} {{/classNames}}">
<a href="{{sortLink}}" class="sort-button">{{label}} <i class="fas fa-caret-{{iconDir}}"></i></a>
{{#sortLink}}
<a href="{{sortLink}}" class="sort-button">{{label}} <i class="fas fa-caret-{{iconDir}}"></i></a>
{{/sortLink}}
{{^sortLink}}
{{label}}
{{/sortLink}}
</th>

7
packages/server/src/views/partials/tableRowItem.mustache
View File

@ -1,3 +1,8 @@
<td class="{{#classNames}}{{.}} {{/classNames}}">
{{#url}}<a href="{{.}}"></span>{{/url}}{{value}}</a>
{{#checkbox}}
<input type="checkbox" name="{{value}}"/>
{{/checkbox}}
{{^checkbox}}
{{#url}}<a href="{{.}}"></span>{{/url}}{{value}}</a>
{{/checkbox}}
</td>

12
packages/turndown/src/commonmark-rules.js
View File

@ -1,4 +1,4 @@
import { repeat, isCodeBlockSpecialCase1, isCodeBlockSpecialCase2, isCodeBlock } from './utilities'
import { repeat, isCodeBlockSpecialCase1, isCodeBlockSpecialCase2, isCodeBlock, getStyleProp } from './utilities'
const Entities = require('html-entities').AllHtmlEntities;
const htmlentities = (new Entities()).encode;
@ -73,7 +73,15 @@ rules.highlight = {
// HTML to avoid any ambiguity.
rules.insert = {
filter: 'ins',
filter: function (node, options) {
// TinyMCE represents this either with an <INS> tag (when pressing the
// toolbar button) or using style "text-decoration" (when using shortcut
// Cmd+U)
//
// https://github.com/laurent22/joplin/issues/5480
if (node.nodeName === 'INS') return true;
return getStyleProp(node, 'text-decoration') === 'underline';
},
replacement: function (content, node, options) {
return '<ins>' + content + '</ins>'

3
packages/turndown/src/turndown.js
View File

@ -23,6 +23,7 @@ export default function TurndownService (options) {
linkReferenceStyle: 'full',
anchorNames: [],
br: ' ',
disableEscapeContent: false,
blankReplacement: function (content, node) {
return node.isBlock ? '\n\n' : ''
},
@ -181,6 +182,8 @@ TurndownService.prototype = {
*/
function process (parentNode, escapeContent = 'auto') {
if (this.options.disableEscapeContent) escapeContent = false;
var self = this
return reduce.call(parentNode.childNodes, function (output, node) {
node = new Node(node)

13
packages/turndown/src/utilities.js
View File

@ -78,3 +78,16 @@ export function isCodeBlock(node) {
node.firstChild.nodeName === 'CODE'
)
}
export function getStyleProp(node, name) {
const style = node.getAttribute('style');
if (!style) return null;
name = name.toLowerCase();
if (!style.toLowerCase().includes(name)) return null;
const o = css.parse('div {' + style + '}');
if (!o.stylesheet.rules.length) return null;
const prop = o.stylesheet.rules[0].declarations.find(d => d.property.toLowerCase() === name);
return prop ? prop.value : null;
}