server_tasks
Laurent Cozic 2021-09-17 20:15:43 +01:00
parent bcb08ac8a2
commit c35c5a5821
23 changed files with 324 additions and 138 deletions

View File

@ -42,6 +42,13 @@ async function handleSqliteInProdNotification(ctx: AppContext) {
}
}
function levelClassName(level: NotificationLevel): string {
if (level === NotificationLevel.Important) return 'is-warning';
if (level === NotificationLevel.Normal) return 'is-info';
if (level === NotificationLevel.Error) return 'is-danger';
throw new Error(`Unknown level: ${level}`);
}
async function makeNotificationViews(ctx: AppContext): Promise<NotificationView[]> {
const markdownIt = new MarkdownIt();
@ -52,7 +59,7 @@ async function makeNotificationViews(ctx: AppContext): Promise<NotificationView[
views.push({
id: n.id,
messageHtml: markdownIt.render(n.message),
level: n.level === NotificationLevel.Important ? 'warning' : 'info',
levelClassName: levelClassName(n.level),
closeUrl: notificationModel.closeUrl(n.id),
});
}

View File

@ -1,8 +1,10 @@
import { Notification, NotificationLevel, Uuid } from '../services/database/types';
import { ErrorUnprocessableEntity } from '../utils/errors';
import uuidgen from '../utils/uuidgen';
import BaseModel, { ValidateOptions } from './BaseModel';
export enum NotificationKey {
Any = 'any',
ConfirmEmail = 'confirmEmail',
PasswordSet = 'passwordSet',
EmailConfirmed = 'emailConfirmed',
@ -52,6 +54,10 @@ export default class NotificationModel extends BaseModel<Notification> {
level: NotificationLevel.Normal,
message: 'Thank you! Your account has been successfully upgraded to Pro.',
},
[NotificationKey.Any]: {
level: NotificationLevel.Normal,
message: '',
},
};
const type = notificationTypes[key];
@ -72,7 +78,9 @@ export default class NotificationModel extends BaseModel<Notification> {
}
}
return this.save({ key, message, level, owner_id: userId });
const actualKey = key === NotificationKey.Any ? `any_${uuidgen()}` : key;
return this.save({ key: actualKey, message, level, owner_id: userId });
}
public async markAsRead(userId: Uuid, key: NotificationKey): Promise<void> {

View File

@ -98,7 +98,7 @@ export default class SubscriptionModel extends BaseModel<Subscription> {
// failed.
//
// We don't update the user can_upload and enabled properties here
// because it's done after a few days from CronService.
// because it's done after a few days from TaskService.
if (!sub.last_payment_failed_time) {
const user = await this.models().user().load(sub.user_id, { fields: ['email', 'id', 'full_name'] });

View File

@ -0,0 +1,135 @@
import { makeUrl, redirect, SubPath, UrlType } from '../../utils/routeUtils';
import Router from '../../utils/Router';
import { RouteType } from '../../utils/types';
import { AppContext } from '../../utils/types';
import { bodyFields } from '../../utils/requestUtils';
import { ErrorBadRequest, ErrorForbidden } from '../../utils/errors';
import defaultView from '../../utils/defaultView';
import { makeTableView, Row, Table } from '../../utils/views/table';
import { yesOrNo } from '../../utils/strings';
import { formatDateTime } from '../../utils/time';
import { createCsrfTag } from '../../utils/csrf';
import { RunType } from '../../services/TaskService';
import { NotificationKey } from '../../models/NotificationModel';
import { NotificationLevel } from '../../services/database/types';
const router: Router = new Router(RouteType.Web);
router.post('tasks', async (_path: SubPath, ctx: AppContext) => {
const user = ctx.joplin.owner;
if (!user.is_admin) throw new ErrorForbidden();
const taskService = ctx.joplin.services.tasks;
const fields: any = await bodyFields(ctx.req);
if (fields.startTaskButton) {
const errors: Error[] = [];
for (const k of Object.keys(fields)) {
if (k.startsWith('checkbox_')) {
const taskId = k.substr(9);
try {
void taskService.runTask(taskId, RunType.Manual);
} catch (error) {
errors.push(error);
}
}
}
if (errors.length) {
await ctx.joplin.models.notification().add(
user.id,
NotificationKey.Any,
NotificationLevel.Error,
`Some tasks could not be started: ${errors.join('. ')}`
);
}
} else {
throw new ErrorBadRequest('Invalid action');
}
return redirect(ctx, makeUrl(UrlType.Tasks));
});
router.get('tasks', async (_path: SubPath, ctx: AppContext) => {
const user = ctx.joplin.owner;
if (!user.is_admin) throw new ErrorForbidden();
const taskService = ctx.joplin.services.tasks;
const taskRows: Row[] = [];
for (const [taskId, task] of Object.entries(taskService.tasks)) {
const state = taskService.taskState(taskId);
taskRows.push([
{
value: `checkbox_${taskId}`,
checkbox: true,
},
{
value: taskId,
},
{
value: task.description,
},
{
value: task.schedule,
},
{
value: yesOrNo(state.running),
},
{
value: state.lastRunTime ? formatDateTime(state.lastRunTime) : '-',
},
{
value: state.lastCompletionTime ? formatDateTime(state.lastCompletionTime) : '-',
},
]);
}
const table: Table = {
headers: [
{
name: 'select',
label: '',
},
{
name: 'id',
label: 'ID',
},
{
name: 'description',
label: 'Description',
},
{
name: 'schedule',
label: 'Schedule',
},
{
name: 'running',
label: 'Running',
},
{
name: 'lastRunTime',
label: 'Last Run',
},
{
name: 'lastCompletionTime',
label: 'Last Completion',
},
],
rows: taskRows,
};
return {
...defaultView('tasks', 'Tasks'),
content: {
itemTable: makeTableView(table),
postUrl: makeUrl(UrlType.Tasks),
csrfTag: await createCsrfTag(ctx),
},
cssFiles: ['index/tasks'],
};
});
export default router;

View File

@ -1,31 +1,32 @@
import { Routers } from '../utils/routeUtils';
import apiBatch from './api/batch';
import apiBatchItems from './api/batch_items';
import apiDebug from './api/debug';
import apiEvents from './api/events';
import apiBatchItems from './api/batch_items';
import apiItems from './api/items';
import apiPing from './api/ping';
import apiSessions from './api/sessions';
import apiUsers from './api/users';
import apiShares from './api/shares';
import apiShareUsers from './api/share_users';
import apiUsers from './api/users';
import indexChanges from './index/changes';
import indexHelp from './index/help';
import indexHome from './index/home';
import indexItems from './index/items';
import indexLogin from './index/login';
import indexLogout from './index/logout';
import indexNotifications from './index/notifications';
import indexPassword from './index/password';
import indexSignup from './index/signup';
import indexShares from './index/shares';
import indexUsers from './index/users';
import indexStripe from './index/stripe';
import indexTerms from './index/terms';
import indexPrivacy from './index/privacy';
import indexShares from './index/shares';
import indexSignup from './index/signup';
import indexStripe from './index/stripe';
import indexTasks from './index/tasks';
import indexTerms from './index/terms';
import indexUpgrade from './index/upgrade';
import indexHelp from './index/help';
import indexUsers from './index/users';
import defaultRoute from './default';
@ -56,6 +57,7 @@ const routes: Routers = {
'privacy': indexPrivacy,
'upgrade': indexUpgrade,
'help': indexHelp,
'tasks': indexTasks,
'': defaultRoute,
};

View File

@ -1,47 +0,0 @@
import Logger from '@joplin/lib/Logger';
import { Models } from '../models/factory';
import { Config, Env } from '../utils/types';
import BaseService from './BaseService';
import TaskService from './TaskService';
const cron = require('node-cron');
const logger = Logger.create('cron');
export default class CronService extends BaseService {
private taskService_: TaskService;
public constructor(env: Env, models: Models, config: Config, taskService: TaskService) {
super(env, models, config);
this.taskService_ = taskService;
}
private async runTask(id: string) {
logger.info(`Run: "${id}"...`);
await this.taskService_.runTask(id);
logger.info(`Done: "${id}"`);
}
public async runInBackground() {
cron.schedule('0 */6 * * *', async () => {
await this.runTask('deleteExpiredTokens');
});
cron.schedule('0 * * * *', async () => {
await this.runTask('updateTotalSizes');
});
cron.schedule('0 12 * * *', async () => {
await this.runTask('handleBetaUserEmails');
});
cron.schedule('0 13 * * *', async () => {
await this.runTask('handleFailedPaymentSubscriptions');
});
cron.schedule('0 14 * * *', async () => {
await this.runTask('handleOversizedAccounts');
});
}
}

View File

@ -1,28 +1,47 @@
import Logger from '@joplin/lib/Logger';
import { Models } from '../models/factory';
import BaseService from './BaseService';
const cron = require('node-cron');
const logger = Logger.create('tasks');
const logger = Logger.create('TaskService');
type TaskId = string;
export enum RunType {
Scheduled = 1,
Manual = 2,
}
const runTypeToString = (runType: RunType) => {
if (runType === RunType.Scheduled) return 'scheduled';
if (runType === RunType.Manual) return 'manual';
throw new Error(`Unknown run type: ${runType}`);
};
export interface Task {
id: TaskId;
description: string;
schedule: string;
run(models: Models): Promise<void>;
}
export type Tasks = Record<TaskId, Task>;
interface TaskState {
running: boolean;
lastRunTime: Date;
lastCompletionTime: Date;
}
const defaultTaskState: TaskState = {
running: false,
lastRunTime: null,
lastCompletionTime: null,
};
export default class TaskService extends BaseService {
private tasks_: Record<TaskId, Task> = {};
private tasks_: Tasks = {};
private taskStates_: Record<TaskId, TaskState> = {};
public registerTask(task: Task) {
@ -35,12 +54,16 @@ export default class TaskService extends BaseService {
for (const task of tasks) this.registerTask(task);
}
private taskState(id: TaskId): TaskState {
public get tasks(): Tasks {
return this.tasks_;
}
public taskState(id: TaskId): TaskState {
if (!this.taskStates_[id]) throw new Error(`No such task: ${id}`);
return this.taskStates_[id];
}
public async runTask(id: TaskId) {
public async runTask(id: TaskId, runType: RunType) {
const state = this.taskState(id);
if (state.running) throw new Error(`Task is already running: ${id}`);
@ -49,10 +72,11 @@ export default class TaskService extends BaseService {
this.taskStates_[id] = {
...this.taskStates_[id],
running: true,
lastRunTime: new Date(),
};
try {
logger.info(`Running "${id}"...`);
logger.info(`Running "${id}" (${runTypeToString(runType)})...`);
await this.tasks_[id].run(this.models);
} catch (error) {
logger.error(`On task "${id}"`, error);
@ -61,9 +85,20 @@ export default class TaskService extends BaseService {
this.taskStates_[id] = {
...this.taskStates_[id],
running: false,
lastCompletionTime: new Date(),
};
logger.info(`Completed "${id}" in ${Date.now() - startTime}ms`);
}
public async runInBackground() {
for (const [taskId, task] of Object.entries(this.tasks_)) {
logger.info(`Scheduling task "${taskId}": ${task.schedule}`);
cron.schedule(task.schedule, async () => {
await this.runTask(taskId, RunType.Scheduled);
});
}
}
}

View File

@ -6,6 +6,7 @@ export enum ItemAddressingType {
}
export enum NotificationLevel {
Error = 5,
Important = 10,
Normal = 20,
}

View File

@ -1,4 +1,3 @@
import CronService from './CronService';
import EmailService from './EmailService';
import MustacheService from './MustacheService';
import ShareService from './ShareService';
@ -7,7 +6,6 @@ import TaskService from './TaskService';
export interface Services {
share: ShareService;
email: EmailService;
cron: CronService;
mustache: MustacheService;
tasks: TaskService;
}

View File

@ -271,6 +271,7 @@ export enum UrlType {
Login = 'login',
Terms = 'terms',
Privacy = 'privacy',
Tasks = 'tasks',
}
export function makeUrl(urlType: UrlType): string {

View File

@ -7,53 +7,15 @@ import routes from '../routes/routes';
import ShareService from '../services/ShareService';
import { Services } from '../services/types';
import EmailService from '../services/EmailService';
import CronService from '../services/CronService';
import MustacheService from '../services/MustacheService';
import TaskService from '../services/TaskService';
function setupTaskService(env: Env, models: Models, config: Config): TaskService {
const taskService = new TaskService(env, models, config);
taskService.registerTasks([
{
id: 'deleteExpiredTokens',
description: 'Delete expired tokens',
run: (models: Models) => models.token().deleteExpiredTokens(),
},
{
id: 'updateTotalSizes',
description: 'Update total sizes',
run: (models: Models) => models.item().updateTotalSizes(),
},
{
id: 'handleBetaUserEmails',
description: 'Process beta user emails',
run: (models: Models) => models.user().handleBetaUserEmails(),
},
{
id: 'handleFailedPaymentSubscriptions',
description: 'Process failed payment subscriptions',
run: (models: Models) => models.user().handleFailedPaymentSubscriptions(),
},
{
id: 'handleOversizedAccounts',
description: 'Process oversized accounts',
run: (models: Models) => models.user().handleOversizedAccounts(),
},
]);
return taskService;
}
import setupTaskService from './setupTaskService';
async function setupServices(env: Env, models: Models, config: Config): Promise<Services> {
const taskService = setupTaskService(env, models, config);
const output: Services = {
share: new ShareService(env, models, config),
email: new EmailService(env, models, config),
cron: new CronService(env, models, config, taskService),
mustache: new MustacheService(config.viewDir, config.baseUrl),
tasks: taskService,
tasks: setupTaskService(env, models, config),
};
await output.mustache.loadPartials();

View File

@ -0,0 +1,49 @@
import { Models } from '../models/factory';
import TaskService, { Task } from '../services/TaskService';
import { Config, Env } from './types';
export default function(env: Env, models: Models, config: Config): TaskService {
const taskService = new TaskService(env, models, config);
let tasks: Task[] = [
{
id: 'deleteExpiredTokens',
description: 'Delete expired tokens',
schedule: '0 */6 * * *',
run: (models: Models) => models.token().deleteExpiredTokens(),
},
{
id: 'updateTotalSizes',
description: 'Update total sizes',
schedule: '0 * * * *',
run: (models: Models) => models.item().updateTotalSizes(),
},
{
id: 'handleOversizedAccounts',
description: 'Process oversized accounts',
schedule: '0 14 * * *',
run: (models: Models) => models.user().handleOversizedAccounts(),
},
];
if (config.isJoplinCloud) {
tasks = tasks.concat([
{
id: 'handleBetaUserEmails',
description: 'Process beta user emails',
schedule: '0 12 * * *',
run: (models: Models) => models.user().handleBetaUserEmails(),
},
{
id: 'handleFailedPaymentSubscriptions',
description: 'Process failed payment subscriptions',
schedule: '0 13 * * *',
run: (models: Models) => models.user().handleFailedPaymentSubscriptions(),
},
]);
}
taskService.registerTasks(tasks);
return taskService;
}

View File

@ -1,3 +0,0 @@
export default function() {
}

View File

@ -3,5 +3,5 @@ import { Services } from '../services/types';
export default async function startServices(services: Services) {
void services.share.runInBackground();
void services.email.runInBackground();
void services.cron.runInBackground();
void services.tasks.runInBackground();
}

View File

@ -29,7 +29,8 @@ export function msleep(ms: number) {
});
}
export function formatDateTime(ms: number): string {
export function formatDateTime(ms: number | Date): string {
ms = ms instanceof Date ? ms.getTime() : ms;
return `${dayjs(ms).format('D MMM YY HH:mm:ss')} (${defaultTimezone()})`;
}

View File

@ -17,7 +17,7 @@ export enum Env {
export interface NotificationView {
id: Uuid;
messageHtml: string;
level: string;
levelClassName: string;
closeUrl: string;
}

View File

@ -4,11 +4,13 @@ import { setQueryParameters } from '../urlUtils';
const defaultSortOrder = PaginationOrderDir.ASC;
function headerIsSelectedClass(name: string, pagination: Pagination): string {
if (!pagination) return '';
const orderBy = pagination.order[0].by;
return name === orderBy ? 'is-selected' : '';
}
function headerSortIconDir(name: string, pagination: Pagination): string {
if (!pagination) return '';
const orderBy = pagination.order[0].by;
const orderDir = orderBy === name ? pagination.order[0].dir : defaultSortOrder;
return orderDir === PaginationOrderDir.ASC ? 'up' : 'down';
@ -35,6 +37,7 @@ interface HeaderView {
interface RowItem {
value: string;
checkbox?: boolean;
url?: string;
stretch?: boolean;
}
@ -45,6 +48,7 @@ interface RowItemView {
value: string;
classNames: string[];
url: string;
checkbox: boolean;
}
type RowView = RowItemView[];
@ -52,10 +56,10 @@ type RowView = RowItemView[];
export interface Table {
headers: Header[];
rows: Row[];
baseUrl: string;
requestQuery: any;
pageCount: number;
pagination: Pagination;
baseUrl?: string;
requestQuery?: any;
pageCount?: number;
pagination?: Pagination;
}
export interface TableView {
@ -77,7 +81,7 @@ export function makeTablePagination(query: any, defaultOrderField: string, defau
function makeHeaderView(header: Header, parentBaseUrl: string, baseUrlQuery: PaginationQueryParams, pagination: Pagination): HeaderView {
return {
label: header.label,
sortLink: setQueryParameters(parentBaseUrl, { ...baseUrlQuery, 'order_by': header.name, 'order_dir': headerNextOrder(header.name, pagination) }),
sortLink: !pagination ? null : setQueryParameters(parentBaseUrl, { ...baseUrlQuery, 'order_by': header.name, 'order_dir': headerNextOrder(header.name, pagination) }),
classNames: [header.stretch ? 'stretch' : 'nowrap', headerIsSelectedClass(header.name, pagination)],
iconDir: headerSortIconDir(header.name, pagination),
};
@ -89,14 +93,21 @@ function makeRowView(row: Row): RowView {
value: rowItem.value,
classNames: [rowItem.stretch ? 'stretch' : 'nowrap'],
url: rowItem.url,
checkbox: rowItem.checkbox,
};
});
}
export function makeTableView(table: Table): TableView {
const baseUrlQuery = filterPaginationQueryParams(table.requestQuery);
const pagination = table.pagination;
const paginationLinks = createPaginationLinks(pagination.page, table.pageCount, setQueryParameters(table.baseUrl, { ...baseUrlQuery, 'page': 'PAGE_NUMBER' }));
let paginationLinks: PageLink[] = [];
let baseUrlQuery: PaginationQueryParams = null;
let pagination: Pagination = null;
if (table.pageCount) {
baseUrlQuery = filterPaginationQueryParams(table.requestQuery);
pagination = table.pagination;
paginationLinks = createPaginationLinks(pagination.page, table.pageCount, setQueryParameters(table.baseUrl, { ...baseUrlQuery, 'page': 'PAGE_NUMBER' }));
}
return {
headers: table.headers.map(h => makeHeaderView(h, table.baseUrl, baseUrlQuery, pagination)),

View File

@ -0,0 +1,11 @@
<form method='POST' action="{{postUrl}}">
{{{csrfTag}}}
{{#itemTable}}
{{>table}}
{{/itemTable}}
<div class="block">
<input class="button is-link" type="submit" value="Start selected tasks" name="startTaskButton"/>
</div>
</form>

View File

@ -16,6 +16,9 @@
{{/global.owner.is_admin}}
<a class="navbar-item" href="{{{global.baseUrl}}}/items">Items</a>
<a class="navbar-item" href="{{{global.baseUrl}}}/changes">Log</a>
{{#global.owner.is_admin}}
<a class="navbar-item" href="{{{global.baseUrl}}}/tasks">Tasks</a>
{{/global.owner.is_admin}}
</div>
<div class="navbar-end">
{{#global.isJoplinCloud}}

View File

@ -1,6 +1,6 @@
{{#global.hasNotifications}}
{{#global.notifications}}
<div class="notification is-{{level}}" id="notification-{{id}}">
<div class="notification {{levelClassName}}" id="notification-{{id}}">
<button data-close-url="{{closeUrl}}" data-id="{{id}}" class="delete close-notification-button"></button>
{{{messageHtml}}}
</div>

View File

@ -1,20 +1,22 @@
<table class="table is-fullwidth is-hoverable">
<thead>
<tr>
{{#headers}}
{{>tableHeader}}
{{/headers}}
</tr>
</thead>
<tbody>
{{#rows}}
<div class="table-container">
<table class="table is-fullwidth is-hoverable ">
<thead>
<tr>
{{#.}}
{{>tableRowItem}}
{{/.}}
{{#headers}}
{{>tableHeader}}
{{/headers}}
</tr>
{{/rows}}
</tbody>
</table>
</thead>
<tbody>
{{#rows}}
<tr>
{{#.}}
{{>tableRowItem}}
{{/.}}
</tr>
{{/rows}}
</tbody>
</table>
</div>
{{>pagination}}

View File

@ -1,3 +1,8 @@
<th class="{{#classNames}}{{.}} {{/classNames}}">
<a href="{{sortLink}}" class="sort-button">{{label}} <i class="fas fa-caret-{{iconDir}}"></i></a>
{{#sortLink}}
<a href="{{sortLink}}" class="sort-button">{{label}} <i class="fas fa-caret-{{iconDir}}"></i></a>
{{/sortLink}}
{{^sortLink}}
{{label}}
{{/sortLink}}
</th>

View File

@ -1,3 +1,8 @@
<td class="{{#classNames}}{{.}} {{/classNames}}">
{{#url}}<a href="{{.}}"></span>{{/url}}{{value}}</a>
{{#checkbox}}
<input type="checkbox" name="{{value}}"/>
{{/checkbox}}
{{^checkbox}}
{{#url}}<a href="{{.}}"></span>{{/url}}{{value}}</a>
{{/checkbox}}
</td>