diff --git a/.eslintignore b/.eslintignore index e58c9ebd51..664f372de7 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1974,9 +1974,15 @@ packages/tools/update-readme-download.js.map packages/tools/update-readme-sponsors.d.ts packages/tools/update-readme-sponsors.js packages/tools/update-readme-sponsors.js.map +packages/tools/updateMarkdownDoc.d.ts +packages/tools/updateMarkdownDoc.js +packages/tools/updateMarkdownDoc.js.map packages/tools/website/build.d.ts packages/tools/website/build.js packages/tools/website/build.js.map +packages/tools/website/updateDownloadPage.d.ts +packages/tools/website/updateDownloadPage.js +packages/tools/website/updateDownloadPage.js.map packages/tools/website/utils/frontMatter.d.ts packages/tools/website/utils/frontMatter.js packages/tools/website/utils/frontMatter.js.map diff --git a/.gitignore b/.gitignore index e3712c7b85..70ce3039b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1957,9 +1957,15 @@ packages/tools/update-readme-download.js.map packages/tools/update-readme-sponsors.d.ts packages/tools/update-readme-sponsors.js packages/tools/update-readme-sponsors.js.map +packages/tools/updateMarkdownDoc.d.ts +packages/tools/updateMarkdownDoc.js +packages/tools/updateMarkdownDoc.js.map packages/tools/website/build.d.ts packages/tools/website/build.js packages/tools/website/build.js.map +packages/tools/website/updateDownloadPage.d.ts +packages/tools/website/updateDownloadPage.js +packages/tools/website/updateDownloadPage.js.map packages/tools/website/utils/frontMatter.d.ts packages/tools/website/utils/frontMatter.js packages/tools/website/utils/frontMatter.js.map diff --git a/package.json b/package.json index d512aaa68f..677235e6b7 100644 --- a/package.json +++ b/package.json @@ -12,12 +12,12 @@ "build": "lerna run build && npm run tsc", "buildApiDoc": "npm start --prefix=packages/app-cli -- apidoc ../../readme/api/references/rest_api.md", "buildCommandIndex": "gulp buildCommandIndex", - "buildDoc": "./packages/tools/build-all.sh", - "buildPluginDoc": "typedoc --name 'Joplin Plugin API Documentation' --mode file -theme './Assets/PluginDocTheme/' --readme './Assets/PluginDocTheme/index.md' --excludeNotExported --excludeExternals --excludePrivate --excludeProtected --out docs/api/references/plugin_api packages/lib/services/plugins/api/", - "buildSettingJsonSchema": "npm start --prefix=packages/app-cli -- settingschema ../../docs/schema/settings.json", + "updateMarkdownDoc": "node ./packages/tools/updateMarkdownDoc", + "buildPluginDoc": "typedoc --name 'Joplin Plugin API Documentation' --mode file -theme './Assets/PluginDocTheme/' --readme './Assets/PluginDocTheme/index.md' --excludeNotExported --excludeExternals --excludePrivate --excludeProtected --out ../joplin-website/docs/api/references/plugin_api packages/lib/services/plugins/api/", + "buildSettingJsonSchema": "npm start --prefix=packages/app-cli -- settingschema ../../../joplin-website/docs/schema/settings.json", "buildTranslations": "npm run tsc && node packages/tools/build-translation.js", "buildTranslationsNoTsc": "node packages/tools/build-translation.js", - "buildWebsite": "npm run buildApiDoc && node ./packages/tools/website/build.js && npm run buildPluginDoc && npm run buildSettingJsonSchema", + "buildWebsite": "node ./packages/tools/website/build.js && npm run buildPluginDoc && npm run buildSettingJsonSchema", "circularDependencyCheck": "madge --warning --circular --extensions js ./", "clean": "lerna clean -y && lerna run clean", "dependencyTree": "madge", @@ -45,7 +45,7 @@ "updateIgnored": "gulp updateIgnoredTypeScriptBuild", "updatePluginTypes": "./packages/generator-joplin/updateTypes.sh", "watch": "lerna run watch --stream --parallel", - "watchWebsite": "nodemon --verbose --watch Assets/WebsiteAssets --watch packages/tools/website/build.js --ext md,ts,js,mustache,css,tsx,gif,png,svg --exec \"node packages/tools/website/build.js && http-server --port 8077 docs -a localhost\"", + "watchWebsite": "nodemon --verbose --watch Assets/WebsiteAssets --watch packages/tools/website/build.js --ext md,ts,js,mustache,css,tsx,gif,png,svg --exec \"node packages/tools/website/build.js && http-server --port 8077 ../joplin-website/docs -a localhost\"", "i": "node packages/tools/lernaInstall" }, "husky": { diff --git a/packages/tools/build-all.sh b/packages/tools/build-all.sh deleted file mode 100755 index fb8edac45a..0000000000 --- a/packages/tools/build-all.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -set -e - -ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -git pull - -echo "---------------------------------------------------" -echo "$ROOT_DIR/update-readme-download.js..." -echo "---------------------------------------------------" -node "$ROOT_DIR/update-readme-download.js" - -echo "---------------------------------------------------" -echo "$ROOT_DIR/build-release-stats.js..." -echo "---------------------------------------------------" -node "$ROOT_DIR/build-release-stats.js" - -echo "---------------------------------------------------" -echo "$ROOT_DIR/build-welcome.js..." -echo "---------------------------------------------------" -node "$ROOT_DIR/build-welcome.js" - -echo "---------------------------------------------------" -echo "$ROOT_DIR/update-readme-sponsors.js..." -echo "---------------------------------------------------" -node "$ROOT_DIR/update-readme-sponsors.js" - -cd "$ROOT_DIR/.." -echo "---------------------------------------------------" -echo "npm run buildWebsite..." -echo "---------------------------------------------------" -npm run buildWebsite - -echo "---------------------------------------------------" -echo "Commit changes..." -echo "---------------------------------------------------" -git add -A && git commit -m "Update website" && git pull && git push \ No newline at end of file diff --git a/packages/tools/build-release-stats.js b/packages/tools/build-release-stats.js index 149e7e7c0d..99b1b717c1 100644 --- a/packages/tools/build-release-stats.js +++ b/packages/tools/build-release-stats.js @@ -4,6 +4,7 @@ const fetch = require('node-fetch'); const fs = require('fs-extra'); const { dirname } = require('@joplin/lib/path-utils'); const markdownUtils = require('@joplin/lib/markdownUtils').default; +const yargParser = require('yargs-parser'); const rootDir = dirname(dirname(__dirname)); @@ -55,6 +56,11 @@ function createChangeLog(releases) { } async function main() { + const argv = yargParser(process.argv); + const types = argv.types ? argv.types.split(',') : ['stats', 'changelog']; + + console.info(`Building docs: ${types.join(', ')}`); + const rows = []; const totals = { @@ -99,8 +105,15 @@ async function main() { pageNum++; } - const changelogText = createChangeLog(rows); - await fs.writeFile(`${rootDir}/readme/changelog.md`, changelogText); + if (types.includes('changelog')) { + console.info('Build stats: Updating changelog...'); + const changelogText = createChangeLog(rows); + await fs.writeFile(`${rootDir}/readme/changelog.md`, changelogText); + } + + if (!types.includes('stats')) return; + + console.info('Build stats: Updating stats...'); const grandTotal = totals.windows_count + totals.mac_count + totals.linux_count; totals.windows_percent = totals.windows_count / grandTotal; diff --git a/packages/tools/release-website.sh b/packages/tools/release-website.sh new file mode 100755 index 0000000000..d5c5bcc016 --- /dev/null +++ b/packages/tools/release-website.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +set -e + +# ------------------------------------------------------------------------------ +# Setup environment +# ------------------------------------------------------------------------------ + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SCRIPT_NAME=`basename "$0"` +JOPLIN_ROOT_DIR="$SCRIPT_DIR/../.." +JOPLIN_WEBSITE_ROOT_DIR="$JOPLIN_ROOT_DIR/../joplin-website" + +# ------------------------------------------------------------------------------ +# Update the Markdown files inside the Joplin directory. This is for example the +# download links README.md or the desktop app changelog. +# ------------------------------------------------------------------------------ + +cd "$JOPLIN_ROOT_DIR" + +# Will fail if there's any local change in the repo, which is what we want +git pull --rebase + +npm install + +# Clean up npm's mess +git reset --hard + +npm run updateMarkdownDoc + +# We commit and push the change. It will be a noop if nothing was actually +# changed + +git add -A + +git commit -m "Doc: Updated Markdown files + +Auto-updated using $SCRIPT_NAME" + +git pull --rebase +git push + +# ------------------------------------------------------------------------------ +# Build and deploy the website +# ------------------------------------------------------------------------------ + +cd "$JOPLIN_WEBSITE_ROOT_DIR" +git pull --rebase + +cd "$JOPLIN_ROOT_DIR" +npm run buildWebsite + +cd "$JOPLIN_WEBSITE_ROOT_DIR" +git add -A +git commit -m "Updated website + +Auto-updated using $SCRIPT_NAME" + +git pull --rebase +git push diff --git a/packages/tools/update-readme-sponsors.ts b/packages/tools/update-readme-sponsors.ts index a9f1bfd7e1..a2746056ce 100644 --- a/packages/tools/update-readme-sponsors.ts +++ b/packages/tools/update-readme-sponsors.ts @@ -1,9 +1,8 @@ import { readFile } from 'fs-extra'; import { insertContentIntoFile, rootDir } from './tool-utils'; import markdownUtils, { MarkdownTableHeader, MarkdownTableJustify, MarkdownTableRow } from '@joplin/lib/markdownUtils'; -const ArrayUtils = require('@joplin/lib/ArrayUtils'); -const { escapeHtml } = require('@joplin/lib/string-utils'); import { GithubSponsor, OrgSponsor, Sponsors } from './website/utils/types'; +const { escapeHtml } = require('@joplin/lib/string-utils'); const readmePath = `${rootDir}/README.md`; const sponsorsPath = `${rootDir}/packages/tools/sponsors.json`; @@ -51,7 +50,7 @@ async function createGitHubSponsorTable(sponsors: GithubSponsor[]): Promise { - sponsors = ArrayUtils.shuffle(sponsors); + // sponsors = ArrayUtils.shuffle(sponsors); const output: string[] = []; diff --git a/packages/tools/updateMarkdownDoc.ts b/packages/tools/updateMarkdownDoc.ts new file mode 100644 index 0000000000..53841f7c6b --- /dev/null +++ b/packages/tools/updateMarkdownDoc.ts @@ -0,0 +1,32 @@ +import { chdir } from 'process'; +import { execCommand2, rootDir, gitRepoCleanTry } from './tool-utils'; +import updateDownloadPage from './website/updateDownloadPage'; + +async function main() { + const doGitOperations = false; + + if (doGitOperations) { + await gitRepoCleanTry(); + await execCommand2(['git', 'pull', '--rebase']); + } + + await execCommand2(['node', `${rootDir}/packages/tools/update-readme-download.js`]); + await execCommand2(['node', `${rootDir}/packages/tools/build-release-stats.js`, '--types=changelog']); + await execCommand2(['node', `${rootDir}/packages/tools/update-readme-sponsors.js`]); + await execCommand2(['node', `${rootDir}/packages/tools/build-welcome.js`]); + chdir(rootDir); + await execCommand2(['npm', 'run', 'buildApiDoc']); + await updateDownloadPage(); + + if (doGitOperations) { + await execCommand2(['git', 'add', '-A']); + await execCommand2(['git', 'commit', '-m', 'Update Markdown doc']); + await execCommand2(['git', 'pull', '--rebase']); + await execCommand2(['git', 'push']); + } +} + +main().catch((error) => { + console.error('Fatal error', error); + process.exit(1); +}); diff --git a/packages/tools/website/build.ts b/packages/tools/website/build.ts index e3ccb387cf..311517cde9 100644 --- a/packages/tools/website/build.ts +++ b/packages/tools/website/build.ts @@ -1,20 +1,24 @@ -import { readFileSync, readFile, mkdirpSync, writeFileSync, remove, copy } from 'fs-extra'; -import { insertContentIntoFile, rootDir } from '../tool-utils'; +import { readFileSync, readFile, mkdirpSync, writeFileSync, remove, copy, pathExistsSync } from 'fs-extra'; +import { rootDir } from '../tool-utils'; import { pressCarouselItems } from './utils/pressCarousel'; import { getMarkdownIt, loadMustachePartials, markdownToPageHtml, renderMustache } from './utils/render'; import { AssetUrls, Env, OrgSponsor, PlanPageParams, Sponsors, TemplateParams } from './utils/types'; import { getPlans, loadStripeConfig } from '@joplin/lib/utils/joplinCloud'; import { shuffle } from '@joplin/lib/array'; import { stripOffFrontMatter } from './utils/frontMatter'; +import { dirname, basename } from 'path'; const moment = require('moment'); -const dirname = require('path').dirname; const glob = require('glob'); const path = require('path'); const md5File = require('md5-file/promise'); const env = Env.Prod; +const docDir = `${dirname(dirname(dirname(dirname(__dirname))))}/joplin-website/docs`; + +if (!pathExistsSync(docDir)) throw new Error(`Doc directory does not exist: ${docDir}`); + const websiteAssetDir = `${rootDir}/Assets/WebsiteAssets`; const mainTemplateHtml = readFileSync(`${websiteAssetDir}/templates/main-new.mustache`, 'utf8'); const frontTemplateHtml = readFileSync(`${websiteAssetDir}/templates/front.mustache`, 'utf8'); @@ -165,37 +169,6 @@ function makeHomePageMd() { return md; } -async function createDownloadButtonsHtml(readmeMd: string): Promise> { - const output: Record = {}; - output['windows'] = readmeMd.match(/()/)[0]; - output['macOs'] = readmeMd.match(/()/)[0]; - output['linux'] = readmeMd.match(/()/)[0]; - output['android'] = readmeMd.match(/(', '', desktopButtonsHtml.join(' ')); - await insertContentIntoFile(`${rootDir}/readme/download.md`, '', '', mobileButtonsHtml.join(' ')); -} - async function loadSponsors(): Promise { const sponsorsPath = `${rootDir}/packages/tools/sponsors.json`; const output: Sponsors = JSON.parse(await readFile(sponsorsPath, 'utf8')); @@ -231,8 +204,8 @@ const isNewsFile = (filePath: string): boolean => { }; async function main() { - await remove(`${rootDir}/docs`); - await copy(websiteAssetDir, `${rootDir}/docs`); + await remove(`${docDir}`); + await copy(websiteAssetDir, `${docDir}`); const sponsors = await loadSponsors(); const partials = await loadMustachePartials(partialDir); @@ -240,20 +213,19 @@ async function main() { const readmeMd = makeHomePageMd(); - const downloadButtonsHtml = await createDownloadButtonsHtml(readmeMd); - await updateDownloadPage(downloadButtonsHtml); + // await updateDownloadPage(readmeMd); // ============================================================= // HELP PAGE // ============================================================= - renderPageToHtml(readmeMd, `${rootDir}/docs/help/index.html`, { sourceMarkdownFile: 'README.md', partials, sponsors, assetUrls }); + renderPageToHtml(readmeMd, `${docDir}/help/index.html`, { sourceMarkdownFile: 'README.md', partials, sponsors, assetUrls }); // ============================================================= // FRONT PAGE // ============================================================= - renderPageToHtml('', `${rootDir}/docs/index.html`, { + renderPageToHtml('', `${docDir}/index.html`, { templateHtml: frontTemplateHtml, partials, pressCarouselRegular: { @@ -290,7 +262,7 @@ async function main() { const planPageContentHtml = renderMustache('', planPageParams); - renderPageToHtml('', `${rootDir}/docs/plans/index.html`, { + renderPageToHtml('', `${docDir}/plans/index.html`, { ...defaultTemplateParams(assetUrls), pageName: 'plans', partials, @@ -310,10 +282,12 @@ async function main() { const donateLinksMd = await getDonateLinks(); const makeTargetFilePath = (input: string): string => { + const filenameNoExt = basename(input, '.md'); + if (isNewsFile(input)) { - return `${input.replace(/\.md/, '').replace(/readme\/news\//, 'docs/news/')}/index.html`; + return `${docDir}/news/${filenameNoExt}/index.html`; // `${input.replace(/\.md/, '').replace(/readme\/news\//, 'docs/news/')}/index.html`; } else { - return `${input.replace(/\.md/, '').replace(/readme\//, 'docs/')}/index.html`; + return `${docDir}/${filenameNoExt}/index.html`; } }; @@ -338,9 +312,10 @@ async function main() { source[2].sourceMarkdownName = path.basename(source[0], path.extname(source[0])); const sourceFilePath = `${rootDir}/${source[0]}`; + const targetFilePath = source[1]; const isNews = isNewsFile(sourceFilePath); - renderFileToHtml(sourceFilePath, `${rootDir}/${source[1]}`, { + renderFileToHtml(sourceFilePath, targetFilePath, { ...source[2], templateHtml: mainTemplateHtml, pageName: isNews ? 'news-item' : '', @@ -355,7 +330,7 @@ async function main() { return a.toLowerCase() > b.toLowerCase() ? -1 : +1; }); - await makeNewsFrontPage(newsFilePaths, `${rootDir}/docs/news/index.html`, { + await makeNewsFrontPage(newsFilePaths, `${docDir}/news/index.html`, { ...defaultTemplateParams(assetUrls), pageName: 'news', partials, diff --git a/packages/tools/website/updateDownloadPage.ts b/packages/tools/website/updateDownloadPage.ts new file mode 100644 index 0000000000..694a5a0f15 --- /dev/null +++ b/packages/tools/website/updateDownloadPage.ts @@ -0,0 +1,41 @@ +import { readFile } from 'fs-extra'; +import { rootDir, insertContentIntoFile } from '../tool-utils'; + +async function getReadmeMd() { + return readFile(`${rootDir}/README.md`, 'utf8'); +} + +async function createDownloadButtonsHtml(readmeMd: string): Promise> { + const output: Record = {}; + output['windows'] = readmeMd.match(/()/)[0]; + output['macOs'] = readmeMd.match(/()/)[0]; + output['linux'] = readmeMd.match(/()/)[0]; + output['android'] = readmeMd.match(/(', '', desktopButtonsHtml.join(' ')); + await insertContentIntoFile(`${rootDir}/readme/download.md`, '', '', mobileButtonsHtml.join(' ')); +}