Filter html from translations (#3665)

* Filter html from translations

* Error when html found

build-scripts/gulp/download_translations.js

@@ -0,0 +1,73 @@
+const del = require("del");
+const gulp = require("gulp");
+const mapStream = require("map-stream");
+
+const inDir = "translations";
+const downloadDir = inDir + "/downloads";
+
+const tasks = [];
+
+function hasHtml(data) {
+  return /<[a-z][\s\S]*>/i.test(data);
+}
+
+function recursiveCheckHasHtml(file, data, errors, recKey) {
+  Object.keys(data).forEach(function(key) {
+    if (typeof data[key] === "object") {
+      nextRecKey = recKey ? `${recKey}.${key}` : key;
+      recursiveCheckHasHtml(file, data[key], errors, nextRecKey);
+    } else if (hasHtml(data[key])) {
+      errors.push(`HTML found in ${file.path} at key ${recKey}.${key}`);
+    }
+  });
+}
+
+function checkHtml() {
+  let errors = [];
+
+  return mapStream(function(file, cb) {
+    const content = file.contents;
+    let error;
+    if (content) {
+      if (hasHtml(String(content))) {
+        const data = JSON.parse(String(content));
+        recursiveCheckHasHtml(file, data, errors);
+        if (errors.length > 0) {
+          error = errors.join("\r\n");
+        }
+      }
+    }
+    cb(error, file);
+  });
+}
+
+let taskName = "clean-downloaded-translations";
+gulp.task(taskName, function() {
+  return del([`${downloadDir}/**`]);
+});
+tasks.push(taskName);
+
+taskName = "check-translations-html";
+gulp.task(taskName, function() {
+  return gulp.src(`${downloadDir}/*.json`).pipe(checkHtml());
+});
+tasks.push(taskName);
+
+taskName = "move-downloaded-translations";
+gulp.task(taskName, function() {
+  return gulp.src(`${downloadDir}/*.json`).pipe(gulp.dest(inDir));
+});
+tasks.push(taskName);
+
+taskName = "check-downloaded-translations";
+gulp.task(
+  taskName,
+  gulp.series(
+    "check-translations-html",
+    "move-downloaded-translations",
+    "clean-downloaded-translations"
+  )
+);
+tasks.push(taskName);
+
+module.exports = tasks;

package.json

@@ -139,6 +139,7 @@
     "husky": "^1.3.1",
     "lint-staged": "^8.1.5",
     "lodash.template": "^4.5.0",
+    "map-stream": "^0.0.7",
     "merge-stream": "^1.0.1",
     "mocha": "^6.0.2",
     "parse5": "^5.1.0",

script/translations_download

@@ -21,7 +21,7 @@ fi
 [ -z "${LOKALISE_TOKEN-}" ] && LOKALISE_TOKEN="$(<.lokalise_token)"
 
 PROJECT_ID="3420425759f6d6d241f598.13594006"
-LOCAL_DIR="$(pwd)/translations"
+LOCAL_DIR="$(pwd)/translations/downloads"
 FILE_FORMAT=json
 
 mkdir -p ${LOCAL_DIR}
@@ -35,3 +35,5 @@ docker run \
      --export_empty skip \
      --type json \
      --unzip_to /opt/dest
+
+./node_modules/.bin/gulp check-downloaded-translations

yarn.lock

@@ -9021,6 +9021,11 @@ map-obj@^1.0.0, map-obj@^1.0.1:
   resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d"
   integrity sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=
 
+map-stream@^0.0.7:
+  version "0.0.7"
+  resolved "https://registry.yarnpkg.com/map-stream/-/map-stream-0.0.7.tgz#8a1f07896d82b10926bd3744a2420009f88974a8"
+  integrity sha1-ih8HiW2CsQkmvTdEokIACfiJdKg=
+
 map-visit@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/map-visit/-/map-visit-1.0.0.tgz#ecdca8f13144e660f1b5bd41f12f3479d98dfb8f"