From a8cfed2170e850c068194f2f665c8ec5aefcb071 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillaume=20Perr=C3=A9al?= <guillaume.perreal@inrae.fr>
Date: Wed, 3 Jun 2020 16:07:33 +0200
Subject: [PATCH] =?UTF-8?q?Met=20en=20place=20une=20ex=C3=A9cution=20paral?=
=?UTF-8?q?l=C3=A8le=20pour=20les=20filtres=20drawio,=20pandoc=20et=20wkht?=
=?UTF-8?q?mltopdf.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
gulpfile.esm.js/plugins/drawio.js | 63 ++++++++++++------------
gulpfile.esm.js/plugins/pandoc.js | 66 +++++++++++++-------------
gulpfile.esm.js/plugins/wkhtmltopdf.js | 8 ++--
gulpfile.esm.js/util/parallel.js | 28 +++++++++++
package-lock.json | 16 +++++--
package.json | 1 +
6 files changed, 107 insertions(+), 75 deletions(-)
create mode 100644 gulpfile.esm.js/util/parallel.js
diff --git a/gulpfile.esm.js/plugins/drawio.js b/gulpfile.esm.js/plugins/drawio.js
index dffd53b..66f3f16 100644
--- a/gulpfile.esm.js/plugins/drawio.js
+++ b/gulpfile.esm.js/plugins/drawio.js
@@ -1,9 +1,8 @@
-import { callbackify } from "util";
import exec from "../util/exec";
import fs from "fs";
import logger from "gulplog";
import { mkTempFile } from "../util/tempdir";
-import { obj } from "through2";
+import parallel from "../util/parallel";
import PluginError from "plugin-error";
import { withBinary } from "../util/optional";
@@ -14,41 +13,39 @@ const drawio = withBinary(
"drawio",
(DRAWIO_BINARY) =>
function drawio() {
- return obj(
- callbackify(async function (input) {
- try {
- const output = await mkTempFile(input);
- output.extname = ".svg";
+ return parallel(async function (input) {
+ try {
+ const output = await mkTempFile(input);
+ output.extname = ".svg";
- const args = [
- "--export",
- "--format",
- "svg",
- "--width",
- "1024",
- "--output",
- output.path,
- input.path,
- ];
+ const args = [
+ "--export",
+ "--format",
+ "svg",
+ "--width",
+ "1024",
+ "--output",
+ output.path,
+ input.path,
+ ];
- await exec(PLUGIN_NAME, DRAWIO_BINARY, args);
+ await exec(PLUGIN_NAME, DRAWIO_BINARY, args);
- logger.info(
- "%s: %s -> %s",
- PLUGIN_NAME,
- input.relative,
- output.relative
- );
+ logger.info(
+ "%s: %s -> %s",
+ PLUGIN_NAME,
+ input.relative,
+ output.relative
+ );
- output.contents = fs.createReadStream(output.path, {
- encoding: "UTF-8",
- });
- this.push(output);
- } catch (error) {
- throw new PluginError(PLUGIN_NAME, error, { showStack: true });
- }
- })
- );
+ output.contents = fs.createReadStream(output.path, {
+ encoding: "UTF-8",
+ });
+ this.push(output);
+ } catch (error) {
+ throw new PluginError(PLUGIN_NAME, error, { showStack: true });
+ }
+ });
}
);
diff --git a/gulpfile.esm.js/plugins/pandoc.js b/gulpfile.esm.js/plugins/pandoc.js
index ae5e66a..8fd6ed8 100644
--- a/gulpfile.esm.js/plugins/pandoc.js
+++ b/gulpfile.esm.js/plugins/pandoc.js
@@ -1,11 +1,11 @@
-import { callbackify, promisify } from "util";
import exec from "../util/exec";
import fs from "fs";
import { merge } from "merge-anything";
import { mkTempFile } from "../util/tempdir";
-import { obj } from "through2";
+import parallel from "../util/parallel";
import path from "path";
import PluginError from "plugin-error";
+import { promisify } from "util";
const writeFile = promisify(fs.writeFile);
const exists = promisify(fs.exists);
@@ -87,41 +87,39 @@ export default function pandoc(options = {}) {
);
const revealJSURLResolver = getRevealJSResolver(revealJSURL);
- return obj(
- callbackify(
- /**
- * @param {import("vinyl")} input
- */
- async function (input) {
- const output = input.clone({ contents: false });
- output.extname = ".html";
- output.contents = null;
+ return parallel(
+ /**
+ * @param {import("vinyl")} input
+ */
+ async function (input) {
+ const output = input.clone({ contents: false });
+ output.extname = ".html";
+ output.contents = null;
- const src = await onDisk(input);
+ const src = await onDisk(input);
- const args = [
- "--from=markdown+backtick_code_blocks+pandoc_title_block+yaml_metadata_block",
- "--to=revealjs",
- "--standalone",
- `--template=${template}`,
- `--slide-level=${slideLevel}`,
- "--toc",
- `--toc-depth=${tocDepth}`,
- `--variable=revealjs-url:${revealJSURLResolver(output)}`,
- ...Object.getOwnPropertyNames(variables).map(
- (name) => `--variable=${name}:${variables[name]}`
- ),
- src.path,
- ];
+ const args = [
+ "--from=markdown+backtick_code_blocks+pandoc_title_block+yaml_metadata_block",
+ "--to=revealjs",
+ "--standalone",
+ `--template=${template}`,
+ `--slide-level=${slideLevel}`,
+ "--toc",
+ `--toc-depth=${tocDepth}`,
+ `--variable=revealjs-url:${revealJSURLResolver(output)}`,
+ ...Object.getOwnPropertyNames(variables).map(
+ (name) => `--variable=${name}:${variables[name]}`
+ ),
+ src.path,
+ ];
- const { stdout } = await exec(PLUGIN_NAME, "pandoc", args, {
- encoding: "buffer",
- cwd: input.cwd,
- });
- output.contents = stdout;
+ const { stdout } = await exec(PLUGIN_NAME, "pandoc", args, {
+ encoding: "buffer",
+ cwd: input.cwd,
+ });
+ output.contents = stdout;
- this.push(output);
- }
- )
+ this.push(output);
+ }
);
}
diff --git a/gulpfile.esm.js/plugins/wkhtmltopdf.js b/gulpfile.esm.js/plugins/wkhtmltopdf.js
index 6280709..420fd8f 100644
--- a/gulpfile.esm.js/plugins/wkhtmltopdf.js
+++ b/gulpfile.esm.js/plugins/wkhtmltopdf.js
@@ -1,8 +1,7 @@
-import { callbackify } from "util";
import { createReadStream } from "fs";
import exec from "../util/exec";
import { mkTempFile } from "../util/tempdir";
-import { obj } from "through2";
+import parallel from "../util/parallel";
import path from "path";
import PluginError from "plugin-error";
import { withBinary } from "../util/optional";
@@ -46,8 +45,7 @@ export const wkhtmltopdf = withBinary(
(WKHTMLTOPDF_BINARY) =>
function wkhtmltopdf(options = {}) {
const { args } = Object.assign({}, DEFAULT_OPTIONS, options);
- return obj(
- callbackify(async function (input) {
+ return parallel(async function (input) {
try {
const output = await mkTempFile(getPDFOutput(input));
const execArgs = [
@@ -64,7 +62,7 @@ export const wkhtmltopdf = withBinary(
throw new PluginError(PLUGIN_NAME, err, { filename: input.path });
}
})
- );
+ ;
}
);
diff --git a/gulpfile.esm.js/util/parallel.js b/gulpfile.esm.js/util/parallel.js
new file mode 100644
index 0000000..e082f94
--- /dev/null
+++ b/gulpfile.esm.js/util/parallel.js
@@ -0,0 +1,28 @@
+
+import { cpus } from "os";
+import { obj } from "through2";
+import pMap from "p-map";
+
+const concurrency = cpus().length;
+
+/**
+ *
+ * @param {(file) => Promise<any>} mapper
+ * @return {import('stream').Transform}
+ */
+export default function parallel(mapper) {
+ const inputs = [];
+
+ return obj(
+ function collect(input, _enc, done) {
+ inputs.push(input);
+ done();
+ },
+ function mapAll(done) {
+ pMap(inputs, mapper.bind(this), { concurrency, stopOnError: false }).then(
+ () => done(),
+ error => done(error)
+ );
+ }
+ );
+}
diff --git a/package-lock.json b/package-lock.json
index 29c4308..2e8758b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -2916,6 +2916,16 @@
"p-map": "^3.0.0",
"rimraf": "^3.0.0",
"slash": "^3.0.0"
+ },
+ "dependencies": {
+ "p-map": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz",
+ "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==",
+ "requires": {
+ "aggregate-error": "^3.0.0"
+ }
+ }
}
},
"delayed-stream": {
@@ -8308,9 +8318,9 @@
}
},
"p-map": {
- "version": "3.0.0",
- "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz",
- "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==",
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
+ "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
"requires": {
"aggregate-error": "^3.0.0"
}
diff --git a/package.json b/package.json
index b7dfd65..8b5c5a5 100644
--- a/package.json
+++ b/package.json
@@ -32,6 +32,7 @@
"gulp-server-livereload": "^1.9.2",
"gulplog": "^1.0.0",
"merge-anything": "^3.0.3",
+ "p-map": "^4.0.0",
"plugin-error": "^1.0.1",
"through2": "^3.0.1",
"vinyl": "^2.2.0",
--
GitLab