use puppeteeer-cluster for parallel crawling

use yargs to parse command-line args
2025-09-23 03:52:16 -04:00 · 2020-09-19 22:19:20 +00:00 · 2020-09-19 22:19:20 +00:00 · 1de577bd78
commit 1de577bd78
parent 7346527a81
5 changed files with 232 additions and 33 deletions
--- a/README.md
+++ b/README.md
@ -1,9 +1,40 @@
 zimit
-===
+=====

-Create ZIM files out of HTTP websites
+This version of Zimit runs a single-site headless-Chrome based crawl in a Docker container and produces a ZIM of the crawled content.

-# Previous version
+The system uses:
+ - `oldwebtoday/chrome` - to install a recent version of Chrome 84
+ - `puppeteer-cluster` - for running Chrome browsers in parallel
+ - `pywb` - in recording mode for capturing the content
+ - `warc2zim` - to convert the crawled WARC files into a ZIM
+
+The driver in `index.js` crawls a given URL using puppeteer-cluster.
+
+After the crawl is done, warc2zim is used to write a zim to the `/output` directory, which can be mounted as a volume.
+
+## Usage
+
+`zimit` is intended to be run in Docker.
+
+The following is an example usage. The `--cap-add` and `--shm-size` flags are needed for Chrome.
+
+The image accepts the following parameters:
+
+- "<URL>" - the url to be crawled (required)
+- `--workers N` - number of crawl workers to be run in parallel
+- `--wait-until` - Puppeteer setting for how long to wait for page load. See [page.goto waitUntil options](https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagegotourl-options). The default is `load`, but for static sites, `--wait-until domcontentloaded` may be used to speed up the crawl (to avoid waiting for ads to load for example).
+
+
+Example command:
+
+```
+docker run -d -e NAME=myzimfile -v /output:/output --cap-add=SYS_ADMIN --cap-add=NET_ADMIN --shm-size=1gb openzim/zimit "<URL>" --workers 2 --wait-until domcontentloaded
+```
+
+<hr>
+
+## Previous version

 A first version of a generic HTTP scraper was created in 2016 during the [Wikimania Esino Lario Hackathon](https://wikimania2016.wikimedia.org/wiki/Programme/Kiwix-dedicated_Hackathon).

--- a/index.js
+++ b/index.js
@ -1,45 +1,80 @@
 const puppeteer = require("puppeteer-core");
+const { Cluster } = require("puppeteer-cluster");

-const PAGE_TIMEOUT = 60000;
+async function run(params) {
+  const args = [
+    "--no-first-run",
+    "--no-xshm",
+    `--proxy-server=http://${process.env.PROXY_HOST}:${process.env.PROXY_PORT}`
+  ];

-async function run() {
-  const defaultViewport = null;
-  const browserURL = `http://localhost:9222`;
-  let browser = null;
+  const puppeteerOptions = {
+    headless: true,
+    executablePath: "/usr/bin/google-chrome",
+    ignoreHTTPSErrors: true,
+    args
+  };

-  console.log("waiting for browser...");
+  const cluster = await Cluster.launch({
+    concurrency: Cluster.CONCURRENCY_PAGE,
+    maxConcurrency: Number(params.workers) || 1,
+    skipDuplicateUrls: true,
+    puppeteerOptions,
+    puppeteer,
+    monitor: true
+  });
+
+  let seenList = new Set();
+  const url = params._[0];
+
+  let { waitUntil, timeout, scope } = params;
+  waitUntil = waitUntil || "load";
+  timeout = timeout || 60000;
+  scope = scope || new URL(url).origin + "/";
+
+  cluster.task(async ({page, data}) => {
+    const {url} = data;

-  while (!browser) {
    try {
-      browser = await puppeteer.connect({browserURL, defaultViewport});
+      await page.goto(url, {waitUntil, timeout});
    } catch (e) {
-      //console.log(e);
-      await new Promise((resolve) => setTimeout(resolve, 500));
+      console.log(`Load timeout for ${url}`);
    }
-  }

-  console.log("connected!");
+    try{
+      const result = await page.evaluate(() => {
+        return [...document.querySelectorAll('a[href]')].map(el => ({ url: el.href}))
+      });

-  const pages = await browser.pages();
-    
-  const page = pages.length ? pages[0] : await browser.newPage();
+      for (data of result) {
+        if (seenList.has(data.url)) {
+          continue;
+        }
+        //console.log(`check ${data.url} in ${allowedDomain}`);
+        if (scope && data.url.startsWith(scope)) {
+          seenList.add(data.url);
+          cluster.queue({url: data.url});
+        }
+      }
+    } catch (e) {
+      console.warn("error");
+      console.warn(e);
+    }
+  });

-  console.log(process.argv);
-  const url = process.argv.length > 2 ? process.argv[2] : "";
+  cluster.queue({url});

-  if (!url) {
-    throw "No URL specified, exiting";
-  }
-
-  await page.goto(url, {"waitUntil": "networkidle0", "timeout": PAGE_TIMEOUT});
-
-  console.log("loaded!");
+  await cluster.idle();
+  await cluster.close();
 }


 async function main() {
+  const params = require('yargs').argv;
+  console.log(params);
+
  try {
-    await run();
+    await run(params);
    process.exit(0);
  } catch(e) {
    console.log(e);
--- a/package.json
+++ b/package.json
@ -6,6 +6,8 @@
  "author": "Ilya Kreymer <ikreymer@gmail.com>",
  "license": "MIT",
  "dependencies": {
-    "puppeteer-core": "^5.3.0"
+    "puppeteer-cluster": "^0.22.0",
+    "puppeteer-core": "^5.3.0",
+    "yargs": "^16.0.3"
  }
 }
--- a/run.sh
+++ b/run.sh
@ -2,15 +2,27 @@
 URL="$1"

 wb-manager init capture
-uwsgi uwsgi.ini &
+uwsgi uwsgi.ini &> /dev/null &

-/browser/browser_entrypoint.sh /browser/run.sh &
+#/browser/browser_entrypoint.sh /browser/run.sh &
+#if [[ -n "$PROXY_CA_FILE" && -f "$PROXY_CA_FILE" && -n "$PROXY_HOST" ]]; then
+#    rm -rf "$HOME/.pki/nssdb"
+#    mkdir -p "$HOME/.pki/nssdb"
+#    certutil -d "$HOME/.pki/nssdb" -N
+#    certutil -d "sql:$HOME/.pki/nssdb" -A -t "C,," -n "Proxy" -i "$PROXY_CA_FILE"
+#    rm "$PROXY_CA_FILE"
+#fi

-node index.js "$URL"
+#mkdir ~/.config/
+#mkdir ~/.config/google-chrome
+#touch ~/.config/google-chrome/First\ Run
+
+export QT_X11_NO_MITSHM=1
+
+node index.js "$@"

 NAME=${NAME:=zimfile}

-stat /output

 warc2zim --url $URL --name $NAME --output=/output ./collections/capture/archive/*.warc.gz

--- a/yarn.lock
+++ b/yarn.lock
@ -2,6 +2,11 @@
 # yarn lockfile v1


+"@types/color-name@^1.1.1":
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0"
+  integrity sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==
+
 "@types/node@*":
  version "14.11.1"
  resolved "https://registry.yarnpkg.com/@types/node/-/node-14.11.1.tgz#56af902ad157e763f9ba63d671c39cda3193c835"
@ -19,6 +24,19 @@ agent-base@5:
  resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-5.1.1.tgz#e8fb3f242959db44d63be665db7a8e739537a32c"
  integrity sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==

+ansi-regex@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
+  integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==
+
+ansi-styles@^4.0.0:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.2.1.tgz#90ae75c424d008d2624c5bf29ead3177ebfcf359"
+  integrity sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==
+  dependencies:
+    "@types/color-name" "^1.1.1"
+    color-convert "^2.0.1"
+
 balanced-match@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
@ -64,6 +82,27 @@ chownr@^1.1.1:
  resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
  integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==

+cliui@^7.0.0:
+  version "7.0.1"
+  resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.1.tgz#a4cb67aad45cd83d8d05128fc9f4d8fbb887e6b3"
+  integrity sha512-rcvHOWyGyid6I1WjT/3NatKj2kDt9OdSHSXpyLXaMWFbKpGACNW8pRhhdPUq9MWUOdwn8Rz9AVETjF4105rZZQ==
+  dependencies:
+    string-width "^4.2.0"
+    strip-ansi "^6.0.0"
+    wrap-ansi "^7.0.0"
+
+color-convert@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3"
+  integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==
+  dependencies:
+    color-name "~1.1.4"
+
+color-name@~1.1.4:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
+  integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
+
 concat-map@0.0.1:
  version "0.0.1"
  resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
@ -81,6 +120,11 @@ devtools-protocol@0.0.799653:
  resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.799653.tgz#86fc95ce5bf4fdf4b77a58047ba9d2301078f119"
  integrity sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg==

+emoji-regex@^8.0.0:
+  version "8.0.0"
+  resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
+  integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
+
 end-of-stream@^1.1.0, end-of-stream@^1.4.1:
  version "1.4.4"
  resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
@ -88,6 +132,11 @@ end-of-stream@^1.1.0, end-of-stream@^1.4.1:
  dependencies:
    once "^1.4.0"

+escalade@^3.0.2:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.0.tgz#e8e2d7c7a8b76f6ee64c2181d6b8151441602d4e"
+  integrity sha512-mAk+hPSO8fLDkhV7V0dXazH5pDc6MrjBTPyD3VeKzxnVFjH1MIxbCdqGZB9O8+EwWakZs3ZCbDS4IpRt79V1ig==
+
 extract-zip@^2.0.0:
  version "2.0.1"
  resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a"
@ -124,6 +173,11 @@ fs.realpath@^1.0.0:
  resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
  integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8=

+get-caller-file@^2.0.5:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
+  integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==
+
 get-stream@^5.1.0:
  version "5.2.0"
  resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3"
@ -169,6 +223,11 @@ inherits@2, inherits@^2.0.3, inherits@^2.0.4:
  resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
  integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==

+is-fullwidth-code-point@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d"
+  integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==
+
 locate-path@^5.0.0:
  version "5.0.0"
  resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0"
@ -264,6 +323,13 @@ pump@^3.0.0:
    end-of-stream "^1.1.0"
    once "^1.3.1"

+puppeteer-cluster@^0.22.0:
+  version "0.22.0"
+  resolved "https://registry.yarnpkg.com/puppeteer-cluster/-/puppeteer-cluster-0.22.0.tgz#4ab214671f414f15ad6a94a4b61ed0b4172e86e6"
+  integrity sha512-hmydtMwfVM+idFIDzS8OXetnujHGre7RY3BGL+3njy9+r8Dcu3VALkZHfuBEPf6byKssTCgzxU1BvLczifXd5w==
+  dependencies:
+    debug "^4.1.1"
+
 puppeteer-core@^5.3.0:
  version "5.3.0"
  resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-5.3.0.tgz#36d2e10132632c9cb73007f3f2880f4e6b655977"
@ -291,6 +357,11 @@ readable-stream@^3.1.1, readable-stream@^3.4.0:
    string_decoder "^1.1.1"
    util-deprecate "^1.0.1"

+require-directory@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
+  integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I=
+
 rimraf@^3.0.2:
  version "3.0.2"
  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a"
@ -303,6 +374,15 @@ safe-buffer@~5.2.0:
  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
  integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==

+string-width@^4.1.0, string-width@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5"
+  integrity sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==
+  dependencies:
+    emoji-regex "^8.0.0"
+    is-fullwidth-code-point "^3.0.0"
+    strip-ansi "^6.0.0"
+
 string_decoder@^1.1.1:
  version "1.3.0"
  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
@ -310,6 +390,13 @@ string_decoder@^1.1.1:
  dependencies:
    safe-buffer "~5.2.0"

+strip-ansi@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532"
+  integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==
+  dependencies:
+    ansi-regex "^5.0.0"
+
 tar-fs@^2.0.0:
  version "2.1.0"
  resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.0.tgz#d1cdd121ab465ee0eb9ccde2d35049d3f3daf0d5"
@ -349,6 +436,15 @@ util-deprecate@^1.0.1:
  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
  integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=

+wrap-ansi@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
+  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
+  dependencies:
+    ansi-styles "^4.0.0"
+    string-width "^4.1.0"
+    strip-ansi "^6.0.0"
+
 wrappy@1:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
@ -359,6 +455,29 @@ ws@^7.2.3:
  resolved "https://registry.yarnpkg.com/ws/-/ws-7.3.1.tgz#d0547bf67f7ce4f12a72dfe31262c68d7dc551c8"
  integrity sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==

+y18n@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.1.tgz#1ad2a7eddfa8bce7caa2e1f6b5da96c39d99d571"
+  integrity sha512-/jJ831jEs4vGDbYPQp4yGKDYPSCCEQ45uZWJHE1AoYBzqdZi8+LDWas0z4HrmJXmKdpFsTiowSHXdxyFhpmdMg==
+
+yargs-parser@^20.0.0:
+  version "20.0.0"
+  resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.0.0.tgz#c65a1daaa977ad63cebdd52159147b789a4e19a9"
+  integrity sha512-8eblPHTL7ZWRkyjIZJjnGf+TijiKJSwA24svzLRVvtgoi/RZiKa9fFQTrlx0OKLnyHSdt/enrdadji6WFfESVA==
+
+yargs@^16.0.3:
+  version "16.0.3"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.0.3.tgz#7a919b9e43c90f80d4a142a89795e85399a7e54c"
+  integrity sha512-6+nLw8xa9uK1BOEOykaiYAJVh6/CjxWXK/q9b5FpRgNslt8s22F2xMBqVIKgCRjNgGvGPBy8Vog7WN7yh4amtA==
+  dependencies:
+    cliui "^7.0.0"
+    escalade "^3.0.2"
+    get-caller-file "^2.0.5"
+    require-directory "^2.1.1"
+    string-width "^4.2.0"
+    y18n "^5.0.1"
+    yargs-parser "^20.0.0"
+
 yauzl@^2.10.0:
  version "2.10.0"
  resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"