fix(js): use pure JS SHA256 library, refactor (#471)

* fix(js): use pure JS SHA256 library, refactor

Closes #458

Additionally, I made a horrifying discovery: Firefox seems to actively
hinder performance if you are using more than one Worker per page. It
does not spread the load out across cores like I expected. Instead it
seems to make that one Worker thrash and have to constantly context
switch, which caused a lot of slowdown.

The benchmarks in #155 continue to be the best contribution ever made to
Anubis. What clued me into there being a problem here was the fact that
the "slow" algorithm was faster than the "fast" algorithm on my laptop.
This made no intuitive sense to me so I dug further.

Either way I think this is a Firefox bug at its core, but for now we
have to work around it by doing the hacky terrible thing that I hate.

I also swapped the SHA256 operations to @aws-crypto/sha256-js on the
advice of a trusted cryptography expert. I don't know what performance
differences this makes, but I'm getting 150-225 kilohashes per second,
which is pretty dang good.

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(js): apply suggestions from code review

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(js): use fast algo for fast worker

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

docs/docs/CHANGELOG.md

@@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Disable Open Graph passthrough by default ([#435](https://github.com/TecharoHQ/anubis/issues/435))
 - Clarify the license of the mascot images ([#442](https://github.com/TecharoHQ/anubis/issues/442))
 - Started Suppressing 'Context canceled' errors from http in the logs ([#446](https://github.com/TecharoHQ/anubis/issues/446))
+- Limit concurrency in Firefox and use a pure-JS SHA256 library to eke out more performance in proof of work checking
 
 ## v1.17.1: Asahi sas Brutus: Echo 1
 

package-lock.json

@@ -8,6 +8,9 @@
       "name": "@techaro/anubis",
       "version": "1.18.0-pre1",
       "license": "ISC",
+      "dependencies": {
+        "@aws-crypto/sha256-js": "^5.2.0"
+      },
       "devDependencies": {
         "cssnano": "^7.0.6",
         "cssnano-preset-advanced": "^7.0.6",
@@ -19,6 +22,44 @@
         "postcss-url": "^10.1.3"
       }
     },
+    "node_modules/@aws-crypto/sha256-js": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz",
+      "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@aws-crypto/util": "^5.2.0",
+        "@aws-sdk/types": "^3.222.0",
+        "tslib": "^2.6.2"
+      },
+      "engines": {
+        "node": ">=16.0.0"
+      }
+    },
+    "node_modules/@aws-crypto/util": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz",
+      "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@aws-sdk/types": "^3.222.0",
+        "@smithy/util-utf8": "^2.0.0",
+        "tslib": "^2.6.2"
+      }
+    },
+    "node_modules/@aws-sdk/types": {
+      "version": "3.804.0",
+      "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.804.0.tgz",
+      "integrity": "sha512-A9qnsy9zQ8G89vrPPlNG9d1d8QcKRGqJKqwyGgS0dclJpwy6d1EWgQLIolKPl6vcFpLoe6avLOLxr+h8ur5wpg==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@smithy/types": "^4.2.0",
+        "tslib": "^2.6.2"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
     "node_modules/@esbuild/aix-ppc64": {
       "version": "0.25.3",
       "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.3.tgz",
@@ -444,6 +485,56 @@
         "node": ">=18"
       }
     },
+    "node_modules/@smithy/is-array-buffer": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz",
+      "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "tslib": "^2.6.2"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
+    "node_modules/@smithy/types": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.2.0.tgz",
+      "integrity": "sha512-7eMk09zQKCO+E/ivsjQv+fDlOupcFUCSC/L2YUPgwhvowVGWbPQHjEFcmjt7QQ4ra5lyowS92SV53Zc6XD4+fg==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "tslib": "^2.6.2"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
+    "node_modules/@smithy/util-buffer-from": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz",
+      "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@smithy/is-array-buffer": "^2.2.0",
+        "tslib": "^2.6.2"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
+    "node_modules/@smithy/util-utf8": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz",
+      "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@smithy/util-buffer-from": "^2.2.0",
+        "tslib": "^2.6.2"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
     "node_modules/@trysound/sax": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz",
@@ -2599,6 +2690,12 @@
         "node": ">=8.0"
       }
     },
+    "node_modules/tslib": {
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+      "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
+      "license": "0BSD"
+    },
     "node_modules/universalify": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",

package.json

@@ -24,5 +24,8 @@
     "postcss-import": "^16.1.0",
     "postcss-import-url": "^7.2.0",
     "postcss-url": "^10.1.3"
+  },
+  "dependencies": {
+    "@aws-crypto/sha256-js": "^5.2.0"
   }
 }

web/build.sh

@@ -32,9 +32,9 @@ THE SOFTWARE.
 for the JavaScript code in this page.
 */'
 
-esbuild js/main.mjs --sourcemap --bundle --minify --outfile=static/js/main.mjs "--banner:js=${LICENSE}"
+for file in js/*.mjs js/worker/*.mjs; do
-gzip -f -k -n static/js/main.mjs
+  esbuild "${file}" --sourcemap --bundle --minify --outfile=static/"${file}" --banner:js="${LICENSE}"
-zstd -f -k --ultra -22 static/js/main.mjs
+  gzip -f -k -n static/${file}
-brotli -fZk static/js/main.mjs
+  zstd -f -k --ultra -22 static/${file}
-
+  brotli -fZk static/${file}
-esbuild js/bench.mjs --sourcemap --bundle --minify --outfile=static/js/bench.mjs
+done

web/js/algorithms/fast.mjs

@@ -0,0 +1,67 @@
+const determineThreadCount = () => {
+  if (navigator.userAgent.includes("Firefox")) {
+    return Math.min(navigator.hardwareConcurrency, 4);
+  }
+
+  if (!!navigator.hardwareConcurrency) {
+    return navigator.hardwareConcurrency;
+  }
+
+  return 1;
+};
+
+export default function process(
+  { basePrefix, version },
+  data,
+  difficulty = 5,
+  signal = null,
+  progressCallback = null,
+  threads = determineThreadCount(),
+) {
+  return new Promise((resolve, reject) => {
+    let webWorkerURL = `${basePrefix}/.within.website/x/cmd/anubis/static/js/worker/fast.mjs?cacheBuster=${version}`;
+
+    const workers = [];
+    const terminate = () => {
+      workers.forEach((w) => w.terminate());
+      if (signal != null) {
+        // clean up listener to avoid memory leak
+        signal.removeEventListener("abort", terminate);
+        if (signal.aborted) {
+          console.log("PoW aborted");
+          reject(false);
+        }
+      }
+    };
+    if (signal != null) {
+      signal.addEventListener("abort", terminate, { once: true });
+    }
+
+    for (let i = 0; i < threads; i++) {
+      let worker = new Worker(webWorkerURL);
+
+      worker.onmessage = (event) => {
+        if (typeof event.data === "number") {
+          progressCallback?.(event.data);
+        } else {
+          terminate();
+          resolve(event.data);
+        }
+      };
+
+      worker.onerror = (event) => {
+        terminate();
+        reject(event);
+      };
+
+      worker.postMessage({
+        data,
+        difficulty,
+        nonce: i,
+        threads,
+      });
+
+      workers.push(worker);
+    }
+  });
+}

web/js/algorithms/slow.mjs

@@ -0,0 +1,48 @@
+// https://dev.to/ratmd/simple-proof-of-work-in-javascript-3kgm
+
+export default function process(
+  { basePrefix, version },
+  data,
+  difficulty = 5,
+  signal = null,
+  progressCallback = null,
+  _threads = 1,
+) {
+  return new Promise((resolve, reject) => {
+    let worker = new Worker(`${basePrefix}/.within.website/x/cmd/anubis/static/js/worker/slow.mjs?cacheBuster=${version}`);
+    const terminate = () => {
+      worker.terminate();
+      if (signal != null) {
+        // clean up listener to avoid memory leak
+        signal.removeEventListener("abort", terminate);
+        if (signal.aborted) {
+          console.log("PoW aborted");
+          reject(false);
+        }
+      }
+    };
+    if (signal != null) {
+      signal.addEventListener("abort", terminate, { once: true });
+    }
+
+    worker.onmessage = (event) => {
+      if (typeof event.data === "number") {
+        progressCallback?.(event.data);
+      } else {
+        terminate();
+        resolve(event.data);
+      }
+    };
+
+    worker.onerror = (event) => {
+      terminate();
+      reject(event);
+    };
+
+    worker.postMessage({
+      data,
+      difficulty
+    });
+  });
+}
+

web/js/bench.mjs

@@ -1,11 +1,12 @@
-import processFast from "./proof-of-work.mjs";
+import processFast from "./algorithms/fast.mjs";
-import processSlow from "./proof-of-work-slow.mjs";
+import processSlow from "./algorithms/slow.mjs";
 
 const defaultDifficulty = 4;
 const algorithms = {
   fast: processFast,
   slow: processSlow,
 };
+const basePrefix = "";
 
 const status = document.getElementById("status");
 const difficultyInput = document.getElementById("difficulty-input");
@@ -42,7 +43,7 @@ const benchmarkTrial = async (stats, difficulty, algorithm, signal) => {
     .join("");
 
   const t0 = performance.now();
-  const { hash, nonce } = await process(challenge, Number(difficulty), signal);
+  const { hash, nonce } = await process({ basePrefix, version: "devel" }, challenge, Number(difficulty), signal);
   const t1 = performance.now();
   console.log({ hash, nonce });
 
@@ -114,6 +115,7 @@ const benchmarkLoop = async (controller) => {
   } catch (e) {
     if (e !== false) {
       status.innerText = e;
+      throw e;
     }
     return;
   }

web/js/main.mjs

@@ -1,6 +1,6 @@
-import processFast from "./proof-of-work.mjs";
+import processFast from "./algorithms/fast.mjs";
-import processSlow from "./proof-of-work-slow.mjs";
+import processSlow from "./algorithms/slow.mjs";
-import { testVideo } from "./video.mjs";
+import { testVideo } from "./tests/video.mjs";
 
 const algorithms = {
   "fast": processFast,
@@ -168,6 +168,7 @@ function showContinueBar(hash, nonce, t0, t1) {
   try {
     const t0 = Date.now();
     const { hash, nonce } = await process(
+      basePrefix,
       challenge,
       rules.difficulty,
       null,

web/js/proof-of-work-slow.mjs

@@ -1,90 +0,0 @@
-// https://dev.to/ratmd/simple-proof-of-work-in-javascript-3kgm
-
-export default function process(
-  data,
-  difficulty = 5,
-  signal = null,
-  progressCallback = null,
-  _threads = 1,
-) {
-  console.debug("slow algo");
-  return new Promise((resolve, reject) => {
-    let webWorkerURL = URL.createObjectURL(new Blob([
-      '(', processTask(), ')()'
-    ], { type: 'application/javascript' }));
-
-    let worker = new Worker(webWorkerURL);
-    const terminate = () => {
-      worker.terminate();
-      if (signal != null) {
-        // clean up listener to avoid memory leak
-        signal.removeEventListener("abort", terminate);
-        if (signal.aborted) {
-          console.log("PoW aborted");
-          reject(false);
-        }
-      }
-    };
-    if (signal != null) {
-      signal.addEventListener("abort", terminate, { once: true });
-    }
-
-    worker.onmessage = (event) => {
-      if (typeof event.data === "number") {
-        progressCallback?.(event.data);
-      } else {
-        terminate();
-        resolve(event.data);
-      }
-    };
-
-    worker.onerror = (event) => {
-      terminate();
-      reject(event);
-    };
-
-    worker.postMessage({
-      data,
-      difficulty
-    });
-
-    URL.revokeObjectURL(webWorkerURL);
-  });
-}
-
-function processTask() {
-  return function () {
-    const sha256 = (text) => {
-      const encoded = new TextEncoder().encode(text);
-      return crypto.subtle.digest("SHA-256", encoded.buffer)
-        .then((result) =>
-          Array.from(new Uint8Array(result))
-            .map((c) => c.toString(16).padStart(2, "0"))
-            .join(""),
-        );
-    };
-
-    addEventListener('message', async (event) => {
-      let data = event.data.data;
-      let difficulty = event.data.difficulty;
-
-      let hash;
-      let nonce = 0;
-      do {
-        if (nonce & 1023 === 0) {
-          postMessage(nonce);
-        }
-        hash = await sha256(data + nonce++);
-      } while (hash.substring(0, difficulty) !== Array(difficulty + 1).join('0'));
-
-      nonce -= 1; // last nonce was post-incremented
-
-      postMessage({
-        hash,
-        data,
-        difficulty,
-        nonce,
-      });
-    });
-  }.toString();
-}

web/js/proof-of-work.mjs

@@ -1,132 +0,0 @@
-export default function process(
-  data,
-  difficulty = 5,
-  signal = null,
-  progressCallback = null,
-  threads = (navigator.hardwareConcurrency || 1),
-) {
-  console.debug("fast algo");
-  return new Promise((resolve, reject) => {
-    let webWorkerURL = URL.createObjectURL(new Blob([
-      '(', processTask(), ')()'
-    ], { type: 'application/javascript' }));
-
-    const workers = [];
-    const terminate = () => {
-      workers.forEach((w) => w.terminate());
-      if (signal != null) {
-        // clean up listener to avoid memory leak
-        signal.removeEventListener("abort", terminate);
-        if (signal.aborted) {
-          console.log("PoW aborted");
-          reject(false);
-        }
-      }
-    };
-    if (signal != null) {
-      signal.addEventListener("abort", terminate, { once: true });
-    }
-
-    for (let i = 0; i < threads; i++) {
-      let worker = new Worker(webWorkerURL);
-
-      worker.onmessage = (event) => {
-        if (typeof event.data === "number") {
-          progressCallback?.(event.data);
-        } else {
-          terminate();
-          resolve(event.data);
-        }
-      };
-
-      worker.onerror = (event) => {
-        terminate();
-        reject(event);
-      };
-
-      worker.postMessage({
-        data,
-        difficulty,
-        nonce: i,
-        threads,
-      });
-
-      workers.push(worker);
-    }
-
-    URL.revokeObjectURL(webWorkerURL);
-  });
-}
-
-function processTask() {
-  return function () {
-    const sha256 = (text) => {
-      const encoded = new TextEncoder().encode(text);
-      return crypto.subtle.digest("SHA-256", encoded.buffer);
-    };
-
-    function uint8ArrayToHexString(arr) {
-      return Array.from(arr)
-        .map((c) => c.toString(16).padStart(2, "0"))
-        .join("");
-    }
-
-    addEventListener('message', async (event) => {
-      let data = event.data.data;
-      let difficulty = event.data.difficulty;
-      let hash;
-      let nonce = event.data.nonce;
-      let threads = event.data.threads;
-
-      const threadId = nonce;
-
-      while (true) {
-        const currentHash = await sha256(data + nonce);
-        const thisHash = new Uint8Array(currentHash);
-        let valid = true;
-
-        for (let j = 0; j < difficulty; j++) {
-          const byteIndex = Math.floor(j / 2); // which byte we are looking at
-          const nibbleIndex = j % 2; // which nibble in the byte we are looking at (0 is high, 1 is low)
-
-          let nibble = (thisHash[byteIndex] >> (nibbleIndex === 0 ? 4 : 0)) & 0x0F; // Get the nibble
-
-          if (nibble !== 0) {
-            valid = false;
-            break;
-          }
-        }
-
-        if (valid) {
-          hash = uint8ArrayToHexString(thisHash);
-          console.log(hash);
-          break;
-        }
-
-        const oldNonce = nonce;
-        nonce += threads;
-
-        // send a progress update every 1024 iterations. since each thread checks
-        // separate values, one simple way to do this is by bit masking the
-        // nonce for multiples of 1024. unfortunately, if the number of threads
-        // is not prime, only some of the threads will be sending the status
-        // update and they will get behind the others. this is slightly more
-        // complicated but ensures an even distribution between threads.
-        if (
-          nonce > oldNonce | 1023 && // we've wrapped past 1024
-          (nonce >> 10) % threads === threadId // and it's our turn
-        ) {
-          postMessage(nonce);
-        }
-      }
-
-      postMessage({
-        hash,
-        data,
-        difficulty,
-        nonce,
-      });
-    });
-  }.toString();
-}
-

web/js/worker/fast.mjs

@@ -0,0 +1,69 @@
+import { Sha256 } from '@aws-crypto/sha256-js';
+
+const sha256 = (text) => {
+  const hash = new Sha256();
+  hash.update(text);
+  return hash.digest();
+};
+
+function uint8ArrayToHexString(arr) {
+  return Array.from(arr)
+    .map((c) => c.toString(16).padStart(2, "0"))
+    .join("");
+}
+
+addEventListener('message', async (event) => {
+  let data = event.data.data;
+  let difficulty = event.data.difficulty;
+  let hash;
+  let nonce = event.data.nonce;
+  let threads = event.data.threads;
+
+  const threadId = nonce;
+
+  while (true) {
+    const currentHash = await sha256(data + nonce);
+    const thisHash = new Uint8Array(currentHash);
+    let valid = true;
+
+    for (let j = 0; j < difficulty; j++) {
+      const byteIndex = Math.floor(j / 2); // which byte we are looking at
+      const nibbleIndex = j % 2; // which nibble in the byte we are looking at (0 is high, 1 is low)
+
+      let nibble = (thisHash[byteIndex] >> (nibbleIndex === 0 ? 4 : 0)) & 0x0F; // Get the nibble
+
+      if (nibble !== 0) {
+        valid = false;
+        break;
+      }
+    }
+
+    if (valid) {
+      hash = uint8ArrayToHexString(thisHash);
+      break;
+    }
+
+    const oldNonce = nonce;
+    nonce += threads;
+
+    // send a progress update every 1024 iterations. since each thread checks
+    // separate values, one simple way to do this is by bit masking the
+    // nonce for multiples of 1024. unfortunately, if the number of threads
+    // is not prime, only some of the threads will be sending the status
+    // update and they will get behind the others. this is slightly more
+    // complicated but ensures an even distribution between threads.
+    if (
+      nonce > oldNonce | 1023 && // we've wrapped past 1024
+      (nonce >> 10) % threads === threadId // and it's our turn
+    ) {
+      postMessage(nonce);
+    }
+  }
+
+  postMessage({
+    hash,
+    data,
+    difficulty,
+    nonce,
+  });
+});

web/js/worker/slow.mjs

@@ -0,0 +1,35 @@
+import { Sha256 } from '@aws-crypto/sha256-js';
+
+const sha256 = (text) => {
+  const hash = new Sha256();
+  hash.update(text);
+  return hash.digest()
+    .then((result) =>
+      Array.from(new Uint8Array(result))
+        .map((c) => c.toString(16).padStart(2, "0"))
+        .join(""),
+    );
+};
+
+addEventListener('message', async (event) => {
+  let data = event.data.data;
+  let difficulty = event.data.difficulty;
+
+  let hash;
+  let nonce = 0;
+  do {
+    if ((nonce & 1023) === 0) {
+      postMessage(nonce);
+    }
+    hash = await sha256(data + nonce++);
+  } while (hash.substring(0, difficulty) !== Array(difficulty + 1).join('0'));
+
+  nonce -= 1; // last nonce was post-incremented
+
+  postMessage({
+    hash,
+    data,
+    difficulty,
+    nonce,
+  });
+});