kiwix/kiwix-js
1
0
Fork 0
mirror of https://github.com/kiwix/kiwix-js.git synced 2025-08-03 11:16:38 -04:00
Code Issues Packages Projects Releases Wiki Activity

Recompile Emscripten binaries, implement WASM with fallback, add API info, add compile scripts (#740)

Browse Source 
Fixes #739, #652 and #511. Instructions and helper scripts are included for easy future recompiling of the binaries with docker.
This commit is contained in:
Jaifroid 2021-08-22 08:54:47 +01:00 committed by GitHub
parent bd0c296198
commit 8579bbeb72
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 595 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitattributes vendored
View File

@ -40,3 +40,4 @@
*.woff binary
*.woff2 binary
*.deb binary
*.wasm binary

6
emscripten/xzdec/compile.sh
View File

@ -1 +1,5 @@
emcc --memory-init-file 0 -O3 -s WASM=0 -s MALLOC="emmalloc" -s TOTAL_MEMORY=83886080 -s NO_FILESYSTEM=1 -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s DOUBLE_MODE=0 -s PRECISE_I64_MATH=0 -s NO_DYNAMIC_EXECUTION=1 -s LEGACY_VM_SUPPORT=1 -s EXPORTED_FUNCTIONS="['_init', '_init_decompression', '_input_empty', '_get_in_buffer', '_set_new_input', '_decompress', '_get_out_pos', '_get_out_buffer', '_out_buffer_cleared', '_release']" -DXZ_USE_CRC64=1 -DXZ_INTERNAL_CRC64=1 *.c -o xzdec.js
echo "Compiling ASM version xzdec-asm.js"
emcc --memory-init-file 0 -O3 --closure 1 -s ENVIRONMENT="web" -s WASM=0 -s MALLOC="emmalloc" -s TOTAL_MEMORY=83886080 -s FILESYSTEM=0 -s DOUBLE_MODE=0 -s DYNAMIC_EXECUTION=0 -s MIN_IE_VERSION=11 -s EXPORT_NAME="XZ" -s MODULARIZE=1 -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s EXPORTED_FUNCTIONS="['_init', '_init_decompression', '_input_empty', '_get_in_buffer', '_set_new_input', '_decompress', '_get_out_pos', '_get_out_buffer', '_out_buffer_cleared', '_release']" -DXZ_USE_CRC64=1 -DXZ_INTERNAL_CRC64=1 ./*.c -o xzdec-asm.js
echo "Compiling WASM version xzdec-wasm.js"
emcc --memory-init-file 0 -O3 --closure 1 -s ENVIRONMENT="web" -s WASM=1 -s MALLOC="emmalloc" -s TOTAL_MEMORY=83886080 -s FILESYSTEM=0 -s DOUBLE_MODE=0 -s DYNAMIC_EXECUTION=0 -s EXPORT_NAME="XZ" -s MODULARIZE=1 -s AGGRESSIVE_VARIABLE_ELIMINATION=1 -s EXPORTED_FUNCTIONS="['_init', '_init_decompression', '_input_empty', '_get_in_buffer', '_set_new_input', '_decompress', '_get_out_pos', '_get_out_buffer', '_out_buffer_cleared', '_release']" -DXZ_USE_CRC64=1 -DXZ_INTERNAL_CRC64=1 ./*.c -o xzdec-wasm.js
echo "Finished."

54
emscripten/xzdec/xzdec-asm.js Normal file
View File

File diff suppressed because one or more lines are too long

32
emscripten/xzdec/xzdec-wasm.js Normal file
View File

@ -0,0 +1,32 @@
var XZ = (function() {
var _scriptDir = typeof document !== 'undefined' && document.currentScript ? document.currentScript.src : undefined;
return (
function(XZ) {
XZ = XZ || {};
var a;a||(a=typeof XZ !== 'undefined' ? XZ : {});var e,f;a.ready=new Promise(function(b,c){e=b;f=c});var h={},k;for(k in a)a.hasOwnProperty(k)&&(h[k]=a[k]);var m="";"undefined"!==typeof document&&document.currentScript&&(m=document.currentScript.src);_scriptDir&&(m=_scriptDir);0!==m.indexOf("blob:")?m=m.substr(0,m.lastIndexOf("/")+1):m="";var n=a.printErr||console.warn.bind(console);for(k in h)h.hasOwnProperty(k)&&(a[k]=h[k]);h=null;var p;a.wasmBinary&&(p=a.wasmBinary);
var noExitRuntime=a.noExitRuntime||!0;"object"!==typeof WebAssembly&&q("no native wasm support detected");var r,t=!1,u,v,w=[],x=[],z=[];function A(){var b=a.preRun.shift();w.unshift(b)}var B=0,C=null,D=null;a.preloadedImages={};a.preloadedAudios={};function q(b){if(a.onAbort)a.onAbort(b);n(b);t=!0;b=new WebAssembly.RuntimeError("abort("+b+"). Build with -s ASSERTIONS=1 for more info.");f(b);throw b;}function E(){return F.startsWith("data:application/octet-stream;base64,")}var F;F="xzdec-wasm.wasm";
if(!E()){var G=F;F=a.locateFile?a.locateFile(G,m):m+G}function H(){var b=F;try{if(b==F&&p)return new Uint8Array(p);throw"both async and sync fetching of the wasm failed";}catch(c){q(c)}}function I(){return p||"function"!==typeof fetch?Promise.resolve().then(function(){return H()}):fetch(F,{credentials:"same-origin"}).then(function(b){if(!b.ok)throw"failed to load wasm binary file at '"+F+"'";return b.arrayBuffer()}).catch(function(){return H()})}
function J(b){for(;0<b.length;){var c=b.shift();if("function"==typeof c)c(a);else{var g=c.u;"number"===typeof g?void 0===c.s?v.get(g)():v.get(g)(c.s):g(void 0===c.s?null:c.s)}}}var K={a:function(b,c,g){u.copyWithin(b,c,c+g)},b:function(){q("OOM")}};
(function(){function b(d){a.asm=d.exports;r=a.asm.c;d=r.buffer;a.HEAP8=new Int8Array(d);a.HEAP16=new Int16Array(d);a.HEAP32=new Int32Array(d);a.HEAPU8=u=new Uint8Array(d);a.HEAPU16=new Uint16Array(d);a.HEAPU32=new Uint32Array(d);a.HEAPF32=new Float32Array(d);a.HEAPF64=new Float64Array(d);v=a.asm.o;x.unshift(a.asm.d);B--;a.monitorRunDependencies&&a.monitorRunDependencies(B);0==B&&(null!==C&&(clearInterval(C),C=null),D&&(d=D,D=null,d()))}function c(d){b(d.instance)}function g(d){return I().then(function(l){return WebAssembly.instantiate(l,
y)}).then(d,function(l){n("failed to asynchronously prepare wasm: "+l);q(l)})}var y={a:K};B++;a.monitorRunDependencies&&a.monitorRunDependencies(B);if(a.instantiateWasm)try{return a.instantiateWasm(y,b)}catch(d){return n("Module.instantiateWasm callback failed with error: "+d),!1}(function(){return p||"function"!==typeof WebAssembly.instantiateStreaming||E()||"function"!==typeof fetch?g(c):fetch(F,{credentials:"same-origin"}).then(function(d){return WebAssembly.instantiateStreaming(d,y).then(c,function(l){n("wasm streaming compile failed: "+
l);n("falling back to ArrayBuffer instantiation");return g(c)})})})().catch(f);return{}})();a.___wasm_call_ctors=function(){return(a.___wasm_call_ctors=a.asm.d).apply(null,arguments)};a._init=function(){return(a._init=a.asm.e).apply(null,arguments)};a._init_decompression=function(){return(a._init_decompression=a.asm.f).apply(null,arguments)};a._input_empty=function(){return(a._input_empty=a.asm.g).apply(null,arguments)};a._get_in_buffer=function(){return(a._get_in_buffer=a.asm.h).apply(null,arguments)};
a._set_new_input=function(){return(a._set_new_input=a.asm.i).apply(null,arguments)};a._decompress=function(){return(a._decompress=a.asm.j).apply(null,arguments)};a._get_out_pos=function(){return(a._get_out_pos=a.asm.k).apply(null,arguments)};a._get_out_buffer=function(){return(a._get_out_buffer=a.asm.l).apply(null,arguments)};a._out_buffer_cleared=function(){return(a._out_buffer_cleared=a.asm.m).apply(null,arguments)};a._release=function(){return(a._release=a.asm.n).apply(null,arguments)};var L;
D=function M(){L||N();L||(D=M)};
function N(){function b(){if(!L&&(L=!0,a.calledRun=!0,!t)){J(x);e(a);if(a.onRuntimeInitialized)a.onRuntimeInitialized();if(a.postRun)for("function"==typeof a.postRun&&(a.postRun=[a.postRun]);a.postRun.length;){var c=a.postRun.shift();z.unshift(c)}J(z)}}if(!(0<B)){if(a.preRun)for("function"==typeof a.preRun&&(a.preRun=[a.preRun]);a.preRun.length;)A();J(w);0<B||(a.setStatus?(a.setStatus("Running..."),setTimeout(function(){setTimeout(function(){a.setStatus("")},1);b()},1)):b())}}a.run=N;
if(a.preInit)for("function"==typeof a.preInit&&(a.preInit=[a.preInit]);0<a.preInit.length;)a.preInit.pop()();N();
return XZ.ready
}
);
})();
if (typeof exports === 'object' && typeof module === 'object')
module.exports = XZ;
else if (typeof define === 'function' && define['amd'])
define([], function() { return XZ; });
else if (typeof exports === 'object')
exports["XZ"] = XZ;

BIN
emscripten/xzdec/xzdec-wasm.wasm Executable file
View File

Binary file not shown.

16
emscripten/xzdec/xzdec.js
View File

File diff suppressed because one or more lines are too long

BIN
emscripten/xzdec/xzdec.js.mem
View File

Binary file not shown.

12
emscripten/xzdec/xzdec_wrapper.js
View File

@ -1,12 +0,0 @@
var dec = require('./xzdec.js');
dec._init();
module.exports.decompress = function(data, callback)
{
var mem = dec._malloc(data.length);
dec.writeArrayToMemory(data, mem);
dec._uncompress(mem, data.length);
var uncompressed = dec.Pointer_stringify(dec._getUncompressedBuffer(), dec._getUncompressedLength());
dec._free(mem);
callback(uncompressed);
};

6
emscripten/zstandard/compile.sh
View File

@ -1 +1,5 @@
emcc --memory-init-file 0 -O3 --closure 1 -s ENVIRONMENT="web" -s WASM=0 -s MALLOC="emmalloc" -s TOTAL_MEMORY=140247040 -s FILESYSTEM=0 -s DOUBLE_MODE=0 -s DYNAMIC_EXECUTION=0 -s MIN_IE_VERSION=11 -s EXPORT_NAME="ZD" -s MODULARIZE=1 -s EXPORTED_FUNCTIONS="['_malloc', '_free', '_ZSTD_createDStream', '_ZSTD_initDStream', '_ZSTD_decompressStream', '_ZSTD_isError', '_ZSTD_getErrorName', '_ZSTD_freeDStream', '_ZSTD_DStreamInSize', '_ZSTD_DStreamOutSize']" -s EXPORTED_RUNTIME_METHODS="['cwrap']" *.c -o zstddec.js
echo "Compiling ASM version zstddec-asm.js"
emcc --memory-init-file 0 -O3 --closure 1 -s ENVIRONMENT="web" -s WASM=0 -s MALLOC="emmalloc" -s TOTAL_MEMORY=150994944 -s FILESYSTEM=0 -s DOUBLE_MODE=0 -s DYNAMIC_EXECUTION=0 -s MIN_IE_VERSION=11 -s EXPORT_NAME="ZD" -s MODULARIZE=1 -s EXPORTED_FUNCTIONS="['_malloc', '_free', '_ZSTD_createDStream', '_ZSTD_initDStream', '_ZSTD_decompressStream', '_ZSTD_isError', '_ZSTD_getErrorName', '_ZSTD_freeDStream', '_ZSTD_DStreamInSize', '_ZSTD_DStreamOutSize']" -s EXPORTED_RUNTIME_METHODS="['cwrap']" ./*.c -o zstddec-asm.js
echo "Compiling WASM version zstddec-wasm.js"
emcc --memory-init-file 0 -O3 --closure 1 -s ENVIRONMENT="web" -s WASM=1 -s MALLOC="emmalloc" -s TOTAL_MEMORY=150994944 -s FILESYSTEM=0 -s DOUBLE_MODE=0 -s DYNAMIC_EXECUTION=0 -s EXPORT_NAME="ZD" -s MODULARIZE=1 -s EXPORTED_FUNCTIONS="['_malloc', '_free', '_ZSTD_createDStream', '_ZSTD_initDStream', '_ZSTD_decompressStream', '_ZSTD_isError', '_ZSTD_getErrorName', '_ZSTD_freeDStream', '_ZSTD_DStreamInSize', '_ZSTD_DStreamOutSize']" -s EXPORTED_RUNTIME_METHODS="['cwrap']" ./*.c -o zstddec-wasm.js
echo "Finished."

2
emscripten/zstandard/readme.md
View File

@ -5,4 +5,6 @@ It has been combined by following the instructions at https://github.com/faceboo
The source code can be compiled with Emscripten by setting up the Emscripten SDK and running the Linux script `./compile.sh` in this directory. This will produce `zstddec.js`. Further optimization and minification of this file may be possible by adjusting the commandline options in `compile.sh`.
You may be able to compile easily using docker. Look in the `/scripts` directory for helper scripts for your platform.
For a fuller Emscripten distribution of the full zstandard library for `node.js`, see https://github.com/yoshihitoh/zstd-codec/tree/develop/js/example.

58
emscripten/zstandard/zstddec-asm.js Normal file
View File

File diff suppressed because one or more lines are too long

36
emscripten/zstandard/zstddec-wasm.js Normal file
View File

@ -0,0 +1,36 @@
var ZD = (function() {
var _scriptDir = typeof document !== 'undefined' && document.currentScript ? document.currentScript.src : undefined;
return (
function(ZD) {
ZD = ZD || {};
var a;a||(a=typeof ZD !== 'undefined' ? ZD : {});var q,r;a.ready=new Promise(function(b,c){q=b;r=c});var t={},w;for(w in a)a.hasOwnProperty(w)&&(t[w]=a[w]);var x="";"undefined"!==typeof document&&document.currentScript&&(x=document.currentScript.src);_scriptDir&&(x=_scriptDir);0!==x.indexOf("blob:")?x=x.substr(0,x.lastIndexOf("/")+1):x="";var y=a.printErr||console.warn.bind(console);for(w in t)t.hasOwnProperty(w)&&(a[w]=t[w]);t=null;var z;a.wasmBinary&&(z=a.wasmBinary);
var noExitRuntime=a.noExitRuntime||!0;"object"!==typeof WebAssembly&&A("no native wasm support detected");var B,C=!1;function D(b){var c=a["_"+b];c||A("Assertion failed: Cannot call unknown function "+(b+", make sure it is exported"));return c}
function aa(b,c,g,n){var e={string:function(d){var m=0;if(null!==d&&void 0!==d&&0!==d){var l=(d.length<<2)+1;m=E(l);var h=m,f=F;if(0<l){l=h+l-1;for(var u=0;u<d.length;++u){var k=d.charCodeAt(u);if(55296<=k&&57343>=k){var ba=d.charCodeAt(++u);k=65536+((k&1023)<<10)|ba&1023}if(127>=k){if(h>=l)break;f[h++]=k}else{if(2047>=k){if(h+1>=l)break;f[h++]=192|k>>6}else{if(65535>=k){if(h+2>=l)break;f[h++]=224|k>>12}else{if(h+3>=l)break;f[h++]=240|k>>18;f[h++]=128|k>>12&63}f[h++]=128|k>>6&63}f[h++]=128|k&63}}f[h]=
0}}return m},array:function(d){var m=E(d.length);H.set(d,m);return m}},p=D(b),G=[];b=0;if(n)for(var v=0;v<n.length;v++){var Q=e[g[v]];Q?(0===b&&(b=I()),G[v]=Q(n[v])):G[v]=n[v]}g=p.apply(null,G);g=function(d){if("string"===c)if(d){for(var m=F,l=d+NaN,h=d;m[h]&&!(h>=l);)++h;if(16<h-d&&m.subarray&&J)d=J.decode(m.subarray(d,h));else{for(l="";d<h;){var f=m[d++];if(f&128){var u=m[d++]&63;if(192==(f&224))l+=String.fromCharCode((f&31)<<6|u);else{var k=m[d++]&63;f=224==(f&240)?(f&15)<<12|u<<6|k:(f&7)<<18|
u<<12|k<<6|m[d++]&63;65536>f?l+=String.fromCharCode(f):(f-=65536,l+=String.fromCharCode(55296|f>>10,56320|f&1023))}}else l+=String.fromCharCode(f)}d=l}}else d="";else d="boolean"===c?!!d:d;return d}(g);0!==b&&K(b);return g}var J="undefined"!==typeof TextDecoder?new TextDecoder("utf8"):void 0,H,F,L,M=[],N=[],O=[];function ca(){var b=a.preRun.shift();M.unshift(b)}var P=0,R=null,S=null;a.preloadedImages={};a.preloadedAudios={};
function A(b){if(a.onAbort)a.onAbort(b);y(b);C=!0;b=new WebAssembly.RuntimeError("abort("+b+"). Build with -s ASSERTIONS=1 for more info.");r(b);throw b;}function T(){return U.startsWith("data:application/octet-stream;base64,")}var U;U="zstddec-wasm.wasm";if(!T()){var V=U;U=a.locateFile?a.locateFile(V,x):x+V}function W(){var b=U;try{if(b==U&&z)return new Uint8Array(z);throw"both async and sync fetching of the wasm failed";}catch(c){A(c)}}
function da(){return z||"function"!==typeof fetch?Promise.resolve().then(function(){return W()}):fetch(U,{credentials:"same-origin"}).then(function(b){if(!b.ok)throw"failed to load wasm binary file at '"+U+"'";return b.arrayBuffer()}).catch(function(){return W()})}function X(b){for(;0<b.length;){var c=b.shift();if("function"==typeof c)c(a);else{var g=c.u;"number"===typeof g?void 0===c.s?L.get(g)():L.get(g)(c.s):g(void 0===c.s?null:c.s)}}}var ea={a:function(b,c,g){F.copyWithin(b,c,c+g)},b:function(){A("OOM")}};
(function(){function b(e){a.asm=e.exports;B=a.asm.c;e=B.buffer;a.HEAP8=H=new Int8Array(e);a.HEAP16=new Int16Array(e);a.HEAP32=new Int32Array(e);a.HEAPU8=F=new Uint8Array(e);a.HEAPU16=new Uint16Array(e);a.HEAPU32=new Uint32Array(e);a.HEAPF32=new Float32Array(e);a.HEAPF64=new Float64Array(e);L=a.asm.o;N.unshift(a.asm.d);P--;a.monitorRunDependencies&&a.monitorRunDependencies(P);0==P&&(null!==R&&(clearInterval(R),R=null),S&&(e=S,S=null,e()))}function c(e){b(e.instance)}function g(e){return da().then(function(p){return WebAssembly.instantiate(p,
n)}).then(e,function(p){y("failed to asynchronously prepare wasm: "+p);A(p)})}var n={a:ea};P++;a.monitorRunDependencies&&a.monitorRunDependencies(P);if(a.instantiateWasm)try{return a.instantiateWasm(n,b)}catch(e){return y("Module.instantiateWasm callback failed with error: "+e),!1}(function(){return z||"function"!==typeof WebAssembly.instantiateStreaming||T()||"function"!==typeof fetch?g(c):fetch(U,{credentials:"same-origin"}).then(function(e){return WebAssembly.instantiateStreaming(e,n).then(c,function(p){y("wasm streaming compile failed: "+
p);y("falling back to ArrayBuffer instantiation");return g(c)})})})().catch(r);return{}})();a.___wasm_call_ctors=function(){return(a.___wasm_call_ctors=a.asm.d).apply(null,arguments)};a._malloc=function(){return(a._malloc=a.asm.e).apply(null,arguments)};a._free=function(){return(a._free=a.asm.f).apply(null,arguments)};a._ZSTD_isError=function(){return(a._ZSTD_isError=a.asm.g).apply(null,arguments)};a._ZSTD_getErrorName=function(){return(a._ZSTD_getErrorName=a.asm.h).apply(null,arguments)};
a._ZSTD_createDStream=function(){return(a._ZSTD_createDStream=a.asm.i).apply(null,arguments)};a._ZSTD_freeDStream=function(){return(a._ZSTD_freeDStream=a.asm.j).apply(null,arguments)};a._ZSTD_DStreamInSize=function(){return(a._ZSTD_DStreamInSize=a.asm.k).apply(null,arguments)};a._ZSTD_DStreamOutSize=function(){return(a._ZSTD_DStreamOutSize=a.asm.l).apply(null,arguments)};a._ZSTD_initDStream=function(){return(a._ZSTD_initDStream=a.asm.m).apply(null,arguments)};
a._ZSTD_decompressStream=function(){return(a._ZSTD_decompressStream=a.asm.n).apply(null,arguments)};var I=a.stackSave=function(){return(I=a.stackSave=a.asm.p).apply(null,arguments)},K=a.stackRestore=function(){return(K=a.stackRestore=a.asm.q).apply(null,arguments)},E=a.stackAlloc=function(){return(E=a.stackAlloc=a.asm.r).apply(null,arguments)};a.cwrap=function(b,c,g,n){g=g||[];var e=g.every(function(p){return"number"===p});return"string"!==c&&e&&!n?D(b):function(){return aa(b,c,g,arguments)}};var Y;
S=function fa(){Y||Z();Y||(S=fa)};
function Z(){function b(){if(!Y&&(Y=!0,a.calledRun=!0,!C)){X(N);q(a);if(a.onRuntimeInitialized)a.onRuntimeInitialized();if(a.postRun)for("function"==typeof a.postRun&&(a.postRun=[a.postRun]);a.postRun.length;){var c=a.postRun.shift();O.unshift(c)}X(O)}}if(!(0<P)){if(a.preRun)for("function"==typeof a.preRun&&(a.preRun=[a.preRun]);a.preRun.length;)ca();X(M);0<P||(a.setStatus?(a.setStatus("Running..."),setTimeout(function(){setTimeout(function(){a.setStatus("")},1);b()},1)):b())}}a.run=Z;
if(a.preInit)for("function"==typeof a.preInit&&(a.preInit=[a.preInit]);0<a.preInit.length;)a.preInit.pop()();Z();
return ZD.ready
}
);
})();
if (typeof exports === 'object' && typeof module === 'object')
module.exports = ZD;
else if (typeof define === 'function' && define['amd'])
define([], function() { return ZD; });
else if (typeof exports === 'object')
exports["ZD"] = ZD;

BIN
emscripten/zstandard/zstddec-wasm.wasm Executable file
View File

Binary file not shown.

66
emscripten/zstandard/zstddec_optimized.js
View File

File diff suppressed because one or more lines are too long

14
scripts/Compile-Xzdec.ps1 Normal file
View File

@ -0,0 +1,14 @@
# This PowerShell script (principally for Windows 10/11) compiles the legacy xzdec.js using docker according to the values set in
# /emscripten/xzdec/compile.sh . Please be sure to review the commandline in compile.sh before running this script, to
# be sure it's doing what you want. If you want to compile with a later version of emnsdk, specify it below after 'emsdk:'.
#
# Prerequisites:
#
# * Ensure WSL 2 is enabled in Windows (WSL = Windows Subsystem for Linux)
# * Install docker desktop on Windows: choose the WSL 2 docker backend (not the legacy Hyper-V one)
# * Open docker and ensure the docker daemon is running correctly (icon should be green)
# * Ensure docker desktop is signed in to your dockerhub account (at least the first time you run this script)
# * Run this script by opening a PowerShell terminal, cd to the kiwix-js repo and run './scripts/Compile-Zstddec.ps1'
$repo_dir = ($PSScriptRoot -replace '[\\/]scripts[\\/]*$', '')
docker container run -v $repo_dir\:/project -w /project emscripten/emsdk:2.0.25 /bin/sh -c 'cd emscripten/xzdec/; ./compile.sh'

14
scripts/Compile-Zstddec.ps1 Normal file
View File

@ -0,0 +1,14 @@
# This PowerShell script (principally for Windows 10/11) compiles zstddec.js using docker according to the values set in
# /emscripten/ztandard/compile.sh . Please be sure to review the commandline in compile.sh before running this script, to
# be sure it's doing what you want. If you want to compile with a later version of emnsdk, specify it below after 'emsdk:'.
#
# Prerequisites:
#
# * Ensure WSL 2 is enabled in Windows (WSL = Windows Subsystem for Linux)
# * Install docker desktop on Windows: choose the WSL 2 docker backend (not the legacy Hyper-V one)
# * Open docker and ensure the docker daemon is running correctly (icon should be green)
# * Ensure docker desktop is signed in to your dockerhub account (at least the first time you run this script)
# * Run this script by opening a PowerShell terminal, cd to the kiwix-js repo and run './scripts/Compile-Zstddec.ps1'
$repo_dir = ($PSScriptRoot -replace '[\\/]scripts[\\/]*$', '')
docker container run -v $repo_dir\:/project -w /project emscripten/emsdk:2.0.25 /bin/sh -c 'cd emscripten/zstandard/; ./compile.sh'

17
scripts/compile_xzdec_with_docker_emscripten.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
# This bash script compiles xzdec.js using emscripten docker image, according to the values set in
# /emscripten/xzdec/compile.sh . Please be sure to review the commandline in compile.sh before running this script, to
# be sure it's doing what you want.
#
# Prerequisites:
#
# * Ensure Docker is installed and its daemon is running
# * If the current user is not in the docker group, you might need to run this script with sudo. See https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user
# Find the repo dir (it's the parent of the dir that contains this script)
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
REPO_DIR="$(dirname "$SCRIPT_DIR")"
# Run the emscripten compilation through docker
docker run -v "$REPO_DIR":/project -w /project -u $(id -u):$(id -g) emscripten/emsdk:2.0.25 /bin/sh -c 'cd emscripten/xzdec/; sh ./compile.sh'

17
scripts/compile_zstddec_with_docker_emscripten.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
# This bash script compiles zstddec.js using emscripten docker image, according to the values set in
# /emscripten/ztandard/compile.sh . Please be sure to review the commandline in compile.sh before running this script, to
# be sure it's doing what you want.
#
# Prerequisites:
#
# * Ensure Docker is installed and its daemon is running
# * If the current user is not in the docker group, you might need to run this script with sudo. See https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user
# Find the repo dir (it's the parent of the dir that contains this script)
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
REPO_DIR="$(dirname "$SCRIPT_DIR")"
# Run the emscripten compilation through docker
docker run -v "$REPO_DIR":/project -w /project -u $(id -u):$(id -g) emscripten/emsdk:2.0.25 /bin/sh -c 'cd emscripten/zstandard/; sh ./compile.sh'

6
www/css/app.css
View File

@ -125,10 +125,14 @@ iframe._invert, iframe._mwInvert {
color : green;
}
.apiUnavailable {
.apiUnavailable, .apiSuboptimal {
color : orange;
}
.apiBroken {
color : red;
}
.loader {
position: absolute;
margin: 0 auto;

1
www/index.html
View File

@ -551,6 +551,7 @@
<div id="serviceWorkerStatus"></div>
<div id="messageChannelStatus"></div>
<div id="settingsStoreStatus"></div>
<div id="decompressorAPIStatus"></div>
</div>
</div>
</div>

13
www/js/app.js
View File

@ -428,7 +428,7 @@ define(['jquery', 'zimArchiveLoader', 'uiUtil', 'settingsStore','abstractFilesys
*/
function refreshAPIStatus() {
var apiStatusPanel = document.getElementById('apiStatusDiv');
apiStatusPanel.classList.remove('card-success', 'card-warning');
apiStatusPanel.classList.remove('card-success', 'card-warning', 'card-danger');
var apiPanelClass = 'card-success';
if (isMessageChannelAvailable()) {
$('#messageChannelStatus').html("MessageChannel API available");
@ -464,6 +464,17 @@ define(['jquery', 'zimArchiveLoader', 'uiUtil', 'settingsStore','abstractFilesys
settingsStoreStatusDiv.classList.remove('apiAvailable', 'apiUnavailable');
settingsStoreStatusDiv.classList.add(params.storeType === 'none' ? 'apiUnavailable' : 'apiAvailable');
apiPanelClass = params.storeType === 'none' ? 'card-warning' : apiPanelClass;
// Update Decompressor API section of panel
var decompAPIStatusDiv = document.getElementById('decompressorAPIStatus');
apiName = params.decompressorAPI.assemblerMachineType;
apiPanelClass = params.decompressorAPI.errorStatus ? 'card-danger' : apiName === 'WASM' ? apiPanelClass : 'card-warning';
decompAPIStatusDiv.className = apiName ? params.decompressorAPI.errorStatus ? 'apiBroken' : apiName === 'WASM' ? 'apiAvailable' : 'apiSuboptimal' : 'apiUnavailable';
// Add the last used decompressor, if known, to the apiName
if (apiName && params.decompressorAPI.decompressorLastUsed) {
apiName += ' [&nbsp;' + params.decompressorAPI.decompressorLastUsed + '&nbsp;]';
}
apiName = params.decompressorAPI.errorStatus || apiName || 'Not initialized';
decompAPIStatusDiv.innerHTML = 'Decompressor API: ' + apiName;
// Add a warning colour to the API Status Panel if any of the above tests failed
apiStatusPanel.classList.add(apiPanelClass);

14
www/js/lib/uiUtil.js
View File

@ -429,6 +429,17 @@ define(rqDef, function() {
});
}
// Reports an error in loading one of the ASM or WASM machines to the UI API Status Panel
// This can't be done in app.js because the error occurs after the API panel is first displayed
function reportAssemblerErrorToAPIStatusPanel(decoderType, error) {
console.error('Could not instantiate any ' + decoderType + ' decoder!', error);
params.decompressorAPI.errorStatus = 'Error loading ' + decoderType + ' decompressor!';
var decompAPI = document.getElementById('decompressorAPIStatus');
decompAPI.innerHTML = 'Decompressor API: ' + params.decompressorAPI.errorStatus;
decompAPI.className = 'apiBroken';
document.getElementById('apiStatusDiv').className = 'card card-danger';
}
// If global variable webpMachine is true (set in init.js), then we need to initialize the WebP Polyfill
if (webpMachine) webpMachine = new webpHero.WebpMachine();
@ -446,6 +457,7 @@ define(rqDef, function() {
htmlEscapeChars: htmlEscapeChars,
removeAnimationClasses: removeAnimationClasses,
applyAnimationToSection: applyAnimationToSection,
applyAppTheme: applyAppTheme
applyAppTheme: applyAppTheme,
reportAssemblerErrorToAPIStatusPanel: reportAssemblerErrorToAPIStatusPanel
};
});

54
www/js/lib/xzdec-asm.js Normal file
View File

File diff suppressed because one or more lines are too long

32
www/js/lib/xzdec-wasm.js Normal file
View File

@ -0,0 +1,32 @@

var XZ = (function () {
var _scriptDir = typeof document !== 'undefined' && document.currentScript ? document.currentScript.src : undefined;
return (
function(XZ) {
XZ = XZ || {};
var a;a||(a=typeof XZ !== 'undefined' ? XZ : {});var e,f;a.ready=new Promise(function(b,c){e=b;f=c});var h={},k;for(k in a)a.hasOwnProperty(k)&&(h[k]=a[k]);var m="";"undefined"!==typeof document&&document.currentScript&&(m=document.currentScript.src);_scriptDir&&(m=_scriptDir);0!==m.indexOf("blob:")?m=m.substr(0,m.lastIndexOf("/")+1):m="";var n=a.printErr||console.warn.bind(console);for(k in h)h.hasOwnProperty(k)&&(a[k]=h[k]);h=null;var p;a.wasmBinary&&(p=a.wasmBinary);
var noExitRuntime=a.noExitRuntime||!0;"object"!==typeof WebAssembly&&q("no native wasm support detected");var r,t=!1,u,v,w=[],x=[],z=[];function A(){var b=a.preRun.shift();w.unshift(b)}var B=0,C=null,D=null;a.preloadedImages={};a.preloadedAudios={};function q(b){if(a.onAbort)a.onAbort(b);n(b);t=!0;b=new WebAssembly.RuntimeError("abort("+b+"). Build with -s ASSERTIONS=1 for more info.");f(b);throw b;}function E(){return F.startsWith("data:application/octet-stream;base64,")}var F;F="xzdec-wasm.wasm";
if(!E()){var G=F;F=a.locateFile?a.locateFile(G,m):m+G}function H(){var b=F;try{if(b==F&&p)return new Uint8Array(p);throw"both async and sync fetching of the wasm failed";}catch(c){q(c)}}function I(){return p||"function"!==typeof fetch?Promise.resolve().then(function(){return H()}):fetch(F,{credentials:"same-origin"}).then(function(b){if(!b.ok)throw"failed to load wasm binary file at '"+F+"'";return b.arrayBuffer()}).catch(function(){return H()})}
function J(b){for(;0<b.length;){var c=b.shift();if("function"==typeof c)c(a);else{var g=c.u;"number"===typeof g?void 0===c.s?v.get(g)():v.get(g)(c.s):g(void 0===c.s?null:c.s)}}}var K={a:function(b,c,g){u.copyWithin(b,c,c+g)},b:function(){q("OOM")}};
(function(){function b(d){a.asm=d.exports;r=a.asm.c;d=r.buffer;a.HEAP8=new Int8Array(d);a.HEAP16=new Int16Array(d);a.HEAP32=new Int32Array(d);a.HEAPU8=u=new Uint8Array(d);a.HEAPU16=new Uint16Array(d);a.HEAPU32=new Uint32Array(d);a.HEAPF32=new Float32Array(d);a.HEAPF64=new Float64Array(d);v=a.asm.o;x.unshift(a.asm.d);B--;a.monitorRunDependencies&&a.monitorRunDependencies(B);0==B&&(null!==C&&(clearInterval(C),C=null),D&&(d=D,D=null,d()))}function c(d){b(d.instance)}function g(d){return I().then(function(l){return WebAssembly.instantiate(l,
y)}).then(d,function(l){n("failed to asynchronously prepare wasm: "+l);q(l)})}var y={a:K};B++;a.monitorRunDependencies&&a.monitorRunDependencies(B);if(a.instantiateWasm)try{return a.instantiateWasm(y,b)}catch(d){return n("Module.instantiateWasm callback failed with error: "+d),!1}(function(){return p||"function"!==typeof WebAssembly.instantiateStreaming||E()||"function"!==typeof fetch?g(c):fetch(F,{credentials:"same-origin"}).then(function(d){return WebAssembly.instantiateStreaming(d,y).then(c,function(l){n("wasm streaming compile failed: "+
l);n("falling back to ArrayBuffer instantiation");return g(c)})})})().catch(f);return{}})();a.___wasm_call_ctors=function(){return(a.___wasm_call_ctors=a.asm.d).apply(null,arguments)};a._init=function(){return(a._init=a.asm.e).apply(null,arguments)};a._init_decompression=function(){return(a._init_decompression=a.asm.f).apply(null,arguments)};a._input_empty=function(){return(a._input_empty=a.asm.g).apply(null,arguments)};a._get_in_buffer=function(){return(a._get_in_buffer=a.asm.h).apply(null,arguments)};
a._set_new_input=function(){return(a._set_new_input=a.asm.i).apply(null,arguments)};a._decompress=function(){return(a._decompress=a.asm.j).apply(null,arguments)};a._get_out_pos=function(){return(a._get_out_pos=a.asm.k).apply(null,arguments)};a._get_out_buffer=function(){return(a._get_out_buffer=a.asm.l).apply(null,arguments)};a._out_buffer_cleared=function(){return(a._out_buffer_cleared=a.asm.m).apply(null,arguments)};a._release=function(){return(a._release=a.asm.n).apply(null,arguments)};var L;
D=function M(){L||N();L||(D=M)};
function N(){function b(){if(!L&&(L=!0,a.calledRun=!0,!t)){J(x);e(a);if(a.onRuntimeInitialized)a.onRuntimeInitialized();if(a.postRun)for("function"==typeof a.postRun&&(a.postRun=[a.postRun]);a.postRun.length;){var c=a.postRun.shift();z.unshift(c)}J(z)}}if(!(0<B)){if(a.preRun)for("function"==typeof a.preRun&&(a.preRun=[a.preRun]);a.preRun.length;)A();J(w);0<B||(a.setStatus?(a.setStatus("Running..."),setTimeout(function(){setTimeout(function(){a.setStatus("")},1);b()},1)):b())}}a.run=N;
if(a.preInit)for("function"==typeof a.preInit&&(a.preInit=[a.preInit]);0<a.preInit.length;)a.preInit.pop()();N();
return XZ.ready
}
);
})();
if (typeof exports === 'object' && typeof module === 'object')
module.exports = XZ;
else if (typeof define === 'function' && define['amd'])
define([], function() { return XZ; });
else if (typeof exports === 'object')
exports["XZ"] = XZ;

BIN
www/js/lib/xzdec-wasm.wasm Normal file
View File

Binary file not shown.

16
www/js/lib/xzdec.js
View File

File diff suppressed because one or more lines are too long

64
www/js/lib/xzdec_wrapper.js
View File

@ -1,7 +1,7 @@
/**
/**
* xzdec_wrapper.js: Javascript wrapper around compiled xz decompressor.
*
* Copyright 2015 Mossroy and contributors
* Copyright 2021 Mossroy and contributors
* License GPL v3:
*
* This file is part of Kiwix.
@ -20,11 +20,59 @@
* along with Kiwix (file LICENSE-GPLv3.txt). If not, see <http://www.gnu.org/licenses/>
*/
'use strict';
define(['xzdec'], function() {
// DEV: xzdec.js emits a global Module variable, which cannot be set in requireJS function line above, though it can be loaded in definition
var xzdec = Module;
xzdec._init();
// DEV: Put your RequireJS definition in the rqDefXZ array below, and any function exports in the function parenthesis of the define statement
// We need to do it this way in order to load the wasm or asm versions of xzdec conditionally. Older browsers can only use the asm version
// because they cannot interpret WebAssembly.
var rqDefXZ = ['uiUtil'];
// Select asm or wasm conditionally
if ('WebAssembly' in self) {
console.debug('Instantiating WASM xz decoder');
params.decompressorAPI.assemblerMachineType = 'WASM';
rqDefXZ.push('xzdec-wasm');
} else {
console.debug('Instantiating ASM xz decoder');
params.decompressorAPI.assemblerMachineType = 'ASM';
rqDefXZ.push('xzdec-asm');
}
define(rqDefXZ, function(uiUtil) {
// DEV: xzdec.js has been compiled with `-s EXPORT_NAME="XZ" -s MODULARIZE=1` to avoid a clash with zstddec.js
// Note that we include xzdec-asm or xzdec-wasm above in requireJS definition, but we cannot change the name in the function list
// There is no longer any need to load it in index.html
// For explanation of loading method below to avoid conflicts, see https://github.com/emscripten-core/emscripten/blob/master/src/settings.js
/**
* @typedef EMSInstance An object type representing an Emscripten instance
*/
/**
* The XZ Decoder instance
* @type EMSInstance
*/
var xzdec;
XZ().then(function (instance) {
xzdec = instance;
}).catch(function (err) {
if (params.decompressorAPI.assemblerMachineType === 'ASM') {
// There is no fallback, because we were attempting to load the ASM machine, so report error immediately
uiUtil.reportAssemblerErrorToAPIStatusPanel('XZ', err);
} else {
console.warn('WASM failed to load, falling back to ASM...', err);
params.decompressorAPI.assemblerMachineType = 'ASM';
XZ = null;
require(['xzdec-asm'], function () {
XZ().then(function (instance) {
xzdec = instance;
}).catch(function (err) {
uiUtil.reportAssemblerErrorToAPIStatusPanel('XZ', err);
});
});
}
});
/**
* Number of milliseconds to wait for the decompressor to be available for another chunk
* @type Integer
@ -54,9 +102,11 @@ define(['xzdec'], function() {
* @returns {Decompressor}
*/
function Decompressor(reader, chunkSize) {
params.decompressorAPI.decompressorLastUsed = 'XZ';
this._chunkSize = chunkSize || 1024 * 5;
this._reader = reader;
};
/**
* Read length bytes, offset into the decompressed stream. Consecutive calls may only
* advance in the stream and may not overlap.

29
www/js/lib/zimfile.js
View File

@ -21,6 +21,35 @@
*/
'use strict';
/**
* Add Polyfill currently required by IE11 to run zstddec-asm and xzdec-asm
* See https://github.com/emscripten-core/emscripten/issues/14700
* If this is resolved upstream, remove this polyfill
* Source: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith
*/
if (!String.prototype.startsWith) {
Object.defineProperty(String.prototype, 'startsWith', {
value: function(search, rawPos) {
var pos = rawPos > 0 ? rawPos|0 : 0;
return this.substring(pos, pos + search.length) === search;
}
});
}
/**
* A global variable to track the assembler machine type and the last used decompressor (for reporting to the API panel)
* This is populated in the Emscripten wrappers
* @type {Object}
* @property {String} assemblerMachineType The assembler machine type supported and/or loaded by this app: 'ASM' or 'WASM'
* @property {String} decompressorLastUsed The decompressor that was last used to decode a compressed cluster (currently 'XZ' or 'ZSTD')
* @property {String} errorStatus A description of any detected error in loading a decompressor
*/
params.decompressorAPI = {
assemblerMachineType: null,
decompressorLastUsed: null,
errorStatus: null
};
define(['xzdec_wrapper', 'zstddec_wrapper', 'util', 'utf8', 'zimDirEntry', 'filecache'], function(xz, zstd, util, utf8, zimDirEntry, FileCache) {
/**

58
www/js/lib/zstddec-asm.js Normal file
View File

File diff suppressed because one or more lines are too long

36
www/js/lib/zstddec-wasm.js Normal file
View File

@ -0,0 +1,36 @@

var ZD = (function() {
var _scriptDir = typeof document !== 'undefined' && document.currentScript ? document.currentScript.src : undefined;
return (
function(ZD) {
ZD = ZD || {};
var a;a||(a=typeof ZD !== 'undefined' ? ZD : {});var q,r;a.ready=new Promise(function(b,c){q=b;r=c});var t={},w;for(w in a)a.hasOwnProperty(w)&&(t[w]=a[w]);var x="";"undefined"!==typeof document&&document.currentScript&&(x=document.currentScript.src);_scriptDir&&(x=_scriptDir);0!==x.indexOf("blob:")?x=x.substr(0,x.lastIndexOf("/")+1):x="";var y=a.printErr||console.warn.bind(console);for(w in t)t.hasOwnProperty(w)&&(a[w]=t[w]);t=null;var z;a.wasmBinary&&(z=a.wasmBinary);
var noExitRuntime=a.noExitRuntime||!0;"object"!==typeof WebAssembly&&A("no native wasm support detected");var B,C=!1;function D(b){var c=a["_"+b];c||A("Assertion failed: Cannot call unknown function "+(b+", make sure it is exported"));return c}
function aa(b,c,g,n){var e={string:function(d){var m=0;if(null!==d&&void 0!==d&&0!==d){var l=(d.length<<2)+1;m=E(l);var h=m,f=F;if(0<l){l=h+l-1;for(var u=0;u<d.length;++u){var k=d.charCodeAt(u);if(55296<=k&&57343>=k){var ba=d.charCodeAt(++u);k=65536+((k&1023)<<10)|ba&1023}if(127>=k){if(h>=l)break;f[h++]=k}else{if(2047>=k){if(h+1>=l)break;f[h++]=192|k>>6}else{if(65535>=k){if(h+2>=l)break;f[h++]=224|k>>12}else{if(h+3>=l)break;f[h++]=240|k>>18;f[h++]=128|k>>12&63}f[h++]=128|k>>6&63}f[h++]=128|k&63}}f[h]=
0}}return m},array:function(d){var m=E(d.length);H.set(d,m);return m}},p=D(b),G=[];b=0;if(n)for(var v=0;v<n.length;v++){var Q=e[g[v]];Q?(0===b&&(b=I()),G[v]=Q(n[v])):G[v]=n[v]}g=p.apply(null,G);g=function(d){if("string"===c)if(d){for(var m=F,l=d+NaN,h=d;m[h]&&!(h>=l);)++h;if(16<h-d&&m.subarray&&J)d=J.decode(m.subarray(d,h));else{for(l="";d<h;){var f=m[d++];if(f&128){var u=m[d++]&63;if(192==(f&224))l+=String.fromCharCode((f&31)<<6|u);else{var k=m[d++]&63;f=224==(f&240)?(f&15)<<12|u<<6|k:(f&7)<<18|
u<<12|k<<6|m[d++]&63;65536>f?l+=String.fromCharCode(f):(f-=65536,l+=String.fromCharCode(55296|f>>10,56320|f&1023))}}else l+=String.fromCharCode(f)}d=l}}else d="";else d="boolean"===c?!!d:d;return d}(g);0!==b&&K(b);return g}var J="undefined"!==typeof TextDecoder?new TextDecoder("utf8"):void 0,H,F,L,M=[],N=[],O=[];function ca(){var b=a.preRun.shift();M.unshift(b)}var P=0,R=null,S=null;a.preloadedImages={};a.preloadedAudios={};
function A(b){if(a.onAbort)a.onAbort(b);y(b);C=!0;b=new WebAssembly.RuntimeError("abort("+b+"). Build with -s ASSERTIONS=1 for more info.");r(b);throw b;}function T(){return U.startsWith("data:application/octet-stream;base64,")}var U;U="zstddec-wasm.wasm";if(!T()){var V=U;U=a.locateFile?a.locateFile(V,x):x+V}function W(){var b=U;try{if(b==U&&z)return new Uint8Array(z);throw"both async and sync fetching of the wasm failed";}catch(c){A(c)}}
function da(){return z||"function"!==typeof fetch?Promise.resolve().then(function(){return W()}):fetch(U,{credentials:"same-origin"}).then(function(b){if(!b.ok)throw"failed to load wasm binary file at '"+U+"'";return b.arrayBuffer()}).catch(function(){return W()})}function X(b){for(;0<b.length;){var c=b.shift();if("function"==typeof c)c(a);else{var g=c.u;"number"===typeof g?void 0===c.s?L.get(g)():L.get(g)(c.s):g(void 0===c.s?null:c.s)}}}var ea={a:function(b,c,g){F.copyWithin(b,c,c+g)},b:function(){A("OOM")}};
(function(){function b(e){a.asm=e.exports;B=a.asm.c;e=B.buffer;a.HEAP8=H=new Int8Array(e);a.HEAP16=new Int16Array(e);a.HEAP32=new Int32Array(e);a.HEAPU8=F=new Uint8Array(e);a.HEAPU16=new Uint16Array(e);a.HEAPU32=new Uint32Array(e);a.HEAPF32=new Float32Array(e);a.HEAPF64=new Float64Array(e);L=a.asm.o;N.unshift(a.asm.d);P--;a.monitorRunDependencies&&a.monitorRunDependencies(P);0==P&&(null!==R&&(clearInterval(R),R=null),S&&(e=S,S=null,e()))}function c(e){b(e.instance)}function g(e){return da().then(function(p){return WebAssembly.instantiate(p,
n)}).then(e,function(p){y("failed to asynchronously prepare wasm: "+p);A(p)})}var n={a:ea};P++;a.monitorRunDependencies&&a.monitorRunDependencies(P);if(a.instantiateWasm)try{return a.instantiateWasm(n,b)}catch(e){return y("Module.instantiateWasm callback failed with error: "+e),!1}(function(){return z||"function"!==typeof WebAssembly.instantiateStreaming||T()||"function"!==typeof fetch?g(c):fetch(U,{credentials:"same-origin"}).then(function(e){return WebAssembly.instantiateStreaming(e,n).then(c,function(p){y("wasm streaming compile failed: "+
p);y("falling back to ArrayBuffer instantiation");return g(c)})})})().catch(r);return{}})();a.___wasm_call_ctors=function(){return(a.___wasm_call_ctors=a.asm.d).apply(null,arguments)};a._malloc=function(){return(a._malloc=a.asm.e).apply(null,arguments)};a._free=function(){return(a._free=a.asm.f).apply(null,arguments)};a._ZSTD_isError=function(){return(a._ZSTD_isError=a.asm.g).apply(null,arguments)};a._ZSTD_getErrorName=function(){return(a._ZSTD_getErrorName=a.asm.h).apply(null,arguments)};
a._ZSTD_createDStream=function(){return(a._ZSTD_createDStream=a.asm.i).apply(null,arguments)};a._ZSTD_freeDStream=function(){return(a._ZSTD_freeDStream=a.asm.j).apply(null,arguments)};a._ZSTD_DStreamInSize=function(){return(a._ZSTD_DStreamInSize=a.asm.k).apply(null,arguments)};a._ZSTD_DStreamOutSize=function(){return(a._ZSTD_DStreamOutSize=a.asm.l).apply(null,arguments)};a._ZSTD_initDStream=function(){return(a._ZSTD_initDStream=a.asm.m).apply(null,arguments)};
a._ZSTD_decompressStream=function(){return(a._ZSTD_decompressStream=a.asm.n).apply(null,arguments)};var I=a.stackSave=function(){return(I=a.stackSave=a.asm.p).apply(null,arguments)},K=a.stackRestore=function(){return(K=a.stackRestore=a.asm.q).apply(null,arguments)},E=a.stackAlloc=function(){return(E=a.stackAlloc=a.asm.r).apply(null,arguments)};a.cwrap=function(b,c,g,n){g=g||[];var e=g.every(function(p){return"number"===p});return"string"!==c&&e&&!n?D(b):function(){return aa(b,c,g,arguments)}};var Y;
S=function fa(){Y||Z();Y||(S=fa)};
function Z(){function b(){if(!Y&&(Y=!0,a.calledRun=!0,!C)){X(N);q(a);if(a.onRuntimeInitialized)a.onRuntimeInitialized();if(a.postRun)for("function"==typeof a.postRun&&(a.postRun=[a.postRun]);a.postRun.length;){var c=a.postRun.shift();O.unshift(c)}X(O)}}if(!(0<P)){if(a.preRun)for("function"==typeof a.preRun&&(a.preRun=[a.preRun]);a.preRun.length;)ca();X(M);0<P||(a.setStatus?(a.setStatus("Running..."),setTimeout(function(){setTimeout(function(){a.setStatus("")},1);b()},1)):b())}}a.run=Z;
if(a.preInit)for("function"==typeof a.preInit&&(a.preInit=[a.preInit]);0<a.preInit.length;)a.preInit.pop()();Z();
return ZD.ready
}
);
})();
if (typeof exports === 'object' && typeof module === 'object')
module.exports = ZD;
else if (typeof define === 'function' && define['amd'])
define([], function() { return ZD; });
else if (typeof exports === 'object')
exports["ZD"] = ZD;

BIN
www/js/lib/zstddec-wasm.wasm Normal file
View File

Binary file not shown.

66
www/js/lib/zstddec.js
View File

File diff suppressed because one or more lines are too long

48
www/js/lib/zstddec_wrapper.js
View File

@ -20,10 +20,26 @@
* along with Kiwix (file LICENSE-GPLv3.txt). If not, see <http://www.gnu.org/licenses/>
*/
'use strict';
define(['zstddec'], function () {
// DEV: zstddec.js has been compiled with `-s EXPORT_NAME="ZD" -s MODULARIZE=1` to avoid a clash with xzdec which uses "Module" as its exported object
// Note that we include zstddec above in requireJS definition, but we cannot change the name in the function list
// There is no longer any need to load it in index.html
// DEV: Put your RequireJS definition in the rqDefZD array below, and any function exports in the function parenthesis of the define statement
// We need to do it this way in order to load the wasm or asm versions of zstddec conditionally. Older browsers can only use the asm version
// because they cannot interpret WebAssembly.
var rqDefZD = ['uiUtil'];
// Select asm or wasm conditionally
if ('WebAssembly' in self) {
console.debug('Instantiating WASM zstandard decoder');
params.decompressorAPI.assemblerMachineType = 'WASM';
rqDefZD.push('zstddec-wasm');
} else {
console.debug('Instantiating ASM zstandard decoder');
params.decompressorAPI.assemblerMachineType = 'ASM';
rqDefZD.push('zstddec-asm');
}
define(rqDefZD, function(uiUtil) {
// DEV: zstddec.js has been compiled with `-s EXPORT_NAME="ZD" -s MODULARIZE=1` to avoid a clash with xzdec.js
// Note that we include zstddec-wasm or zstddec-asm above in requireJS definition, but we cannot change the name in the function list
// For explanation of loading method below to avoid conflicts, see https://github.com/emscripten-core/emscripten/blob/master/src/settings.js
/**
@ -39,7 +55,8 @@ define(['zstddec'], function () {
* @type EMSInstanceExt
*/
var zd;
ZD().then(function (instance) {
var instantiateDecoder = function (instance) {
// Instantiate the zd object
zd = instance;
// Create JS API by wrapping C++ functions
@ -82,6 +99,26 @@ define(['zstddec'], function () {
zd._outBuffer.ptr = mallocOrDie(3 << 2); // 3 x 32bit bytes
// Reserve w/asm memory for the outBuffer data steam
zd._outBuffer.dst = mallocOrDie(zd._outBuffer.size);
};
ZD().then(function (inst) {
instantiateDecoder(inst);
}).catch(function (err) {
if (params.decompressorAPI.assemblerMachineType === 'ASM') {
// There is no fallback, because we were attempting to load the ASM machine, so report error immediately
uiUtil.reportAssemblerErrorToAPIStatusPanel('ZSTD', err);
} else {
console.warn('WASM failed to load, falling back to ASM...', err);
params.decompressorAPI.assemblerMachineType = 'ASM';
ZD = null;
require(['zstddec-asm'], function () {
ZD().then(function (inst) {
instantiateDecoder(inst);
}).catch(function (err) {
uiUtil.reportAssemblerErrorToAPIStatusPanel('ZSTD', err);
});
});
}
});
/**
@ -111,6 +148,7 @@ define(['zstddec'], function () {
* @param {FileReader} reader The reader used to extract file slices (defined in zimfile.js)
*/
function Decompressor(reader) {
params.decompressorAPI.decompressorLastUsed = 'ZSTD';
this._reader = reader;
}