diff --git a/www/js/lib/xzdec_wrapper.js b/www/js/lib/xzdec_wrapper.js index 04ba6b1f..83dd6b37 100644 --- a/www/js/lib/xzdec_wrapper.js +++ b/www/js/lib/xzdec_wrapper.js @@ -24,6 +24,18 @@ define(['q'], function(q) { var xzdec = Module; //@todo including via requirejs seems to not work xzdec._init(); + /** + * Number of milliseconds to wait for the decompressor to be available for another chunk + * @type Integer + */ + var DELAY_WAITING_IDLE_DECOMPRESSOR = 50; + + /** + * Is the decompressor already working? + * @type Boolean + */ + var busy = false; + /** * @typedef Decompressor * @property {Integer} _chunkSize @@ -51,6 +63,7 @@ define(['q'], function(q) { * @param {Integer} length */ Decompressor.prototype.readSlice = function(offset, length) { + busy = true; var that = this; this._inStreamPos = 0; this._outStreamPos = 0; @@ -59,10 +72,40 @@ define(['q'], function(q) { this._outBufferPos = 0; return this._readLoop(offset, length).then(function(data) { xzdec._release(that._decHandle); + busy = false; return data; }); }; + /** + * Read length bytes, offset into the decompressed stream. + * This function ensures that only one decompression runs at a time. + * + * @param {Integer} offset + * @param {Integer} length + */ + Decompressor.prototype.readSliceSingleThread = function(offset, length) { + if (!busy) { + //console.log("decompressor not busy : let's read the slice " + offset + " " + length); + return this.readSlice(offset, length); + } + else { + //console.log("decompressor busy : let's wait before reading the slice " + offset + " " + length); + // The decompressor is already in progress. + // To avoid using too much memory, we wait until it has finished + // before using it for another decompression. + // Inspired by https://codereview.stackexchange.com/questions/145563/angularjs-recursive-function-call-with-timeout + var that = this; + var deferred = q.defer(); + + setTimeout(function(){ + that.readSliceSingleThread(offset, length).then(deferred.resolve, deferred.reject); + }, DELAY_WAITING_IDLE_DECOMPRESSOR); + + return deferred.promise; + } + }; + /** * * @param {Integer} offset diff --git a/www/js/lib/zimArchive.js b/www/js/lib/zimArchive.js index 2c345662..b7b108c9 100644 --- a/www/js/lib/zimArchive.js +++ b/www/js/lib/zimArchive.js @@ -23,13 +23,6 @@ define(['zimfile', 'zimDirEntry', 'util', 'utf8'], function(zimfile, zimDirEntry, util, utf8) { - // DEV: This controls the number of jobs sent to the decompressor at any one time. The value should be set in - // init.js and will need fine-turning according to your build of xzdec, and the memory constraints of your target - // environment(s). Some low-end mobiles can only support a value of 1. - var MAX_DECOMPRESSOR_JOBS = params.xzMaxJobs || 4; - // Counter to track the number of decompression-type jobs sent to xz - var xzJobs = 0; - /** * ZIM Archive * @@ -222,22 +215,6 @@ define(['zimfile', 'zimDirEntry', 'util', 'utf8'], this._file.dirEntryByUrlIndex(dirEntry.redirectTarget).then(callback); }; - /** - * Utility queue and store for dirEntries awaiting the decompressor - * See head of file for information about MAX_DECOMPRESSOR_JOBS - * - * @param {DirEntry} dirEntry A directory entry to queue - * @param {callbackDirEntry} callback The function to call when the decompressor is freed up - * @returns {callback} Resumes the calling function - */ - function xzAwait (dirEntry, callback) { - if (xzJobs < MAX_DECOMPRESSOR_JOBS) { - return callback(dirEntry); - } else { - setTimeout(xzAwait, 100, dirEntry, callback); - } - } - /** * @callback callbackStringContent * @param {String} content String content @@ -249,13 +226,9 @@ define(['zimfile', 'zimDirEntry', 'util', 'utf8'], * @param {callbackStringContent} callback */ ZIMArchive.prototype.readUtf8File = function(dirEntry, callback) { - xzAwait(dirEntry, function(dirEntry) { - xzJobs++; - return dirEntry.readData().then(function(data) { - xzJobs--; + dirEntry.readData().then(function(data) { callback(dirEntry, utf8.parse(data)); }); - }); }; /** @@ -269,19 +242,9 @@ define(['zimfile', 'zimDirEntry', 'util', 'utf8'], * @param {callbackBinaryContent} callback */ ZIMArchive.prototype.readBinaryFile = function(dirEntry, callback) { - if (/\.svg$|\.css$/i.test(dirEntry.url)) { - xzAwait(dirEntry, function(dirEntry) { - xzJobs++; return dirEntry.readData().then(function(data) { - xzJobs--; callback(dirEntry, data); }); - }); - } else { - return dirEntry.readData().then(function(data) { - callback(dirEntry, data); - }); - } }; /** diff --git a/www/js/lib/zimfile.js b/www/js/lib/zimfile.js index f81b3b97..e32a46d1 100644 --- a/www/js/lib/zimfile.js +++ b/www/js/lib/zimfile.js @@ -97,7 +97,7 @@ define(['xzdec_wrapper', 'util', 'utf8', 'q', 'zimDirEntry'], function(xz, util, return readRequests[0]; } else { // Wait until all are resolved and concatenate. - console.log("Concatenating split ZIM fileset..."); + console.log("CONCAT"); return Q.all(readRequests).then(function(arrays) { var concatenated = new Uint8Array(size); var sizeSum = 0; @@ -193,16 +193,16 @@ define(['xzdec_wrapper', 'util', 'utf8', 'q', 'zimDirEntry'], function(xz, util, }; if (compressionType[0] === 0 || compressionType[0] === 1) { // uncompressed - decompressor = { readSlice: plainBlobReader }; + decompressor = { readSliceSingleThread: plainBlobReader }; } else if (compressionType[0] === 4) { decompressor = new xz.Decompressor(plainBlobReader); } else { return new Uint8Array(); // unsupported compression type } - return decompressor.readSlice(blob * 4, 8).then(function(data) { + return decompressor.readSliceSingleThread(blob * 4, 8).then(function(data) { var blobOffset = readInt(data, 0, 4); var nextBlobOffset = readInt(data, 4, 4); - return decompressor.readSlice(blobOffset, nextBlobOffset - blobOffset); + return decompressor.readSliceSingleThread(blobOffset, nextBlobOffset - blobOffset); }); }); });