New non-parallel decompression code

Former-commit-id: 8f450fb19ac82d74c3b4bce9580976ddfce6a888 [formerly 5df60d1c810c7b98316e7a1ba5e0133cef329d25]
Former-commit-id: 275e5245e40972f2bb4ae1ce3a6ed34389497421

www/js/lib/xzdec_wrapper.js

@@ -24,6 +24,18 @@ define(['q'], function(q) {
     var xzdec = Module; //@todo including via requirejs seems to not work
     xzdec._init();
     
+    /**
+     * Number of milliseconds to wait for the decompressor to be available for another chunk
+     * @type Integer
+     */
+    var DELAY_WAITING_IDLE_DECOMPRESSOR = 50;
+    
+    /**
+     * Is the decompressor already working?
+     * @type Boolean
+     */
+    var busy = false;
+    
     /**
      * @typedef Decompressor
      * @property {Integer} _chunkSize
@@ -51,6 +63,7 @@ define(['q'], function(q) {
      * @param {Integer} length
      */
     Decompressor.prototype.readSlice = function(offset, length) {
+        busy = true;
         var that = this;
         this._inStreamPos = 0;
         this._outStreamPos = 0;
@@ -59,10 +72,40 @@ define(['q'], function(q) {
         this._outBufferPos = 0;
         return this._readLoop(offset, length).then(function(data) {
             xzdec._release(that._decHandle);
+            busy = false;
             return data;
         });
     };
 
+    /**
+     * Read length bytes, offset into the decompressed stream.
+     * This function ensures that only one decompression runs at a time.
+     * 
+     * @param {Integer} offset
+     * @param {Integer} length
+     */
+    Decompressor.prototype.readSliceSingleThread = function(offset, length) {
+        if (!busy) {
+            //console.log("decompressor not busy : let's read the slice " + offset + " " + length);
+            return this.readSlice(offset, length);
+        }
+        else {
+            //console.log("decompressor busy : let's wait before reading the slice " + offset + " " + length);
+            // The decompressor is already in progress.
+            // To avoid using too much memory, we wait until it has finished
+            // before using it for another decompression.
+            // Inspired by https://codereview.stackexchange.com/questions/145563/angularjs-recursive-function-call-with-timeout
+            var that = this;
+            var deferred = q.defer();
+
+            setTimeout(function(){
+                that.readSliceSingleThread(offset, length).then(deferred.resolve, deferred.reject);
+            }, DELAY_WAITING_IDLE_DECOMPRESSOR);
+
+            return deferred.promise;
+        }
+    };
+
     /**
      * 
      * @param {Integer} offset

www/js/lib/zimArchive.js

@@ -23,13 +23,6 @@
 define(['zimfile', 'zimDirEntry', 'util', 'utf8'],
     function(zimfile, zimDirEntry, util, utf8) {
     
-    // DEV: This controls the number of jobs sent to the decompressor at any one time. The value should be set in
-    // init.js and will need fine-turning according to your build of xzdec, and the memory constraints of your target
-    // environment(s). Some low-end mobiles can only support a value of 1.
-    var MAX_DECOMPRESSOR_JOBS = params.xzMaxJobs || 4;
-    // Counter to track the number of decompression-type jobs sent to xz
-    var xzJobs = 0;
-
     /**
      * ZIM Archive
      * 
@@ -222,22 +215,6 @@ define(['zimfile', 'zimDirEntry', 'util', 'utf8'],
         this._file.dirEntryByUrlIndex(dirEntry.redirectTarget).then(callback);
     };
     
-    /**
-     * Utility queue and store for dirEntries awaiting the decompressor
-     * See head of file for information about MAX_DECOMPRESSOR_JOBS
-     * 
-     * @param {DirEntry} dirEntry A directory entry to queue
-     * @param {callbackDirEntry} callback The function to call when the decompressor is freed up
-     * @returns {callback} Resumes the calling function
-     */
-    function xzAwait (dirEntry, callback) {
-        if (xzJobs < MAX_DECOMPRESSOR_JOBS) {
-            return callback(dirEntry);
-        } else {
-            setTimeout(xzAwait, 100, dirEntry, callback);
-        }
-    }
-
     /**
      * @callback callbackStringContent
      * @param {String} content String content
@@ -249,13 +226,9 @@ define(['zimfile', 'zimDirEntry', 'util', 'utf8'],
      * @param {callbackStringContent} callback
      */
     ZIMArchive.prototype.readUtf8File = function(dirEntry, callback) {
-        xzAwait(dirEntry, function(dirEntry) {
+        dirEntry.readData().then(function(data) {
-            xzJobs++;
-            return dirEntry.readData().then(function(data) {
-                xzJobs--;
             callback(dirEntry, utf8.parse(data));
         });
-        });
     };
 
     /**
@@ -269,19 +242,9 @@ define(['zimfile', 'zimDirEntry', 'util', 'utf8'],
      * @param {callbackBinaryContent} callback
      */
     ZIMArchive.prototype.readBinaryFile = function(dirEntry, callback) {
-        if (/\.svg$|\.css$/i.test(dirEntry.url)) {
-            xzAwait(dirEntry, function(dirEntry) {
-                xzJobs++;
-        return dirEntry.readData().then(function(data) {
-                    xzJobs--;
-            callback(dirEntry, data);
-        });
-            });
-        } else {
         return dirEntry.readData().then(function(data) {
             callback(dirEntry, data);
         });
-        }
     };
     
     /**

www/js/lib/zimfile.js

@@ -97,7 +97,7 @@ define(['xzdec_wrapper', 'util', 'utf8', 'q', 'zimDirEntry'], function(xz, util,
             return readRequests[0];
         } else {
             // Wait until all are resolved and concatenate.
-            console.log("Concatenating split ZIM fileset...");
+            console.log("CONCAT");
             return Q.all(readRequests).then(function(arrays) {
                 var concatenated = new Uint8Array(size);
                 var sizeSum = 0;
@@ -193,16 +193,16 @@ define(['xzdec_wrapper', 'util', 'utf8', 'q', 'zimDirEntry'], function(xz, util,
                 };
                 if (compressionType[0] === 0 || compressionType[0] === 1) {
                     // uncompressed
-                    decompressor = { readSlice: plainBlobReader };
+                    decompressor = { readSliceSingleThread: plainBlobReader };
                 } else if (compressionType[0] === 4) {
                     decompressor = new xz.Decompressor(plainBlobReader);
                 } else {
                     return new Uint8Array(); // unsupported compression type
                 }
-                return decompressor.readSlice(blob * 4, 8).then(function(data) {
+                return decompressor.readSliceSingleThread(blob * 4, 8).then(function(data) {
                     var blobOffset = readInt(data, 0, 4);
                     var nextBlobOffset = readInt(data, 4, 4);
-                    return decompressor.readSlice(blobOffset, nextBlobOffset - blobOffset);
+                    return decompressor.readSliceSingleThread(blobOffset, nextBlobOffset - blobOffset);
                 });
             });
         });