kiwix/kiwix-js-pwa
1
0
Fork 0
mirror of https://github.com/kiwix/kiwix-js-pwa.git synced 2025-09-09 04:06:27 -04:00
Code Issues Packages Projects Releases Wiki Activity

New non-parallel decompression code

Browse Source 
Former-commit-id: 8f450fb19ac82d74c3b4bce9580976ddfce6a888 [formerly 5df60d1c810c7b98316e7a1ba5e0133cef329d25]
Former-commit-id: 275e5245e40972f2bb4ae1ce3a6ed34389497421
This commit is contained in:
Jaifroid 2018-07-08 12:48:41 +01:00
parent 4f7e3fd4a5
commit 19293b062d
3 changed files with 48 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
www/js/lib/xzdec_wrapper.js
View File

@ -24,6 +24,18 @@ define(['q'], function(q) {
var xzdec = Module; //@todo including via requirejs seems to not work
xzdec._init();
/**
* Number of milliseconds to wait for the decompressor to be available for another chunk
* @type Integer
*/
var DELAY_WAITING_IDLE_DECOMPRESSOR = 50;
/**
* Is the decompressor already working?
* @type Boolean
*/
var busy = false;
/**
* @typedef Decompressor
* @property {Integer} _chunkSize
@ -51,6 +63,7 @@ define(['q'], function(q) {
* @param {Integer} length
*/
Decompressor.prototype.readSlice = function(offset, length) {
busy = true;
var that = this;
this._inStreamPos = 0;
this._outStreamPos = 0;
@ -59,10 +72,40 @@ define(['q'], function(q) {
this._outBufferPos = 0;
return this._readLoop(offset, length).then(function(data) {
xzdec._release(that._decHandle);
busy = false;
return data;
});
};
/**
* Read length bytes, offset into the decompressed stream.
* This function ensures that only one decompression runs at a time.
*
* @param {Integer} offset
* @param {Integer} length
*/
Decompressor.prototype.readSliceSingleThread = function(offset, length) {
if (!busy) {
//console.log("decompressor not busy : let's read the slice " + offset + " " + length);
return this.readSlice(offset, length);
}
else {
//console.log("decompressor busy : let's wait before reading the slice " + offset + " " + length);
// The decompressor is already in progress.
// To avoid using too much memory, we wait until it has finished
// before using it for another decompression.
// Inspired by https://codereview.stackexchange.com/questions/145563/angularjs-recursive-function-call-with-timeout
var that = this;
var deferred = q.defer();
setTimeout(function(){
that.readSliceSingleThread(offset, length).then(deferred.resolve, deferred.reject);
}, DELAY_WAITING_IDLE_DECOMPRESSOR);
return deferred.promise;
}
};
/**
*
* @param {Integer} offset

39
www/js/lib/zimArchive.js
View File

@ -23,13 +23,6 @@
define(['zimfile', 'zimDirEntry', 'util', 'utf8'],
function(zimfile, zimDirEntry, util, utf8) {
// DEV: This controls the number of jobs sent to the decompressor at any one time. The value should be set in
// init.js and will need fine-turning according to your build of xzdec, and the memory constraints of your target
// environment(s). Some low-end mobiles can only support a value of 1.
var MAX_DECOMPRESSOR_JOBS = params.xzMaxJobs || 4;
// Counter to track the number of decompression-type jobs sent to xz
var xzJobs = 0;
/**
* ZIM Archive
*
@ -222,22 +215,6 @@ define(['zimfile', 'zimDirEntry', 'util', 'utf8'],
this._file.dirEntryByUrlIndex(dirEntry.redirectTarget).then(callback);
};
/**
* Utility queue and store for dirEntries awaiting the decompressor
* See head of file for information about MAX_DECOMPRESSOR_JOBS
*
* @param {DirEntry} dirEntry A directory entry to queue
* @param {callbackDirEntry} callback The function to call when the decompressor is freed up
* @returns {callback} Resumes the calling function
*/
function xzAwait (dirEntry, callback) {
if (xzJobs < MAX_DECOMPRESSOR_JOBS) {
return callback(dirEntry);
} else {
setTimeout(xzAwait, 100, dirEntry, callback);
}
}
/**
* @callback callbackStringContent
* @param {String} content String content
@ -249,13 +226,9 @@ define(['zimfile', 'zimDirEntry', 'util', 'utf8'],
* @param {callbackStringContent} callback
*/
ZIMArchive.prototype.readUtf8File = function(dirEntry, callback) {
xzAwait(dirEntry, function(dirEntry) {
xzJobs++;
return dirEntry.readData().then(function(data) {
xzJobs--;
dirEntry.readData().then(function(data) {
callback(dirEntry, utf8.parse(data));
});
});
};
/**
@ -269,19 +242,9 @@ define(['zimfile', 'zimDirEntry', 'util', 'utf8'],
* @param {callbackBinaryContent} callback
*/
ZIMArchive.prototype.readBinaryFile = function(dirEntry, callback) {
if (/\.svg$|\.css$/i.test(dirEntry.url)) {
xzAwait(dirEntry, function(dirEntry) {
xzJobs++;
return dirEntry.readData().then(function(data) {
xzJobs--;
callback(dirEntry, data);
});
});
} else {
return dirEntry.readData().then(function(data) {
callback(dirEntry, data);
});
}
};
/**

8
www/js/lib/zimfile.js
View File

@ -97,7 +97,7 @@ define(['xzdec_wrapper', 'util', 'utf8', 'q', 'zimDirEntry'], function(xz, util,
return readRequests[0];
} else {
// Wait until all are resolved and concatenate.
console.log("Concatenating split ZIM fileset...");
console.log("CONCAT");
return Q.all(readRequests).then(function(arrays) {
var concatenated = new Uint8Array(size);
var sizeSum = 0;
@ -193,16 +193,16 @@ define(['xzdec_wrapper', 'util', 'utf8', 'q', 'zimDirEntry'], function(xz, util,
};
if (compressionType[0] === 0 || compressionType[0] === 1) {
// uncompressed
decompressor = { readSlice: plainBlobReader };
decompressor = { readSliceSingleThread: plainBlobReader };
} else if (compressionType[0] === 4) {
decompressor = new xz.Decompressor(plainBlobReader);
} else {
return new Uint8Array(); // unsupported compression type
}
return decompressor.readSlice(blob * 4, 8).then(function(data) {
return decompressor.readSliceSingleThread(blob * 4, 8).then(function(data) {
var blobOffset = readInt(data, 0, 4);
var nextBlobOffset = readInt(data, 4, 4);
return decompressor.readSlice(blobOffset, nextBlobOffset - blobOffset);
return decompressor.readSliceSingleThread(blobOffset, nextBlobOffset - blobOffset);
});
});
});