From 8c186867e9e746172f0d056a3a24c045201a296b Mon Sep 17 00:00:00 2001 From: mossroy Date: Wed, 6 Jan 2016 16:33:17 +0100 Subject: [PATCH 1/4] Oups, revert last commit --- service-worker.js | 1 - 1 file changed, 1 deletion(-) diff --git a/service-worker.js b/service-worker.js index 1b3f7f40..1171954e 100644 --- a/service-worker.js +++ b/service-worker.js @@ -130,7 +130,6 @@ function(util, utf8) { else if (regexpCSS.test(titleName)) { contentType = 'image/css'; } - reject("temporarily refuse javascript and css dependencies"); } // Let's instanciate a new messageChannel, to allow app.s to give us the content From 6f05a76c66153abac93467f3894a4f9b88102727 Mon Sep 17 00:00:00 2001 From: peter-x Date: Wed, 6 Jan 2016 11:10:11 +0100 Subject: [PATCH 2/4] Fixed typos and added some documentation. --- service-worker.js | 4 ++-- www/js/app.js | 4 ++-- www/js/lib/archive.js | 1 + www/js/lib/zimArchive.js | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/service-worker.js b/service-worker.js index 1171954e..b0675788 100644 --- a/service-worker.js +++ b/service-worker.js @@ -68,9 +68,9 @@ self.addEventListener('activate', function(event) { require({ baseUrl: "./www/js/lib/" }, -["util", "utf8"], +["util"], -function(util, utf8) { +function(util) { console.log("ServiceWorker startup"); diff --git a/www/js/app.js b/www/js/app.js index 11130688..0c98d84f 100644 --- a/www/js/app.js +++ b/www/js/app.js @@ -777,11 +777,11 @@ define(['jquery', 'abstractBackend', 'util', 'cookies','geometry','osabstraction console.log("Reading binary file..."); selectedArchive.readBinaryFile(title, function(readableTitleName, content) { messagePort.postMessage({'action': 'giveContent', 'titleName' : titleName, 'content': content}); - console.log("content sent to ServiceWorker)"); + console.log("content sent to ServiceWorker"); }); } } - console.log("Fetching tile " + titleName); + console.log("Fetching title " + titleName); selectedArchive.getTitleByName(titleName).then(readFile).fail(function() { console.log("could not find title:" + arguments); messagePort.postMessage({'action': 'giveContent', 'titleName' : titleName, 'content': new UInt8Array()}); diff --git a/www/js/lib/archive.js b/www/js/lib/archive.js index 2d2ba606..3dd103a6 100644 --- a/www/js/lib/archive.js +++ b/www/js/lib/archive.js @@ -338,6 +338,7 @@ define(['normalize_string', 'geometry', 'title', 'util', 'titleIterators', 'q'], * Look for a title by its name, and call the callbackFunction with this Title * If the title is not found, the callbackFunction is called with parameter null * @param {String} titleName + * @return {Promise} resolving to the title object or null if not found. */ LocalArchive.prototype.getTitleByName = function(titleName) { var that = this; diff --git a/www/js/lib/zimArchive.js b/www/js/lib/zimArchive.js index 07114f03..46e0e5a4 100644 --- a/www/js/lib/zimArchive.js +++ b/www/js/lib/zimArchive.js @@ -198,8 +198,9 @@ define(['zimfile', 'zimDirEntry', 'util', 'utf8'], }; /** - * + * Searches a title (article / page) by name. * @param {String} titleName + * @return {Promise} resolving to the title object or null if not found. */ ZIMArchive.prototype.getTitleByName = function(titleName) { var that = this; From b679948db98afbe4197cb6e4bbc0f09511f1b4f7 Mon Sep 17 00:00:00 2001 From: peter-x Date: Thu, 7 Jan 2016 16:05:40 +0100 Subject: [PATCH 3/4] Fix streaming bug in xz decompressor. The problem was that in some cases, the input was already read in too far, so we just re-start the decompressor for each read (the full contents of a file are read in one go anyway). --- www/js/lib/xzdec_wrapper.js | 47 ++++++++++++++++--------------------- www/js/lib/zimfile.js | 8 ++----- 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/www/js/lib/xzdec_wrapper.js b/www/js/lib/xzdec_wrapper.js index 148ad582..59d4179d 100644 --- a/www/js/lib/xzdec_wrapper.js +++ b/www/js/lib/xzdec_wrapper.js @@ -43,10 +43,6 @@ define(['q'], function(q) { function Decompressor(reader, chunkSize) { this._chunkSize = chunkSize || 1024 * 5; this._reader = reader; - this._decHandle = xzdec._init_decompression(this._chunkSize); - this._inStreamPos = 0; - this._outStreamPos = 0; - this._outBuffer = null; }; /** * Read length bytes, offset into the decompressed stream. Consecutive calls may only @@ -55,15 +51,16 @@ define(['q'], function(q) { * @param {Integer} length */ Decompressor.prototype.readSlice = function(offset, length) { + var that = this; + this._inStreamPos = 0; + this._outStreamPos = 0; + this._decHandle = xzdec._init_decompression(this._chunkSize); this._outBuffer = new Int8Array(new ArrayBuffer(length)); this._outBufferPos = 0; - return this._readLoop(offset, length); - }; - /** - * Finish the decompressing and release resources. - */ - Decompressor.prototype.end = function() { - xzdec._release(this._decHandle); + return this._readLoop(offset, length).then(function(data) { + xzdec._release(that._decHandle); + return data; + }); }; /** @@ -88,23 +85,19 @@ define(['q'], function(q) { } var outPos = xzdec._get_out_pos(that._decHandle); - if (finished || outPos === that._chunkSize || that._outStreamPos + outPos >= offset + length) { - if (that._outStreamPos + outPos >= offset) - { - var outBuffer = xzdec._get_out_buffer(that._decHandle); - var copyStart = offset - that._outStreamPos; - if (copyStart < 0) - copyStart = 0; - for (var i = copyStart; i < outPos && that._outBufferPos < that._outBuffer.length; i++) - that._outBuffer[that._outBufferPos++] = xzdec.HEAP8[outBuffer + i]; - } - if (outPos === that._chunkSize) - { - that._outStreamPos += outPos; - xzdec._out_buffer_cleared(that._decHandle); - } + if (outPos > 0 && that._outStreamPos + outPos >= offset) + { + var outBuffer = xzdec._get_out_buffer(that._decHandle); + var copyStart = offset - that._outStreamPos; + if (copyStart < 0) + copyStart = 0; + for (var i = copyStart; i < outPos && that._outBufferPos < that._outBuffer.length; i++) + that._outBuffer[that._outBufferPos++] = xzdec.HEAP8[outBuffer + i]; } - if (finished || that._outStreamPos + outPos >= offset + length) + that._outStreamPos += outPos; + if (outPos > 0) + xzdec._out_buffer_cleared(that._decHandle); + if (finished || that._outStreamPos >= offset + length) return that._outBuffer; else return that._readLoop(offset, length); diff --git a/www/js/lib/zimfile.js b/www/js/lib/zimfile.js index b0ff2f55..c7f5d196 100644 --- a/www/js/lib/zimfile.js +++ b/www/js/lib/zimfile.js @@ -154,7 +154,6 @@ define(['xzdec_wrapper', 'util', 'utf8'], function(xz, util, utf8) { ZIMFile.prototype.blob = function(cluster, blob) { var that = this; - //@todo decompress in a streaming way, otherwise we have to "guess" the sizes return this._readSlice(this.clusterPtrPos + cluster * 8, 16).then(function(clusterOffsets) { var clusterOffset = readInt(clusterOffsets, 0, 8); @@ -166,7 +165,7 @@ define(['xzdec_wrapper', 'util', 'utf8'], function(xz, util, utf8) { }; if (compressionType[0] === 0 || compressionType[0] === 1) { // uncompressed - decompressor = { readSlice: plainBlobReader, end: function() {} }; + decompressor = { readSlice: plainBlobReader }; } else if (compressionType[0] === 4) { decompressor = new xz.Decompressor(plainBlobReader); } else { @@ -175,10 +174,7 @@ define(['xzdec_wrapper', 'util', 'utf8'], function(xz, util, utf8) { return decompressor.readSlice(blob * 4, 8).then(function(data) { var blobOffset = readInt(data, 0, 4); var nextBlobOffset = readInt(data, 4, 4); - return decompressor.readSlice(blobOffset, nextBlobOffset - blobOffset).then(function(data) { - decompressor.end(); - return data; - }); + return decompressor.readSlice(blobOffset, nextBlobOffset - blobOffset); }); }); }); From 79dbcbf3b1f0360b94d07f1f68f55321fdc85b27 Mon Sep 17 00:00:00 2001 From: peter-x Date: Thu, 7 Jan 2016 16:08:16 +0100 Subject: [PATCH 4/4] Tests. --- tests/tests.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/tests.js b/tests/tests.js index dbfa2d90..9b8b1b35 100644 --- a/tests/tests.js +++ b/tests/tests.js @@ -532,5 +532,19 @@ define(['jquery', 'title', 'archive', 'zimArchive', 'zimDirEntry', 'util', 'geom }); }); }); + asyncTest("Image 's/style.css' can be loaded", function() { + expect(4); + localZimArchive.getTitleByName("-/s/style.css").then(function(title) { + ok(title !== null, "Title found"); + equal(title.url, "-/s/style.css", "URL is correct."); + localZimArchive.readBinaryFile(title, function(title, data) { + equal(data.length, 104495, "Data length is correct."); + data = utf8.parse(data); + var beginning = "\n/* start http://en.wikipedia.org/w/load.php?debug=false&lang=en&modules=site&only=styles&skin=vector"; + equal(data.slice(0, beginning.length), beginning, "Content starts correctly."); + start(); + }); + }); + }); }; });