kiwix/kiwix-js
1
0
Fork 0
mirror of https://github.com/kiwix/kiwix-js.git synced 2025-09-22 12:01:15 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/mossroy/evopedia-html5

Browse Source 
Conflicts:
	www/index.html
	www/js/app.js
This commit is contained in:
mossroy 2013-05-28 11:13:00 +02:00
commit d3a30a40de
4 changed files with 340 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/tests.js
View File

@ -113,7 +113,7 @@ define(function (require) {
});
asyncTest("check readArticle", function(){
var callbackFunction = function(htmlArticle) {
var callbackFunction = function(title, htmlArticle) {
ok(htmlArticle && htmlArticle.length>0,"Article not empty");
// Remove new lines
htmlArticle = htmlArticle.replace(/[\r\n]/g, " ");

191
www/index.html
View File

@ -1,96 +1,95 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Evopedia</title>
<meta name="description" content="Offline wikipedia reader">
<meta name="viewport" content="width=device-width">
<!--
Port of Evopedia (offline wikipedia reader) in HTML5/Javascript, with Firefox OS as the primary target
The original application is at http://www.evopedia.info/
It uses wikipedia dumps located at http://dumpathome.evopedia.info/dumps/finished
Author : Mossroy - mossroy@free.fr
License:
This program is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 3 of the License, or (at your
option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public
License along with this program; if not, see
<http://www.gnu.org/licenses/>.
-->
<link rel="stylesheet" href="css/app.css">
</head>
<body>
<!-- Use this installation button to install locally without going
through the marketpace (see app.js) -->
<!-- <button id="install-btn">Install</button> -->
<!-- Write your application here -->
<h1>Evopedia</h1>
<input type="button" id="showHideAbout" value="About" />
<div id="about">
This is a preliminary work on the port of Evopedia (offline wikipedia reader) in HTML5/Javascript, with Firefox OS as the primary target
<br />
The original application is at <a href="http://www.evopedia.info/">http://www.evopedia.info/</a>
<br />
<br />
To use it, you have to first download locally a dump from <a href="http://dumpathome.evopedia.info/dumps/finished">http://dumpathome.evopedia.info/dumps/finished</a> (with a Bittorrent client), and select some of the dowloaded files below.
<br />
I have tested it with the <a href="http://evopedia.info/dumps/wikipedia_small_2010-08-14.torrent">small dump (2010-08-14)</a>, the <a href="http://evopedia.info/dumps/wikipedia_fr_2013-02-16.torrent">French dump (2013-02-16)</a>, the <a href="http://evopedia.info/dumps/wikipedia_frwiktionary_2011-03-16.torrent">French wiktionary dump (2011-03-16)</a> and the <a href="http://evopedia.info/dumps/wikipedia_en_2012-02-11.torrent">English dump (2012-02-11)</a>
<br />
<br />
<ul>
<li>On desktops, it works on recent Firefox and Chrome, and maybe on other browsers</li>
<li>On the Firefos OS simulator, you have (for now) to put the French dump files in a "fake-sdcard/evopedia" folder of your firefox profile (ex : ~/.mozilla/firefox/xxxx.default/extensions/r2d2b2g@mozilla.org/profile/fake-sdcard). It looks for evopedia/wikipedia_fr_2013-02-16/titles.idx in it. You also need to install the application from the dashboard of the simulator instead of accessing via the browser (due to security restrictions in Firefox OS : only certified webapps can access the sdcard)</li>
<li>On a real Firefox OS device, you also have (for now) to put the French dump files in an "evopedia" directory at the root of your sdcard, so that it finds a file /evopedia/wikipedia_fr_2013-02-16/titles.idx on it</li>
</ul>
<br />
It's only a proof of concept so far : there are many many ways this could be enhanced (suggestions and patches are welcome : the source code is on <a href="https://github.com/mossroy/evopedia-html5">github</a>). In particular :
<ul>
<li>The performance has to be optimized when reading an article</li>
<li>Some searches (for example with prefix "a" on the French dump) do not give any result even if they should</li>
<li>In some cases, the links inside an article do not work, or do not lead to the right article</li>
<li>On a real device, reading an article sometimes crashes because it loads too many things in memory</li>
<li>It is hardly usable on a device because the buttons and inputs are too small</li>
<li>Following the links in an article does not populate the history of the browser, which prevents the use of the back button</li>
</ul>
<br />
</div>
<div id="openLocalFiles" style="display: none;">
<br /> Please select the file titles.idx :<br /> <input type="file"
id="titleFile" /><br /> Please select the files wikipedia_*.dat
from the same dump :<br /> <input type="file" id="dataFiles" multiple />
</div>
<br /> Find titles starting with :
<input type="text" id="prefix" value="" />&nbsp;
<input type="button" id="searchTitles" value="Search titles" />
<br /> Choose a title from the filtered list :
<select id="titleList"></select>
<br />
<input type="button" id="readData" value="Read article from dump" />
<div id="articleContent">&nbsp;</div>
<hr />
<!-- Using require.js, a module system for javascript, include the
js files. This loads "main.js", which in turn can load other
files, all handled by require.js:
http://requirejs.org/docs/api.html#jsfiles -->
<script type="text/javascript"
data-main="js/init.js"
src="js/lib/require.js"></script>
</body>
</html>
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Evopedia</title>
<meta name="description" content="Offline wikipedia reader">
<meta name="viewport" content="width=device-width">
<!--
Port of Evopedia (offline wikipedia reader) in HTML5/Javascript, with Firefox OS as the primary target
The original application is at http://www.evopedia.info/
It uses wikipedia dumps located at http://dumpathome.evopedia.info/dumps/finished
Author : Mossroy - mossroy@free.fr
License:
This program is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 3 of the License, or (at your
option) any later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public
License along with this program; if not, see
<http://www.gnu.org/licenses/>.
-->
<link rel="stylesheet" href="css/app.css">
</head>
<body>
<!-- Use this installation button to install locally without going
through the marketpace (see app.js) -->
<!-- <button id="install-btn">Install</button> -->
<!-- Write your application here -->
<h1>Evopedia</h1>
<input type="button" id="showHideAbout" value="About" />
<div id="about">
This is a preliminary work on the port of Evopedia (offline wikipedia reader) in HTML5/Javascript, with Firefox OS as the primary target
<br />
The original application is at <a href="http://www.evopedia.info/">http://www.evopedia.info/</a>
<br />
<br />
To use it, you have to first download locally a dump from <a href="http://dumpathome.evopedia.info/dumps/finished">http://dumpathome.evopedia.info/dumps/finished</a> (with a Bittorrent client), and select some of the dowloaded files below.
<br />
I have tested it with the <a href="http://evopedia.info/dumps/wikipedia_small_2010-08-14.torrent">small dump (2010-08-14)</a>, the <a href="http://evopedia.info/dumps/wikipedia_fr_2013-02-16.torrent">French dump (2013-02-16)</a>, the <a href="http://evopedia.info/dumps/wikipedia_frwiktionary_2011-03-16.torrent">French wiktionary dump (2011-03-16)</a> and the <a href="http://evopedia.info/dumps/wikipedia_en_2012-02-11.torrent">English dump (2012-02-11)</a>
<br />
<br />
<ul>
<li>On desktops, it works on recent Firefox and Chrome, and maybe on other browsers</li>
<li>On the Firefos OS simulator, you have (for now) to put the French dump files in a "fake-sdcard/evopedia" folder of your firefox profile (ex : ~/.mozilla/firefox/xxxx.default/extensions/r2d2b2g@mozilla.org/profile/fake-sdcard). It looks for evopedia/wikipedia_fr_2013-02-16/titles.idx in it. You also need to install the application from the dashboard of the simulator instead of accessing via the browser (due to security restrictions in Firefox OS : only certified webapps can access the sdcard)</li>
<li>On a real Firefox OS device, you also have (for now) to put the French dump files in an "evopedia" directory at the root of your sdcard, so that it finds a file /evopedia/wikipedia_fr_2013-02-16/titles.idx on it</li>
</ul>
<br />
It's only a proof of concept so far : there are many many ways this could be enhanced (suggestions and patches are welcome : the source code is on <a href="https://github.com/mossroy/evopedia-html5">github</a>). In particular :
<ul>
<li>The performance has to be optimized when reading an article</li>
<li>Some searches (for example with prefix "a" on the French dump) do not give any result even if they should</li>
<li>In some cases, the links inside an article do not work, or do not lead to the right article</li>
<li>On a real device, reading an article sometimes crashes because it loads too many things in memory</li>
<li>It is hardly usable on a device because the buttons and inputs are too small</li>
</ul>
<br />
</div>
<div id="openLocalFiles" style="display: none;">
<br /> Please select the file titles.idx :<br /> <input type="file"
id="titleFile" /><br /> Please select the files wikipedia_*.dat
from the same dump :<br /> <input type="file" id="dataFiles" multiple />
</div>
<br /> Find titles starting with :
<input type="text" id="prefix" value="" />&nbsp;
<input type="button" id="searchTitles" value="Search titles" />
<br /> Choose a title from the filtered list :
<select id="titleList"></select>
<br />
<input type="button" id="readData" value="Read article from dump" />
<div id="articleContent">&nbsp;</div>
<hr />
<!-- Using require.js, a module system for javascript, include the
js files. This loads "main.js", which in turn can load other
files, all handled by require.js:
http://requirejs.org/docs/api.html#jsfiles -->
<script type="text/javascript"
data-main="js/init.js"
src="js/lib/require.js"></script>
</body>
</html>

402
www/js/app.js
View File

@ -1,191 +1,211 @@
// This uses require.js to structure javascript:
// http://requirejs.org/docs/api.html#define
define(function(require) {
// Zepto provides nice js and DOM methods (very similar to jQuery,
// and a lot smaller):
// http://zeptojs.com/
var $ = require('zepto');
// Need to verify receipts? This library is included by default.
// https://github.com/mozilla/receiptverifier
require('receiptverifier');
// Want to install the app locally? This library hooks up the
// installation button. See <button class="install-btn"> in
// index.html
require('./install-button');
// Evopedia javascript dependencies
var evopedia = require('evopedia');
var localArchive = null;
// Define behavior of HTML elements
$('#about').hide();
$('#showHideAbout').on('click', function(e) {
$('#about').toggle();
});
$('#searchTitles').on('click', function(e) {
searchTitlesFromPrefix($('#prefix').val());
});
$('#readData').on('click', function(e) {
findTitleFromTitleIdAndLaunchArticleRead($('#titleList').val());
});
$('#prefix').on('keyup', function(e) {
onKeyUpPrefix(e);
});
// Detect if DeviceStorage is available
var storage = null;
if ($.isFunction(navigator.getDeviceStorage)) {
storage = navigator.getDeviceStorage('sdcard');
}
if (storage != null) {
//var directory = 'evopedia/wikipedia_small_2010-08-14';
var directory = 'evopedia/wikipedia_fr_2013-02-16';
localArchive = new evopedia.LocalArchive();
localArchive.readTitleFile(storage, directory);
localArchive.readDataFiles(storage, directory, 0);
}
else {
displayFileSelect();
setLocalArchiveFromFileSelect();
}
/**
* Displays the zone to select files from the dump
*/
function displayFileSelect() {
$('#openLocalFiles').show();
$('#dataFiles').on('change', setLocalArchiveFromFileSelect);
$('#titleFile').on('change', setLocalArchiveFromFileSelect);
}
function setLocalArchiveFromFileSelect() {
dataFiles=document.getElementById('dataFiles').files;
titleFile=document.getElementById('titleFile').files[0];
localArchive = new evopedia.LocalArchive();
localArchive.dataFiles = dataFiles;
localArchive.titleFile = titleFile;
}
/**
* Handle Enter key in the prefix input zone
*/
function onKeyUpPrefix(evt) {
if (evt.keyCode == 13) {
document.getElementById("searchTitles").click();
}
}
/**
* Search the index for titles that start with the given prefix (implemented
* with a binary search inside the index file)
*/
function searchTitlesFromPrefix(prefix) {
if (localArchive.titleFile) {
localArchive.findTitlesWithPrefix(prefix, populateDropDownListOfTitles);
} else {
alert("Title file not set");
}
}
/**
* Populate the drop-down list of titles with the given list
*/
function populateDropDownListOfTitles(titleList) {
var comboTitleList = document.getElementById('titleList');
// Remove previous results
comboTitleList.options.length = 0;
for (var i=0; i<titleList.length; i++) {
var title = titleList[i];
comboTitleList.options[i] = new Option (title.name, title.toStringId());
}
}
/**
* Creates an instance of title from given titleId (including resolving redirects),
* and call the function to read the corresponding article
*/
function findTitleFromTitleIdAndLaunchArticleRead(titleId) {
$("#articleContent").html("Loading article from dump...");
if (localArchive.dataFiles && localArchive.dataFiles.length>0) {
var title = evopedia.Title.parseTitleId(localArchive,titleId);
if (title.fileNr == 255) {
localArchive.resolveRedirect(title, readArticle);
}
else {
readArticle(title);
}
}
else {
alert("Data files not set");
}
}
/**
* Read the article corresponding to the given title
*/
function readArticle(title) {
if ($.isArray(title)) {
title = title[0];
if (title.fileNr == 255) {
localArchive.resolveRedirect(title, readArticle);
return;
}
}
localArchive.readArticle(title, displayArticleInForm);
}
/**
* Display the the given HTML article in the web page,
* and convert links to javascript calls
*/
function displayArticleInForm(htmlArticle) {
// Display the article inside the web page.
$('#articleContent').html(htmlArticle);
// Convert links into javascript calls
$('#articleContent').find('a').each(function(){
// Store current link's url
var url = $(this).attr("href");
if(url.slice(0, 1) == "#") {
// It's an anchor link : do nothing
}
else if (url.substring(0,4) === "http") {
// It's an external link : do nothing
}
else if (url.substring(0,2) === ".." || url.substring(0,4) === "./..") {
// It's a link to another language : TODO redirect to the online article?
}
else {
// It's a link to another article : add an onclick event to go to this article
// instead of following the link
$(this).on('click', function(e) {
goToArticle(decodeURIComponent($(this).attr("href")));
return false;
});
}
});
}
/**
* Replace article content with the one of the given title
*/
function goToArticle(title) {
$("#articleContent").html("Loading article from dump...");
localArchive.getTitleByName(title, readArticle);
}
});
// This uses require.js to structure javascript:
// http://requirejs.org/docs/api.html#define
define(function(require) {
// Zepto provides nice js and DOM methods (very similar to jQuery,
// and a lot smaller):
// http://zeptojs.com/
var $ = require('zepto');
// Need to verify receipts? This library is included by default.
// https://github.com/mozilla/receiptverifier
require('receiptverifier');
// Want to install the app locally? This library hooks up the
// installation button. See <button class="install-btn"> in
// index.html
require('./install-button');
// Evopedia javascript dependencies
var evopedia = require('evopedia');
var localArchive = null;
// Define behavior of HTML elements
$('#about').hide();
$('#showHideAbout').on('click', function(e) {
$('#about').toggle();
});
$('#searchTitles').on('click', function(e) {
searchTitlesFromPrefix($('#prefix').val());
});
$('#readData').on('click', function(e) {
var titleId = $('#titleList').val();
findTitleFromTitleIdAndLaunchArticleRead(titleId);
var title = evopedia.Title.parseTitleId(localArchive,titleId);
pushBrowserHistoryState(title.name);
});
$('#prefix').on('keyup', function(e) {
onKeyUpPrefix(e);
});
// Detect if DeviceStorage is available
var storage = null;
if ($.isFunction(navigator.getDeviceStorage)) {
storage = navigator.getDeviceStorage('sdcard');
}
if (storage != null) {
//var directory = 'evopedia/wikipedia_small_2010-08-14';
var directory = 'evopedia/wikipedia_fr_2013-02-16';
localArchive = new evopedia.LocalArchive();
localArchive.readTitleFile(storage, directory);
localArchive.readDataFiles(storage, directory, 0);
}
else {
displayFileSelect();
setLocalArchiveFromFileSelect();
}
// Display the article when the user goes back in the browser history
window.onpopstate = function(event) {
var titleName = event.state.titleName;
goToArticle(titleName);
};
/**
* Displays the zone to select files from the dump
*/
function displayFileSelect() {
$('#openLocalFiles').show();
$('#dataFiles').on('change', setLocalArchiveFromFileSelect);
$('#titleFile').on('change', setLocalArchiveFromFileSelect);
}
function setLocalArchiveFromFileSelect() {
dataFiles=document.getElementById('dataFiles').files;
titleFile=document.getElementById('titleFile').files[0];
localArchive = new evopedia.LocalArchive();
localArchive.dataFiles = dataFiles;
localArchive.titleFile = titleFile;
}
/**
* Handle Enter key in the prefix input zone
*/
function onKeyUpPrefix(evt) {
if (evt.keyCode == 13) {
document.getElementById("searchTitles").click();
}
}
/**
* Search the index for titles that start with the given prefix (implemented
* with a binary search inside the index file)
*/
function searchTitlesFromPrefix(prefix) {
if (localArchive.titleFile) {
localArchive.findTitlesWithPrefix(prefix, populateDropDownListOfTitles);
} else {
alert("Title file not set");
}
}
/**
* Populate the drop-down list of titles with the given list
*/
function populateDropDownListOfTitles(titleList) {
var comboTitleList = document.getElementById('titleList');
// Remove previous results
comboTitleList.options.length = 0;
for (var i=0; i<titleList.length; i++) {
var title = titleList[i];
comboTitleList.options[i] = new Option (title.name, title.toStringId());
}
}
/**
* Creates an instance of title from given titleId (including resolving redirects),
* and call the function to read the corresponding article
*/
function findTitleFromTitleIdAndLaunchArticleRead(titleId) {
if (localArchive.dataFiles && localArchive.dataFiles.length>0) {
var title = evopedia.Title.parseTitleId(localArchive,titleId);
$("#articleContent").html("Loading from dump article " + title.name + " ...");
if (title.fileNr == 255) {
localArchive.resolveRedirect(title, readArticle);
}
else {
readArticle(title);
}
}
else {
alert("Data files not set");
}
}
/**
* Read the article corresponding to the given title
*/
function readArticle(title) {
if ($.isArray(title)) {
title = title[0];
if (title.fileNr == 255) {
localArchive.resolveRedirect(title, readArticle);
return;
}
}
localArchive.readArticle(title, displayArticleInForm);
}
/**
* Display the the given HTML article in the web page,
* and convert links to javascript calls
*/
function displayArticleInForm(title, htmlArticle) {
// Display the article inside the web page.
$('#articleContent').html(htmlArticle);
// Convert links into javascript calls
$('#articleContent').find('a').each(function(){
// Store current link's url
var url = $(this).attr("href");
if(url.slice(0, 1) == "#") {
// It's an anchor link : do nothing
}
else if (url.substring(0,4) === "http") {
// It's an external link : do nothing
}
else if (url.substring(0,2) === ".." || url.substring(0,4) === "./..") {
// It's a link to another language : TODO redirect to the online article?
}
else {
// It's a link to another article : add an onclick event to go to this article
// instead of following the link
$(this).on('click', function(e) {
var titleName = decodeURIComponent($(this).attr("href"));
pushBrowserHistoryState(titleName);
goToArticle(titleName);
return false;
});
}
});
}
/**
* Changes the URL of the browser page
*/
function pushBrowserHistoryState(titleName) {
if (titleName) {
var stateObj = { titleName: titleName};
window.history.pushState(stateObj,"Wikipedia Article : " + titleName,"#" + titleName);
}
}
/**
* Replace article content with the one of the given title
*/
function goToArticle(titleName) {
$("#articleContent").html("Loading from dump article " + titleName + " ...");
localArchive.getTitleByName(titleName, readArticle);
}
});

55
www/js/lib/evopedia.js
View File

@ -336,30 +336,41 @@ define(function(require) {
alert('Data file read cancelled');
};
reader.onload = function(e) {
var compressedArticles = e.target.result;
var htmlArticles;
try {
htmlArticles = bzip2.simple(bzip2.array(new Uint8Array(
compressedArticles)));
} catch (e) {
// TODO : rethrow exception if we reach the end of the file
currentLocalArchiveInstance.readArticleChunk(title, dataFile, reader, readLength + CHUNK_SIZE,
callbackFunction);
return;
var compressedArticles = e.target.result;
var htmlArticles;
try {
htmlArticles = bzip2.simple(bzip2.array(new Uint8Array(
compressedArticles)));
} catch (e) {
// TODO : there must be a better way to differentiate real exceptions
// and exceptions due to the fact that the article is too long to fit in the chunk
if (e != "No magic number found") {
currentLocalArchiveInstance.readArticleChunk(title, dataFile, reader, readLength + CHUNK_SIZE,
callbackFunction);
return;
}
else {
throw e;
}
}
// Start reading at offset, and keep length characters
var htmlArticle = htmlArticles.substring(title.blockOffset,
title.blockOffset + title.articleLength);
if (htmlArticle.length >= title.articleLength) {
// Keep only length characters
htmlArticle = htmlArticle.substring(0, title.articleLength);
// Decode UTF-8 encoding
htmlArticle = decodeURIComponent(escape(htmlArticle));
callbackFunction(title, htmlArticle);
} else {
// TODO : throw exception if we reach the end of the file
currentLocalArchiveInstance.readArticleChunk(title, dataFile, reader, readLength + CHUNK_SIZE,
callbackFunction);
}
}
// Start reading at offset, and keep length characters
var htmlArticle = htmlArticles.substring(title.blockOffset,
title.blockOffset + title.articleLength);
if (htmlArticle.length >= title.articleLength) {
// Keep only length characters
htmlArticle = htmlArticle.substring(0, title.articleLength);
// Decode UTF-8 encoding
htmlArticle = decodeURIComponent(escape(htmlArticle));
callbackFunction(htmlArticle);
} else {
// TODO : throw exception if we reach the end of the file
currentLocalArchiveInstance.readArticleChunk(title, dataFile, reader, readLength + CHUNK_SIZE,
callbackFunction);
catch (e) {
callbackFunction("Error : " + e);
}
};
var blob = dataFile.slice(title.blockStart, title.blockStart