kiwix/kiwix-js
1
0
Fork 0
mirror of https://github.com/kiwix/kiwix-js.git synced 2025-09-24 04:54:51 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/mossroy/evopedia-html5

Browse Source 
Conflicts:
	www/index.html
	www/js/app.js
This commit is contained in:
mossroy 2013-05-28 11:13:00 +02:00
commit d3a30a40de
4 changed files with 340 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/tests.js
View File

@ -113,7 +113,7 @@ define(function (require) {
}); });
asyncTest("check readArticle", function(){ asyncTest("check readArticle", function(){
var callbackFunction = function(htmlArticle) { var callbackFunction = function(title, htmlArticle) {
ok(htmlArticle && htmlArticle.length>0,"Article not empty"); ok(htmlArticle && htmlArticle.length>0,"Article not empty");
// Remove new lines // Remove new lines
htmlArticle = htmlArticle.replace(/[\r\n]/g, " "); htmlArticle = htmlArticle.replace(/[\r\n]/g, " ");

191
www/index.html
View File

@ -1,96 +1,95 @@
<!DOCTYPE html> <!DOCTYPE html>
<html> <html>
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
<title>Evopedia</title> <title>Evopedia</title>
<meta name="description" content="Offline wikipedia reader"> <meta name="description" content="Offline wikipedia reader">
<meta name="viewport" content="width=device-width"> <meta name="viewport" content="width=device-width">
<!-- <!--
Port of Evopedia (offline wikipedia reader) in HTML5/Javascript, with Firefox OS as the primary target Port of Evopedia (offline wikipedia reader) in HTML5/Javascript, with Firefox OS as the primary target
The original application is at http://www.evopedia.info/ The original application is at http://www.evopedia.info/
It uses wikipedia dumps located at http://dumpathome.evopedia.info/dumps/finished It uses wikipedia dumps located at http://dumpathome.evopedia.info/dumps/finished
Author : Mossroy - mossroy@free.fr Author : Mossroy - mossroy@free.fr
License: License:
This program is free software; you can redistribute it and/or modify it This program is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 3 of the License, or (at your Free Software Foundation; either version 3 of the License, or (at your
option) any later version. option) any later version.
This program is distributed in the hope that it will be useful, but This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details. General Public License for more details.
You should have received a copy of the GNU General Public You should have received a copy of the GNU General Public
License along with this program; if not, see License along with this program; if not, see
<http://www.gnu.org/licenses/>. <http://www.gnu.org/licenses/>.
--> -->
<link rel="stylesheet" href="css/app.css"> <link rel="stylesheet" href="css/app.css">
</head> </head>
<body> <body>
<!-- Use this installation button to install locally without going <!-- Use this installation button to install locally without going
through the marketpace (see app.js) --> through the marketpace (see app.js) -->
<!-- <button id="install-btn">Install</button> --> <!-- <button id="install-btn">Install</button> -->
<!-- Write your application here --> <!-- Write your application here -->
<h1>Evopedia</h1> <h1>Evopedia</h1>
<input type="button" id="showHideAbout" value="About" /> <input type="button" id="showHideAbout" value="About" />
<div id="about"> <div id="about">
This is a preliminary work on the port of Evopedia (offline wikipedia reader) in HTML5/Javascript, with Firefox OS as the primary target This is a preliminary work on the port of Evopedia (offline wikipedia reader) in HTML5/Javascript, with Firefox OS as the primary target
<br /> <br />
The original application is at <a href="http://www.evopedia.info/">http://www.evopedia.info/</a> The original application is at <a href="http://www.evopedia.info/">http://www.evopedia.info/</a>
<br /> <br />
<br /> <br />
To use it, you have to first download locally a dump from <a href="http://dumpathome.evopedia.info/dumps/finished">http://dumpathome.evopedia.info/dumps/finished</a> (with a Bittorrent client), and select some of the dowloaded files below. To use it, you have to first download locally a dump from <a href="http://dumpathome.evopedia.info/dumps/finished">http://dumpathome.evopedia.info/dumps/finished</a> (with a Bittorrent client), and select some of the dowloaded files below.
<br /> <br />
I have tested it with the <a href="http://evopedia.info/dumps/wikipedia_small_2010-08-14.torrent">small dump (2010-08-14)</a>, the <a href="http://evopedia.info/dumps/wikipedia_fr_2013-02-16.torrent">French dump (2013-02-16)</a>, the <a href="http://evopedia.info/dumps/wikipedia_frwiktionary_2011-03-16.torrent">French wiktionary dump (2011-03-16)</a> and the <a href="http://evopedia.info/dumps/wikipedia_en_2012-02-11.torrent">English dump (2012-02-11)</a> I have tested it with the <a href="http://evopedia.info/dumps/wikipedia_small_2010-08-14.torrent">small dump (2010-08-14)</a>, the <a href="http://evopedia.info/dumps/wikipedia_fr_2013-02-16.torrent">French dump (2013-02-16)</a>, the <a href="http://evopedia.info/dumps/wikipedia_frwiktionary_2011-03-16.torrent">French wiktionary dump (2011-03-16)</a> and the <a href="http://evopedia.info/dumps/wikipedia_en_2012-02-11.torrent">English dump (2012-02-11)</a>
<br /> <br />
<br /> <br />
<ul> <ul>
<li>On desktops, it works on recent Firefox and Chrome, and maybe on other browsers</li> <li>On desktops, it works on recent Firefox and Chrome, and maybe on other browsers</li>
<li>On the Firefos OS simulator, you have (for now) to put the French dump files in a "fake-sdcard/evopedia" folder of your firefox profile (ex : ~/.mozilla/firefox/xxxx.default/extensions/r2d2b2g@mozilla.org/profile/fake-sdcard). It looks for evopedia/wikipedia_fr_2013-02-16/titles.idx in it. You also need to install the application from the dashboard of the simulator instead of accessing via the browser (due to security restrictions in Firefox OS : only certified webapps can access the sdcard)</li> <li>On the Firefos OS simulator, you have (for now) to put the French dump files in a "fake-sdcard/evopedia" folder of your firefox profile (ex : ~/.mozilla/firefox/xxxx.default/extensions/r2d2b2g@mozilla.org/profile/fake-sdcard). It looks for evopedia/wikipedia_fr_2013-02-16/titles.idx in it. You also need to install the application from the dashboard of the simulator instead of accessing via the browser (due to security restrictions in Firefox OS : only certified webapps can access the sdcard)</li>
<li>On a real Firefox OS device, you also have (for now) to put the French dump files in an "evopedia" directory at the root of your sdcard, so that it finds a file /evopedia/wikipedia_fr_2013-02-16/titles.idx on it</li> <li>On a real Firefox OS device, you also have (for now) to put the French dump files in an "evopedia" directory at the root of your sdcard, so that it finds a file /evopedia/wikipedia_fr_2013-02-16/titles.idx on it</li>
</ul> </ul>
<br /> <br />
It's only a proof of concept so far : there are many many ways this could be enhanced (suggestions and patches are welcome : the source code is on <a href="https://github.com/mossroy/evopedia-html5">github</a>). In particular : It's only a proof of concept so far : there are many many ways this could be enhanced (suggestions and patches are welcome : the source code is on <a href="https://github.com/mossroy/evopedia-html5">github</a>). In particular :
<ul> <ul>
<li>The performance has to be optimized when reading an article</li> <li>The performance has to be optimized when reading an article</li>
<li>Some searches (for example with prefix "a" on the French dump) do not give any result even if they should</li> <li>Some searches (for example with prefix "a" on the French dump) do not give any result even if they should</li>
<li>In some cases, the links inside an article do not work, or do not lead to the right article</li> <li>In some cases, the links inside an article do not work, or do not lead to the right article</li>
<li>On a real device, reading an article sometimes crashes because it loads too many things in memory</li> <li>On a real device, reading an article sometimes crashes because it loads too many things in memory</li>
<li>It is hardly usable on a device because the buttons and inputs are too small</li> <li>It is hardly usable on a device because the buttons and inputs are too small</li>
<li>Following the links in an article does not populate the history of the browser, which prevents the use of the back button</li> </ul>
</ul> <br />
<br /> </div>
</div> <div id="openLocalFiles" style="display: none;">
<div id="openLocalFiles" style="display: none;"> <br /> Please select the file titles.idx :<br /> <input type="file"
<br /> Please select the file titles.idx :<br /> <input type="file" id="titleFile" /><br /> Please select the files wikipedia_*.dat
id="titleFile" /><br /> Please select the files wikipedia_*.dat from the same dump :<br /> <input type="file" id="dataFiles" multiple />
from the same dump :<br /> <input type="file" id="dataFiles" multiple /> </div>
</div> <br /> Find titles starting with :
<br /> Find titles starting with : <input type="text" id="prefix" value="" />&nbsp;
<input type="text" id="prefix" value="" />&nbsp; <input type="button" id="searchTitles" value="Search titles" />
<input type="button" id="searchTitles" value="Search titles" /> <br /> Choose a title from the filtered list :
<br /> Choose a title from the filtered list : <select id="titleList"></select>
<select id="titleList"></select> <br />
<br /> <input type="button" id="readData" value="Read article from dump" />
<input type="button" id="readData" value="Read article from dump" /> <div id="articleContent">&nbsp;</div>
<div id="articleContent">&nbsp;</div> <hr />
<hr />
<!-- Using require.js, a module system for javascript, include the
<!-- Using require.js, a module system for javascript, include the js files. This loads "main.js", which in turn can load other
js files. This loads "main.js", which in turn can load other files, all handled by require.js:
files, all handled by require.js: http://requirejs.org/docs/api.html#jsfiles -->
http://requirejs.org/docs/api.html#jsfiles --> <script type="text/javascript"
<script type="text/javascript" data-main="js/init.js"
data-main="js/init.js" src="js/lib/require.js"></script>
src="js/lib/require.js"></script> </body>
</body> </html>
</html>

402
www/js/app.js
View File

@ -1,191 +1,211 @@
// This uses require.js to structure javascript: // This uses require.js to structure javascript:
// http://requirejs.org/docs/api.html#define // http://requirejs.org/docs/api.html#define
define(function(require) { define(function(require) {
// Zepto provides nice js and DOM methods (very similar to jQuery, // Zepto provides nice js and DOM methods (very similar to jQuery,
// and a lot smaller): // and a lot smaller):
// http://zeptojs.com/ // http://zeptojs.com/
var $ = require('zepto'); var $ = require('zepto');
// Need to verify receipts? This library is included by default. // Need to verify receipts? This library is included by default.
// https://github.com/mozilla/receiptverifier // https://github.com/mozilla/receiptverifier
require('receiptverifier'); require('receiptverifier');
// Want to install the app locally? This library hooks up the // Want to install the app locally? This library hooks up the
// installation button. See <button class="install-btn"> in // installation button. See <button class="install-btn"> in
// index.html // index.html
require('./install-button'); require('./install-button');
// Evopedia javascript dependencies // Evopedia javascript dependencies
var evopedia = require('evopedia'); var evopedia = require('evopedia');
var localArchive = null; var localArchive = null;
// Define behavior of HTML elements // Define behavior of HTML elements
$('#about').hide(); $('#about').hide();
$('#showHideAbout').on('click', function(e) { $('#showHideAbout').on('click', function(e) {
$('#about').toggle(); $('#about').toggle();
}); });
$('#searchTitles').on('click', function(e) { $('#searchTitles').on('click', function(e) {
searchTitlesFromPrefix($('#prefix').val()); searchTitlesFromPrefix($('#prefix').val());
}); });
$('#readData').on('click', function(e) { $('#readData').on('click', function(e) {
findTitleFromTitleIdAndLaunchArticleRead($('#titleList').val()); var titleId = $('#titleList').val();
}); findTitleFromTitleIdAndLaunchArticleRead(titleId);
$('#prefix').on('keyup', function(e) { var title = evopedia.Title.parseTitleId(localArchive,titleId);
onKeyUpPrefix(e); pushBrowserHistoryState(title.name);
}); });
$('#prefix').on('keyup', function(e) {
onKeyUpPrefix(e);
// Detect if DeviceStorage is available });
var storage = null;
if ($.isFunction(navigator.getDeviceStorage)) {
storage = navigator.getDeviceStorage('sdcard'); // Detect if DeviceStorage is available
} var storage = null;
if ($.isFunction(navigator.getDeviceStorage)) {
if (storage != null) { storage = navigator.getDeviceStorage('sdcard');
//var directory = 'evopedia/wikipedia_small_2010-08-14'; }
var directory = 'evopedia/wikipedia_fr_2013-02-16';
localArchive = new evopedia.LocalArchive(); if (storage != null) {
localArchive.readTitleFile(storage, directory); //var directory = 'evopedia/wikipedia_small_2010-08-14';
localArchive.readDataFiles(storage, directory, 0); var directory = 'evopedia/wikipedia_fr_2013-02-16';
} localArchive = new evopedia.LocalArchive();
else { localArchive.readTitleFile(storage, directory);
displayFileSelect(); localArchive.readDataFiles(storage, directory, 0);
setLocalArchiveFromFileSelect(); }
} else {
displayFileSelect();
/** setLocalArchiveFromFileSelect();
* Displays the zone to select files from the dump }
*/
function displayFileSelect() { // Display the article when the user goes back in the browser history
$('#openLocalFiles').show(); window.onpopstate = function(event) {
$('#dataFiles').on('change', setLocalArchiveFromFileSelect); var titleName = event.state.titleName;
$('#titleFile').on('change', setLocalArchiveFromFileSelect); goToArticle(titleName);
} };
function setLocalArchiveFromFileSelect() { /**
dataFiles=document.getElementById('dataFiles').files; * Displays the zone to select files from the dump
titleFile=document.getElementById('titleFile').files[0]; */
localArchive = new evopedia.LocalArchive(); function displayFileSelect() {
localArchive.dataFiles = dataFiles; $('#openLocalFiles').show();
localArchive.titleFile = titleFile; $('#dataFiles').on('change', setLocalArchiveFromFileSelect);
} $('#titleFile').on('change', setLocalArchiveFromFileSelect);
}
/**
* Handle Enter key in the prefix input zone function setLocalArchiveFromFileSelect() {
*/ dataFiles=document.getElementById('dataFiles').files;
function onKeyUpPrefix(evt) { titleFile=document.getElementById('titleFile').files[0];
if (evt.keyCode == 13) { localArchive = new evopedia.LocalArchive();
document.getElementById("searchTitles").click(); localArchive.dataFiles = dataFiles;
} localArchive.titleFile = titleFile;
} }
/**
* Handle Enter key in the prefix input zone
/** */
* Search the index for titles that start with the given prefix (implemented function onKeyUpPrefix(evt) {
* with a binary search inside the index file) if (evt.keyCode == 13) {
*/ document.getElementById("searchTitles").click();
function searchTitlesFromPrefix(prefix) { }
if (localArchive.titleFile) { }
localArchive.findTitlesWithPrefix(prefix, populateDropDownListOfTitles);
} else {
alert("Title file not set");
} /**
} * Search the index for titles that start with the given prefix (implemented
* with a binary search inside the index file)
/** */
* Populate the drop-down list of titles with the given list function searchTitlesFromPrefix(prefix) {
*/ if (localArchive.titleFile) {
function populateDropDownListOfTitles(titleList) { localArchive.findTitlesWithPrefix(prefix, populateDropDownListOfTitles);
var comboTitleList = document.getElementById('titleList'); } else {
// Remove previous results alert("Title file not set");
comboTitleList.options.length = 0; }
for (var i=0; i<titleList.length; i++) { }
var title = titleList[i];
comboTitleList.options[i] = new Option (title.name, title.toStringId()); /**
} * Populate the drop-down list of titles with the given list
} */
function populateDropDownListOfTitles(titleList) {
var comboTitleList = document.getElementById('titleList');
/** // Remove previous results
* Creates an instance of title from given titleId (including resolving redirects), comboTitleList.options.length = 0;
* and call the function to read the corresponding article for (var i=0; i<titleList.length; i++) {
*/ var title = titleList[i];
function findTitleFromTitleIdAndLaunchArticleRead(titleId) { comboTitleList.options[i] = new Option (title.name, title.toStringId());
$("#articleContent").html("Loading article from dump..."); }
if (localArchive.dataFiles && localArchive.dataFiles.length>0) { }
var title = evopedia.Title.parseTitleId(localArchive,titleId);
if (title.fileNr == 255) {
localArchive.resolveRedirect(title, readArticle); /**
} * Creates an instance of title from given titleId (including resolving redirects),
else { * and call the function to read the corresponding article
readArticle(title); */
} function findTitleFromTitleIdAndLaunchArticleRead(titleId) {
} if (localArchive.dataFiles && localArchive.dataFiles.length>0) {
else { var title = evopedia.Title.parseTitleId(localArchive,titleId);
alert("Data files not set"); $("#articleContent").html("Loading from dump article " + title.name + " ...");
} if (title.fileNr == 255) {
} localArchive.resolveRedirect(title, readArticle);
}
/** else {
* Read the article corresponding to the given title readArticle(title);
*/ }
function readArticle(title) { }
if ($.isArray(title)) { else {
title = title[0]; alert("Data files not set");
if (title.fileNr == 255) { }
localArchive.resolveRedirect(title, readArticle); }
return;
} /**
} * Read the article corresponding to the given title
localArchive.readArticle(title, displayArticleInForm); */
} function readArticle(title) {
if ($.isArray(title)) {
/** title = title[0];
* Display the the given HTML article in the web page, if (title.fileNr == 255) {
* and convert links to javascript calls localArchive.resolveRedirect(title, readArticle);
*/ return;
function displayArticleInForm(htmlArticle) { }
// Display the article inside the web page. }
$('#articleContent').html(htmlArticle); localArchive.readArticle(title, displayArticleInForm);
}
// Convert links into javascript calls
$('#articleContent').find('a').each(function(){ /**
// Store current link's url * Display the the given HTML article in the web page,
var url = $(this).attr("href"); * and convert links to javascript calls
*/
if(url.slice(0, 1) == "#") { function displayArticleInForm(title, htmlArticle) {
// It's an anchor link : do nothing // Display the article inside the web page.
} $('#articleContent').html(htmlArticle);
else if (url.substring(0,4) === "http") {
// It's an external link : do nothing // Convert links into javascript calls
} $('#articleContent').find('a').each(function(){
else if (url.substring(0,2) === ".." || url.substring(0,4) === "./..") { // Store current link's url
// It's a link to another language : TODO redirect to the online article? var url = $(this).attr("href");
}
else { if(url.slice(0, 1) == "#") {
// It's a link to another article : add an onclick event to go to this article // It's an anchor link : do nothing
// instead of following the link }
$(this).on('click', function(e) { else if (url.substring(0,4) === "http") {
goToArticle(decodeURIComponent($(this).attr("href"))); // It's an external link : do nothing
return false; }
}); else if (url.substring(0,2) === ".." || url.substring(0,4) === "./..") {
} // It's a link to another language : TODO redirect to the online article?
}); }
} else {
// It's a link to another article : add an onclick event to go to this article
// instead of following the link
/** $(this).on('click', function(e) {
* Replace article content with the one of the given title var titleName = decodeURIComponent($(this).attr("href"));
*/ pushBrowserHistoryState(titleName);
function goToArticle(title) { goToArticle(titleName);
$("#articleContent").html("Loading article from dump..."); return false;
localArchive.getTitleByName(title, readArticle); });
} }
});
}); }
/**
* Changes the URL of the browser page
*/
function pushBrowserHistoryState(titleName) {
if (titleName) {
var stateObj = { titleName: titleName};
window.history.pushState(stateObj,"Wikipedia Article : " + titleName,"#" + titleName);
}
}
/**
* Replace article content with the one of the given title
*/
function goToArticle(titleName) {
$("#articleContent").html("Loading from dump article " + titleName + " ...");
localArchive.getTitleByName(titleName, readArticle);
}
});

55
www/js/lib/evopedia.js
View File

@ -336,30 +336,41 @@ define(function(require) {
alert('Data file read cancelled'); alert('Data file read cancelled');
}; };
reader.onload = function(e) { reader.onload = function(e) {
var compressedArticles = e.target.result;
var htmlArticles;
try { try {
htmlArticles = bzip2.simple(bzip2.array(new Uint8Array( var compressedArticles = e.target.result;
compressedArticles))); var htmlArticles;
} catch (e) { try {
// TODO : rethrow exception if we reach the end of the file htmlArticles = bzip2.simple(bzip2.array(new Uint8Array(
currentLocalArchiveInstance.readArticleChunk(title, dataFile, reader, readLength + CHUNK_SIZE, compressedArticles)));
callbackFunction); } catch (e) {
return; // TODO : there must be a better way to differentiate real exceptions
// and exceptions due to the fact that the article is too long to fit in the chunk
if (e != "No magic number found") {
currentLocalArchiveInstance.readArticleChunk(title, dataFile, reader, readLength + CHUNK_SIZE,
callbackFunction);
return;
}
else {
throw e;
}
}
// Start reading at offset, and keep length characters
var htmlArticle = htmlArticles.substring(title.blockOffset,
title.blockOffset + title.articleLength);
if (htmlArticle.length >= title.articleLength) {
// Keep only length characters
htmlArticle = htmlArticle.substring(0, title.articleLength);
// Decode UTF-8 encoding
htmlArticle = decodeURIComponent(escape(htmlArticle));
callbackFunction(title, htmlArticle);
} else {
// TODO : throw exception if we reach the end of the file
currentLocalArchiveInstance.readArticleChunk(title, dataFile, reader, readLength + CHUNK_SIZE,
callbackFunction);
}
} }
// Start reading at offset, and keep length characters catch (e) {
var htmlArticle = htmlArticles.substring(title.blockOffset, callbackFunction("Error : " + e);
title.blockOffset + title.articleLength);
if (htmlArticle.length >= title.articleLength) {
// Keep only length characters
htmlArticle = htmlArticle.substring(0, title.articleLength);
// Decode UTF-8 encoding
htmlArticle = decodeURIComponent(escape(htmlArticle));
callbackFunction(htmlArticle);
} else {
// TODO : throw exception if we reach the end of the file
currentLocalArchiveInstance.readArticleChunk(title, dataFile, reader, readLength + CHUNK_SIZE,
callbackFunction);
} }
}; };
var blob = dataFile.slice(title.blockStart, title.blockStart var blob = dataFile.slice(title.blockStart, title.blockStart