From 6957a8cb1e9338a4ee367a952b26090024a9c115 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Wed, 7 Dec 2022 17:16:17 +0400 Subject: [PATCH 01/17] Enter docs/ This commit introduces an embryo of sphinx-based documentation. The content of documentation in this commit is limited to the command line options of kiwix-serve. --- docs/conf.py | 50 ++++++++++++++++++ docs/index.rst | 8 +++ docs/kiwix-serve.rst | 114 ++++++++++++++++++++++++++++++++++++++++++ docs/meson.build | 7 +++ docs/requirements.txt | 2 + meson.build | 3 ++ meson_options.txt | 2 + 7 files changed, 186 insertions(+) create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/kiwix-serve.rst create mode 100644 docs/meson.build create mode 100644 docs/requirements.txt diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..bb81550 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,50 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'kiwix-tools' +copyright = '2022, kiwix-team' +author = 'kiwix-team' + + +# -- General configuration --------------------------------------------------- + +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +if not on_rtd: + html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = [] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..a4076af --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,8 @@ +Welcome to kiwix-tools' documentation! +====================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + kiwix-serve diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst new file mode 100644 index 0000000..1227619 --- /dev/null +++ b/docs/kiwix-serve.rst @@ -0,0 +1,114 @@ +*********** +kiwix-serve +*********** + +Usage +===== + +.. code-block:: sh + + kiwix-serve --library [OPTIONS] LIBRARY_FILE_PATH + kiwix-serve [OPTIONS] ZIM_FILE_PATH ... + + +Mandatory arguments +------------------- + +``LIBRARY_FILE_PATH``: XML library file path listing ZIM file to serve. To be +used only with the ``--library`` option. + +``ZIM_FILE_PATH``: ZIM file path (multiple arguments are allowed). + +Options +------- + +-i ADDR, --address=ADDR + + Listen only on this IP address. By default the server listens on all + available IP addresses. + + +-p PORT, --port=PORT + + TCP port on which to listen for HTTP requests (default: 80). + + +-r ROOT, --urlRootLocation=ROOT + + URL prefix on which the content should be made available (default: /). + + +-d, --daemon + + Detach the HTTP server daemon from the main process. + + +-a PID, --attachToProcess=PID + + Exit when the process with id PID stops running. + + +-M, --monitorLibrary + + Monitor the XML library file and reload it automatically when it changes. + + +-m, --nolibrarybutton + + Disable the library home button in the ZIM viewer toolbar. + + +-n, --nosearchbar + + Disable the searchbox in the ZIM viewer toolbar. + + +-b, --blockexternal + + Prevent the users from directly navigating to external resources via such + links in ZIM content. + + +-t N, --threads=N + + Number of threads to run in parallel (default: 4). + + +-s N, --searchLimit=N + + Maximum number of ZIM files in a fulltext multizim search (default: No limit). + + +-z, --nodatealiases + + Create URL aliases for each content by removing the date embedded in the file + name. The expected format of the date in the filename is ``_YYYY-MM``. For + example, ZIM file ``wikipedia_en_all_2020-08.zim`` will be accessible both as + ``wikipedia_en_all_2020-08`` and ``wikipedia_en_all``. + + +-c PATH, --customIndex=PATH + + Override the welcome page with a custom HTML file. + + +-L N, --ipConnectionLimit=N + + Max number of (concurrent) connections per IP (default: infinite, + recommended: >= 6). + + +-v, --verbose + + Print debug log to STDOUT. + + +-V, --version + + Print the software version. + + +-h, --help + + Print the help text. + diff --git a/docs/meson.build b/docs/meson.build new file mode 100644 index 0000000..71d4fc5 --- /dev/null +++ b/docs/meson.build @@ -0,0 +1,7 @@ + +sphinx = find_program('sphinx-build', native:true) + +sphinx_target = run_target('doc', + command: [sphinx, '-bhtml', + meson.current_source_dir(), + meson.current_build_dir()]) diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..24bbc56 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +Sphinx==5.3.0 +sphinx-rtd-theme==1.1.1 diff --git a/meson.build b/meson.build index 0c7baaa..bab9221 100644 --- a/meson.build +++ b/meson.build @@ -26,3 +26,6 @@ if static_linkage endif subdir('src') +if get_option('doc') + subdir('docs') +endif diff --git a/meson_options.txt b/meson_options.txt index 3b0daed..04d2079 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -1,2 +1,4 @@ option('static-linkage', type : 'boolean', value : false, description : 'Create statically linked binaries.') +option('doc', type : 'boolean', value : false, + description : 'Build the documentations.') From 614f15913fd36341c728ad69c6ffaef3bc0ac497 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 9 Dec 2022 17:05:18 +0400 Subject: [PATCH 02/17] Added introduction to kiwix-serve's documentation --- docs/kiwix-serve.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 1227619..842bce5 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -2,6 +2,22 @@ kiwix-serve *********** +Introduction +============ + +``kiwix-serve`` is a tool for serving ZIM file content over HTTP. It supports +serving a library containing multiple ZIM files. In a large library served by a +``kiwix-serve`` instance clients can look up/filter ZIM files of interest by +words in their titles and/or descriptions, language, tags, etc. + +``kiwix-serve`` provides a ZIM file viewer for displaying inidividual pages +from a ZIM file inside the user's web browser (without downloading the full ZIM +file). + +Clients can also remotely search inside those ZIM files that contain a full-text +search database. + + Usage ===== From 76023e9231bdea74c66db845d46cf8a8c8462178 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 8 Dec 2022 16:14:42 +0400 Subject: [PATCH 03/17] Started documenting kiwix-serve HTTP API --- docs/kiwix-serve.rst | 106 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 842bce5..806b90a 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -51,7 +51,7 @@ Options -r ROOT, --urlRootLocation=ROOT - URL prefix on which the content should be made available (default: /). + URL prefix on which the content should be made available (default: empty). -d, --daemon @@ -128,3 +128,107 @@ Options Print the help text. + +HTTP API +======== + + +``kiwix-serve`` serves content at/under ``http://ADDR:PORT/ROOT`` where +``ADDR``, ``PORT`` and ``ROOT`` are the values supplied to the +``--address``/``-i``, ``--port``/``-p`` and ``--urlRootLocation``/``-r`` +options, respectively. + +HTTP API endpoints presented below are relative to that location, i.e. +``/foo/bar`` must be actually accessed as ``http://ADDR:PORT/ROOT/foo/bar``. + +``/`` +----- + +Welcome page is served under ``/``. By default this is the library page, where +books are listed and can be looked up/filtered interactively. However, the +welcome page can be overriden through the ``--customIndex``/``-c`` command line +option of ``kiwix-serve``. + + +``/catalog`` +------------ + +Blablabla + + +``/catch`` +---------- + +Blablabla + + +``/content`` +------------ + +ZIM file content is served under the ``/content`` endpoint as described below. + + +``/content/ZIMNAME/PATH/IN/ZIMFILE`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Returns the entry with path ``PATH/IN/ZIMFILE`` from ZIM file with name +``ZIMNAME``. + + +``/content/ZIMNAME`` +^^^^^^^^^^^^^^^^^^^^ + +``/content/ZIMNAME`` redirects to the main page of the ZIM file with name +``ZIMNAME`` (unless that ZIM file contains an entry with an empty path or path +equal to ``/``, in which case that entry is returned). + + +``/random`` +----------- + +Blablabla + + +``/raw`` +-------- + +Blablabla + + +``/search`` +----------- + +Blablabla + + +``/skin`` +----------- + +Blablabla + + +``/suggest`` +------------ + +Blablabla + + +``/viewer`` +----------- + +ZIM file viewer. The ZIM file and entry therein must be specified via the hash +component of the URL as ``/viewer#ZIMNAME/PATH/IN/ZIMFILE``. + + +``/viewer_settings.js`` +----------------------- + +Settings of the ZIM file viewer that are configurable via certain command line +options of ``kiwix-serve`` (e.g. ``--nolibrarybutton``). + + +/ANYTHING/ELSE +-------------- + +Any other URL is considered as an attempt to access ZIM file content using the +legacy URL scheme and is redirected to ``/content/ANYTHING/ELSE``. From 2aef2d4a926fd56ca46db8f2e9d11593ed0971e7 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 9 Dec 2022 15:39:18 +0400 Subject: [PATCH 04/17] Expanded the /catalog endpoint --- docs/kiwix-serve.rst | 76 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 806b90a..129c324 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -150,8 +150,80 @@ welcome page can be overriden through the ``--customIndex``/``-c`` command line option of ``kiwix-serve``. -``/catalog`` ------------- +``/catalog/v2`` (New API) +------------------------- + +Blablabla + + +``/catalog/v2/root.xml`` +^^^^^^^^^^^^^^^^^^^^^^^^ + +Blablabla + + +``/catalog/v2/searchdescription.xml`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Blablabla + + +``/catalog/v2/categories`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Blablabla + + +``/catalog/v2/entries`` +^^^^^^^^^^^^^^^^^^^^^^^ + +Blablabla + + +``/catalog/v2/entry`` +^^^^^^^^^^^^^^^^^^^^^ + +Blablabla + + +``/catalog/v2/illustration`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Blablabla + + +``/catalog/v2/languages`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Blablabla + + +``/catalog/v2/partial_entries`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Blablabla + + +``/catalog`` (Legacy API) +------------------------- + +Blablabla + + +``/catalog/root.xml`` +^^^^^^^^^^^^^^^^^^^^^ + +Blablabla + + +``/catalog/searchdescription.xml`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Blablabla + + +``/catalog/search`` +^^^^^^^^^^^^^^^^^^^ Blablabla From 7b5b0cd7f74fd99d6e3906531abe8bc6592fb571 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 9 Dec 2022 15:44:16 +0400 Subject: [PATCH 05/17] Documented the /skin endpoint --- docs/kiwix-serve.rst | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 129c324..99ad84b 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -141,6 +141,8 @@ options, respectively. HTTP API endpoints presented below are relative to that location, i.e. ``/foo/bar`` must be actually accessed as ``http://ADDR:PORT/ROOT/foo/bar``. +.. _welcome-page: + ``/`` ----- @@ -276,7 +278,26 @@ Blablabla ``/skin`` ----------- -Blablabla +Static front-end resources (such as CSS, javascript and images) are all grouped +under ``/skin``. + +**Usage:** + ``/skin/PATH/TO/RESOURCE[?cacheid=CACHEID]`` + +`Cache busting +`_ of +static resources is supported via the optional param ``cacheid``. By default, +i.e. when the ``cacheid`` parameter is not specified while accessing the +``/skin`` endpoint, static resources are served as if they were dynamic (i.e. +could be different for an immediately repeated request). Specifying the +``cacheid`` parameter with a correct value (matching the value embedded in the +``kiwix-serve`` instance), makes the returned resource to be presented as +immutable. However, if the value of the ``cacheid`` parameter mismatches then +``kiwix-serve`` responds with a 404 HTTP error. + +``kiwix-serve``'s default front-end (the :ref:`welcome page ` and +the :ref:`ZIM file viewer `) access all underlying static +resources by using explicit ``cacheid`` s. ``/suggest`` @@ -285,6 +306,8 @@ Blablabla Blablabla +.. _zim-file-viewer: + ``/viewer`` ----------- From 11ef036174eb1d026c609c478343637a1980d39c Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 9 Dec 2022 15:57:07 +0400 Subject: [PATCH 06/17] Documented the /random endpoint --- docs/kiwix-serve.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 99ad84b..66ec149 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -260,7 +260,16 @@ equal to ``/``, in which case that entry is returned). ``/random`` ----------- -Blablabla +**Usage:** + + ``/random?content=ZIMNAME`` + +Generates a HTTP redirect to a randomly selected article/page from the +specified ZIM file. + +**Parameters:** + + ``content``: name of the ZIM file. ``/raw`` From 244009f3b12231b136aee6430ef06fa3f479cd24 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sat, 10 Dec 2022 14:58:45 +0400 Subject: [PATCH 07/17] Marked options with directives This will enable cross-referencing them. --- docs/kiwix-serve.rst | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 66ec149..1587d15 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -38,64 +38,64 @@ used only with the ``--library`` option. Options ------- --i ADDR, --address=ADDR +.. option:: -i ADDR, --address=ADDR Listen only on this IP address. By default the server listens on all available IP addresses. --p PORT, --port=PORT +.. option:: -p PORT, --port=PORT TCP port on which to listen for HTTP requests (default: 80). --r ROOT, --urlRootLocation=ROOT +.. option:: -r ROOT, --urlRootLocation=ROOT URL prefix on which the content should be made available (default: empty). --d, --daemon +.. option:: -d, --daemon Detach the HTTP server daemon from the main process. --a PID, --attachToProcess=PID +.. option:: -a PID, --attachToProcess=PID Exit when the process with id PID stops running. --M, --monitorLibrary +.. option:: -M, --monitorLibrary Monitor the XML library file and reload it automatically when it changes. --m, --nolibrarybutton +.. option:: -m, --nolibrarybutton Disable the library home button in the ZIM viewer toolbar. --n, --nosearchbar +.. option:: -n, --nosearchbar Disable the searchbox in the ZIM viewer toolbar. --b, --blockexternal +.. option:: -b, --blockexternal Prevent the users from directly navigating to external resources via such links in ZIM content. --t N, --threads=N +.. option:: -t N, --threads=N Number of threads to run in parallel (default: 4). --s N, --searchLimit=N +.. option:: -s N, --searchLimit=N Maximum number of ZIM files in a fulltext multizim search (default: No limit). --z, --nodatealiases +.. option:: -z, --nodatealiases Create URL aliases for each content by removing the date embedded in the file name. The expected format of the date in the filename is ``_YYYY-MM``. For @@ -103,28 +103,28 @@ Options ``wikipedia_en_all_2020-08`` and ``wikipedia_en_all``. --c PATH, --customIndex=PATH +.. option:: -c PATH, --customIndex=PATH Override the welcome page with a custom HTML file. --L N, --ipConnectionLimit=N +.. option:: -L N, --ipConnectionLimit=N Max number of (concurrent) connections per IP (default: infinite, recommended: >= 6). --v, --verbose +.. option:: -v, --verbose Print debug log to STDOUT. --V, --version +.. option:: -V, --version Print the software version. --h, --help +.. option:: -h, --help Print the help text. From ec2effe25708e8ffd723015d02d7c83c2c5c6fcf Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sat, 10 Dec 2022 15:36:56 +0400 Subject: [PATCH 08/17] Cross-referenced a few options --- docs/kiwix-serve.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 1587d15..ca750be 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -135,8 +135,8 @@ HTTP API ``kiwix-serve`` serves content at/under ``http://ADDR:PORT/ROOT`` where ``ADDR``, ``PORT`` and ``ROOT`` are the values supplied to the -``--address``/``-i``, ``--port``/``-p`` and ``--urlRootLocation``/``-r`` -options, respectively. +:option:`--address`/:option:`-i`, :option:`--port`/:option:`-p` and +:option:`--urlRootLocation`/:option:`-r` options, respectively. HTTP API endpoints presented below are relative to that location, i.e. ``/foo/bar`` must be actually accessed as ``http://ADDR:PORT/ROOT/foo/bar``. @@ -148,8 +148,8 @@ HTTP API endpoints presented below are relative to that location, i.e. Welcome page is served under ``/``. By default this is the library page, where books are listed and can be looked up/filtered interactively. However, the -welcome page can be overriden through the ``--customIndex``/``-c`` command line -option of ``kiwix-serve``. +welcome page can be overriden through the :option:`--customIndex`/:option:`-c` +command line option of ``kiwix-serve``. ``/catalog/v2`` (New API) From ecf5e67759bc31353028f42518b6263f300bb533 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sat, 10 Dec 2022 15:11:41 +0400 Subject: [PATCH 09/17] Documented the --library option --- docs/kiwix-serve.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index ca750be..f5c4afd 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -27,17 +27,27 @@ Usage kiwix-serve [OPTIONS] ZIM_FILE_PATH ... -Mandatory arguments -------------------- +Arguments +--------- -``LIBRARY_FILE_PATH``: XML library file path listing ZIM file to serve. To be -used only with the ``--library`` option. +.. _cli-arg-library-file-path: + +``LIBRARY_FILE_PATH``: path of an XML library file listing ZIM files to serve. +To be used only with the :option:`--library` option. Multiple library files can +be provided as a semicolon (``;``) separated list. ``ZIM_FILE_PATH``: ZIM file path (multiple arguments are allowed). Options ------- +.. option:: --library + + By default, ``kiwix-serve`` expects a list of ZIM files as command line + arguments. Providing the :option:`--library` option tells ``kiwix-serve`` + that the command line argument is rather a :ref:`library XML file + `. + .. option:: -i ADDR, --address=ADDR Listen only on this IP address. By default the server listens on all From 591a347647a838b95d466dd1a16a2e39ccdc8952 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 9 Dec 2022 17:06:03 +0400 Subject: [PATCH 10/17] Added glossary to kiwix-serve's documentation --- docs/kiwix-serve.rst | 56 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index f5c4afd..49306a8 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -8,7 +8,7 @@ Introduction ``kiwix-serve`` is a tool for serving ZIM file content over HTTP. It supports serving a library containing multiple ZIM files. In a large library served by a ``kiwix-serve`` instance clients can look up/filter ZIM files of interest by -words in their titles and/or descriptions, language, tags, etc. +words in their :term:`titles ` and/or descriptions, language, tags, etc. ``kiwix-serve`` provides a ZIM file viewer for displaying inidividual pages from a ZIM file inside the user's web browser (without downloading the full ZIM @@ -142,7 +142,6 @@ Options HTTP API ======== - ``kiwix-serve`` serves content at/under ``http://ADDR:PORT/ROOT`` where ``ADDR``, ``PORT`` and ``ROOT`` are the values supplied to the :option:`--address`/:option:`-i`, :option:`--port`/:option:`-p` and @@ -255,16 +254,16 @@ ZIM file content is served under the ``/content`` endpoint as described below. ``/content/ZIMNAME/PATH/IN/ZIMFILE`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Returns the entry with path ``PATH/IN/ZIMFILE`` from ZIM file with name -``ZIMNAME``. +Returns the entry with path ``PATH/IN/ZIMFILE`` from ZIM file with :term:`name +` ``ZIMNAME``. ``/content/ZIMNAME`` ^^^^^^^^^^^^^^^^^^^^ -``/content/ZIMNAME`` redirects to the main page of the ZIM file with name -``ZIMNAME`` (unless that ZIM file contains an entry with an empty path or path -equal to ``/``, in which case that entry is returned). +``/content/ZIMNAME`` redirects to the main page of the ZIM file with :term:`name +` ``ZIMNAME`` (unless that ZIM file contains an entry with an empty +path or path equal to ``/``, in which case that entry is returned). ``/random`` @@ -279,7 +278,7 @@ specified ZIM file. **Parameters:** - ``content``: name of the ZIM file. + ``content``: :term:`name ` of the ZIM file. ``/raw`` @@ -346,3 +345,44 @@ options of ``kiwix-serve`` (e.g. ``--nolibrarybutton``). Any other URL is considered as an attempt to access ZIM file content using the legacy URL scheme and is redirected to ``/content/ANYTHING/ELSE``. + + +Glossary +======== + +.. glossary:: + + ZIM filename + + Name of a ZIM file on the server filesystem. + + ZIM name + + Identifier of a ZIM file in the server's library (used for referring to a + particular ZIM file in requests). + + For a ``kiwix-serve`` started with a list of ZIM files, ZIM names are + derived from the filename by dropping the extension and replacing certain + characters (spaces are replaced with underscores, and ``+`` symbols are + replaced with the text ``plus``). Presence of the + :option:`-z`/:option:`--nodatealiases` option will create additional names + (aliases) for filenames with dates. + + For a ``kiwix-serve`` started with the :option:`--library` option, ZIM + names come from the library XML file. + + ZIM names are expected to be unique across the library. Any name conflicts + (including those caused by the usage of the + :option:`-z`/:option:`--nodatealiases` option) are reported on STDERR but, + otherwise, are ignored. + + ZIM title + + Title of a ZIM file. This can be any text (with whitespace). It is never + used as a way of referring to a ZIM file. + + ZIM UUID + + This is a unique identifier of a ZIM file designated at its creation time + and embedded in the ZIM file. Certain ``kiwix-serve`` operations may + require that a ZIM file be referenced through its UUID rather than name. From 490ba0f3f292e9782bc299b315dba60e9d07e002 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 11 Dec 2022 16:36:44 +0400 Subject: [PATCH 11/17] Documented the OPDS API --- docs/kiwix-serve.rst | 120 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 97 insertions(+), 23 deletions(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 49306a8..8d63034 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -161,82 +161,156 @@ welcome page can be overriden through the :option:`--customIndex`/:option:`-c` command line option of ``kiwix-serve``. -``/catalog/v2`` (New API) -------------------------- +.. _new-opds-api: -Blablabla +``/catalog/v2`` (OPDS API) +------------------------------ + +The new OPDS API of ``kiwix-serve`` is based on the `OPDS Catalog specification +v1.2 `_. All of its endpoints are grouped under +``/catalog/v2``. + +:ref:`Legacy OPDS API ` is preserved for backward +compatibility. ``/catalog/v2/root.xml`` ^^^^^^^^^^^^^^^^^^^^^^^^ -Blablabla +The OPDS Catalog Root links to the OPDS acquisition and navigation feeds +accessible through the other endpoints of the OPDS API. ``/catalog/v2/searchdescription.xml`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Blablabla +Describes the `/catalog/v2/entries`_ endpoint in `OpenSearch description format +`_. + ``/catalog/v2/categories`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^ -Blablabla +Returns the full list of ZIM file categories as an `OPDS Navigation Feed +`_. ``/catalog/v2/entries`` ^^^^^^^^^^^^^^^^^^^^^^^ -Blablabla +Returns the full or filtered list of ZIM files as an `OPDS acquisition feed +`_ with `complete entries +`_. + +By default, all entries in the library are returned. A subset can be requested +by providing one or more filtering criteria, whereupon only entries matching +*all* of the criteria are included in the response. The filtering criteria must +be specified as URL search parameters. + +* ``lang`` - filter by language (specified as a 3-letter language code). + +* ``category`` - filter by categories associated with the library entries. + +* ``tag`` - filter by tags associated with the library entries. Multiple tags + can be provided as a semicolon separated list (e.g + ``tag=wikipedia;_videos:no``). The result will contain only those entries + that contain *all* of the requested tags. + +* ``notag`` - filter out (exclude) entries with *any* of the specified tags + (example - ``notag=ted;youtube``). + +* ``maxsize`` - include in the results only entries whose size (in bytes) + doesn't exceed the provided value. + +* ``q`` - include in the results only entries that contain the specified text + in the title or description. + +* ``name`` - include in the results only the entry with the specified + :term:`name `. + +* ``start=s`` and ``count=n`` - these parameters enable pagination of the + search/filtering results - the feed will contain (at most) ``n`` results + starting from the result # ``s`` (0-based). -``/catalog/v2/entry`` -^^^^^^^^^^^^^^^^^^^^^ +**Examples:** -Blablabla +.. code:: sh + + # List only books in Italian (lang=ita) but + # return only results ## 100-149 (start=100&count=50) + $ curl 'http://localhost:8080/catalog/v2/entries?lang=ita&start=100&count=50' + + # List only books with category of 'wikipedia' AND containing the word + # 'science' in the title or description + $ curl 'http://localhost:8080/catalog/v2/entries?q=science&category=wikipedia' + +``/catalog/v2/entry/ZIMID`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Returns full info about the library entry with :term:`UUID ` +``ZIMID``. -``/catalog/v2/illustration`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``/catalog/v2/illustration/ZIMID`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Blablabla +**Usage:** + + ``/catalog/v2/illustration/ZIMID?size=N`` + +Returns the illustration of size ``NxN`` pixels for the library entry with +:term:`UUID ` ``ZIMID``. + +If no illustration of requested size is found a HTTP 404 error is returned. ``/catalog/v2/languages`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^ -Blablabla +Returns the full list of ZIM file languages as an `OPDS Navigation Feed +`_. ``/catalog/v2/partial_entries`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Blablabla +Returns the full or filtered list of ZIM files as an `OPDS acquisition feed +`_ with `partial entries +`_. + +Supported filters are the same as for the `/catalog/v2/entries`_ endpoint. -``/catalog`` (Legacy API) -------------------------- +.. _legacy-opds-api: -Blablabla +``/catalog`` (Legacy OPDS API) +------------------------------ + +The legacy OPDS API is preserved for backward compatibility and is deprecated. +:ref:`New OPDS API ` should be used instead. ``/catalog/root.xml`` ^^^^^^^^^^^^^^^^^^^^^ -Blablabla +Full library OPDS catalog (list of all ZIM files). ``/catalog/searchdescription.xml`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Blablabla +Describes the `/catalog/search`_ endpoint in `OpenSearch description format +`_. ``/catalog/search`` ^^^^^^^^^^^^^^^^^^^ -Blablabla +Returns the list of ZIM files (in OPDS catalog format) matching the +search/filtering criteria. Supported filters are the same as for the +`/catalog/v2/entries`_ endpoint. ``/catch`` From 436f1b83f6753587e98bb266a2e6aa57ad9e718d Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Wed, 14 Dec 2022 15:08:44 +0400 Subject: [PATCH 12/17] Documented library reloading via SIGHUP --- docs/kiwix-serve.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 8d63034..f06d03b 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -78,6 +78,10 @@ Options Monitor the XML library file and reload it automatically when it changes. + Library reloading can be forced anytime by sending a SIGHUP signal to the + ``kiwix-serve`` process (this works regardless of the presence of the + :option:`--monitorLibrary`/:option:`-M` option). + .. option:: -m, --nolibrarybutton From 7f863175373442fb692cefa632ffd9539be773b5 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Wed, 14 Dec 2022 15:23:32 +0400 Subject: [PATCH 13/17] Documented the /catch/external endpoint --- docs/kiwix-serve.rst | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index f06d03b..04534cd 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -317,10 +317,28 @@ search/filtering criteria. Supported filters are the same as for the `/catalog/v2/entries`_ endpoint. -``/catch`` ----------- +``/catch/external`` +------------------- + +**Usage:** + + ``/catch/external?source=URL`` + +Generates a HTML page with a link leading to (the decoded version of) ``URL`` +and a warning that following that link will load an external (out of ZIM) +resource. + +**Parameters:** + + ``source``: URL of the external resource (must be URL-encoded). + +**Example:** + +.. code:: sh + + # Intercept an external link to https://example.com?query=abcd + $ curl 'http://localhost:8080/catch/external?source=https%3A%2F%2Fexample.com%3Fquery%3Dabcd' -Blablabla ``/content`` From 68d3c9d81d0b0695c70fa06a27391ad7c995bb3b Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Wed, 14 Dec 2022 16:21:38 +0400 Subject: [PATCH 14/17] Documented the /suggest endpoint --- docs/kiwix-serve.rst | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 04534cd..3162ad2 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -417,7 +417,49 @@ resources by using explicit ``cacheid`` s. ``/suggest`` ------------ -Blablabla +**Usage:** + + ``/suggest?content=ZIMNAME[&term=QUERY][&count=N][&start=S]`` + +Returns suggestions (in JSON format) for a text string that is assumed to be a +partially typed search for a page inside a particular ZIM file. + +Suggestions are obtained as matches of the query text against the page titles +in the ZIM file using the title index database generated during the creation of +the ZIM file. + +In case of a multi-word query the order of the words matters in two ways: + +1. the last word is considered as partially typed, unless followed by a space; +2. ranking of the matches. + +If the ZIM file doesn't contain a title index then suggestions are generated by +listing page titles starting *exactly* (in a case sensitive manner) with the +query text. Otherwise, suggestions are case-insensitive. + +If the ZIM file contains a full text search index, then an extra suggestion is +added as an option to perform a full text search in the said ZIM file. + +**Parameters:** + + ``content`` (mandatory): :term:`name ` of the ZIM file. + + ``term`` (optional; defaults to an empty string): query text. + + ``count`` (optional, default: 10): maximum number of page suggestions in the + response (i.e. the extra option to perform a full text search is not included + in this count). + + ``start`` (optional, default: 0): this parameter enables pagination of + results. The response will include up to ``count`` entries starting with + entry # ``start`` from the full list of page suggestions (the first result is + assumed to have index 0). + +**Example:** + +.. code:: sh + + $ curl 'http://localhost/suggest?content=stackoverflow_en&term=pyth&count=50' .. _zim-file-viewer: From 7179244c478214077659c240c98acad61b889fc3 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Wed, 14 Dec 2022 16:41:13 +0400 Subject: [PATCH 15/17] Documented the /raw endpoint --- docs/kiwix-serve.rst | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 3162ad2..3fc5503 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -377,10 +377,24 @@ specified ZIM file. ``content``: :term:`name ` of the ZIM file. -``/raw`` --------- +``/raw/ZIMNAME/content/PATH/IN/ZIMFILE`` +---------------------------------------- -Blablabla +Returns the entry with path ``PATH/IN/ZIMFILE`` from the ZIM file with +:term:`name ` ``ZIMNAME``. Currently, this endpoint almost duplicates +(with some subtle technical differences) the newer endpoint +`/content/ZIMNAME/PATH/IN/ZIMFILE`_. The important difference is that the +``/raw`` endpoint guarantees that no server-side processing will be applied to +the returned content, whereas content obtained via the ``/content`` endpoint +may in the future undergo some processing intended to improve the operation of +the viewer (e.g. compensating for certain bugs in ZIM creation). + + +``/raw/ZIMNAME/meta/METADATAID`` +-------------------------------- + +Returns the metadata item ``METADATAID`` from the ZIM file with :term:`name +` ``ZIMNAME``. ``/search`` From 996ea031fc4d1356cfd64c494cb03230851479c5 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Wed, 14 Dec 2022 18:16:48 +0400 Subject: [PATCH 16/17] Documented the /search endpoint --- docs/kiwix-serve.rst | 63 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/docs/kiwix-serve.rst b/docs/kiwix-serve.rst index 3fc5503..e430987 100644 --- a/docs/kiwix-serve.rst +++ b/docs/kiwix-serve.rst @@ -400,7 +400,68 @@ Returns the metadata item ``METADATAID`` from the ZIM file with :term:`name ``/search`` ----------- -Blablabla +Performs a full text search on one or more ZIM files and returns an HTML page +with a list of links to matching pages along with snippets of the matching +portions of those pages. + +.. _multi-zim-search-constraints: + +A multi-ZIM search request must comply with the following constraints: + +* the number of ZIM files participating in the search operation must not exceed + the limit imposed by the :option:`--searchLimit` option of ``kiwix-serve``. + +* all of the ZIM files participating in the same search operation must be in + the same language. + +**Parameters:** + + + ZIM file selection parameters: + + At least one of the following parameters must be provided in order to + specify in which ZIM file(s) to search. Parameters appearing earlier in + below list take precedence over subsequent ones (the later ones, even if + present in the request, are simply ignored). + + ``content``: :term:`name ` of the ZIM file (for a single-ZIM + search). This is a legacy parameter. ``books.name`` should be used instead. + + ``books.id``: :term:`UUID ` of the ZIM file. Can be repeated for + a multi-ZIM search, however must respect the :ref:`multi-ZIM search + constraints `. + + ``books.name``: :term:`name ` of the ZIM file. Can be repeated + for a multi-ZIM search, however must respect the :ref:`multi-ZIM search + constraints `. + + Query parameters: + + ``pattern`` (optional; defaults to an empty string): text to search for. + + ``latitude``, ``longitude`` & ``distance`` (optional): geospatial query + parameters. If all of these are provided, then the results will be + restricted to geotagged pages that are within ``distance`` metres from the + location on Earth with coordinates ``latitude`` and ``longitude``. + + Pagination parameters: + + ``pageLength`` (optional, default: 25): maximum number of search results in + the response. Capped at 140. + + ``start`` (optional, default: 1): this parameter enables pagination of + results. The response will include up to ``pageLength`` results starting + with entry # ``start`` from the full list of search results (the first + result is assumed to have index 1). + + + +``/search/searchdescription.xml`` +--------------------------------- + +Describes the `/search`_ endpoint in `OpenSearch description format +`_. + ``/skin`` From 7c7cd42e834369f3811ab0bf4dffa5c6a95af887 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Tue, 3 Jan 2023 17:55:57 +0400 Subject: [PATCH 17/17] Added docs badge to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4ffd409..9da2d5e 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ command line tools: [![Docker (kiwix-serve)](https://img.shields.io/docker/v/kiwix/kiwix-serve?arch=amd64&label=Docker%20(kiwix-serve)&sort=date)](https://hub.docker.com/r/kiwix/kiwix-serve) [![Sandstorm](https://img.shields.io/badge/Sandstorm-kiwix-blue)](https://apps.sandstorm.io/app/5uh349d0kky2zp5whrh2znahn27gwha876xze3864n0fu9e5220h) [![Build Status](https://github.com/kiwix/kiwix-tools/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/kiwix/kiwix-tools/actions?query=branch%3Amain) +[![Doc](https://readthedocs.org/projects/kiwix-tools/badge/?style=flat)](https://kiwix-tools.readthedocs.org/en/latest/?badge=latest) [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-tools/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-tools) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)