Support for catalog-only mode

This commit is contained in:
Veloman Yunkan 2025-09-04 18:42:43 +04:00
parent 8a3c4c92e0
commit 25e03ce597
8 changed files with 65 additions and 10 deletions

View File

@ -50,7 +50,7 @@ class HumanReadableNameMapper : public NameMapper {
std::map<std::string, std::string> m_nameToId; std::map<std::string, std::string> m_nameToId;
public: public:
HumanReadableNameMapper(kiwix::Library& library, bool withAlias); HumanReadableNameMapper(const kiwix::Library& library, bool withAlias);
virtual ~HumanReadableNameMapper() = default; virtual ~HumanReadableNameMapper() = default;
virtual std::string getNameForId(const std::string& id) const; virtual std::string getNameForId(const std::string& id) const;
virtual std::string getIdForName(const std::string& name) const; virtual std::string getIdForName(const std::string& name) const;

View File

@ -63,6 +63,7 @@ namespace kiwix
{ m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; } { m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
void setBlockExternalLinks(bool blockExternalLinks) void setBlockExternalLinks(bool blockExternalLinks)
{ m_blockExternalLinks = blockExternalLinks; } { m_blockExternalLinks = blockExternalLinks; }
void setCatalogOnlyMode(bool enable) { m_catalogOnlyMode = enable; }
void setIpMode(IpMode mode) { m_ipMode = mode; } void setIpMode(IpMode mode) { m_ipMode = mode; }
int getPort() const; int getPort() const;
IpAddress getAddress() const; IpAddress getAddress() const;
@ -83,6 +84,7 @@ namespace kiwix
bool m_blockExternalLinks = false; bool m_blockExternalLinks = false;
IpMode m_ipMode = IpMode::AUTO; IpMode m_ipMode = IpMode::AUTO;
int m_ipConnectionLimit = 0; int m_ipConnectionLimit = 0;
bool m_catalogOnlyMode = false;
std::unique_ptr<InternalServer> mp_server; std::unique_ptr<InternalServer> mp_server;
}; };
} }

View File

@ -24,8 +24,8 @@
namespace kiwix { namespace kiwix {
HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool withAlias) { HumanReadableNameMapper::HumanReadableNameMapper(const kiwix::Library& library, bool withAlias) {
for (auto& bookId: library.filter(kiwix::Filter().local(true).valid(true))) { for (auto& bookId: library.filter(kiwix::Filter())) {
auto& currentBook = library.getBookById(bookId); auto& currentBook = library.getBookById(bookId);
auto bookName = currentBook.getHumanReadableIdFromPath(); auto bookName = currentBook.getHumanReadableIdFromPath();
m_idToName[bookId] = bookName; m_idToName[bookId] = bookName;

View File

@ -53,7 +53,8 @@ bool Server::start() {
m_blockExternalLinks, m_blockExternalLinks,
m_ipMode, m_ipMode,
m_indexTemplateString, m_indexTemplateString,
m_ipConnectionLimit)); m_ipConnectionLimit,
m_catalogOnlyMode));
return mp_server->start(); return mp_server->start();
} }

View File

@ -125,9 +125,12 @@ std::string getSearchComponent(const RequestContext& request)
return query.empty() ? query : "?" + query; return query.empty() ? query : "?" + query;
} }
Filter get_search_filter(const RequestContext& request, const std::string& prefix="") Filter get_search_filter(const RequestContext& request, const std::string& prefix="", bool catalogOnlyMode = false)
{ {
auto filter = kiwix::Filter().valid(true).local(true); auto filter = kiwix::Filter();
if ( !catalogOnlyMode ) {
filter.valid(true).local(true);
}
try { try {
filter.query(request.get_argument(prefix+"q")); filter.query(request.get_argument(prefix+"q"));
} catch (const std::out_of_range&) {} } catch (const std::out_of_range&) {}
@ -432,7 +435,8 @@ InternalServer::InternalServer(LibraryPtr library,
bool blockExternalLinks, bool blockExternalLinks,
IpMode ipMode, IpMode ipMode,
std::string indexTemplateString, std::string indexTemplateString,
int ipConnectionLimit) : int ipConnectionLimit,
bool catalogOnlyMode) :
m_addr(addr), m_addr(addr),
m_port(port), m_port(port),
m_root(normalizeRootUrl(root)), m_root(normalizeRootUrl(root)),
@ -451,7 +455,8 @@ InternalServer::InternalServer(LibraryPtr library,
mp_nameMapper(nameMapper ? nameMapper : std::shared_ptr<NameMapper>(&defaultNameMapper, NoDelete())), mp_nameMapper(nameMapper ? nameMapper : std::shared_ptr<NameMapper>(&defaultNameMapper, NoDelete())),
searchCache(getEnvVar<int>("KIWIX_SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)), searchCache(getEnvVar<int>("KIWIX_SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
suggestionSearcherCache(getEnvVar<int>("KIWIX_SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))), suggestionSearcherCache(getEnvVar<int>("KIWIX_SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))),
m_customizedResources(new CustomizedResources) m_customizedResources(new CustomizedResources),
m_catalogOnlyMode(catalogOnlyMode)
{ {
m_root = urlEncode(m_root); m_root = urlEncode(m_root);
} }
@ -1103,7 +1108,7 @@ std::vector<std::string>
InternalServer::search_catalog(const RequestContext& request, InternalServer::search_catalog(const RequestContext& request,
kiwix::OPDSDumper& opdsDumper) kiwix::OPDSDumper& opdsDumper)
{ {
const auto filter = get_search_filter(request); const auto filter = get_search_filter(request, "", m_catalogOnlyMode);
std::vector<std::string> bookIdsToDump = mp_library->filter(filter); std::vector<std::string> bookIdsToDump = mp_library->filter(filter);
const auto totalResults = bookIdsToDump.size(); const auto totalResults = bookIdsToDump.size();
const long count = request.get_optional_param("count", 10L); const long count = request.get_optional_param("count", 10L);

View File

@ -106,7 +106,8 @@ class InternalServer {
bool blockExternalLinks, bool blockExternalLinks,
IpMode ipMode, IpMode ipMode,
std::string indexTemplateString, std::string indexTemplateString,
int ipConnectionLimit); int ipConnectionLimit,
bool catalogOnlyMode);
virtual ~InternalServer(); virtual ~InternalServer();
MHD_Result handlerCallback(struct MHD_Connection* connection, MHD_Result handlerCallback(struct MHD_Connection* connection,
@ -192,6 +193,8 @@ class InternalServer {
class CustomizedResources; class CustomizedResources;
std::unique_ptr<CustomizedResources> m_customizedResources; std::unique_ptr<CustomizedResources> m_customizedResources;
const bool m_catalogOnlyMode;
}; };
} }

View File

@ -150,6 +150,30 @@ std::string maskVariableOPDSFeedData(std::string s)
"125952"\ "125952"\
) )
#define INACCESSIBLEZIMFILE_CATALOG_ENTRY \
" <entry>\n" \
" <id>urn:uuid:inaccessiblezim</id>\n" \
" <title>Catalog of all catalogs</title>\n" \
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n" \
" <summary>Testing that running kiwix-serve without access to ZIM files doesn&apos;t lead to a catastrophe</summary>\n" \
" <language>cat</language>\n" \
" <name>catalog_of_all_catalogs</name>\n" \
" <flavour></flavour>\n" \
" <category>cats</category>\n" \
" <tags>unittest;_category:cats</tags>\n" \
" <articleCount>12107</articleCount>\n" \
" <mediaCount>8</mediaCount>\n" \
" <link type=\"text/html\" href=\"/ROOT%23%3F/content/nosuchzimfile\" />\n" \
" <author>\n" \
" <name>Catherine of Catalonia</name>\n" \
" </author>\n" \
" <publisher>\n" \
" <name>Caterpillar</name>\n" \
" </publisher>\n" \
" <dc:issued>2025-09-04T00:00:00Z</dc:issued>\n" \
" <link rel=\"http://opds-spec.org/acquisition/open-access\" type=\"application/x-zim\" href=\"https://github.com/kiwix/libkiwix/raw/master/test/data/nosuchzimfile.zim\" length=\"20736925696\" />\n" \
" </entry>\n"
TEST_F(LibraryServerTest, catalog_root_xml) TEST_F(LibraryServerTest, catalog_root_xml)
{ {
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/root.xml"); const auto r = zfs1_->GET("/ROOT%23%3F/catalog/root.xml");
@ -697,6 +721,24 @@ TEST_F(LibraryServerTest, catalog_v2_entries)
); );
} }
TEST_F(LibraryServerTest, catalog_v2_entries_catalog_only_mode)
{
resetServer(ZimFileServer::CATALOG_ONLY_MODE);
const auto r = zfs1_->GET("/ROOT%23%3F/catalog/v2/entries");
EXPECT_EQ(r->status, 200);
EXPECT_EQ(maskVariableOPDSFeedData(r->body),
CATALOG_V2_ENTRIES_PREAMBLE("")
" <title>All Entries</title>\n"
" <updated>YYYY-MM-DDThh:mm:ssZ</updated>\n"
"\n"
CHARLES_RAY_CATALOG_ENTRY
INACCESSIBLEZIMFILE_CATALOG_ENTRY
RAY_CHARLES_CATALOG_ENTRY
UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY
"</feed>\n"
);
}
TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range) TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range)
{ {
{ {

View File

@ -62,6 +62,7 @@ public: // types
WITH_LIBRARY_BUTTON = 1 << 2, WITH_LIBRARY_BUTTON = 1 << 2,
BLOCK_EXTERNAL_LINKS = 1 << 3, BLOCK_EXTERNAL_LINKS = 1 << 3,
NO_NAME_MAPPER = 1 << 4, NO_NAME_MAPPER = 1 << 4,
CATALOG_ONLY_MODE = 1 << 5,
WITH_TASKBAR_AND_LIBRARY_BUTTON = WITH_TASKBAR | WITH_LIBRARY_BUTTON, WITH_TASKBAR_AND_LIBRARY_BUTTON = WITH_TASKBAR | WITH_LIBRARY_BUTTON,
@ -149,6 +150,7 @@ void ZimFileServer::run(int serverPort, std::string indexTemplateString)
server->setTaskbar(cfg.options & WITH_TASKBAR, cfg.options & WITH_LIBRARY_BUTTON); server->setTaskbar(cfg.options & WITH_TASKBAR, cfg.options & WITH_LIBRARY_BUTTON);
server->setBlockExternalLinks(cfg.options & BLOCK_EXTERNAL_LINKS); server->setBlockExternalLinks(cfg.options & BLOCK_EXTERNAL_LINKS);
server->setMultiZimSearchLimit(3); server->setMultiZimSearchLimit(3);
server->setCatalogOnlyMode(cfg.options & CATALOG_ONLY_MODE);
if (!indexTemplateString.empty()) { if (!indexTemplateString.empty()) {
server->setIndexTemplateString(indexTemplateString); server->setIndexTemplateString(indexTemplateString);
} }