kiwix/java-libkiwix
1
0
Fork 0
mirror of https://github.com/kiwix/java-libkiwix.git synced 2025-09-08 14:45:54 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #87 from kiwix/fds_archive

Browse Source 
Allow `zim::Archive` to be created with a set of File descriptor.
This commit is contained in:
Kelson 2024-04-22 15:12:17 +02:00 committed by GitHub
commit 6109a7a962
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 194 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/build.gradle
View File

@ -354,7 +354,8 @@ String getLibzimFiles() {
"${projectDir}/src/main/java/org/kiwix/libzim/SuggestionSearcher.java " +
"${projectDir}/src/main/java/org/kiwix/libzim/SuggestionSearch.java " +
"${projectDir}/src/main/java/org/kiwix/libzim/ZimFileFormatException.java " +
"${projectDir}/src/main/java/org/kiwix/libzim/EntryNotFoundException.java"
"${projectDir}/src/main/java/org/kiwix/libzim/EntryNotFoundException.java " +
"${projectDir}/src/main/java/org/kiwix/libzim/FdInput.java"
}
task buildLinuxBinding(type: Exec) {

71
lib/src/main/cpp/libzim/archive.cpp
View File

@ -62,6 +62,23 @@ int jni2fd(const jobject& fdObj, JNIEnv* env)
return env->GetIntField(fdObj, field_fd);
}
zim::FdInput jni2fdInput(const jobject& fdInputObj, JNIEnv* env)
{
jclass class_fdesc = env->FindClass("org/kiwix/libzim/FdInput");
jfieldID field_id = env->GetFieldID(class_fdesc, "fd", "Ljava/io/FileDescriptor;");
jobject fdObj = env->GetObjectField(fdInputObj, field_id);
int fd = jni2fd(fdObj, env);
field_id = env->GetFieldID(class_fdesc, "offset", "J");
long offset = env->GetLongField(fdInputObj, field_id);
field_id = env->GetFieldID(class_fdesc, "size", "J");
long size = env->GetLongField(fdInputObj, field_id);
return zim::FdInput(fd, offset, size);
}
} // unnamed namespace
JNIEXPORT void JNICALL Java_org_kiwix_libzim_Archive_setNativeArchiveByFD(
@ -71,13 +88,8 @@ JNIEXPORT void JNICALL Java_org_kiwix_libzim_Archive_setNativeArchiveByFD(
int fd = jni2fd(fdObj, env);
LOG("Attempting to create reader with fd: %d", fd);
try {
auto archive = std::make_shared<zim::Archive>(fd);
SET_PTR(archive);
} catch (std::exception& e) {
LOG("Error opening ZIM file");
LOG("%s", e.what());
}
auto archive = std::make_shared<zim::Archive>(fd);
SET_PTR(archive);
#else
jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
env->ThrowNew(exception, "org.kiwix.libzim.Archive.setNativeArchiveByFD() is not supported under Windows");
@ -91,13 +103,46 @@ JNIEXPORT void JNICALL Java_org_kiwix_libzim_Archive_setNativeArchiveEmbedded(
int fd = jni2fd(fdObj, env);
LOG("Attempting to create reader with fd: %d", fd);
try {
auto archive = std::make_shared<zim::Archive>(fd, offset, size);
SET_PTR(archive);
} catch (std::exception& e) {
LOG("Error opening ZIM file");
LOG("%s", e.what());
auto archive = std::make_shared<zim::Archive>(fd, offset, size);
SET_PTR(archive);
#else
jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
env->ThrowNew(exception, "org.kiwix.libzim.Archive.setNativeArchiveEmbedded() is not supported under Windows");
#endif
} CATCH_EXCEPTION()
JNIEXPORT void JNICALL Java_org_kiwix_libzim_Archive_setNativeArchiveEmbeddedFd(
JNIEnv* env, jobject thisObj, jobject fdObj) try
{
#ifndef _WIN32
auto fdInput = jni2fdInput(fdObj, env);
auto archive = std::make_shared<zim::Archive>(fdInput);
SET_PTR(archive);
#else
jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
env->ThrowNew(exception, "org.kiwix.libzim.Archive.setNativeArchiveEmbedded() is not supported under Windows");
#endif
} CATCH_EXCEPTION()
JNIEXPORT void JNICALL Java_org_kiwix_libzim_Archive_setNativeArchiveEmbeddedFds(
JNIEnv* env, jobject thisObj, jobjectArray fdsObj) try
{
#ifndef _WIN32
jsize length = env->GetArrayLength(fdsObj);
std::vector<zim::FdInput> v;
int i;
for(i = 0; i<length; i++) {
jobject fdObj = env->GetObjectArrayElement(fdsObj, i);
auto fdInput = jni2fdInput(fdObj, env);
v.push_back(fdInput);
}
auto archive = std::make_shared<zim::Archive>(v);
SET_PTR(archive);
#else
jclass exception = env->FindClass("java/lang/UnsupportedOperationException");
env->ThrowNew(exception, "org.kiwix.libzim.Archive.setNativeArchiveEmbedded() is not supported under Windows");

15
lib/src/main/java/org/kiwix/libzim/Archive.java
View File

@ -23,6 +23,7 @@ import org.kiwix.libzim.ZimFileFormatException;
import org.kiwix.libzim.Entry;
import org.kiwix.libzim.Item;
import org.kiwix.libzim.EntryIterator;
import org.kiwix.libzim.FdInput;
import java.io.FileDescriptor;
public class Archive
@ -44,6 +45,18 @@ public class Archive
setNativeArchiveEmbedded(fd, offset, size);
}
public Archive(FdInput fd)
throws ZimFileFormatException
{
setNativeArchiveEmbeddedFd(fd);
}
public Archive(FdInput[] fds)
throws ZimFileFormatException
{
setNativeArchiveEmbeddedFds(fds);
}
public native String getFilename();
public native long getFilesize();
public native int getAllEntryCount();
@ -94,6 +107,8 @@ public class Archive
private native void setNativeArchive(String filename);
private native void setNativeArchiveByFD(FileDescriptor fd);
private native void setNativeArchiveEmbedded(FileDescriptor fd, long offset, long size);
private native void setNativeArchiveEmbeddedFd(FdInput fd);
private native void setNativeArchiveEmbeddedFds(FdInput[] fds);
@Override
protected void finalize() { dispose(); }

35
lib/src/main/java/org/kiwix/libzim/FdInput.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
package org.kiwix.libzim;
import java.io.FileDescriptor;
public class FdInput
{
public FileDescriptor fd;
public long offset;
public long size;
public FdInput(FileDescriptor fd_, long offset_, long size_) {
fd = fd_;
offset = offset_;
size = size_;
}
}

12
lib/src/test/org/kiwix/test/libzim/TestArchive.java
View File

@ -47,6 +47,18 @@ public class TestArchive
inner = new Archive(fd, offset, size);
}
public TestArchive(FdInput fd)
throws ZimFileFormatException
{
inner = new Archive(fd);
}
public TestArchive(FdInput[] fds)
throws ZimFileFormatException
{
inner = new Archive(fds);
}
public String getFilename() { return inner.getFilename(); }
public long getFilesize() { return inner.getFilesize(); }
public int getAllEntryCount() { return inner.getAllEntryCount(); }

74
lib/src/test/test.java
View File

@ -83,11 +83,9 @@ public class test {
assertTrue(Arrays.equals(faviconData, readData));
// Checking all metadata
assertFalse(archive.isMultiPart());
assertTrue(archive.hasNewNamespaceScheme());
assertTrue(archive.hasChecksum());
assertEquals("4a2709fddbee8c27db708c20b4952a06", archive.getChecksum());
assertTrue(archive.hasTitleIndex());
assertTrue(archive.hasFulltextIndex());
assertTrue(archive.hasMainEntry());
long[] illuSizes = {48};
@ -221,6 +219,8 @@ public class test {
{
TestArchive archive = new TestArchive("small.zim");
testArchive(archive);
assertFalse(archive.isMultiPart());
assertTrue(archive.hasTitleIndex());
assertTrue(archive.check());
assertEquals("small.zim", archive.getFilename());
}
@ -261,6 +261,26 @@ public class test {
FileInputStream fis = new FileInputStream("small.zim");
TestArchive archive = new TestArchive(fis.getFD());
testArchive(archive);
assertFalse(archive.isMultiPart());
assertTrue(archive.hasTitleIndex());
assertTrue(archive.check());
assertEquals("", archive.getFilename());
}
System.gc();
System.runFinalization();
}
@Test
public void testArchiveByFdInput()
throws JNIKiwixException, IOException, ZimFileFormatException, EntryNotFoundException {
{
File plainArchive = new File("small.zim");
FileInputStream fis = new FileInputStream("small.zim");
FdInput fd = new FdInput(fis.getFD(), 0, plainArchive.length());
TestArchive archive = new TestArchive(fd);
testArchive(archive);
assertFalse(archive.isMultiPart());
assertTrue(archive.hasTitleIndex());
assertTrue(archive.check());
assertEquals("", archive.getFilename());
}
@ -278,6 +298,56 @@ public class test {
// This fails. See https://github.com/openzim/libzim/issues/812
//assertTrue(archive.check());
testArchive(archive);
assertFalse(archive.isMultiPart());
assertTrue(archive.hasTitleIndex());
assertEquals("", archive.getFilename());
}
System.gc();
System.runFinalization();
}
@Test
public void testArchiveWithAnEmbeddedArchiveFdInputNaive()
throws JNIKiwixException, IOException, ZimFileFormatException, EntryNotFoundException {
{
File plainArchive = new File("small.zim");
FileInputStream fis = new FileInputStream("small.zim.embedded");
FdInput fd1 = new FdInput(fis.getFD(), 8, plainArchive.length() / 2);
FdInput fd2 = new FdInput(fis.getFD(), fd1.offset + fd1.size, plainArchive.length() - fd1.size);
FdInput fds[] = {fd1, fd2};
TestArchive archive = new TestArchive(fds);
// This fails. See https://github.com/openzim/libzim/issues/812
//assertTrue(archive.check());
testArchive(archive);
assertTrue(archive.isMultiPart());
//Naive split cut the title index in the middle. libzim cannot read it.
assertFalse(archive.hasTitleIndex());
assertEquals("", archive.getFilename());
}
System.gc();
System.runFinalization();
}
@Test
public void testArchiveWithAnEmbeddedArchiveFdInput()
throws JNIKiwixException, IOException, ZimFileFormatException, EntryNotFoundException {
{
File plainArchive = new File("small.zim");
FileInputStream fis = new FileInputStream("small.zim.embedded");
FdInput fd1 = new FdInput(fis.getFD(), 8, plainArchive.length() / 10);
FdInput fd2 = new FdInput(fis.getFD(), fd1.offset + fd1.size, plainArchive.length() - fd1.size);
FdInput fds[] = {fd1, fd2};
TestArchive archive = new TestArchive(fds);
// This fails. See https://github.com/openzim/libzim/issues/812
//assertTrue(archive.check());
testArchive(archive);
assertTrue(archive.isMultiPart());
//If we don't cut in the middle of xapian db, we can read it.
assertTrue(archive.hasTitleIndex());
assertEquals("", archive.getFilename());
}
System.gc();