add support for ft-index in zim

Kiwix-iOS/Info.plist

@@ -49,7 +49,7 @@
 		</dict>
 	</array>
 	<key>CFBundleVersion</key>
-	<string>1.7.719</string>
+	<string>1.7.728</string>
 	<key>ITSAppUsesNonExemptEncryption</key>
 	<false/>
 	<key>LSRequiresIPhoneOS</key>

Kiwix-iOSWidgets/Bookmarks/Info.plist

@@ -21,7 +21,7 @@
 	<key>CFBundleSignature</key>
 	<string>????</string>
 	<key>CFBundleVersion</key>
-	<string>1.7.816</string>
+	<string>1.7.833</string>
 	<key>NSExtension</key>
 	<dict>
 		<key>NSExtensionMainStoryboard</key>

Kiwix.xcodeproj/project.pbxproj

@@ -132,8 +132,6 @@
 		975227A11D020C00001D1DDE /* resourceTools.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 975227961D020C00001D1DDE /* resourceTools.cpp */; };
 		975227A21D020C00001D1DDE /* stringTools.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 975227971D020C00001D1DDE /* stringTools.cpp */; settings = {COMPILER_FLAGS = "-w"; }; };
 		975227A31D020C00001D1DDE /* stringTools.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 975227971D020C00001D1DDE /* stringTools.cpp */; settings = {COMPILER_FLAGS = "-w"; }; };
-		975227A81D020C2E001D1DDE /* htmlparse.cc in Sources */ = {isa = PBXBuildFile; fileRef = 975227A61D020C2E001D1DDE /* htmlparse.cc */; };
-		975227A91D020C2E001D1DDE /* myhtmlparse.cc in Sources */ = {isa = PBXBuildFile; fileRef = 975227A71D020C2E001D1DDE /* myhtmlparse.cc */; };
 		975227AC1D020D83001D1DDE /* ZimIndexer.mm in Sources */ = {isa = PBXBuildFile; fileRef = 975227AB1D020D83001D1DDE /* ZimIndexer.mm */; };
 		975227AE1D0213D3001D1DDE /* xapianIndexer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 975227AD1D0213D3001D1DDE /* xapianIndexer.cpp */; settings = {COMPILER_FLAGS = "-w"; }; };
 		975227B01D021539001D1DDE /* IndexerController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 975227AF1D021539001D1DDE /* IndexerController.swift */; };
@@ -373,8 +371,6 @@
 		973DD4271D36E3E4009D45DB /* SettingSingleSwitchTBVC.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = SettingSingleSwitchTBVC.swift; path = "Kiwix-iOS/Controller/Setting/SettingSingleSwitchTBVC.swift"; sourceTree = SOURCE_ROOT; };
 		97497B5A1D07487000ECD691 /* indexer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = indexer.h; path = "Kiwix/libkiwix/C&C++/indexer.h"; sourceTree = "<group>"; };
 		97497B5B1D07487000ECD691 /* resourceTools.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = resourceTools.h; path = "Kiwix/libkiwix/C&C++/resourceTools.h"; sourceTree = "<group>"; };
-		97497B5C1D074FB800ECD691 /* htmlparse.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = htmlparse.h; path = "Kiwix/libkiwix/C&C++/xapian/htmlparse.h"; sourceTree = "<group>"; };
-		97497B5D1D074FB800ECD691 /* myhtmlparse.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = myhtmlparse.h; path = "Kiwix/libkiwix/C&C++/xapian/myhtmlparse.h"; sourceTree = "<group>"; };
 		97497B5E1D07584100ECD691 /* xapianIndexer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = xapianIndexer.h; path = "Kiwix/libkiwix/C&C++/xapianIndexer.h"; sourceTree = "<group>"; };
 		974F42811D47E19A00F8074C /* SettingWidgetBookmarksTBVC.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = SettingWidgetBookmarksTBVC.swift; path = "Kiwix-iOS/Controller/Setting/SettingWidgetBookmarksTBVC.swift"; sourceTree = SOURCE_ROOT; };
 		975227811D020560001D1DDE /* Indexer.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = Indexer.storyboard; path = "Kiwix-OSX/StoryBoards/Indexer.storyboard"; sourceTree = SOURCE_ROOT; };
@@ -384,8 +380,6 @@
 		975227951D020C00001D1DDE /* reader.cpp */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 2; lastKnownFileType = sourcecode.cpp.cpp; name = reader.cpp; path = "Kiwix/libkiwix/C&C++/reader.cpp"; sourceTree = "<group>"; tabWidth = 2; };
 		975227961D020C00001D1DDE /* resourceTools.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = resourceTools.cpp; path = "Kiwix/libkiwix/C&C++/resourceTools.cpp"; sourceTree = "<group>"; };
 		975227971D020C00001D1DDE /* stringTools.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = stringTools.cpp; path = "Kiwix/libkiwix/C&C++/stringTools.cpp"; sourceTree = "<group>"; };
-		975227A61D020C2E001D1DDE /* htmlparse.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = htmlparse.cc; path = "Kiwix/libkiwix/C&C++/xapian/htmlparse.cc"; sourceTree = "<group>"; };
-		975227A71D020C2E001D1DDE /* myhtmlparse.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = myhtmlparse.cc; path = "Kiwix/libkiwix/C&C++/xapian/myhtmlparse.cc"; sourceTree = "<group>"; };
 		975227AA1D020D83001D1DDE /* ZimIndexer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ZimIndexer.h; path = Kiwix/libkiwix/ZimIndexer.h; sourceTree = "<group>"; };
 		975227AB1D020D83001D1DDE /* ZimIndexer.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; name = ZimIndexer.mm; path = Kiwix/libkiwix/ZimIndexer.mm; sourceTree = "<group>"; };
 		975227AD1D0213D3001D1DDE /* xapianIndexer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 2; lastKnownFileType = sourcecode.cpp.cpp; name = xapianIndexer.cpp; path = "Kiwix/libkiwix/C&C++/xapianIndexer.cpp"; sourceTree = "<group>"; tabWidth = 2; };
@@ -824,10 +818,6 @@
 				975227921D020C00001D1DDE /* indexer.cpp */,
 				97497B5E1D07584100ECD691 /* xapianIndexer.h */,
 				975227AD1D0213D3001D1DDE /* xapianIndexer.cpp */,
-				97497B5C1D074FB800ECD691 /* htmlparse.h */,
-				975227A61D020C2E001D1DDE /* htmlparse.cc */,
-				97497B5D1D074FB800ECD691 /* myhtmlparse.h */,
-				975227A71D020C2E001D1DDE /* myhtmlparse.cc */,
 				975227931D020C00001D1DDE /* otherTools.cpp */,
 			);
 			name = indexer;
@@ -1565,7 +1555,6 @@
 				975B912F1CEB9B0F00D13906 /* Preference.swift in Sources */,
 				971187941CEB541A00B9909D /* Article.swift in Sources */,
 				9752279D1D020C00001D1DDE /* pathTools.cpp in Sources */,
-				975227A91D020C2E001D1DDE /* myhtmlparse.cc in Sources */,
 				975227B01D021539001D1DDE /* IndexerController.swift in Sources */,
 				975334D01CEB6AE3007ED50B /* PreferenceGeneralController.swift in Sources */,
 				979CB6C81D05CF37005E1BA1 /* SearchResultController.swift in Sources */,
@@ -1579,7 +1568,6 @@
 				975334D21CEB6B01007ED50B /* LibraryController.swift in Sources */,
 				975227AC1D020D83001D1DDE /* ZimIndexer.mm in Sources */,
 				975227A11D020C00001D1DDE /* resourceTools.cpp in Sources */,
-				975227A81D020C2E001D1DDE /* htmlparse.cc in Sources */,
 				971187971CEB542500B9909D /* Article+CoreDataProperties.swift in Sources */,
 				975227A31D020C00001D1DDE /* stringTools.cpp in Sources */,
 				979C518D1CECAE4C001707F2 /* PreferenceWindowController.swift in Sources */,

Kiwix/libkiwix/ZimReader.mm

@@ -32,11 +32,24 @@
         }
         
         try {
-            NSString *zimPath = [url absoluteString];
+            zim::File zimFileHandle = *_reader->getZimFileHandler();
-            zimPath = [zimPath stringByReplacingOccurrencesOfString:@".zimaa" withString:@".zim"];
+            zim::Article xapianArticle = zimFileHandle.getArticle('Z', "/Z/fulltextIndex/xapian");
-            NSURL *xapianURl = [[NSURL fileURLWithPath:zimPath] URLByAppendingPathExtension:@"idx"];
+            if (xapianArticle.good()) {
-            _db = new Xapian::Database([xapianURl fileSystemRepresentation]);
+                zim::offset_type dbOffset = xapianArticle.getOffset();
-        } catch (const Xapian::DatabaseOpeningError &e) {}
+                int databasefd = open([url fileSystemRepresentation], O_RDONLY);
+                lseek(databasefd, dbOffset, SEEK_SET);
+                _db = new Xapian::Database(databasefd);
+            } else {
+                throw "xapian db not in zim";
+            }
+        } catch (...) {
+            try {
+                NSString *zimPath = [url absoluteString];
+                zimPath = [zimPath stringByReplacingOccurrencesOfString:@".zimaa" withString:@".zim"];
+                NSURL *xapianURl = [[NSURL fileURLWithPath:zimPath] URLByAppendingPathExtension:@"idx"];
+                _db = new Xapian::Database([xapianURl fileSystemRepresentation]);
+            } catch (const Xapian::DatabaseOpeningError &e) {}
+        }
         
         self.fileURL = url;
     }

Kiwix/libkiwix/build/build-xapian.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
 
+# regarding export MACOSX_DEPLOYMENT_TARGET="10.4" in build_iOS, see reason why do it like this
+# http://stackoverflow.com/questions/32622284/building-c-static-libraries-using-configure-make-with-fembed-bitcode-fails
+
 ROOT=$(pwd)
 
 ########################  XAPIAN  ########################
 
-XAPIANPATH=$ROOT/xapian-core-1.4.0
-cd $XAPIANPATH
-
 build_iOS()
 {
 	ARCH=$1
@@ -18,19 +18,24 @@ build_iOS()
 		SDKROOT="$(xcodebuild -version -sdk iphoneos | grep -E '^Path' | sed 's/Path: //')"
 	fi
 
-	export CC="$(xcrun -sdk iphoneos -find clang)"
+    export MACOSX_DEPLOYMENT_TARGET="10.4"
+
+	export CC="$(xcrun -find clang)"
 	export CFLAGS="-fembed-bitcode -isysroot $SDKROOT -arch ${ARCH} -miphoneos-version-min=9.0"
 
 	export CPP="$CC -E"
 	export CPPFLAGS="$CFLAGS"
 
+	export CXX="$(xcrun -find clang++)"
+	export CXXFLAGS="$CFLAGS -stdlib=libc++ -std=c++11"
+
 	export LDFLAGS="-arch ${ARCH} -isysroot $SDKROOT"
 
 	if [ $ARCH == "i386" ] || [ $ARCH == "x86_64" ];
 	then
-		./configure --prefix=$(pwd)/build/iOS/$ARCH --host=i686-apple-darwin11 --enable-static --disable-shared
+		./configure --prefix=$(pwd)/build/iOS/$ARCH --host=i686-apple-darwin11 --enable-static --enable-shared
 	else
-		./configure --prefix=$(pwd)/build/iOS/$ARCH --host=arm-apple-darwin --enable-static --disable-shared
+		./configure --prefix=$(pwd)/build/iOS/$ARCH --host=arm-apple-darwin --enable-static --enable-shared
 	fi
 
 	make && make install && make clean
@@ -44,10 +49,13 @@ build_OSX()
 
 	export MACOSX_DEPLOYMENT_TARGET="10.10"
 
-	export CC="$(xcrun -sdk macosx10.11 -find clang)"
+	export CC="$(xcrun -find clang)"
 	export CFLAGS="-fembed-bitcode -isysroot $SDKROOT -arch ${ARCH} -mmacosx-version-min=10.10"
 
-	export CXX="$(xcrun -sdk macosx10.11 -find clang++)"
+	export CPP="$CC -E"
+	export CPPFLAGS="$CFLAGS"
+
+	export CXX="$(xcrun -find clang++)"
 	export CXXFLAGS="$CFLAGS -stdlib=libc++ -std=c++11"
 
 	export LDFLAGS="-arch ${ARCH} -isysroot $SDKROOT"
@@ -57,6 +65,28 @@ build_OSX()
 	make && make install && make clean
 }
 
+distribute_iOS() {
+	iOSBUILDDir=$(pwd)/build/iOS
+	cd $iOSBUILDDir
+	mkdir -p universal/lib
+
+	cd armv7/lib
+	for file in *.a
+	do
+		cd $iOSBUILDDir
+		lipo -create armv7/lib/$file armv7s/lib/$file arm64/lib/$file x86_64/lib/$file i386/lib/$file -output universal/lib/$file
+	done
+
+	cd armv7/lib
+	for file in *.dylib
+	do
+		cd $iOSBUILDDir
+		lipo -create armv7/lib/$file armv7s/lib/$file arm64/lib/$file x86_64/lib/$file i386/lib/$file -output universal/lib/$file
+	done
+
+	cp -r armv7/include universal
+}
+
 distribute() {
 	cd $XAPIANPATH/build
 	mkdir -p Universal/iOS/lib
@@ -86,9 +116,10 @@ distribute() {
 # build_iOS armv7
 # build_iOS armv7s
 # build_iOS arm64
+distribute_iOS
 
 # build_OSX i386
 # build_OSX x86_64
 
-distribute
+# distribute
 

Kiwix/libkiwix/build/build-zim.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# regarding export MACOSX_DEPLOYMENT_TARGET="10.4" in build_iOS, see reason why do it like this
+# http://stackoverflow.com/questions/32622284/building-c-static-libraries-using-configure-make-with-fembed-bitcode-fails
+
 ROOT=$(pwd)
 
 ##########################  ZIM  ##########################
@@ -7,11 +10,7 @@ ROOT=$(pwd)
 # git clone https://gerrit.wikimedia.org/r/p/openzim.git
 # modify ffstream.cpp, replace stat64 with stat
 
-LZMAHEADERPATH="/Volumes/Data/Developer/build/xz-5.2.2/build/Universal/include"
+LZMAHEADERPATH="/Volumes/Data/Developer/Kiwix/Kiwix/libkiwix/include"
-ZIMREADERPATH=$ROOT/openzim/zimlib
-cd $ZIMREADERPATH
-
-# ./autogen.sh
 
 build_iOS()
 {
@@ -24,12 +23,17 @@ build_iOS()
 		SDKROOT="$(xcodebuild -version -sdk iphoneos | grep -E '^Path' | sed 's/Path: //')"
 	fi
 
-	export CC="$(xcrun -sdk iphoneos -find clang)"
+	export MACOSX_DEPLOYMENT_TARGET="10.4"
+
+	export CC="$(xcrun -find clang)"
 	export CFLAGS="-fembed-bitcode -isysroot $SDKROOT -arch ${ARCH} -miphoneos-version-min=9.0 -I$LZMAHEADERPATH"
 
 	export CPP="$CC -E"
 	export CPPFLAGS="$CFLAGS"
 
+	export CXX="$(xcrun -find clang++)"
+	export CXXFLAGS="$CFLAGS -stdlib=libc++ -std=c++11"
+
 	export LDFLAGS="-arch ${ARCH} -isysroot $SDKROOT"
 
 	if [ $ARCH == "i386" ] || [ $ARCH == "x86_64" ];
@@ -63,6 +67,28 @@ build_OSX()
 	make && make install && make clean
 }
 
+distribute_iOS() {
+	iOSBUILDDir=$(pwd)/build/iOS
+	cd $iOSBUILDDir
+	mkdir -p universal/lib
+
+	cd armv7/lib
+	for file in *.a
+	do
+		cd $iOSBUILDDir
+		lipo -create armv7/lib/$file armv7s/lib/$file arm64/lib/$file x86_64/lib/$file i386/lib/$file -output universal/lib/$file
+	done
+
+	# cd armv7/lib
+	# for file in *.dylib
+	# do
+	# 	cd $iOSBUILDDir
+	# 	lipo -create armv7/lib/$file armv7s/lib/$file arm64/lib/$file x86_64/lib/$file i386/lib/$file -output universal/lib/$file
+	# done
+
+	cp -r armv7/include universal
+}
+
 distribute() {
 	cd $ZIMREADERPATH/build
 	mkdir -p Universal/iOS/lib
@@ -92,9 +118,10 @@ distribute() {
 # build_iOS armv7
 # build_iOS armv7s
 # build_iOS arm64
+distribute_iOS
 
 # build_OSX i386
 # build_OSX x86_64
 
-distribute
+# distribute
 

Podfile.lock

@@ -1,6 +1,6 @@
 PODS:
   - DateTools (1.7.0)
-  - DZNEmptyDataSet (1.8)
+  - DZNEmptyDataSet (1.8.1)
   - Operations (3.0.0):
     - Operations/Standard (= 3.0.0)
   - Operations/Standard (3.0.0)
@@ -14,7 +14,7 @@ DEPENDENCIES:
 
 SPEC CHECKSUMS:
   DateTools: 53288ee8b905fdc75897a1e6b5cc0144b14cba60
-  DZNEmptyDataSet: d2351b2e8daefa40433ef292e246e21f6be31a7b
+  DZNEmptyDataSet: 9525833b9e68ac21c30253e1d3d7076cc828eaa7
   Operations: c05cc4b2ad060201ab98cbacd5217876658b5f2c
   SwiftyUserDefaults: e7c8197a9d4068702e7f2406a968cae413b8de27