kiwix/kiwix-android
1
0
Fork 0
mirror of https://github.com/kiwix/kiwix-android.git synced 2025-09-08 23:07:26 -04:00
Code Issues Packages Projects Releases Wiki Activity

Improve download stablity and rename partial download fragments

Browse Source
This commit is contained in:
mhutti1 2018-07-10 23:52:48 +01:00 committed by Isaac Hutt
parent d16217e2db
commit f7fe922fd4
7 changed files with 156 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/src/main/java/org/kiwix/kiwixmobile/KiwixMobileActivity.java
View File

@ -116,6 +116,7 @@ import static android.os.Build.VERSION.SDK_INT;
import static android.os.Build.VERSION_CODES;
import static org.kiwix.kiwixmobile.TableDrawerAdapter.DocumentSection;
import static org.kiwix.kiwixmobile.TableDrawerAdapter.TableClickListener;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.getFileName;
import static org.kiwix.kiwixmobile.search.SearchActivity.EXTRA_SEARCH_IN_TEXT;
import static org.kiwix.kiwixmobile.utils.Constants.BOOKMARK_CHOSEN_REQUEST;
import static org.kiwix.kiwixmobile.utils.Constants.CONTACT_EMAIL_ADDRESS;
@ -480,7 +481,7 @@ public class KiwixMobileActivity extends BaseActivity implements WebViewCallback
getCurrentWebView().loadUrl(i.getStringExtra(EXTRA_CHOSE_X_TITLE));
}
if (i.hasExtra(EXTRA_ZIM_FILE)) {
File file = new File(FileUtils.getFileName(i.getStringExtra(EXTRA_ZIM_FILE)));
File file = new File(getFileName(i.getStringExtra(EXTRA_ZIM_FILE)));
LibraryFragment.mService.cancelNotification(i.getIntExtra(EXTRA_NOTIFICATION_ID, 0));
Uri uri = Uri.fromFile(file);

8
app/src/main/java/org/kiwix/kiwixmobile/database/BookDao.java
View File

@ -30,6 +30,8 @@ import java.util.ArrayList;
import javax.inject.Inject;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.hasParts;
/**
* Dao class for books
*/
@ -86,11 +88,11 @@ public class BookDao {
while (bookCursor.moveToNext()) {
Book book = new Book();
setBookDetails(book, bookCursor);
if (!FileUtils.hasPart(book.file)) {
if (!hasParts(book.file)) {
if (book.file.exists()) {
books.add(book);
} else {
mDb.deleteWhere(BookDatabaseEntity.class, BookDatabaseEntity.URL.eq(book.file.getPath()));
mDb.deleteWhere(BookDatabaseEntity.class, BookDatabaseEntity.URL.eq(book.file));
}
}
}
@ -107,7 +109,7 @@ public class BookDao {
Book book = new Book();
setBookDetails(book, bookCursor);
book.remoteUrl = bookCursor.get(BookDatabaseEntity.REMOTE_URL);
if (FileUtils.hasPart(book.file)) {
if (hasParts(book.file)) {
books.add(book);
}
}

81
app/src/main/java/org/kiwix/kiwixmobile/downloader/ChunkUtils.java
View File

@ -17,8 +17,11 @@
*/
package org.kiwix.kiwixmobile.downloader;
import org.kiwix.kiwixmobile.library.entity.LibraryNetworkEntity;
import org.kiwix.kiwixmobile.utils.StorageUtils;
import java.io.File;
import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.List;
@ -26,9 +29,81 @@ public class ChunkUtils {
public static final String ALPHABET = "abcdefghijklmnopqrstuvwxyz";
public static final String ZIM_EXTENSION = ".zim";
public static final String PART = ".part.part";
// Chuck Part
private static final String CPART = ".cpart";
// Total Part
private static final String TPART = ".tpart";
public static final long CHUNK_SIZE = 1024L * 1024L * 1024L * 2L;
public static String baseNameFromParts(File file) {
return file.getName().replace(CPART, "").replace(TPART, "")
.replaceAll("\\.zim..", ".zim");
}
public static File completedChunk(String name) {
return new File(name + TPART);
}
public static boolean isPresent(String name) {
return new File(name).exists() || new File(name + TPART).exists()
|| new File(name + CPART + TPART).exists();
}
public static boolean hasParts(File file) {
return file.getParentFile().listFiles((file1, s) ->
s.startsWith(baseNameFromParts(file)) && s.endsWith(TPART)).length > 0;
}
public static String getFileName(String fileName) {
if (isPresent(fileName)) {
return fileName;
} else {
return fileName + "aa";
}
}
public static File initialChunk(String name) {
return new File(name + CPART + TPART);
}
public static void completeChunk(File chunk) {
chunk.renameTo(new File(chunk.getPath().replace(CPART, "")));
}
public static void completeDownload(File file) {
final String baseName = baseNameFromParts(file);
File directory =file.getParentFile();
File[] parts = directory.listFiles((file1, s) -> s.startsWith(baseName) && s.endsWith(TPART));
for (File part : parts) {
part.renameTo(new File(part.getPath().replace(TPART, "")));
}
}
public static long getCurrentSize(LibraryNetworkEntity.Book book) {
long size = 0;
File[] files = getAllZimParts(book.file);
for (File file : files) {
size += file.length();
}
return size;
}
private static File[] getAllZimParts(File file) {
final String baseName = baseNameFromParts(file);
File directory = new File(file.getPath()).getParentFile();
File[] parts = directory.listFiles((file1, s) -> s.matches(baseName + ".*"));
return parts;
}
public static void deleteAllParts(File file) {
final String baseName = baseNameFromParts(file);
File directory = file.getParentFile();
File[] parts = directory.listFiles((file1, s) -> s.matches(baseName + ".*"));
for (File part : parts) {
part.delete();
}
}
public static List<Chunk> getChunks(String url, long contentLength, int notificationID) {
int fileCount = getZimChunkFileCount(contentLength);
String filename = StorageUtils.getFileNameFromUrl(url);
@ -63,7 +138,7 @@ public class ChunkUtils {
private static String[] getZimChunkFileNames(String fileName, int count) {
if (count == 1) {
return new String[] { fileName + PART};
return new String[] { fileName };
}
int position = fileName.lastIndexOf(".");
String baseName = position > 0 ? fileName.substring(0, position) : fileName;
@ -73,7 +148,7 @@ public class ChunkUtils {
char first = ALPHABET.charAt(i / 26);
char second = ALPHABET.charAt(i % 26);
String chunkExtension = String.valueOf(first) + second;
fileNames[i] = baseName + ZIM_EXTENSION + chunkExtension + PART;
fileNames[i] = baseName + ZIM_EXTENSION + chunkExtension;
}
return fileNames;
}

3
app/src/main/java/org/kiwix/kiwixmobile/downloader/DownloadFragment.java
View File

@ -58,6 +58,7 @@ import java.util.Locale;
import javax.inject.Inject;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.getFileName;
import static org.kiwix.kiwixmobile.utils.StyleUtils.dialogStyle;
public class DownloadFragment extends BaseFragment {
@ -189,7 +190,7 @@ public class DownloadFragment extends BaseFragment {
}
ImageView pause = viewGroup.findViewById(R.id.pause);
pause.setEnabled(false);
String fileName = FileUtils.getFileName(mDownloadFiles.get(mKeys[position]));
String fileName = getFileName(mDownloadFiles.get(mKeys[position]));
{
Snackbar completeSnack = Snackbar.make(mainLayout, getResources().getString(R.string.download_complete_snackbar), Snackbar.LENGTH_LONG);
completeSnack.setAction(getResources().getString(R.string.open), v -> zimManageActivity.finishResult(fileName)).setActionTextColor(getResources().getColor(R.color.white)).show();

149
app/src/main/java/org/kiwix/kiwixmobile/downloader/DownloadService.java
View File

@ -63,20 +63,26 @@ import javax.inject.Inject;
import io.reactivex.Observable;
import io.reactivex.android.schedulers.AndroidSchedulers;
import io.reactivex.functions.Action;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import okio.BufferedSource;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.ALPHABET;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.PART;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.ZIM_EXTENSION;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.completeChunk;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.completeDownload;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.completedChunk;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.deleteAllParts;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.getCurrentSize;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.initialChunk;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.isPresent;
import static org.kiwix.kiwixmobile.utils.Constants.EXTRA_BOOK;
import static org.kiwix.kiwixmobile.utils.Constants.EXTRA_LIBRARY;
import static org.kiwix.kiwixmobile.utils.Constants.EXTRA_NOTIFICATION_ID;
import static org.kiwix.kiwixmobile.utils.Constants.EXTRA_ZIM_FILE;
import static org.kiwix.kiwixmobile.utils.Constants.ONGOING_DOWNLOAD_CHANNEL_ID;
import static org.kiwix.kiwixmobile.utils.files.FileUtils.getCurrentSize;
public class DownloadService extends Service {
@ -291,11 +297,11 @@ public class DownloadService extends Service {
KIWIX_ROOT + StorageUtils.getFileNameFromUrl(book.getUrl()));
}
TestingUtils.bindResource(DownloadService.class);
if (book.file != null && (book.file.exists() || new File(book.file.getPath() + ".part").exists())) {
if (book.file != null && isPresent(book.file.getPath())) {
// Calculate initial download progress
int initial = (int) (getCurrentSize(book) / (Long.valueOf(book.getSize()) * BOOK_SIZE_OFFSET));
notification.get(notificationID).setProgress(100, initial, false);
updateDownloadFragmentProgress(initial, notificationID);
updateDownloadFragmentProgress(initial, notificationID, book);
notificationManager.notify(notificationID, notification.get(notificationID).build());
}
kiwixService.getMetaLinks(url)
@ -304,53 +310,29 @@ public class DownloadService extends Service {
.flatMap(metaLink -> getMetaLinkContentLength(metaLink.getRelevantUrl().getValue()))
.flatMap(pair -> Observable.fromIterable(ChunkUtils.getChunks(pair.first, pair.second, notificationID)))
.concatMap(this::downloadChunk)
.distinctUntilChanged().doOnComplete(() -> updateDownloadFragmentComplete(notificationID))
.subscribe(progress -> {
if (progress == 100) {
notification.get(notificationID).setOngoing(false);
notification.get(notificationID).setContentTitle(notificationTitle + " " + getResources().getString(R.string.zim_file_downloaded));
notification.get(notificationID).setContentText(getString(R.string.zim_file_downloaded));
final Intent target = new Intent(this, KiwixMobileActivity.class);
target.putExtra(EXTRA_ZIM_FILE, KIWIX_ROOT + StorageUtils.getFileNameFromUrl(book.getUrl()));
//Remove the extra ".part" from files
String filename = book.file.getPath();
if(filename.endsWith(ZIM_EXTENSION)) {
filename = filename + PART;
File partFile = new File(filename);
if(partFile.exists()) {
partFile.renameTo(new File(partFile.getPath().replaceAll(".part", "")));
}
} else {
for (int i = 0; true; i++) {
char first = ALPHABET.charAt(i / 26);
char second = ALPHABET.charAt(i % 26);
String chunkExtension = String.valueOf(first) + second;
filename = book.file.getPath();
filename = filename.replaceAll(".zim([a-z][a-z]){0,1}$", ".zim");
filename = filename + chunkExtension + ".part";
File partFile = new File(filename);
if(partFile.exists()) {
partFile.renameTo(new File(partFile.getPath().replaceAll(".part$", "")));
} else {
File lastChunkFile = new File(filename + ".part");
if(lastChunkFile.exists()) {
lastChunkFile.renameTo(new File(partFile.getPath().replaceAll(".part", "")));
} else {
break;
}
}
}
}
target.putExtra(EXTRA_NOTIFICATION_ID, notificationID);
PendingIntent pendingIntent = PendingIntent.getActivity
(getBaseContext(), 0,
target, PendingIntent.FLAG_CANCEL_CURRENT);
book.downloaded = true;
bookDao.deleteBook(book.id);
notification.get(notificationID).setContentIntent(pendingIntent);
notification.get(notificationID).mActions.clear();
TestingUtils.unbindResource(DownloadService.class);
}
.distinctUntilChanged().doOnComplete(() -> updateDownloadFragmentComplete(notificationID, book)).doOnComplete(() -> {
notification.get(notificationID).setOngoing(false);
notification.get(notificationID).setContentTitle(notificationTitle + " " + getResources().getString(R.string.zim_file_downloaded));
notification.get(notificationID).setContentText(getString(R.string.zim_file_downloaded));
final Intent target = new Intent(this, KiwixMobileActivity.class);
target.putExtra(EXTRA_ZIM_FILE, KIWIX_ROOT + StorageUtils.getFileNameFromUrl(book.getUrl()));
File filec = book.file;
completeDownload(filec);
target.putExtra(EXTRA_NOTIFICATION_ID, notificationID);
PendingIntent pendingIntent = PendingIntent.getActivity
(getBaseContext(), 0,
target, PendingIntent.FLAG_CANCEL_CURRENT);
book.downloaded = true;
bookDao.deleteBook(book.id);
notification.get(notificationID).setContentIntent(pendingIntent);
notification.get(notificationID).mActions.clear();
TestingUtils.unbindResource(DownloadService.class);
notification.get(notificationID).setProgress(100, 100, false);
notificationManager.notify(notificationID, notification.get(notificationID).build());
updateForeground();
updateDownloadFragmentProgress(100, notificationID, book);
stopSelf();
}).subscribe(progress -> {
notification.get(notificationID).setProgress(100, progress, false);
if (progress != 100 && timeRemaining.get(notificationID) != -1)
notification.get(notificationID).setContentText(DownloadFragment.toHumanReadableTime(timeRemaining.get(notificationID)));
@ -359,30 +341,35 @@ public class DownloadService extends Service {
// Tells android to not kill the service
updateForeground();
}
updateDownloadFragmentProgress(progress, notificationID);
if (progress == 100) {
stopSelf();
}
updateDownloadFragmentProgress(progress, notificationID, book);
}, Throwable::printStackTrace);
}
private void updateDownloadFragmentProgress(int progress, int notificationID) {
if (DownloadFragment.mDownloads != null && DownloadFragment.mDownloads.get(notificationID) != null) {
handler.post(() -> {
if (DownloadFragment.mDownloads.get(notificationID) != null) {
DownloadFragment.downloadAdapter.updateProgress(progress, notificationID);
}
});
private void updateDownloadFragmentProgress(int progress, int notificationID, LibraryNetworkEntity.Book book) {
if (DownloadFragment.mDownloads != null) {
if (DownloadFragment.mDownloads.get(notificationID) != null) {
handler.post(() -> {
if (DownloadFragment.mDownloads.get(notificationID) != null) {
DownloadFragment.downloadAdapter.updateProgress(progress, notificationID);
}
});
} else {
DownloadFragment.mDownloads.put(notificationID, book);
}
}
}
private void updateDownloadFragmentComplete(int notificationID) {
if (DownloadFragment.mDownloads != null && DownloadFragment.mDownloads.get(notificationID) != null) {
handler.post(() -> {
if (DownloadFragment.mDownloads.get(notificationID) != null) {
DownloadFragment.downloadAdapter.complete(notificationID);
}
});
private void updateDownloadFragmentComplete(int notificationID, LibraryNetworkEntity.Book book) {
if (DownloadFragment.mDownloads != null) {
if (DownloadFragment.mDownloads.get(notificationID) != null) {
handler.post(() -> {
if (DownloadFragment.mDownloads.get(notificationID) != null) {
DownloadFragment.downloadAdapter.complete(notificationID);
}
});
} else {
DownloadFragment.mDownloads.put(notificationID, book);
}
}
}
@ -443,9 +430,9 @@ public class DownloadService extends Service {
}
// Create chunk file
File file = new File(KIWIX_ROOT, chunk.getFileName());
file.getParentFile().mkdirs();
File fullFile = new File(file.getPath().substring(0, file.getPath().length() - PART.length()));
File fullFile = new File(KIWIX_ROOT, chunk.getFileName());
fullFile.getParentFile().mkdirs();
File file = initialChunk(fullFile.getPath());
long downloaded = Long.parseLong(chunk.getRangeHeader().split("-")[0]);
if (fullFile.exists() && fullFile.length() == chunk.getSize()) {
@ -540,7 +527,6 @@ public class DownloadService extends Service {
double speed = (downloaded - lastSize) / (timeDiff / 1000.0);
lastSize = downloaded;
int secondsLeft = (int) ((chunk.getContentLength() - downloaded) / speed);
timeRemaining.put(chunk.getNotificationID(), secondsLeft);
}
@ -567,20 +553,13 @@ public class DownloadService extends Service {
if (input != null) {
input.close();
}
// If download is canceled clean up else remove .part from file name
if (downloadStatus.get(chunk.getNotificationID()) == CANCEL) {
String path = file.getPath();
Log.i(KIWIX_TAG, "Download Cancelled, deleting file: " + path);
if (path.substring(path.length() - (ZIM_EXTENSION + PART).length()).equals(ZIM_EXTENSION + PART)) {
path = path.substring(0, path.length() - PART.length() + 1);
FileUtils.deleteZimFile(path);
} else {
path = path.substring(0, path.length() - (ZIM_EXTENSION + PART).length() + 2) + "aa";
FileUtils.deleteZimFile(path);
}
File path = file;
Log.i(KIWIX_TAG, "Download Cancelled, deleting file: " + file.getPath());
deleteAllParts(path);
} else {
Log.i(KIWIX_TAG, "Download completed, renaming file ([" + file.getPath() + "] -> .zim.part)");
file.renameTo(new File(file.getPath().replaceAll(".part$", "")));
Log.i(KIWIX_TAG, "Chunk download completed, competing chunk rename");
completeChunk(file);
}
// Mark chunk status as downloaded
chunk.isDownloaded = true;

111
app/src/main/java/org/kiwix/kiwixmobile/utils/files/FileUtils.java
View File

@ -37,6 +37,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static org.kiwix.kiwixmobile.downloader.ChunkUtils.deleteAllParts;
import static org.kiwix.kiwixmobile.utils.Constants.TAG_KIWIX;
public class FileUtils {
@ -61,44 +62,8 @@ public class FileUtils {
}
}
public static synchronized void deleteZimFile(String path) {
if (path.substring(path.length() - ChunkUtils.PART.length()).equals(ChunkUtils.PART)) {
path = path.substring(0, path.length() - ChunkUtils.PART.length());
}
Log.i("kiwix", "Deleting file: " + path);
File file = new File(path);
if (!file.getPath().substring(file.getPath().length() - 3).equals("zim")) {
fileloop:
for (char alphabetFirst = 'a'; alphabetFirst <= 'z'; alphabetFirst++) {
for (char alphabetSecond = 'a'; alphabetSecond <= 'z'; alphabetSecond++) {
String chunkPath = path.substring(0, path.length() - 2) + alphabetFirst + alphabetSecond;
File fileChunk = new File(chunkPath);
if (fileChunk.exists()) {
fileChunk.delete();
} else if (!deleteZimFileParts(chunkPath)) {
break fileloop;
}
}
}
} else {
file.delete();
deleteZimFileParts(path);
}
}
private static synchronized boolean deleteZimFileParts(String path) {
File file = new File(path + ChunkUtils.PART);
if (file.exists()) {
file.delete();
return true;
} else {
File singlePart = new File(path + ".part");
if (singlePart.exists()) {
singlePart.delete();
return true;
}
}
return false;
public static synchronized void deleteZimFile(File file) {
deleteAllParts(file);
}
/**
@ -218,79 +183,11 @@ public class FileUtils {
return readCsv(content);
}
private static List<File> getAllZimParts(Book book) {
List<File> files = new ArrayList<>();
if(book.file.getPath().endsWith(".zim") || book.file.getPath().endsWith(".zim.part")) {
if(book.file.exists()) {
files.add(book.file);
} else {
files.add(new File(book.file + ".part"));
}
return files;
}
String path = book.file.getPath();
for(char alphabetFirst = 'a'; alphabetFirst <= 'z'; alphabetFirst++) {
for(char alphabetSecond = 'a'; alphabetSecond <= 'z'; alphabetSecond++) {
path = path.substring(0, path.length() - 2) + alphabetFirst + alphabetSecond;
if(new File(path).exists()) {
files.add(new File(path));
} else if(new File(path + ".part").exists()) {
files.add(new File(path + ".part"));
} else {
return files;
}
}
}
return files;
}
private static ArrayList<String> readCsv(String csv) {
String[] csvArray = csv.split(",");
return new ArrayList<>(Arrays.asList(csvArray));
}
public static boolean hasPart(File file) {
file = new File(getFileName(file.getPath()));
if (file.getPath().endsWith(".zim")) {
return false;
}
if (file.getPath().endsWith(".part")) {
return true;
}
String path = file.getPath();
for (char alphabetFirst = 'a'; alphabetFirst <= 'z'; alphabetFirst++) {
for (char alphabetSecond = 'a'; alphabetSecond <= 'z'; alphabetSecond++) {
String chunkPath = path.substring(0, path.length() - 2) + alphabetFirst + alphabetSecond;
File fileChunk = new File(chunkPath + ".part");
if (fileChunk.exists()) {
return true;
} else if (!new File(chunkPath).exists()) {
return false;
}
}
}
return false;
}
public static String getFileName (String fileName) {
if (new File(fileName).exists()) {
return fileName;
} else if (new File(fileName + ".part").exists()) {
return fileName + ".part";
} else {
return fileName + "aa";
}
}
public static long getCurrentSize(Book book) {
long size = 0;
List<File> files = getAllZimParts(book);
for (File file : files) {
size += file.length();
}
return size;
}
}

2
app/src/main/java/org/kiwix/kiwixmobile/zim_manager/fileselect_view/ZimFileSelectFragment.java
View File

@ -294,7 +294,7 @@ public class ZimFileSelectFragment extends BaseFragment
public boolean deleteSpecificZimFile(int position) {
File file = mFiles.get(position).file;
FileUtils.deleteZimFile(file.getPath());
FileUtils.deleteZimFile(file);
if (file.exists()) {
return false;
}