Merge pull request #1457 from kiwix/feature/macgills/#1449-database-migration

Feature/macgills/#1449 database migration
This commit is contained in:
Seán Mac Gillicuddy 2019-09-11 08:57:32 +01:00 committed by GitHub
commit 6570b43b63
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 136 additions and 161 deletions

View File

@ -188,9 +188,9 @@ def branchName = System.getenv('TRAVIS_PULL_REQUEST') ?: "false" == "false"
def buildNumber = System.getenv('TRAVIS_BUILD_NUMBER') ?: "dev"
ext {
versionMajor = 2
versionMinor = 5
versionPatch = 1
versionMajor = 3
versionMinor = 0
versionPatch = 0
}
private String generateVersionName() {

View File

@ -51,7 +51,7 @@ public class KiwixDatabaseTest {
@Test
@SuppressWarnings("CharsetObjectCanBeUsed") // Standard charset throws exception on < API19
public void testMigrateDatabase() throws IOException {
KiwixDatabase kiwixDatabase = new KiwixDatabase(context, null, null);
KiwixDatabase kiwixDatabase = new KiwixDatabase(context, null, null, null, null);
kiwixDatabase.recreate();
String testId = "8ce5775a-10a9-bbf3-178a-9df69f23263c";
String[] testBookmarks = new String[] { "Test1", "Test2", "Test3" };

View File

@ -23,9 +23,7 @@ import com.yahoo.squidb.sql.Query;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.kiwix.kiwixmobile.data.ZimContentProvider;
import org.kiwix.kiwixmobile.data.local.KiwixDatabase;
import org.kiwix.kiwixmobile.data.local.entity.RecentSearch;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
@ -58,10 +56,6 @@ public class RecentSearchDaoTest {
recentSearchDao.getRecentSearches();
// verify ordering is in descending order of search ID and the results are limited to 5 only
verify(kiwixDatabase).query(any(),
eq(Query.selectDistinct(RecentSearch.SEARCH_STRING).where(RecentSearch.ZIM_I_D.eq(
ZimContentProvider.getId()))
.orderBy(RecentSearch.ID.desc())
.limit(5)));
verify(kiwixDatabase).query(any(), eq(Query.select()));
}
}

View File

@ -32,6 +32,7 @@ import dagger.android.HasActivityInjector;
import java.io.File;
import java.io.IOException;
import javax.inject.Inject;
import org.kiwix.kiwixmobile.data.local.KiwixDatabase;
import org.kiwix.kiwixmobile.di.components.ApplicationComponent;
import org.kiwix.kiwixmobile.di.components.DaggerApplicationComponent;
@ -46,7 +47,8 @@ public class KiwixApplication extends MultiDexApplication implements HasActivity
@Inject
DispatchingAndroidInjector<Activity> activityInjector;
private File logFile;
@Inject
KiwixDatabase kiwixDatabase;
public static KiwixApplication getInstance() {
return application;
@ -73,41 +75,36 @@ public class KiwixApplication extends MultiDexApplication implements HasActivity
public void onCreate() {
super.onCreate();
AndroidThreeTen.init(this);
writeLogFile();
applicationComponent.inject(this);
kiwixDatabase.forceMigration();
if (BuildConfig.DEBUG) {
StrictMode.setThreadPolicy(buildThreadPolicy(new StrictMode.ThreadPolicy.Builder()));
StrictMode.setVmPolicy(buildVmPolicy(new StrictMode.VmPolicy.Builder()));
}
}
private void writeLogFile() {
if (isExternalStorageWritable()) {
File appDirectory = new File(Environment.getExternalStorageDirectory() + "/Kiwix");
logFile = new File(appDirectory, "logcat.txt");
File logFile = new File(appDirectory, "logcat.txt");
Log.d("KIWIX", "Writing all logs into [" + logFile.getPath() + "]");
// create app folder
if (!appDirectory.exists()) {
appDirectory.mkdir();
}
// create log folder
if (!appDirectory.exists()) {
appDirectory.mkdir();
}
if (logFile.exists() && logFile.isFile()) {
logFile.delete();
}
// clear the previous logcat and then write the new one to the file
try {
logFile.createNewFile();
Process process = Runtime.getRuntime().exec("logcat -c");
process = Runtime.getRuntime().exec("logcat -f " + logFile.getPath() + " -s kiwix");
Runtime.getRuntime().exec("logcat -c");
Runtime.getRuntime().exec("logcat -f " + logFile.getPath() + " -s kiwix");
} catch (IOException e) {
Log.e("KIWIX", "Error while writing logcat.txt", e);
}
}
Log.d("KIWIX", "Started KiwixApplication");
applicationComponent.inject(this);
if (BuildConfig.DEBUG) {
StrictMode.setThreadPolicy(buildThreadPolicy(new StrictMode.ThreadPolicy.Builder()));
StrictMode.setVmPolicy(buildVmPolicy(new StrictMode.VmPolicy.Builder()));
}
}
private StrictMode.ThreadPolicy buildThreadPolicy(StrictMode.ThreadPolicy.Builder builder) {

View File

@ -7,10 +7,10 @@ data class BookmarkItem(
val databaseId: Long = 0L,
val zimId: String,
val zimName: String,
val zimFilePath: String,
val zimFilePath: String?,
val bookmarkUrl: String,
val bookmarkTitle: String,
val favicon: String
val favicon: String?
) {
constructor(entity: BookmarkEntity) : this(
entity.id,

View File

@ -29,39 +29,48 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.kiwix.kiwixmobile.data.ZimContentProvider;
import org.kiwix.kiwixmobile.data.local.dao.BookDao;
import org.kiwix.kiwixmobile.data.local.dao.BookmarksDao;
import org.kiwix.kiwixmobile.data.local.dao.NetworkLanguageDao;
import org.kiwix.kiwixmobile.data.local.dao.RecentSearchDao;
import org.kiwix.kiwixmobile.data.local.entity.BookDatabaseEntity;
import org.kiwix.kiwixmobile.data.local.entity.Bookmark;
import org.kiwix.kiwixmobile.data.local.entity.LibraryDatabaseEntity;
import org.kiwix.kiwixmobile.data.local.entity.NetworkLanguageDatabaseEntity;
import org.kiwix.kiwixmobile.data.local.entity.RecentSearch;
import org.kiwix.kiwixmobile.database.newdb.dao.NewBookDao;
import org.kiwix.kiwixmobile.database.newdb.dao.NewBookmarksDao;
import org.kiwix.kiwixmobile.database.newdb.dao.NewLanguagesDao;
import org.kiwix.kiwixmobile.library.entity.LibraryNetworkEntity;
import org.kiwix.kiwixmobile.database.newdb.dao.NewRecentSearchDao;
import org.kiwix.kiwixmobile.utils.UpdateUtils;
import static org.kiwix.kiwixmobile.utils.Constants.TAG_KIWIX;
@Singleton
@Deprecated //delete once migrations are no longer needed
public class KiwixDatabase extends SquidDatabase {
private static final int VERSION = 17;
private static final int TWO_POINT_FIVE_POINT_THREE = 16;
private static final int FINAL = 17;//3.0.0
private final Context context;
private final NewBookDao bookDao;
private final NewLanguagesDao languagesDao;
private final NewBookmarksDao bookmarksDao;
private final NewRecentSearchDao recentSearchDao;
@Inject
public KiwixDatabase(Context context, NewBookDao bookDao, NewLanguagesDao languagesDao) {
public KiwixDatabase(Context context, NewBookDao bookDao, NewLanguagesDao languagesDao,
NewBookmarksDao bookmarksDao,
NewRecentSearchDao recentSearchDao) {
super(context);
this.context = context;
this.bookDao = bookDao;
this.languagesDao = languagesDao;
this.bookmarksDao = bookmarksDao;
this.recentSearchDao = recentSearchDao;
}
@Override
@ -74,23 +83,14 @@ public class KiwixDatabase extends SquidDatabase {
return new Table[] {
RecentSearch.TABLE,
Bookmark.TABLE,
BookDatabaseEntity.TABLE,
NetworkLanguageDatabaseEntity.TABLE
};
}
@Override
protected boolean onUpgrade(SQLiteDatabaseWrapper db, int oldVersion, int newVersion) {
if (newVersion >= 16) { //2.5 attempt reading values from old db before they get dropped
try {
bookDao.migrationInsert(new BookDao(this).getBooks());
} catch (Exception e) {
e.printStackTrace();
}
try {
languagesDao.insert(new NetworkLanguageDao(this).getFilteredLanguages());
} catch (Exception e) {
e.printStackTrace();
}
}
Log.e("UPGRADE", "oldversion: " + oldVersion + " newVersion: " + newVersion);
switch (oldVersion) {
case 1:
case 2:
@ -132,40 +132,41 @@ public class KiwixDatabase extends SquidDatabase {
tryCreateTable(BookDatabaseEntity.TABLE);
case 14:
case 15:
tryAddColumn(Bookmark.ZIM_FILE_PATH);
tryAddColumn(Bookmark.FAVICON);
migrateBookmarksVersion16();
try {
bookDao.migrationInsert(new BookDao(this).getBooks());
} catch (Exception e) {
e.printStackTrace();
}
try {
languagesDao.insert(new NetworkLanguageDao(this).getFilteredLanguages());
} catch (Exception e) {
e.printStackTrace();
}
tryDropTable(BookDatabaseEntity.TABLE);
tryDropTable(NetworkLanguageDatabaseEntity.TABLE);
tryDropTable(LibraryDatabaseEntity.TABLE);
case 16:
new BookmarksDao(this).processBookmark(UpdateUtils::reformatProviderUrl);
//TODO MIGRATIONS BEFORE 3.0
case TWO_POINT_FIVE_POINT_THREE:
try {
final BookmarksDao oldBookmarksDao = new BookmarksDao(this);
oldBookmarksDao.processBookmark(UpdateUtils::reformatProviderUrl);
this.bookmarksDao.migrationInsert(oldBookmarksDao.getBookmarks(false), bookDao);
} catch (Exception e) {
e.printStackTrace();
}
tryDropTable(Bookmark.TABLE);
try {
recentSearchDao.migrationInsert(new RecentSearchDao(this).getRecentSearches());
} catch (Exception e) {
e.printStackTrace();
}
tryDropTable(RecentSearch.TABLE);
}
return true;
}
private void migrateBookmarksVersion16() {
BookmarksDao bookmarksDao = new BookmarksDao(this);
BookDao bookDao = new BookDao(this);
List<Bookmark> bookmarks = bookmarksDao.getBookmarks(false);
List<LibraryNetworkEntity.Book> books = bookDao.getBooks();
for (Bookmark bookmark : bookmarks) {
if (bookmark.getZimId() != null) {
for (LibraryNetworkEntity.Book book : books) {
if (bookmark.getZimId().equals(book.getId())) {
bookmark.setZimFilePath(book.getUrl()).setFavicon(book.getFavicon());
bookmarksDao.saveBookmark(bookmark);
break;
}
}
}
}
}
@Override
protected int getVersion() {
return VERSION;
return FINAL;
}
public void migrateBookmarksVersion6() {
@ -199,6 +200,14 @@ public class KiwixDatabase extends SquidDatabase {
}
}
}
/* Now that the database is no longer used
* we need to make a migration happen with an explicit call
*/
public void forceMigration() {
beginTransaction();
endTransaction();
}
}

View File

@ -40,36 +40,6 @@ public class BookmarksDao {
this.kiwixDatabase = kiwixDatabase;
}
/**
* @return Url of the bookmarks from the current Zim file.
*/
public List<String> getCurrentZimBookmarksUrl() {
ArrayList<String> bookmarksUrl = new ArrayList<>();
try (SquidCursor<Bookmark> bookmarkCursor = kiwixDatabase.query(Bookmark.class,
Query.selectDistinct(Bookmark.BOOKMARK_URL)
.where(Bookmark.ZIM_ID.eq(ZimContentProvider.getId())
.or(Bookmark.ZIM_NAME.eq(ZimContentProvider.getName())))
.orderBy(Bookmark.BOOKMARK_TITLE.asc()))) {
while (bookmarkCursor.moveToNext()) {
bookmarksUrl.add(bookmarkCursor.get(Bookmark.BOOKMARK_URL));
}
}
return bookmarksUrl;
}
public void saveBookmark(Bookmark bookmark) {
kiwixDatabase.deleteWhere(Bookmark.class, Bookmark.BOOKMARK_URL.eq(bookmark.getBookmarkUrl())
.and(Bookmark.ZIM_ID.eq(bookmark.getZimId())));
kiwixDatabase.persist(new Bookmark()
.setZimId(bookmark.getZimId())
.setZimName(bookmark.getZimName())
.setZimFilePath(bookmark.getZimFilePath())
.setFavicon(bookmark.getFavicon())
.setBookmarkUrl(bookmark.getBookmarkUrl())
.setBookmarkTitle(bookmark.getBookmarkTitle()));
}
public List<Bookmark> getBookmarks(boolean fromCurrentBook) {
ArrayList<Bookmark> bookmarks = new ArrayList<>();
Query query = Query.select();
@ -80,32 +50,16 @@ public class BookmarksDao {
.query(Bookmark.class, query.orderBy(Bookmark.BOOKMARK_TITLE.asc()))) {
while (squidCursor.moveToNext()) {
Bookmark bookmark = new Bookmark();
bookmark.setZimId(squidCursor.get(Bookmark.ZIM_ID));
bookmark.setZimName(squidCursor.get(Bookmark.ZIM_NAME));
bookmark.setZimFilePath(squidCursor.get(Bookmark.ZIM_FILE_PATH));
bookmark.setFavicon(squidCursor.get(Bookmark.FAVICON));
bookmark.setBookmarkTitle(squidCursor.get(Bookmark.BOOKMARK_TITLE));
bookmark.setBookmarkUrl(squidCursor.get(Bookmark.BOOKMARK_URL));
bookmarks.add(bookmark);
}
}
return bookmarks;
}
public void deleteBookmarks(List<Bookmark> bookmarks) {
for (Bookmark bookmark : bookmarks) {
kiwixDatabase.deleteWhere(Bookmark.class, Bookmark.BOOKMARK_URL.eq(bookmark.getBookmarkUrl())
.and(Bookmark.ZIM_ID.eq(bookmark.getZimId())));
}
}
public void deleteBookmark(Bookmark bookmark) {
kiwixDatabase.deleteWhere(Bookmark.class, Bookmark.BOOKMARK_URL.eq(bookmark.getBookmarkUrl())
.and(Bookmark.ZIM_ID.eq(bookmark.getZimId())));
}
public void processBookmark(StringOperation operation) {
try (SquidCursor<Bookmark> bookmarkCursor = kiwixDatabase.query(Bookmark.class,
Query.select(Bookmark.ID, Bookmark.BOOKMARK_URL))) {

View File

@ -22,7 +22,6 @@ import com.yahoo.squidb.sql.Query;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import org.kiwix.kiwixmobile.data.ZimContentProvider;
import org.kiwix.kiwixmobile.data.local.KiwixDatabase;
import org.kiwix.kiwixmobile.data.local.entity.RecentSearch;
@ -43,43 +42,14 @@ public class RecentSearchDao {
/**
* Returns a distinct enumeration of the {@code NUM_RECENT_RESULTS} most recent searches.
*/
public List<String> getRecentSearches() {
SquidCursor<RecentSearch> searchCursor = mDb.query(
RecentSearch.class,
Query.selectDistinct(RecentSearch.SEARCH_STRING)
.where(RecentSearch.ZIM_I_D.eq(ZimContentProvider.getId()))
.orderBy(RecentSearch.ID.desc())
.limit(NUM_RECENT_RESULTS));
List<String> result = new ArrayList<>();
try {
public List<RecentSearch> getRecentSearches() {
List<RecentSearch> result = new ArrayList<>();
try (SquidCursor<RecentSearch> searchCursor = mDb.query(
RecentSearch.class, Query.select())) {
while (searchCursor.moveToNext()) {
result.add(searchCursor.get(RecentSearch.SEARCH_STRING));
result.add(new RecentSearch(searchCursor));
}
} finally {
searchCursor.close();
}
return result;
}
/**
* Save {@code searchString} as the most recent search.
*/
public void saveSearch(String searchString) {
mDb.persist(
new RecentSearch().setSearchString(searchString).setZimID(ZimContentProvider.getId()));
}
/**
* Delete all entries that exactly matches {@code searchString}
*/
public void deleteSearchString(String searchString) {
mDb.deleteWhere(RecentSearch.class, RecentSearch.SEARCH_STRING.eq(searchString));
}
/**
* Deletes all entries.
*/
public void deleteSearchHistory() {
mDb.deleteWhere(RecentSearch.class, RecentSearch.ID.isNotNull());
}
}

View File

@ -28,8 +28,6 @@ public class BookmarksSpec {
@ColumnSpec(constraints = "NOT NULL")
public String ZimId;
public String ZimName;
public String zimFilePath;
public String bookmarkUrl;
public String bookmarkTitle;
public String favicon;
}

View File

@ -20,6 +20,7 @@ package org.kiwix.kiwixmobile.database.newdb.dao
import io.objectbox.Box
import io.objectbox.kotlin.inValues
import io.objectbox.kotlin.query
import org.kiwix.kiwixmobile.data.local.entity.Bookmark
import org.kiwix.kiwixmobile.database.newdb.entities.BookOnDiskEntity
import org.kiwix.kiwixmobile.database.newdb.entities.BookOnDiskEntity_
import org.kiwix.kiwixmobile.library.entity.LibraryNetworkEntity.Book
@ -62,4 +63,11 @@ class NewBookDao @Inject constructor(private val box: Box<BookOnDiskEntity>) {
private fun delete(books: List<BookOnDiskEntity>) {
box.remove(books)
}
fun getFavIconAndZimFile(it: Bookmark): Pair<String?, String?> {
val bookOnDiskEntity = box.query {
equal(BookOnDiskEntity_.bookId, it.zimId)
}.find().getOrNull(0)
return bookOnDiskEntity?.let { Pair(it.favIcon, it.file.path) } ?: Pair(null, null)
}
}

View File

@ -21,6 +21,7 @@ import io.objectbox.Box
import io.objectbox.kotlin.query
import org.kiwix.kiwixmobile.bookmark.BookmarkItem
import org.kiwix.kiwixmobile.data.ZimContentProvider
import org.kiwix.kiwixmobile.data.local.entity.Bookmark
import org.kiwix.kiwixmobile.database.newdb.entities.BookmarkEntity
import org.kiwix.kiwixmobile.database.newdb.entities.BookmarkEntity_
import javax.inject.Inject
@ -58,4 +59,11 @@ class NewBookmarksDao @Inject constructor(val box: Box<BookmarkEntity>) {
equal(BookmarkEntity_.bookmarkUrl, bookmarkUrl)
}.remove()
}
fun migrationInsert(
bookmarks: MutableList<Bookmark>,
bookDao: NewBookDao
) {
box.put(bookmarks.zip(bookmarks.map { bookDao.getFavIconAndZimFile(it) }).map(::BookmarkEntity))
}
}

View File

@ -20,6 +20,7 @@ package org.kiwix.kiwixmobile.database.newdb.dao
import io.objectbox.Box
import io.objectbox.kotlin.query
import org.kiwix.kiwixmobile.data.ZimContentProvider
import org.kiwix.kiwixmobile.data.local.entity.RecentSearch
import org.kiwix.kiwixmobile.database.newdb.entities.RecentSearchEntity
import org.kiwix.kiwixmobile.database.newdb.entities.RecentSearchEntity_
import javax.inject.Inject
@ -51,6 +52,10 @@ class NewRecentSearchDao @Inject constructor(val box: Box<RecentSearchEntity>) {
box.removeAll()
}
fun migrationInsert(recentSearches: MutableList<RecentSearch>) {
box.put(recentSearches.map(::RecentSearchEntity))
}
companion object {
const val NUM_RECENT_RESULTS = 5
}

View File

@ -20,16 +20,17 @@ package org.kiwix.kiwixmobile.database.newdb.entities
import io.objectbox.annotation.Entity
import io.objectbox.annotation.Id
import org.kiwix.kiwixmobile.bookmark.BookmarkItem
import org.kiwix.kiwixmobile.data.local.entity.Bookmark
@Entity
data class BookmarkEntity(
@Id var id: Long = 0,
val zimId: String,
var zimName: String,
var zimFilePath: String,
var zimFilePath: String?,
var bookmarkUrl: String,
var bookmarkTitle: String,
var favicon: String
var favicon: String?
) {
constructor(item: BookmarkItem) : this(
item.databaseId,
@ -40,4 +41,20 @@ data class BookmarkEntity(
item.bookmarkTitle,
item.favicon
)
private constructor(bookmark: Bookmark, zimFilePath: String?, favicon: String?) : this(
0,
bookmark.zimId,
bookmark.zimName,
zimFilePath,
bookmark.bookmarkUrl,
bookmark.bookmarkTitle,
favicon
)
constructor(bookmarkWithFavIconAndFile: Pair<Bookmark, Pair<String?, String?>>) : this(
bookmarkWithFavIconAndFile.first,
bookmarkWithFavIconAndFile.second.first,
bookmarkWithFavIconAndFile.second.second
)
}

View File

@ -20,6 +20,7 @@ package org.kiwix.kiwixmobile.database.newdb.entities
import io.objectbox.annotation.Entity
import io.objectbox.annotation.Id
import org.kiwix.kiwixmobile.data.ZimContentProvider
import org.kiwix.kiwixmobile.data.local.entity.RecentSearch
@Entity
data class RecentSearchEntity(
@ -31,4 +32,10 @@ data class RecentSearchEntity(
searchTerm = searchTerm,
zimId = ZimContentProvider.getId()
)
constructor(recentSearch: RecentSearch) : this(
0,
recentSearch.searchString,
recentSearch.zimID
)
}

View File

@ -33,8 +33,16 @@ import javax.inject.Singleton
@Module
class DatabaseModule {
@Provides @Singleton fun providesBoxStore(context: Context): BoxStore =
MyObjectBox.builder().androidContext(context.applicationContext).build()
companion object {
var boxStore: BoxStore? = null
}
@Provides @Singleton fun providesBoxStore(context: Context): BoxStore {
if (boxStore == null) {
boxStore = MyObjectBox.builder().androidContext(context.applicationContext).build()
}
return boxStore!!
}
@Provides @Singleton fun providesNewDownloadDao(boxStore: BoxStore): NewDownloadDao =
NewDownloadDao(boxStore.boxFor())

View File

@ -15,6 +15,6 @@ fun ImageView.setBitmap(base64String: Base64String) {
// methods that accept inline classes as parameters are not allowed to be called from java
// hence this facade
fun ImageView.setBitmapFromString(string: String) {
fun ImageView.setBitmapFromString(string: String?) {
setBitmap(Base64String(string))
}