Merge pull request #670 from kiwix/668-fix-root-cause-of-language-problems

Fix by reverting to 2 letter lang codes in DB

Model/CategoriesToLanguage.swift

@@ -7,23 +7,23 @@ import Foundation
 import Defaults
 
 struct CategoriesToLanguages {
-
+    
     private let dictionary: [Category: Set<String>] = Defaults[.categoriesToLanguages]
-
+    
     func has(category: Category, inLanguages langCodes: Set<String>) -> Bool {
-        guard !langCodes.isEmpty else {
+        guard !langCodes.isEmpty, !dictionary.isEmpty else {
-            return true // no languages provided, do not filter
+            return true // no languages or category filters provided, do not filter
         }
         guard let languages = dictionary[category] else {
             return false
         }
         return !languages.isDisjoint(with: langCodes)
     }
-
+    
     static func save(_ dictionary: [Category: Set<String>]) {
         Defaults[.categoriesToLanguages] = dictionary
     }
-
+    
     static func allCategories() -> [Category] {
         let categoriesToLanguages = CategoriesToLanguages()
         let contentLanguages = Defaults[.libraryLanguageCodes]

Model/Entities/ZimFileMetaData/ZimFileMetaData.mm

@@ -63,7 +63,13 @@
 }
 
 - (NSString *)getLanguageCodesFromBook:(kiwix::Book *)book {
-    return [NSString stringWithUTF8String:book->getCommaSeparatedLanguages().c_str()];
+    NSString* string = [NSString stringWithUTF8String:book->getCommaSeparatedLanguages().c_str()];
+    NSArray* components = [string componentsSeparatedByString: @","];
+    NSMutableArray* langCodes = [NSMutableArray array];
+    [components enumerateObjectsUsingBlock:^(id  _Nonnull codeString, NSUInteger idx, BOOL * _Nonnull stop) {
+        [langCodes addObject:[NSLocale canonicalLanguageIdentifierFromString: codeString]];
+    }];
+    return [langCodes componentsJoinedByString: @","];
 }
 
 - (NSString *)getCategoryFromBook:(kiwix::Book *)book {

Tests/LibraryRefreshViewModelTest.swift

@@ -193,7 +193,8 @@ final class LibraryRefreshViewModelTest: XCTestCase {
         XCTAssertEqual(zimFile.hasVideos, false)
         XCTAssertEqual(zimFile.includedInSearch, true)
         XCTAssertEqual(zimFile.isMissing, false)
-        XCTAssertEqual(zimFile.languageCode, "eng")
+        // !important make sure the language code is put into the DB as a 2 letter string
+        XCTAssertEqual(zimFile.languageCode, "en")
         XCTAssertEqual(zimFile.mediaCount, 566835)
         XCTAssertEqual(zimFile.name, "Best of Wikipedia")
         XCTAssertEqual(zimFile.persistentID, "wikipedia_en_top")

Tests/OPDSParserTests.swift

@@ -70,7 +70,8 @@ final class OPDSParserTests: XCTestCase {
         XCTAssertEqual(metadata.groupIdentifier, "wikipedia_en_top")
         XCTAssertEqual(metadata.title, "Best of Wikipedia")
         XCTAssertEqual(metadata.fileDescription, "A selection of the best 50,000 Wikipedia articles")
-        XCTAssertEqual(metadata.languageCodes, "eng")
+        // !important make sure the language code is put into the DB as a 2 letter string
+        XCTAssertEqual(metadata.languageCodes, "en")
         XCTAssertEqual(metadata.category, "wikipedia")
         XCTAssertEqual(metadata.creationDate, try! Date("2023-01-07T00:00:00Z", strategy: .iso8601))
         XCTAssertEqual(metadata.size, 6515656704)

ViewModel/LibraryViewModel.swift

@@ -89,46 +89,36 @@ public class LibraryViewModel: ObservableObject {
         CategoriesToLanguages.save(dictionary)
     }
 
-    /// Make sure we remove the incompatible "en" if it was already set
-    static func cleanUpDefaultLanguages() {
-        Defaults[.libraryLanguageCodes] = Defaults[.libraryLanguageCodes].filter { langCode in
-            langCode != "en"
-        }
-    }
-
     /// The fetched content is filtered by the languages set in settings.
     /// Try to set it to the device language, making sure we have content to display.
     /// Falls back to English, where most of the content is.
     /// This is only affecting the "fresh-install" defaults.
     /// The user can always set the prefered content languages in settings.
     private func setDefaultContentFilterLanguage() async {
-        Self.cleanUpDefaultLanguages()
         guard Defaults[.libraryLanguageCodes].isEmpty else {
             return // it was already set earlier (either by default or the user)
         }
-
+        let fallbackToEnglish = "en"
-        let defaultLangCode: String
+        let deviceLang: String?
-
+        // In both cases we store the 2 letter version in DB, that is our current
+        // standard, even though the feed values are 3 letter, those are also converted to 2 letter values
         if #available(iOS 16, macOS 13, *) {
-            let languages = await Languages.fetch()
+            deviceLang = Locale.current.language.languageCode?.identifier(.alpha2)
-            // Double check if the current device language is on the list of languages,
-            // and there is content in that language
-            if let deviceLang = Locale.current.language.languageCode?.identifier(.alpha3),
-               languages.contains(where: { (lang: Language) in
-                   lang.code == deviceLang && lang.count > 0
-               }) {
-                defaultLangCode  = deviceLang
-            } else {
-                defaultLangCode = "eng"
-            }
         } else {
-            // Locale.current.languageCode is returning a 2 char lang code, eg: "en"
+            deviceLang = Locale.current.languageCode
-            // we want a 3 char value, eg: "eng", otherwise we filter out every results
-            // and end up with an empty list in the categories
-            defaultLangCode = "eng"
         }
+        // if the device language code cannot be determined, we fall back to English
+        let defaultLangCode: String = deviceLang ?? fallbackToEnglish
+        let languages = await Languages.fetch()
 
-        Defaults[.libraryLanguageCodes] = [defaultLangCode]
+        // make sure the language we default to is on the list of Languages comming from the feed
+        if languages.contains(where: { (lang: Language) in
+            lang.code == defaultLangCode
+        }) {
+            Defaults[.libraryLanguageCodes] = [defaultLangCode]
+        } else {
+            Defaults[.libraryLanguageCodes] = [fallbackToEnglish]
+        }
     }
 
     private func fetchData() async throws -> Data? {

Views/Settings/Settings.swift

@@ -247,8 +247,6 @@ private struct SelectedLanaguageLabel: View {
             } else if languageCodes.count > 1 {
                 Text("\(languageCodes.count)").foregroundColor(.secondary)
             }
-        }.task {
-            LibraryViewModel.cleanUpDefaultLanguages()
         }
     }
 }