prepareSearchResultsJapaneseLanguage function

Implementation

Future<int?> prepareSearchResultsJapaneseLanguage(
    DictionarySearchParams params) async {
  int bestLength = 0;
  String searchTerm = params.searchTerm.trim();
  const KanaKit kanaKit = KanaKit();
  if (kanaKit.isRomaji(searchTerm)) {
    searchTerm = kanaKit.toHiragana(searchTerm);
  }

  if (searchTerm.length > 20) {
    searchTerm = searchTerm.substring(0, 20);
  }

  int maximumHeadings = params.maximumDictionarySearchResults;

  if (searchTerm.isEmpty) {
    return null;
  }

  final Isar database = await Isar.open(
    globalSchemas,
    directory: params.directoryPath,
    maxSizeMiB: 8192,
  );

  Map<int, DictionaryHeading> uniqueHeadingsById = {};

  int limit() {
    return maximumHeadings - uniqueHeadingsById.length;
  }

  bool shouldSearchWildcards = params.searchWithWildcards &&
      (searchTerm.contains('※') ||
          searchTerm.contains('？') ||
          searchTerm.contains('*') ||
          searchTerm.contains('?'));

  if (shouldSearchWildcards) {
    bool noExactMatches = database.dictionaryHeadings
        .where()
        .termEqualTo(searchTerm)
        .isEmptySync();

    if (noExactMatches) {
      String matchesTerm = searchTerm
          .replaceAll('※', '*')
          .replaceAll('？', '?')
          .replaceAll('?', '???');

      List<DictionaryHeading> termMatchHeadings = [];
      List<DictionaryHeading> readingMatchHeadings = [];
      String withoutWildcards =
          matchesTerm.replaceAll('?', '').replaceAll('*', '');
      bool matchTermIsKana = kanaKit.isKana(withoutWildcards);
      bool matchTermIsKatakana = kanaKit.isKatakana(withoutWildcards);

      bool questionMarkOnly = !matchesTerm.contains('*');
      String noAsterisks = searchTerm
          .replaceAll('※', '*')
          .replaceAll('？', '?')
          .replaceAll('*', '');

      if (params.maximumDictionaryTermsInResult > uniqueHeadingsById.length) {
        if (questionMarkOnly) {
          termMatchHeadings = database.dictionaryHeadings
              .where()
              .termLengthEqualTo(searchTerm.length)
              .filter()
              .termMatches(matchesTerm, caseSensitive: false)
              .and()
              .entriesIsNotEmpty()
              .limit(maximumHeadings - uniqueHeadingsById.length)
              .findAllSync();
        } else {
          termMatchHeadings = database.dictionaryHeadings
              .where()
              .termLengthGreaterThan(noAsterisks.length, include: true)
              .filter()
              .termMatches(matchesTerm, caseSensitive: false)
              .and()
              .entriesIsNotEmpty()
              .limit(maximumHeadings - uniqueHeadingsById.length)
              .findAllSync();
        }
      }

      uniqueHeadingsById.addEntries(
        termMatchHeadings.map(
          (heading) => MapEntry(heading.id, heading),
        ),
      );

      if (params.maximumDictionaryTermsInResult > uniqueHeadingsById.length) {
        if (matchTermIsKana) {
          readingMatchHeadings = database.dictionaryHeadings
              .where()
              .filter()
              .entriesIsNotEmpty()
              .readingMatches(matchesTerm)
              .or()
              .optional(matchTermIsKatakana,
                  (q) => q.readingMatches(kanaKit.toHiragana(matchesTerm)))
              .and()
              .entriesIsNotEmpty()
              .findAllSync();

          uniqueHeadingsById.addEntries(
            readingMatchHeadings.map(
              (heading) => MapEntry(heading.id, heading),
            ),
          );
        }
      }
    }
  } else {
    List<String> deinflectionsAlreadySearched = [];

    bool startsWithAdded = false;
    for (int i = 0; i < searchTerm.length; i++) {
      String partialTerm = searchTerm.substring(0, searchTerm.length - i);

      bool partialTermIsKana = kanaKit.isKana(partialTerm);
      bool partialTermIsHiragana = kanaKit.isHiragana(partialTerm);
      bool partialTermIsKatakana = kanaKit.isKatakana(partialTerm);

      List<String> possibleDeinflections = Deinflector.deinflect(partialTerm)
          .map((e) => e.term)
          .where((e) => !deinflectionsAlreadySearched.contains(e))
          .toList();
      possibleDeinflections.toSet().addAll(deinflectionsAlreadySearched);

      List<DictionaryHeading> termExactResults = [];
      List<DictionaryHeading> termDeinflectedResults = [];
      List<DictionaryHeading> readingExactResults = [];
      List<DictionaryHeading> readingDeinflectedResults = [];
      List<DictionaryHeading> termExactKatakanaResults = [];

      if (params.maximumDictionaryTermsInResult > uniqueHeadingsById.length) {
        /// If an exception is caught, skip the iteration entirely. Something
        /// has gone wrong with building a query for this term and there is
        /// no point continuing for the rest of the queries.
        try {
          termExactResults = database.dictionaryHeadings
              .where()
              .termEqualTo(partialTerm)
              .or()
              .optional(partialTermIsKatakana,
                  (q) => q.termEqualTo(kanaKit.toHiragana(partialTerm)))
              .filter()
              .entriesIsNotEmpty()
              .findAllSync();
        } catch (e) {
          await Future.delayed(const Duration(milliseconds: 200), () {});
          continue;
        }

        uniqueHeadingsById.addEntries(
          termExactResults.map(
            (heading) => MapEntry(heading.id, heading),
          ),
        );
      }

      if (params.maximumDictionaryTermsInResult > uniqueHeadingsById.length) {
        if (partialTermIsKana) {
          readingExactResults = database.dictionaryHeadings
              .where()
              .readingEqualTo(partialTerm)
              .or()
              .optional(partialTermIsKatakana,
                  (q) => q.readingEqualTo(kanaKit.toHiragana(partialTerm)))
              .filter()
              .entriesIsNotEmpty()
              .findAllSync();

          uniqueHeadingsById.addEntries(
            readingExactResults.map(
              (heading) => MapEntry(heading.id, heading),
            ),
          );
        }
      }

      if (params.maximumDictionaryTermsInResult > uniqueHeadingsById.length) {
        termDeinflectedResults = database.dictionaryHeadings
            .where()
            .anyOf<String, String>(
                possibleDeinflections, (q, term) => q.termEqualTo(term))
            .or()
            .optional(
                partialTermIsKatakana,
                (q) => q.anyOf<String, String>(
                    Deinflector.deinflect(kanaKit.toHiragana(partialTerm))
                        .map((e) => e.term)
                        .toList(),
                    (q, term) => q.termEqualTo(term)))
            .filter()
            .entriesIsNotEmpty()
            .findAllSync();

        for (DictionaryHeading result in termDeinflectedResults) {
          result.entries.loadSync();
        }
        termDeinflectedResults
            .sort((a, b) => b.popularitySum.compareTo(a.popularitySum));

        uniqueHeadingsById.addEntries(
          termDeinflectedResults.map(
            (heading) => MapEntry(heading.id, heading),
          ),
        );
      }

      if (params.maximumDictionaryTermsInResult > uniqueHeadingsById.length) {
        if (partialTermIsKana) {
          readingDeinflectedResults = database.dictionaryHeadings
              .where()
              .anyOf<String, String>(possibleDeinflections,
                  (q, reading) => q.readingEqualTo(reading))
              .or()
              .optional(
                  partialTermIsKatakana,
                  (q) => q.anyOf<String, String>(
                      Deinflector.deinflect(kanaKit.toHiragana(partialTerm))
                          .map((e) => e.term)
                          .toList(),
                      (q, term) => q.readingEqualTo(term)))
              .filter()
              .entriesIsNotEmpty()
              .limit(limit())
              .findAllSync();

          uniqueHeadingsById.addEntries(
            readingDeinflectedResults.map(
              (heading) => MapEntry(heading.id, heading),
            ),
          );
        }
      }

      if (params.maximumDictionaryTermsInResult > uniqueHeadingsById.length) {
        if (partialTermIsHiragana) {
          termExactKatakanaResults = database.dictionaryHeadings
              .where()
              .termEqualTo(kanaKit.toKatakana(partialTerm))
              .filter()
              .entriesIsNotEmpty()
              .findAllSync();

          uniqueHeadingsById.addEntries(
            termExactKatakanaResults.map(
              (heading) => MapEntry(heading.id, heading),
            ),
          );
        }
      }

      if (termExactResults.isNotEmpty && bestLength < partialTerm.length) {
        bestLength = partialTerm.length;
      }
      if (termDeinflectedResults.isNotEmpty &&
          bestLength < partialTerm.length) {
        bestLength = partialTerm.length;
      }
      if (readingExactResults.isNotEmpty && bestLength < partialTerm.length) {
        bestLength = partialTerm.length;
      }
      if (readingDeinflectedResults.isNotEmpty &&
          bestLength < partialTerm.length) {
        bestLength = partialTerm.length;
      }
      if (termExactKatakanaResults.isNotEmpty &&
          bestLength < partialTerm.length) {
        bestLength = partialTerm.length;
      }

      if (params.maximumDictionaryTermsInResult > uniqueHeadingsById.length) {
        if (params.searchWithWildcards) {
          if (i == 0) {
            startsWithAdded = true;

            List<DictionaryHeading> startsWithToAdd = database
                .dictionaryHeadings
                .where()
                .termStartsWith(searchTerm)
                .filter()
                .entriesIsNotEmpty()
                .sortByTermLength()
                .findAllSync();

            uniqueHeadingsById.addEntries(startsWithToAdd.map(
              (heading) => MapEntry(heading.id, heading),
            ));
          }
        } else {
          if (!startsWithAdded && uniqueHeadingsById.isNotEmpty) {
            startsWithAdded = true;

            List<DictionaryHeading> startsWithToAdd = database
                .dictionaryHeadings
                .where()
                .termStartsWith(searchTerm)
                .filter()
                .entriesIsNotEmpty()
                .sortByTermLength()
                .findAllSync();

            uniqueHeadingsById.addEntries(startsWithToAdd.map(
              (heading) => MapEntry(heading.id, heading),
            ));
          }
        }
      }
    }
  }

  List<DictionaryHeading> headings =
      uniqueHeadingsById.values.where((e) => e.entries.isNotEmpty).toList();
  Map<DictionaryHeading, int> headingOrders = Map.fromEntries(
    headings.mapIndexed(
      (index, id) => MapEntry(headings[index], index),
    ),
  );

  if (headings.isEmpty) {
    return null;
  }

  DictionarySearchResult unsortedResult = DictionarySearchResult(
    searchTerm: searchTerm,
    bestLength: bestLength,
  );
  unsortedResult.headings.addAll(headings);

  late int resultId;
  database.writeTxnSync(() async {
    database.dictionarySearchResults.deleteBySearchTermSync(searchTerm);
    resultId = database.dictionarySearchResults.putSync(unsortedResult);
  });

  preloadResultSync(resultId);

  List<Dictionary> dictionaries = database.dictionarys.where().findAllSync();

  Map<String, int> dictionaryNamesByOrder = Map<String, int>.fromEntries(
      dictionaries.map((e) => MapEntry(e.name, e.order)));

  headings.sort((a, b) {
    if (a.term == b.term ||
        (a.reading.isNotEmpty && b.reading.isNotEmpty) &&
            a.reading == b.reading) {
      int hasPopularTag = (a.tags.any((e) => e.name == 'P') ? -1 : 1)
          .compareTo(b.tags.any((e) => e.name == 'P') ? -1 : 1);
      if (hasPopularTag != 0) {
        return hasPopularTag;
      }

      if (a.term != b.term) {
        int popularityCompare = (b.popularitySum).compareTo(a.popularitySum);
        if (popularityCompare != 0) {
          return popularityCompare;
        }
      }

      List<DictionaryFrequency> aFrequencies = a.frequencies.toList();
      List<DictionaryFrequency> bFrequencies = b.frequencies.toList();
      aFrequencies.sort((a, b) =>
          dictionaryNamesByOrder[a.dictionary.value!.name]!
              .compareTo(dictionaryNamesByOrder[b.dictionary.value!.name]!));
      bFrequencies.sort((a, b) =>
          dictionaryNamesByOrder[a.dictionary.value!.name]!
              .compareTo(dictionaryNamesByOrder[b.dictionary.value!.name]!));

      Map<Dictionary, List<DictionaryFrequency>> aFrequenciesByDictionary =
          groupBy<DictionaryFrequency, Dictionary>(
              aFrequencies, (frequency) => frequency.dictionary.value!);
      Map<Dictionary, List<DictionaryFrequency>> bFrequenciesByDictionary =
          groupBy<DictionaryFrequency, Dictionary>(
              bFrequencies, (frequency) => frequency.dictionary.value!);
      Map<Dictionary, double> aValues = aFrequenciesByDictionary
          .map((k, v) => MapEntry(k, v.map((e) => e.value).min));
      Map<Dictionary, double> bValues = bFrequenciesByDictionary
          .map((k, v) => MapEntry(k, v.map((e) => e.value).min));

      Set<Dictionary> sharedDictionaries =
          aValues.keys.toSet().intersection(bValues.keys.toSet());

      if (sharedDictionaries.isNotEmpty) {
        for (Dictionary dictionary in sharedDictionaries) {
          int freqCompare =
              aValues[dictionary]!.compareTo(bValues[dictionary]!);
          if (freqCompare != 0) {
            return freqCompare;
          }
        }
      } else {
        int popularityCompare = (b.popularitySum).compareTo(a.popularitySum);
        if (popularityCompare != 0) {
          return popularityCompare;
        }
      }

      int entriesCompare = b.entries.length.compareTo(a.entries.length);
      if (entriesCompare != 0) {
        return entriesCompare;
      }
    }

    return headingOrders[a]!.compareTo(headingOrders[b]!);
  });

  /// Prioritise kanji match.
  if (searchTerm.length == 1 && kanaKit.isKanji(searchTerm)) {
    DictionaryHeading? heading = database.dictionaryHeadings
        .getSync(DictionaryHeading.hash(term: searchTerm, reading: ''));
    if (heading != null && heading.entries.isNotEmpty) {
      headings.remove(heading);
      headings.insert(0, heading);
    }
  }

  headings = headings.sublist(
      0, min(headings.length, params.maximumDictionaryTermsInResult));

  List<int> headingIds = headings.map((e) => e.id).toList();

  DictionarySearchResult result = DictionarySearchResult(
    id: resultId,
    searchTerm: searchTerm,
    bestLength: bestLength,
    headingIds: headingIds,
  );

  database.writeTxnSync(() async {
    resultId = database.dictionarySearchResults.putSync(result);

    int countInSameHistory = database.dictionarySearchResults.countSync();

    if (params.maximumDictionarySearchResults < countInSameHistory) {
      int surplus = countInSameHistory - params.maximumDictionarySearchResults;
      database.dictionarySearchResults
          .where()
          .limit(surplus)
          .build()
          .deleteAllSync();
    }
  });

  return resultId;
}