static void latinime_BinaryDictionary_getWordProperty(JNIEnv *env, jclass clazz,
        jlong dict, jintArray word, jboolean isBeginningOfSentence, jintArray outCodePoints,
        jbooleanArray outFlags, jintArray outProbabilityInfo, jobject outBigramTargets,
        jobject outBigramProbabilityInfo, jobject outShortcutTargets,
        jobject outShortcutProbabilities) {
    Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
    if (!dictionary) return;
    const jsize wordLength = env->GetArrayLength(word);
    if (wordLength > MAX_WORD_LENGTH) {
        AKLOGE("Invalid wordLength: %d", wordLength);
        return;
    }
    int wordCodePoints[MAX_WORD_LENGTH];
    env->GetIntArrayRegion(word, 0, wordLength, wordCodePoints);
    int codePointCount = wordLength;
    if (isBeginningOfSentence) {
        codePointCount = CharUtils::attachBeginningOfSentenceMarker(
                wordCodePoints, wordLength, MAX_WORD_LENGTH);
        if (codePointCount < 0) {
            AKLOGE("Cannot attach Beginning-of-Sentence marker.");
            return;
        }
    }
    const WordProperty wordProperty = dictionary->getWordProperty(wordCodePoints, codePointCount);
    wordProperty.outputProperties(env, outCodePoints, outFlags, outProbabilityInfo,
            outBigramTargets, outBigramProbabilityInfo, outShortcutTargets,
            outShortcutProbabilities);
}

/* static */ bool DictFileWritingUtils::flushAllHeaderAndBodyToFile(const char *const filePath,
        BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody) {
    const int tmpFileNameBufSize = strlen(filePath)
            + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1 /* terminator */;
    // Name of a temporary file used for writing that is a connected string of original name and
    // TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE.
    char tmpFileName[tmpFileNameBufSize];
    snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", filePath,
            TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
    FILE *const file = fopen(tmpFileName, "wb");
    if (!file) {
        AKLOGE("Dictionary file %s cannnot be opened.", tmpFileName);
        ASSERT(false);
        return false;
    }
    // Write the dictionary header.
    if (!writeBufferToFile(file, dictHeader)) {
        remove(tmpFileName);
        AKLOGE("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition());
        ASSERT(false);
        return false;
    }
    // Write the dictionary body.
    if (!writeBufferToFile(file, dictBody)) {
        remove(tmpFileName);
        AKLOGE("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition());
        ASSERT(false);
        return false;
    }
    fclose(file);
    rename(tmpFileName, filePath);
    return true;
}

bool ShortcutDictContent::runGC(
        const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
        const ShortcutDictContent *const originalShortcutDictContent) {
   for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
           it != terminalIdMap->end(); ++it) {
       const int originalShortcutListPos =
               originalShortcutDictContent->getShortcutListHeadPos(it->first);
       if (originalShortcutListPos == NOT_A_DICT_POS) {
           continue;
       }
       const int shortcutListPos = getContentBuffer()->getTailPosition();
       // Copy shortcut list from original content.
       if (!copyShortcutListFromDictContent(originalShortcutListPos, originalShortcutDictContent,
               shortcutListPos)) {
           AKLOGE("Cannot copy shortcut list during GC. original pos: %d, pos: %d",
                   originalShortcutListPos, shortcutListPos);
           return false;
       }
       // Set shortcut list position to the lookup table.
       if (!getUpdatableAddressLookupTable()->set(it->second, shortcutListPos)) {
           AKLOGE("Cannot set shortcut list position. terminal id: %d, pos: %d",
                   it->second, shortcutListPos);
           return false;
       }
   }
   return true;
}

/* static */ bool DictFileWritingUtils::flushBufferToFile(const char *const filePath,
        const BufferWithExtendableBuffer *const buffer) {
    const int fd = open(filePath, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
    if (fd == -1) {
        AKLOGE("File %s cannot be opened. errno: %d", filePath, errno);
        ASSERT(false);
        return false;
    }
    FILE *const file = fdopen(fd, "wb");
    if (!file) {
        AKLOGE("fdopen failed for the file %s. errno: %d", filePath, errno);
        ASSERT(false);
        return false;
    }
    if (!writeBufferToFile(file, buffer)) {
        fclose(file);
        remove(filePath);
        AKLOGE("Buffer cannot be written to the file %s. size: %d", filePath,
                buffer->getTailPosition());
        ASSERT(false);
        return false;
    }
    fclose(file);
    return true;
}

bool Ver4PtNodeArrayReader::readPtNodeArrayInfoAndReturnIfValid(const int ptNodeArrayPos,
        int *const outPtNodeCount, int *const outFirstPtNodePos) const {
    if (ptNodeArrayPos < 0 || ptNodeArrayPos >= mBuffer->getTailPosition()) {
        // Reading invalid position because of a bug or a broken dictionary.
        AKLOGE("Reading PtNode array info from invalid dictionary position: %d, dict size: %d",
                ptNodeArrayPos, mBuffer->getTailPosition());
        ASSERT(false);
        return false;
    }
    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(ptNodeArrayPos);
    const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer);
    int readingPos = ptNodeArrayPos;
    if (usesAdditionalBuffer) {
        readingPos -= mBuffer->getOriginalBufferSize();
    }
    const int ptNodeCountInArray = PatriciaTrieReadingUtils::getPtNodeArraySizeAndAdvancePosition(
            dictBuf, &readingPos);
    if (usesAdditionalBuffer) {
        readingPos += mBuffer->getOriginalBufferSize();
    }
    if (ptNodeCountInArray < 0) {
        AKLOGE("Invalid PtNode count in an array: %d.", ptNodeCountInArray);
        return false;
    }
    *outPtNodeCount = ptNodeCountInArray;
    *outFirstPtNodePos = readingPos;
    return true;
}

bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbabilityAndGetNeedsToKeepPtNodeAfterGC(
        const PtNodeParams *const toBeUpdatedPtNodeParams, bool *const outNeedsToKeepPtNode) {
    if (!toBeUpdatedPtNodeParams->isTerminal()) {
        AKLOGE("updatePtNodeProbabilityAndGetNeedsToSaveForGC is called for non-terminal PtNode.");
        return false;
    }
    const ProbabilityEntry originalProbabilityEntry =
            mBuffers->getLanguageModelDictContent()->getProbabilityEntry(
                    toBeUpdatedPtNodeParams->getTerminalId());
    if (originalProbabilityEntry.hasHistoricalInfo()) {
        const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave(
                originalProbabilityEntry.getHistoricalInfo(), mHeaderPolicy);
        const ProbabilityEntry probabilityEntry =
                originalProbabilityEntry.createEntryWithUpdatedHistoricalInfo(&historicalInfo);
        if (!mBuffers->getMutableLanguageModelDictContent()->setProbabilityEntry(
                toBeUpdatedPtNodeParams->getTerminalId(), &probabilityEntry)) {
            AKLOGE("Cannot write updated probability entry. terminalId: %d",
                    toBeUpdatedPtNodeParams->getTerminalId());
            return false;
        }
        const bool isValid = ForgettingCurveUtils::needsToKeep(&historicalInfo, mHeaderPolicy);
        if (!isValid) {
            if (!markPtNodeAsWillBecomeNonTerminal(toBeUpdatedPtNodeParams)) {
                AKLOGE("Cannot mark PtNode as willBecomeNonTerminal.");
                return false;
            }
        }
        *outNeedsToKeepPtNode = isValid;
    } else {
        // No need to update probability.
        *outNeedsToKeepPtNode = true;
    }
    return true;
}

bool ProbabilityDictContent::writeEntry(const ProbabilityEntry *const probabilityEntry,
        const int entryPos) {
    BufferWithExtendableBuffer *const bufferToWrite = getWritableBuffer();
    int writingPos = entryPos;
    if (!bufferToWrite->writeUintAndAdvancePosition(probabilityEntry->getFlags(),
            Ver4DictConstants::FLAGS_IN_PROBABILITY_FILE_SIZE, &writingPos)) {
        AKLOGE("Cannot write flags in probability dict content. pos: %d", writingPos);
        return false;
    }
    if (!bufferToWrite->writeUintAndAdvancePosition(probabilityEntry->getProbability(),
            Ver4DictConstants::PROBABILITY_SIZE, &writingPos)) {
        AKLOGE("Cannot write probability in probability dict content. pos: %d", writingPos);
        return false;
    }
    if (mHasHistoricalInfo) {
        const HistoricalInfo *const historicalInfo = probabilityEntry->getHistoricalInfo();
        if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getTimestamp(),
                Ver4DictConstants::TIME_STAMP_FIELD_SIZE, &writingPos)) {
            AKLOGE("Cannot write timestamp in probability dict content. pos: %d", writingPos);
            return false;
        }
        if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getLevel(),
                Ver4DictConstants::WORD_LEVEL_FIELD_SIZE, &writingPos)) {
            AKLOGE("Cannot write level in probability dict content. pos: %d", writingPos);
            return false;
        }
        if (!bufferToWrite->writeUintAndAdvancePosition(historicalInfo->getCount(),
                Ver4DictConstants::WORD_COUNT_FIELD_SIZE, &writingPos)) {
            AKLOGE("Cannot write count in probability dict content. pos: %d", writingPos);
            return false;
        }
    }
    return true;
}

/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
        DictionaryStructureWithBufferPolicyFactory::newPolicyForFileDict(
                const char *const path, const int bufOffset, const int size) {
    // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
    // MmappedBufferPtr if the instance has the responsibility.
    MmappedBuffer::MmappedBufferPtr mmappedBuffer(
            MmappedBuffer::openBuffer(path, bufOffset, size, false /* isUpdatable */));
    if (!mmappedBuffer) {
        return nullptr;
    }
    switch (FormatUtils::detectFormatVersion(mmappedBuffer->getReadOnlyByteArrayView())) {
        case FormatUtils::VERSION_2:
        case FormatUtils::VERSION_201:
            AKLOGE("Dictionary versions 2 and 201 are incompatible with this version");
            break;
        case FormatUtils::VERSION_202:
            return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(
                    new PatriciaTriePolicy(std::move(mmappedBuffer)));
        case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
        case FormatUtils::VERSION_402:
        case FormatUtils::VERSION_403:
            AKLOGE("Given path is a file but the format is version 4. path: %s", path);
            break;
        default:
            AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path);
            break;
    }
    ASSERT(false);
    return nullptr;
}

bool BigramDictContent::runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,
        const BigramDictContent *const originalBigramDictContent,
        int *const outBigramEntryCount) {
    for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin();
            it != terminalIdMap->end(); ++it) {
        const int originalBigramListPos =
                originalBigramDictContent->getBigramListHeadPos(it->first);
        if (originalBigramListPos == NOT_A_DICT_POS) {
            // This terminal does not have a bigram list.
            continue;
        }
        const int bigramListPos = getContentBuffer()->getTailPosition();
        int bigramEntryCount = 0;
        // Copy bigram list with GC from original content.
        if (!runGCBigramList(originalBigramListPos, originalBigramDictContent, bigramListPos,
                terminalIdMap, &bigramEntryCount)) {
            AKLOGE("Cannot complete GC for the bigram list. original pos: %d, pos: %d",
                    originalBigramListPos, bigramListPos);
            return false;
        }
        if (bigramEntryCount == 0) {
            // All bigram entries are useless. This terminal does not have a bigram list.
            continue;
        }
        *outBigramEntryCount += bigramEntryCount;
        // Set bigram list position to the lookup table.
        if (!getUpdatableAddressLookupTable()->set(it->second, bigramListPos)) {
            AKLOGE("Cannot set bigram list position. terminal id: %d, pos: %d",
                    it->second, bigramListPos);
            return false;
        }
    }
    return true;
}

/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers(
        const char *const dictPath, MmappedBuffer::MmappedBufferPtr &&headerBuffer,
        const FormatUtils::FORMAT_VERSION formatVersion) {
    if (!headerBuffer) {
        ASSERT(false);
        AKLOGE("The header buffer must be valid to open ver4 dict buffers.");
        return Ver4DictBuffersPtr(nullptr);
    }
    // TODO: take only dictDirPath, and open both header and trie files in the constructor below
    const bool isUpdatable = headerBuffer->isUpdatable();
    MmappedBuffer::MmappedBufferPtr bodyBuffer = MmappedBuffer::openBuffer(dictPath,
            Ver4DictConstants::BODY_FILE_EXTENSION, isUpdatable);
    if (!bodyBuffer) {
        return Ver4DictBuffersPtr(nullptr);
    }
    std::vector<ReadWriteByteArrayView> buffers;
    const ReadWriteByteArrayView buffer = bodyBuffer->getReadWriteByteArrayView();
    int position = 0;
    while (position < static_cast<int>(buffer.size())) {
        const int bufferSize = ByteArrayUtils::readUint32AndAdvancePosition(
                buffer.data(), &position);
        buffers.push_back(buffer.subView(position, bufferSize));
        position += bufferSize;
        if (bufferSize < 0 || position < 0 || position > static_cast<int>(buffer.size())) {
            AKLOGE("The dict body file is corrupted.");
            return Ver4DictBuffersPtr(nullptr);
        }
    }
    if (buffers.size() != Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE) {
        AKLOGE("The dict body file is corrupted.");
        return Ver4DictBuffersPtr(nullptr);
    }
    return Ver4DictBuffersPtr(new Ver4DictBuffers(std::move(headerBuffer), std::move(bodyBuffer),
            formatVersion, buffers));
}

/* static */ MmappedBuffer::MmappedBufferPtr MmappedBuffer::openBuffer(
        const char *const path, const int bufferOffset, const int bufferSize,
        const bool isUpdatable) {
    const int mmapFd = open(path, O_RDONLY);
    if (mmapFd < 0) {
        AKLOGE("DICT: Can't open the source. path=%s errno=%d", path, errno);
        return nullptr;
    }
    const int pagesize = sysconf(_SC_PAGESIZE);
    const int offset = bufferOffset % pagesize;
    int alignedOffset = bufferOffset - offset;
    int alignedSize = bufferSize + offset;
    const int protMode = isUpdatable ? PROT_READ | PROT_WRITE : PROT_READ;
    void *const mmappedBuffer = mmap(0, alignedSize, protMode, MAP_PRIVATE, mmapFd,
            alignedOffset);
    if (mmappedBuffer == MAP_FAILED) {
        AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno);
        close(mmapFd);
        return nullptr;
    }
    uint8_t *const buffer = static_cast<uint8_t *>(mmappedBuffer) + offset;
    if (!buffer) {
        AKLOGE("DICT: buffer is null");
        close(mmapFd);
        return nullptr;
    }
    return MmappedBufferPtr(new MmappedBuffer(buffer, bufferSize, mmappedBuffer, alignedSize,
            mmapFd, isUpdatable));
}

bool Ver4PatriciaTriePolicy::updateEntriesForWordWithNgramContext(
        const NgramContext *const ngramContext, const CodePointArrayView wordCodePoints,
        const bool isValidWord, const HistoricalInfo historicalInfo) {
    if (!mBuffers->isUpdatable()) {
        AKLOGI("Warning: updateEntriesForWordWithNgramContext() is called for non-updatable "
                "dictionary.");
        return false;
    }
    const bool updateAsAValidWord = ngramContext->isNthPrevWordBeginningOfSentence(1 /* n */) ?
            false : isValidWord;
    int wordId = getWordId(wordCodePoints, false /* tryLowerCaseSearch */);
    if (wordId == NOT_A_WORD_ID) {
        // The word is not in the dictionary.
        const UnigramProperty unigramProperty(false /* representsBeginningOfSentence */,
                false /* isNotAWord */, false /* isBlacklisted */, false /* isPossiblyOffensive */,
                NOT_A_PROBABILITY, HistoricalInfo(historicalInfo.getTimestamp(), 0 /* level */,
                0 /* count */));
        if (!addUnigramEntry(wordCodePoints, &unigramProperty)) {
            AKLOGE("Cannot add unigarm entry in updateEntriesForWordWithNgramContext().");
            return false;
        }
        if (!isValidWord) {
            return true;
        }
        wordId = getWordId(wordCodePoints, false /* tryLowerCaseSearch */);
    }

    WordIdArray<MAX_PREV_WORD_COUNT_FOR_N_GRAM> prevWordIdArray;
    const WordIdArrayView prevWordIds = ngramContext->getPrevWordIds(this, &prevWordIdArray,
            false /* tryLowerCaseSearch */);
    if (ngramContext->isNthPrevWordBeginningOfSentence(1 /* n */)) {
        if (prevWordIds.firstOrDefault(NOT_A_WORD_ID) == NOT_A_WORD_ID) {
            const UnigramProperty beginningOfSentenceUnigramProperty(
                    true /* representsBeginningOfSentence */,
                    true /* isNotAWord */, false /* isPossiblyOffensive */, NOT_A_PROBABILITY,
                    HistoricalInfo(historicalInfo.getTimestamp(), 0 /* level */, 0 /* count */));
            if (!addUnigramEntry(ngramContext->getNthPrevWordCodePoints(1 /* n */),
                    &beginningOfSentenceUnigramProperty)) {
                AKLOGE("Cannot add BoS entry in updateEntriesForWordWithNgramContext().");
                return false;
            }
            // Refresh word ids.
            ngramContext->getPrevWordIds(this, &prevWordIdArray, false /* tryLowerCaseSearch */);
        }
        // Update entries for beginning of sentence.
        if (!mBuffers->getMutableLanguageModelDictContent()->updateAllEntriesOnInputWord(
                prevWordIds.skip(1 /* n */), prevWordIds[0], true /* isVaild */, historicalInfo,
                mHeaderPolicy, &mEntryCounters)) {
            return false;
        }
    }
    if (!mBuffers->getMutableLanguageModelDictContent()->updateAllEntriesOnInputWord(prevWordIds,
            wordId, updateAsAValidWord, historicalInfo, mHeaderPolicy, &mEntryCounters)) {
        return false;
    }
    return true;
}

MmappedBuffer::~MmappedBuffer() {
    if (mAlignedSize == 0) {
        return;
    }
    int ret = munmap(mMmappedBuffer, mAlignedSize);
    if (ret != 0) {
        AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno);
    }
    ret = close(mMmapFd);
    if (ret != 0) {
        AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno);
    }
}

static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd) {
#ifdef USE_MMAP_FOR_DICTIONARY
    int ret = munmap(const_cast<void *>(dictBuf), length);
    if (ret != 0) {
        AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno);
    }
    ret = close(fd);
    if (ret != 0) {
        AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno);
    }
#else // USE_MMAP_FOR_DICTIONARY
    free(const_cast<void *>(dictBuf));
#endif // USE_MMAP_FOR_DICTIONARY
}

// Follow the forward link and read the next node array if exists.
void DynamicPatriciaTrieReadingHelper::followForwardLink() {
    if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) {
        // Reading invalid position because of bug or broken dictionary.
        AKLOGE("Reading forward link from invalid dictionary position: %d, dict size: %d",
                mReadingState.mPos, mBuffer->getTailPosition());
        ASSERT(false);
        mIsError = true;
        mReadingState.mPos = NOT_A_DICT_POS;
        return;
    }
    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(mReadingState.mPos);
    const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer);
    if (usesAdditionalBuffer) {
        mReadingState.mPos -= mBuffer->getOriginalBufferSize();
    }
    const int forwardLinkPosition =
            DynamicPatriciaTrieReadingUtils::getForwardLinkPosition(dictBuf, mReadingState.mPos);
    if (usesAdditionalBuffer) {
        mReadingState.mPos += mBuffer->getOriginalBufferSize();
    }
    mReadingState.mPosOfLastForwardLinkField = mReadingState.mPos;
    if (DynamicPatriciaTrieReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) {
        // Follow the forward link.
        mReadingState.mPos += forwardLinkPosition;
        nextPtNodeArray();
    } else {
        // All node arrays have been read.
        mReadingState.mPos = NOT_A_DICT_POS;
    }
}

bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const int length0,
        const int *const word1, const int length1) {
    if (!mBuffer->isUpdatable()) {
        AKLOGI("Warning: removeBigramWords() is called for non-updatable dictionary.");
        return false;
    }
    if (mBufferWithExtendableBuffer.getTailPosition()
            >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) {
        AKLOGE("The dictionary is too large to dynamically update.");
        return false;
    }
    const int word0Pos = getTerminalNodePositionOfWord(word0, length0,
            false /* forceLowerCaseSearch */);
    if (word0Pos == NOT_A_DICT_POS) {
        return false;
    }
    const int word1Pos = getTerminalNodePositionOfWord(word1, length1,
            false /* forceLowerCaseSearch */);
    if (word1Pos == NOT_A_DICT_POS) {
        return false;
    }
    DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer,
            &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict());
    if (writingHelper.removeBigramWords(word0Pos, word1Pos)) {
        mBigramCount--;
        return true;
    } else {
        return false;
    }
}

/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr
        DictionaryStructureWithBufferPolicyFactory:: newPolicyForOnMemoryDict(
                const int formatVersion, const std::vector<int> &locale,
                const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) {
    FormatUtils::FORMAT_VERSION dictFormatVersion = FormatUtils::getFormatVersion(formatVersion);
    switch (dictFormatVersion) {
        case FormatUtils::VERSION_402: {
            return newPolicyForOnMemoryV4Dict<backward::v402::Ver4DictConstants,
                    backward::v402::Ver4DictBuffers,
                    backward::v402::Ver4DictBuffers::Ver4DictBuffersPtr,
                    backward::v402::Ver4PatriciaTriePolicy>(
                            dictFormatVersion, locale, attributeMap);
        }
        case FormatUtils::VERSION_4_ONLY_FOR_TESTING:
        case FormatUtils::VERSION_403: {
            return newPolicyForOnMemoryV4Dict<Ver4DictConstants, Ver4DictBuffers,
                    Ver4DictBuffers::Ver4DictBuffersPtr, Ver4PatriciaTriePolicy>(
                            dictFormatVersion, locale, attributeMap);
        }
        default:
            AKLOGE("DICT: dictionary format %d is not supported for on memory dictionary",
                    formatVersion);
            break;
    }
    return nullptr;
}

bool LanguageModelDictContent::updateAllProbabilityEntriesForGCInner(const int bitmapEntryIndex,
        const int prevWordCount, const HeaderPolicy *const headerPolicy,
        const bool needsToHalveCounters, MutableEntryCounters *const outEntryCounters) {
    for (const auto &entry : mTrieMap.getEntriesInSpecifiedLevel(bitmapEntryIndex)) {
        if (prevWordCount > MAX_PREV_WORD_COUNT_FOR_N_GRAM) {
            AKLOGE("Invalid prevWordCount. prevWordCount: %d, MAX_PREV_WORD_COUNT_FOR_N_GRAM: %d.",
                    prevWordCount, MAX_PREV_WORD_COUNT_FOR_N_GRAM);
            return false;
        }
        const ProbabilityEntry probabilityEntry =
                ProbabilityEntry::decode(entry.value(), mHasHistoricalInfo);
        if (prevWordCount > 0 && probabilityEntry.isValid()
                && !mTrieMap.getRoot(entry.key()).mIsValid) {
            // The entry is related to a word that has been removed. Remove the entry.
            if (!mTrieMap.remove(entry.key(), bitmapEntryIndex)) {
                return false;
            }
            continue;
        }
        if (mHasHistoricalInfo && probabilityEntry.isValid()) {
            const HistoricalInfo *originalHistoricalInfo = probabilityEntry.getHistoricalInfo();
            if (DynamicLanguageModelProbabilityUtils::shouldRemoveEntryDuringGC(
                    *originalHistoricalInfo)) {
                // Remove the entry.
                if (!mTrieMap.remove(entry.key(), bitmapEntryIndex)) {
                    return false;
                }
                continue;
            }
            if (needsToHalveCounters) {
                const int updatedCount = originalHistoricalInfo->getCount() / 2;
                if (updatedCount == 0) {
                    // Remove the entry.
                    if (!mTrieMap.remove(entry.key(), bitmapEntryIndex)) {
                        return false;
                    }
                    continue;
                }
                const HistoricalInfo historicalInfoToSave(originalHistoricalInfo->getTimestamp(),
                        originalHistoricalInfo->getLevel(), updatedCount);
                const ProbabilityEntry updatedEntry(probabilityEntry.getFlags(),
                        &historicalInfoToSave);
                if (!mTrieMap.put(entry.key(), updatedEntry.encode(mHasHistoricalInfo),
                        bitmapEntryIndex)) {
                    return false;
                }
            }
        }
        outEntryCounters->incrementNgramCount(
                NgramUtils::getNgramTypeFromWordCount(prevWordCount + 1));
        if (!entry.hasNextLevelMap()) {
            continue;
        }
        if (!updateAllProbabilityEntriesForGCInner(entry.getNextLevelBitmapEntryIndex(),
                prevWordCount + 1, headerPolicy, needsToHalveCounters, outEntryCounters)) {
            return false;
        }
    }
    return true;
}

int Ver4PatriciaTriePolicy::getNextWordAndNextToken(const int token, int *const outCodePoints,
        int *const outCodePointCount) {
    *outCodePointCount = 0;
    if (token == 0) {
        mTerminalPtNodePositionsForIteratingWords.clear();
        DynamicPtReadingHelper::TraversePolicyToGetAllTerminalPtNodePositions traversePolicy(
                &mTerminalPtNodePositionsForIteratingWords);
        DynamicPtReadingHelper readingHelper(&mNodeReader, &mPtNodeArrayReader);
        readingHelper.initWithPtNodeArrayPos(getRootPosition());
        readingHelper.traverseAllPtNodesInPostorderDepthFirstManner(&traversePolicy);
    }
    const int terminalPtNodePositionsVectorSize =
            static_cast<int>(mTerminalPtNodePositionsForIteratingWords.size());
    if (token < 0 || token >= terminalPtNodePositionsVectorSize) {
        AKLOGE("Given token %d is invalid.", token);
        return 0;
    }
    const int terminalPtNodePos = mTerminalPtNodePositionsForIteratingWords[token];
    const PtNodeParams ptNodeParams =
            mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(terminalPtNodePos);
    *outCodePointCount = getCodePointsAndReturnCodePointCount(ptNodeParams.getTerminalId(),
            MAX_WORD_LENGTH, outCodePoints);
    const int nextToken = token + 1;
    if (nextToken >= terminalPtNodePositionsVectorSize) {
        // All words have been iterated.
        mTerminalPtNodePositionsForIteratingWords.clear();
        return 0;
    }
    return nextToken;
}

static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
        jlong dictOffset, jlong dictSize, jboolean isUpdatable) {
    PROF_OPEN;
    PROF_START(66);
    const jsize sourceDirUtf8Length = env->GetStringUTFLength(sourceDir);
    if (sourceDirUtf8Length <= 0) {
        AKLOGE("DICT: Can't get sourceDir string");
        return 0;
    }
    char sourceDirChars[sourceDirUtf8Length + 1];
    env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars);
    sourceDirChars[sourceDirUtf8Length] = '\0';
    DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy =
            DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy(
                    sourceDirChars, static_cast<int>(dictOffset), static_cast<int>(dictSize),
                    isUpdatable == JNI_TRUE);
    if (!dictionaryStructureWithBufferPolicy) {
        return 0;
    }

    Dictionary *const dictionary = new Dictionary(env, dictionaryStructureWithBufferPolicy);
    PROF_END(66);
    PROF_CLOSE;
    return reinterpret_cast<jlong>(dictionary);
}