From 503db52d1026f425599f1441de3bfe42941df6cd Mon Sep 17 00:00:00 2001 From: VanCZ1 <92382185+VanCZ1@users.noreply.github.com> Date: Sun, 9 Aug 2026 17:03:00 +0800 Subject: [PATCH] perf: optimize directory cache and animation initialization --- src/OpenAnimationReplacer.cpp | 17 ++++---- src/Parsing.cpp | 79 +++++++++++++---------------------- src/Parsing.h | 2 +- src/ReplacerMods.cpp | 72 ++++++++++++++++++++++++------- src/ReplacerMods.h | 27 ++++++++++-- 5 files changed, 120 insertions(+), 77 deletions(-) diff --git a/src/OpenAnimationReplacer.cpp b/src/OpenAnimationReplacer.cpp index 948dd49..57a4cad 100644 --- a/src/OpenAnimationReplacer.cpp +++ b/src/OpenAnimationReplacer.cpp @@ -560,7 +560,8 @@ void OpenAnimationReplacer::CreateReplacerMods() auto endTime = std::chrono::high_resolution_clock::now(); logger::info("Time spent creating replacer mods:"); - logger::info(" Parsing: {}ms", std::chrono::duration_cast(endOfParsingTime - startTime).count()); + logger::info(" Waiting cache: {}ms", parseResults.waitCacheDuration.count()); + logger::info(" Parsing directory: {}ms", std::chrono::duration_cast(endOfParsingTime - startTime - parseResults.waitCacheDuration).count()); logger::info(" Adding mods: {}ms", std::chrono::duration_cast(endOfModsTime - endOfParsingTime).count()); logger::info(" Adding legacy mods: {}ms", std::chrono::duration_cast(endOfLegacyModsTime - endOfModsTime).count()); logger::info(" Checking for problems: {}ms", std::chrono::duration_cast(endTime - endOfLegacyModsTime).count()); @@ -609,8 +610,9 @@ void OpenAnimationReplacer::CreateReplacementAnimations([[maybe_unused]] const c } for (const auto& subMod : search->second) { - subMod->AddReplacementAnimation(originalAnimationPathString, static_cast(i), projectData, a_stringData); - subModsToUpdate.emplace(subMod); + if (subMod->AddReplacementAnimation(originalAnimationPathString, static_cast(i), projectData, a_stringData)) { + subModsToUpdate.emplace(subMod); + } } } } @@ -618,9 +620,8 @@ void OpenAnimationReplacer::CreateReplacementAnimations([[maybe_unused]] const c auto endOfParsingTime = std::chrono::high_resolution_clock::now(); for (auto& subMod : subModsToUpdate) { - subMod->LoadReplacementAnimationDatas(); - subMod->HandleDeprecatedSettings(); - subMod->UpdateAnimations(); + subMod->AddReplacerProject(projectData); + subMod->SortReplacementAnimationsByPath(); } auto endOfUpdatingTime = std::chrono::high_resolution_clock::now(); @@ -640,8 +641,8 @@ void OpenAnimationReplacer::CreateReplacementAnimations([[maybe_unused]] const c auto endTime = std::chrono::high_resolution_clock::now(); logger::info("Time spent creating replacement animations for {}:", a_path); logger::info(" Parsing: {}ms", std::chrono::duration_cast(endOfParsingTime - startTime).count()); - logger::info(" Updating animations in submods: {}ms", std::chrono::duration_cast(endOfUpdatingTime - endOfParsingTime).count()); - logger::info(" Initializing replacment animations: {}ms", std::chrono::duration_cast(endTime - endOfUpdatingTime).count()); + logger::info(" Updating submods: {}ms", std::chrono::duration_cast(endOfUpdatingTime - endOfParsingTime).count()); + logger::info(" Final initialization: {}ms", std::chrono::duration_cast(endTime - endOfUpdatingTime).count()); logger::info(" Total: {}ms", std::chrono::duration_cast(endTime - startTime).count()); } diff --git a/src/Parsing.cpp b/src/Parsing.cpp index 71e6b6c..2abb343 100644 --- a/src/Parsing.cpp +++ b/src/Parsing.cpp @@ -840,8 +840,9 @@ namespace Parsing } if (!cachedMod.subModDirectories.empty()) { - a_outParseResults.modParseResultFutures.emplace_back(a_workerPool.Enqueue([cachedMod]() { - return ParseModDirectory(cachedMod); + const CachedModDirectory* cachedModPtr = &cachedMod; + a_outParseResults.modParseResultFutures.emplace_back(a_workerPool.Enqueue([cachedModPtr]() { + return ParseModDirectory(*cachedModPtr); })); continue; } @@ -867,8 +868,9 @@ namespace Parsing if (detailedEntry.isCustomConditions) { for (const auto& cachedSubMod : detailedEntry.subMods) { - a_outParseResults.legacyParseResultFutures.emplace_back(a_workerPool.Enqueue([cachedSubMod]() { - return ParseLegacyCustomConditionsDirectory(cachedSubMod); + const CachedLegacySubMod* cachedSubModPtr = &cachedSubMod; + a_outParseResults.legacyParseResultFutures.emplace_back(a_workerPool.Enqueue([cachedSubModPtr]() { + return ParseLegacyCustomConditionsDirectory(*cachedSubModPtr); })); } } else { @@ -918,15 +920,17 @@ namespace Parsing if (!IsDirectoryCacheReady()) { logger::info("Waiting for directory cache to complete..."); + auto startWaitingTime = std::chrono::high_resolution_clock::now(); WaitForDirectoryCache(); + auto endWaitingTime = std::chrono::high_resolution_clock::now(); + a_outParseResults.waitCacheDuration = std::chrono::duration_cast(endWaitingTime - startWaitingTime); } const auto workerCount = std::clamp(Settings::uParsingWorkerCount, 1u, 32u); logger::info("Using {} parsing worker(s).", workerCount); - WorkerPool workerPool(workerCount); - { std::shared_lock lock(g_directoryCache.cacheLock); + WorkerPool workerPool(workerCount); for (const auto& cachedOAR : g_directoryCache.oarDirectories) { ParseCachedOARDirectory(cachedOAR, a_outParseResults, workerPool); } @@ -1668,38 +1672,19 @@ namespace Parsing return true; } - static void PrecacheAnimationHashes(const std::filesystem::path& a_directory) + static void PrecacheAnimationHash(const std::filesystem::path& a_path) { if (!Settings::bFilterOutDuplicateAnimations) { return; } - try { - for (const auto& entry : std::filesystem::recursive_directory_iterator(a_directory)) { - if (!entry.is_regular_file()) { - continue; - } - - const auto& path = entry.path(); - if (!path.has_extension() || !Utils::CompareStringsIgnoreCase(path.extension().string(), ".hkx"sv) || !IsPathValid(path)) { - continue; - } - - auto filename = TryConvertPathToString(path); - if (!filename) { - continue; - } - - if (IsHiddenDirectoryName(*filename)) { - continue; - } - - AnimationFileHashCache::CalculateHash(*filename); - g_precachedHashCount.fetch_add(1, std::memory_order_relaxed); - } - } catch (const std::filesystem::filesystem_error& e) { - logger::warn("Error pre-caching animation hashes in {}: {}", a_directory.string(), e.what()); + auto filename = TryConvertPathToString(a_path); + if (!filename) { + return; } + + AnimationFileHashCache::CalculateHash(*filename); + g_precachedHashCount.fetch_add(1, std::memory_order_relaxed); } static bool IsDirectoryCacheReady() @@ -1736,6 +1721,7 @@ namespace Parsing { if (!a_bIsLegacy) { std::vector variantDirectoryNames; + std::vector regularAnimationFiles; for (const auto& entry : std::filesystem::directory_iterator(a_directory)) { if (!IsPathValid(entry.path())) { @@ -1743,6 +1729,11 @@ namespace Parsing } if (!Utils::IsDirectory(entry)) { + if (Utils::IsRegularFile(entry) && Utils::CompareStringsIgnoreCase(entry.path().extension().string(), ".hkx"sv)) { + CachedAnimationFile cachedAnim; + cachedAnim.path = entry.path(); + regularAnimationFiles.push_back(std::move(cachedAnim)); + } continue; } @@ -1759,6 +1750,7 @@ namespace Parsing Utils::IsRegularFile(variantEntry) && Utils::CompareStringsIgnoreCase(variantEntry.path().extension().string(), ".hkx"sv)) { cachedAnim.variantPaths.push_back(variantEntry.path()); + PrecacheAnimationHash(variantEntry.path()); } } @@ -1775,19 +1767,14 @@ namespace Parsing } } - for (const auto& entry : std::filesystem::directory_iterator(a_directory)) { - if (!IsPathValid(entry.path()) || !Utils::IsRegularFile(entry) || !Utils::CompareStringsIgnoreCase(entry.path().extension().string(), ".hkx"sv)) { - continue; - } - - std::string filename = entry.path().filename().string(); + for (auto& cachedAnim : regularAnimationFiles) { + std::string filename = cachedAnim.path.filename().string(); const bool bSkip = std::ranges::any_of(variantDirectoryNames, [&](const auto& name) { return filename == name; }); if (!bSkip) { - CachedAnimationFile cachedAnim; - cachedAnim.path = entry.path(); + PrecacheAnimationHash(cachedAnim.path); a_outCached.animationFiles.push_back(std::move(cachedAnim)); } } @@ -1798,6 +1785,7 @@ namespace Parsing Utils::CompareStringsIgnoreCase(entry.path().extension().string(), ".hkx"sv)) { CachedAnimationFile cachedAnim; cachedAnim.path = entry.path(); + PrecacheAnimationHash(cachedAnim.path); a_outCached.animationFiles.push_back(std::move(cachedAnim)); } } @@ -1827,14 +1815,10 @@ namespace Parsing continue; } - cachedMod.entries.push_back({ subEntry.path(), Utils::IsDirectory(subEntry) }); - if (Utils::IsDirectory(subEntry)) { CachedSubModDirectory cachedSubMod; CacheSubModDirectoryContents(subEntry.path(), cachedSubMod, false); cachedMod.subModDirectories.push_back(std::move(cachedSubMod)); - - PrecacheAnimationHashes(subEntry.path()); } } @@ -1852,6 +1836,7 @@ namespace Parsing Utils::CompareStringsIgnoreCase(entry.path().extension().string(), ".hkx"sv)) { CachedAnimationFile cachedAnim; cachedAnim.path = entry.path(); + PrecacheAnimationHash(cachedAnim.path); a_outCached.animationFiles.push_back(std::move(cachedAnim)); } } @@ -1879,8 +1864,6 @@ namespace Parsing CachedLegacySubMod cachedSubMod; CacheLegacyAnimationFiles(customConditionEntry.path(), cachedSubMod); detailedEntry.subMods.push_back(std::move(cachedSubMod)); - - PrecacheAnimationHashes(customConditionEntry.path()); } } } else { @@ -1889,8 +1872,6 @@ namespace Parsing CachedLegacySubMod cachedSubMod; CacheLegacyAnimationFiles(formIdEntry.path(), cachedSubMod); detailedEntry.subMods.push_back(std::move(cachedSubMod)); - - PrecacheAnimationHashes(formIdEntry.path()); } } } @@ -1927,7 +1908,7 @@ namespace Parsing try { for (std::filesystem::recursive_directory_iterator i(meshesDir), end; i != end; ++i) { - auto entry = *i; + const auto& entry = *i; if (!Utils::IsDirectory(entry)) { continue; } diff --git a/src/Parsing.h b/src/Parsing.h index efa40dc..8120f02 100644 --- a/src/Parsing.h +++ b/src/Parsing.h @@ -208,6 +208,7 @@ namespace Parsing { std::vector> modParseResultFutures; std::vector> legacyParseResultFutures; + std::chrono::milliseconds waitCacheDuration{ 0 }; }; [[nodiscard]] std::unique_ptr ParseConditionsTxt(const std::filesystem::path& a_txtPath); @@ -245,7 +246,6 @@ namespace Parsing struct CachedModDirectory { std::filesystem::path path; - std::vector entries; std::vector subModDirectories; }; diff --git a/src/ReplacerMods.cpp b/src/ReplacerMods.cpp index 74b8c89..5e95fb1 100644 --- a/src/ReplacerMods.cpp +++ b/src/ReplacerMods.cpp @@ -92,30 +92,52 @@ bool SubMod::AddReplacementAnimation(std::string_view a_animPath, uint16_t a_ori { WriteLocker locker(_dataLock); _replacementAnimations.emplace_back(newReplacementAnimation.get()); - - // sort replacement animations by path - std::ranges::sort(_replacementAnimations, [](const auto& a_lhs, const auto& a_rhs) { - return a_lhs->_path < a_rhs->_path; - }); } // load anim data - const auto animDataSearch = std::ranges::find_if(_replacementAnimDatas, [&](const ReplacementAnimData& a_replacementAnimData) { - return a_replacementAnimData.projectName == projectName && a_replacementAnimData.path == animFile.fullPath; - }); - - if (animDataSearch != _replacementAnimDatas.end()) { - newReplacementAnimation->LoadAnimData(*animDataSearch); + for (const auto& replacementAnimData : _replacementAnimDatas) { + if (replacementAnimData.projectName == projectName && Utils::ComparePaths(replacementAnimData.path, animFile.fullPath)) { + newReplacementAnimation->LoadAnimData(replacementAnimData); + } } + InitializeReplacementAnimation(newReplacementAnimation.get()); a_replacerProjectData->AddReplacementAnimation(a_stringData, a_originalIndex, newReplacementAnimation); - AddReplacerProject(a_replacerProjectData); } } return bAdded; } +void SubMod::InitializeReplacementAnimation(ReplacementAnimation* a_replacementAnimation) const +{ + if (_bKeepRandomResultsOnLoop_DEPRECATED || _bShareRandomResults_DEPRECATED) { + if (a_replacementAnimation->HasVariants()) { + auto& variants = a_replacementAnimation->GetVariants(); + if (_bKeepRandomResultsOnLoop_DEPRECATED) { + variants.SetShouldResetRandomOnLoopOrEcho(false); + } + if (_bShareRandomResults_DEPRECATED) { + variants.SetVariantStateScope(Conditions::StateDataScope::kSubMod); + } + } + } + + a_replacementAnimation->UpdateVariantCache(); + if (_synchronizedConditionSet) { + a_replacementAnimation->SetSynchronizedConditionSet(_synchronizedConditionSet.get()); + } +} + +void SubMod::SortReplacementAnimationsByPath() +{ + WriteLocker locker(_dataLock); + + std::ranges::sort(_replacementAnimations, [](const auto& a_lhs, const auto& a_rhs) { + return a_lhs->_path < a_rhs->_path; + }); +} + void SubMod::SetAnimationFiles(const std::vector& a_animationFiles) { Parsing::ScopedTimer timer(Parsing::TimingBucket::kSetAnimationFiles); @@ -1461,6 +1483,25 @@ void AnimationReplacements::MarkAsSynchronizedAnimation(bool a_bSynchronized) } } +ReplacerProjectData::ReplacerProjectData(RE::hkbCharacterStringData* a_stringData, RE::BShkbHkxDB::ProjectDBData* a_projectDBData) : + stringData(a_stringData), + projectDBData(a_projectDBData) +{ + if (stringData) { + const auto animationCount = static_cast(stringData->animationNames.size()); + _animationPathToIndexMap.reserve(animationCount); + uint16_t pathIndex = 0; + for (const auto& animationName : stringData->animationNames) { + if (pathIndex == std::numeric_limits::max()) { + break; + } + + _animationPathToIndexMap.try_emplace(animationName.data(), pathIndex); + ++pathIndex; + } + } +} + ReplacementAnimation* ReplacerProjectData::EvaluateConditionsAndGetReplacementAnimation(RE::hkbClipGenerator* a_clipGenerator, uint16_t a_originalIndex, RE::TESObjectREFR* a_refr) const { if (const auto replacementAnimations = GetAnimationReplacements(a_originalIndex)) { @@ -1502,10 +1543,8 @@ uint16_t ReplacerProjectData::TryAddAnimationToAnimationBundleNames(std::string_ } // Check if the animation is already in the list and return the index if it is - for (uint16_t i = 0; i < stringData->animationNames.size(); i++) { - if (stringData->animationNames[i].data() == a_path) { - return i; - } + if (const auto search = _animationPathToIndexMap.find(a_path); search != _animationPathToIndexMap.end()) { + return search->second; } // Check if the animation can be added to the list @@ -1522,6 +1561,7 @@ uint16_t ReplacerProjectData::TryAddAnimationToAnimationBundleNames(std::string_ if (Settings::bFilterOutDuplicateAnimations && hash) { _fileHashToIndexMap[*hash].emplace_back(a_path, newIndex); } + _animationPathToIndexMap.try_emplace(std::string(a_path), newIndex); return newIndex; } diff --git a/src/ReplacerMods.h b/src/ReplacerMods.h index 9d2779e..5265251 100644 --- a/src/ReplacerMods.h +++ b/src/ReplacerMods.h @@ -32,6 +32,8 @@ class SubMod : public IStateDataContainerHolder void StateDataClearData() override; bool AddReplacementAnimation(std::string_view a_animPath, uint16_t a_originalIndex, class ReplacerProjectData* a_replacerProjectData, RE::hkbCharacterStringData* a_stringData); + void InitializeReplacementAnimation(ReplacementAnimation* a_replacementAnimation) const; + void SortReplacementAnimationsByPath(); void SetAnimationFiles(const std::vector& a_animationFiles); void LoadParseResult(const Parsing::SubModParseResult& a_parseResult); @@ -311,9 +313,7 @@ class AnimationReplacements class ReplacerProjectData { public: - ReplacerProjectData(RE::hkbCharacterStringData* a_stringData, RE::BShkbHkxDB::ProjectDBData* a_projectDBData) : - stringData(a_stringData), - projectDBData(a_projectDBData) {} + ReplacerProjectData(RE::hkbCharacterStringData* a_stringData, RE::BShkbHkxDB::ProjectDBData* a_projectDBData); ReplacementAnimation* EvaluateConditionsAndGetReplacementAnimation(RE::hkbClipGenerator* a_clipGenerator, uint16_t a_originalIndex, RE::TESObjectREFR* a_refr) const; [[nodiscard]] uint16_t GetOriginalAnimationIndex(uint16_t a_currentIndex) const; @@ -340,6 +340,26 @@ class ReplacerProjectData uint16_t synchronizedClipIDOffset = 0; protected: + struct AnimationPathHash + { + using is_transparent = void; + + size_t operator()(std::string_view a_path) const + { + return std::hash{}(a_path); + } + }; + + struct AnimationPathEqual + { + using is_transparent = void; + + bool operator()(std::string_view a_lhs, std::string_view a_rhs) const + { + return a_lhs == a_rhs; + } + }; + struct DuplicateHashCandidate { DuplicateHashCandidate(std::string_view a_path, uint16_t a_index) : @@ -352,5 +372,6 @@ class ReplacerProjectData }; std::unordered_map> _fileHashToIndexMap; + std::unordered_map _animationPathToIndexMap; uint32_t _filteredDuplicates = 0; };