Skip to content

perf: optimize directory cache and animation initialization #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
VanCZ1 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

perf: optimize directory cache and animation initialization #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions src/OpenAnimationReplacer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -560,7 +560,8 @@ void OpenAnimationReplacer::CreateReplacerMods()
auto endTime = std::chrono::high_resolution_clock::now();

logger::info("Time spent creating replacer mods:");
logger::info(" Parsing: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(endOfParsingTime - startTime).count());
logger::info(" Waiting cache: {}ms", parseResults.waitCacheDuration.count());
logger::info(" Parsing directory: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(endOfParsingTime - startTime - parseResults.waitCacheDuration).count());
logger::info(" Adding mods: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(endOfModsTime - endOfParsingTime).count());
logger::info(" Adding legacy mods: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(endOfLegacyModsTime - endOfModsTime).count());
logger::info(" Checking for problems: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(endTime - endOfLegacyModsTime).count());
Expand Down Expand Up @@ -609,18 +610,18 @@ void OpenAnimationReplacer::CreateReplacementAnimations([[maybe_unused]] const c
}

for (const auto& subMod : search->second) {
subMod->AddReplacementAnimation(originalAnimationPathString, static_cast<uint16_t>(i), projectData, a_stringData);
subModsToUpdate.emplace(subMod);
if (subMod->AddReplacementAnimation(originalAnimationPathString, static_cast<uint16_t>(i), projectData, a_stringData)) {
subModsToUpdate.emplace(subMod);
}
}
}
}

auto endOfParsingTime = std::chrono::high_resolution_clock::now();

for (auto& subMod : subModsToUpdate) {
subMod->LoadReplacementAnimationDatas();
subMod->HandleDeprecatedSettings();
subMod->UpdateAnimations();
subMod->AddReplacerProject(projectData);
subMod->SortReplacementAnimationsByPath();
}

auto endOfUpdatingTime = std::chrono::high_resolution_clock::now();
Expand All @@ -640,8 +641,8 @@ void OpenAnimationReplacer::CreateReplacementAnimations([[maybe_unused]] const c
auto endTime = std::chrono::high_resolution_clock::now();
logger::info("Time spent creating replacement animations for {}:", a_path);
logger::info(" Parsing: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(endOfParsingTime - startTime).count());
logger::info(" Updating animations in submods: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(endOfUpdatingTime - endOfParsingTime).count());
logger::info(" Initializing replacment animations: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(endTime - endOfUpdatingTime).count());
logger::info(" Updating submods: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(endOfUpdatingTime - endOfParsingTime).count());
logger::info(" Final initialization: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(endTime - endOfUpdatingTime).count());
logger::info(" Total: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(endTime - startTime).count());
}

Expand Down
79 changes: 30 additions & 49 deletions src/Parsing.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -840,8 +840,9 @@ namespace Parsing
}

if (!cachedMod.subModDirectories.empty()) {
a_outParseResults.modParseResultFutures.emplace_back(a_workerPool.Enqueue([cachedMod]() {
return ParseModDirectory(cachedMod);
const CachedModDirectory* cachedModPtr = &cachedMod;
a_outParseResults.modParseResultFutures.emplace_back(a_workerPool.Enqueue([cachedModPtr]() {
return ParseModDirectory(*cachedModPtr);
}));
continue;
}
Expand All @@ -867,8 +868,9 @@ namespace Parsing

if (detailedEntry.isCustomConditions) {
for (const auto& cachedSubMod : detailedEntry.subMods) {
a_outParseResults.legacyParseResultFutures.emplace_back(a_workerPool.Enqueue([cachedSubMod]() {
return ParseLegacyCustomConditionsDirectory(cachedSubMod);
const CachedLegacySubMod* cachedSubModPtr = &cachedSubMod;
a_outParseResults.legacyParseResultFutures.emplace_back(a_workerPool.Enqueue([cachedSubModPtr]() {
return ParseLegacyCustomConditionsDirectory(*cachedSubModPtr);
}));
}
} else {
Expand Down Expand Up @@ -918,15 +920,17 @@ namespace Parsing

if (!IsDirectoryCacheReady()) {
logger::info("Waiting for directory cache to complete...");
auto startWaitingTime = std::chrono::high_resolution_clock::now();
WaitForDirectoryCache();
auto endWaitingTime = std::chrono::high_resolution_clock::now();
a_outParseResults.waitCacheDuration = std::chrono::duration_cast<std::chrono::milliseconds>(endWaitingTime - startWaitingTime);
}

const auto workerCount = std::clamp(Settings::uParsingWorkerCount, 1u, 32u);
logger::info("Using {} parsing worker(s).", workerCount);
WorkerPool workerPool(workerCount);

{
std::shared_lock lock(g_directoryCache.cacheLock);
WorkerPool workerPool(workerCount);
for (const auto& cachedOAR : g_directoryCache.oarDirectories) {
ParseCachedOARDirectory(cachedOAR, a_outParseResults, workerPool);
}
Expand Down Expand Up @@ -1668,38 +1672,19 @@ namespace Parsing
return true;
}

static void PrecacheAnimationHashes(const std::filesystem::path& a_directory)
static void PrecacheAnimationHash(const std::filesystem::path& a_path)
{
if (!Settings::bFilterOutDuplicateAnimations) {
return;
}

try {
for (const auto& entry : std::filesystem::recursive_directory_iterator(a_directory)) {
if (!entry.is_regular_file()) {
continue;
}

const auto& path = entry.path();
if (!path.has_extension() || !Utils::CompareStringsIgnoreCase(path.extension().string(), ".hkx"sv) || !IsPathValid(path)) {
continue;
}

auto filename = TryConvertPathToString(path);
if (!filename) {
continue;
}

if (IsHiddenDirectoryName(*filename)) {
continue;
}

AnimationFileHashCache::CalculateHash(*filename);
g_precachedHashCount.fetch_add(1, std::memory_order_relaxed);
}
} catch (const std::filesystem::filesystem_error& e) {
logger::warn("Error pre-caching animation hashes in {}: {}", a_directory.string(), e.what());
auto filename = TryConvertPathToString(a_path);
if (!filename) {
return;
}

AnimationFileHashCache::CalculateHash(*filename);
g_precachedHashCount.fetch_add(1, std::memory_order_relaxed);
}

static bool IsDirectoryCacheReady()
Expand Down Expand Up @@ -1736,13 +1721,19 @@ namespace Parsing
{
if (!a_bIsLegacy) {
std::vector<std::string> variantDirectoryNames;
std::vector<CachedAnimationFile> regularAnimationFiles;

for (const auto& entry : std::filesystem::directory_iterator(a_directory)) {
if (!IsPathValid(entry.path())) {
continue;
}

if (!Utils::IsDirectory(entry)) {
if (Utils::IsRegularFile(entry) && Utils::CompareStringsIgnoreCase(entry.path().extension().string(), ".hkx"sv)) {
CachedAnimationFile cachedAnim;
cachedAnim.path = entry.path();
regularAnimationFiles.push_back(std::move(cachedAnim));
}
continue;
}

Expand All @@ -1759,6 +1750,7 @@ namespace Parsing
Utils::IsRegularFile(variantEntry) &&
Utils::CompareStringsIgnoreCase(variantEntry.path().extension().string(), ".hkx"sv)) {
cachedAnim.variantPaths.push_back(variantEntry.path());
PrecacheAnimationHash(variantEntry.path());
}
}

Expand All @@ -1775,19 +1767,14 @@ namespace Parsing
}
}

for (const auto& entry : std::filesystem::directory_iterator(a_directory)) {
if (!IsPathValid(entry.path()) || !Utils::IsRegularFile(entry) || !Utils::CompareStringsIgnoreCase(entry.path().extension().string(), ".hkx"sv)) {
continue;
}

std::string filename = entry.path().filename().string();
for (auto& cachedAnim : regularAnimationFiles) {
std::string filename = cachedAnim.path.filename().string();
const bool bSkip = std::ranges::any_of(variantDirectoryNames, [&](const auto& name) {
return filename == name;
});

if (!bSkip) {
CachedAnimationFile cachedAnim;
cachedAnim.path = entry.path();
PrecacheAnimationHash(cachedAnim.path);
a_outCached.animationFiles.push_back(std::move(cachedAnim));
}
}
Expand All @@ -1798,6 +1785,7 @@ namespace Parsing
Utils::CompareStringsIgnoreCase(entry.path().extension().string(), ".hkx"sv)) {
CachedAnimationFile cachedAnim;
cachedAnim.path = entry.path();
PrecacheAnimationHash(cachedAnim.path);
a_outCached.animationFiles.push_back(std::move(cachedAnim));
}
}
Expand Down Expand Up @@ -1827,14 +1815,10 @@ namespace Parsing
continue;
}

cachedMod.entries.push_back({ subEntry.path(), Utils::IsDirectory(subEntry) });

if (Utils::IsDirectory(subEntry)) {
CachedSubModDirectory cachedSubMod;
CacheSubModDirectoryContents(subEntry.path(), cachedSubMod, false);
cachedMod.subModDirectories.push_back(std::move(cachedSubMod));

PrecacheAnimationHashes(subEntry.path());
}
}

Expand All @@ -1852,6 +1836,7 @@ namespace Parsing
Utils::CompareStringsIgnoreCase(entry.path().extension().string(), ".hkx"sv)) {
CachedAnimationFile cachedAnim;
cachedAnim.path = entry.path();
PrecacheAnimationHash(cachedAnim.path);
a_outCached.animationFiles.push_back(std::move(cachedAnim));
}
}
Expand Down Expand Up @@ -1879,8 +1864,6 @@ namespace Parsing
CachedLegacySubMod cachedSubMod;
CacheLegacyAnimationFiles(customConditionEntry.path(), cachedSubMod);
detailedEntry.subMods.push_back(std::move(cachedSubMod));

PrecacheAnimationHashes(customConditionEntry.path());
}
}
} else {
Expand All @@ -1889,8 +1872,6 @@ namespace Parsing
CachedLegacySubMod cachedSubMod;
CacheLegacyAnimationFiles(formIdEntry.path(), cachedSubMod);
detailedEntry.subMods.push_back(std::move(cachedSubMod));

PrecacheAnimationHashes(formIdEntry.path());
}
}
}
Expand Down Expand Up @@ -1927,7 +1908,7 @@ namespace Parsing

try {
for (std::filesystem::recursive_directory_iterator i(meshesDir), end; i != end; ++i) {
auto entry = *i;
const auto& entry = *i;
if (!Utils::IsDirectory(entry)) {
continue;
}
Expand Down
2 changes: 1 addition & 1 deletion src/Parsing.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ namespace Parsing
{
std::vector<std::future<ModParseResult>> modParseResultFutures;
std::vector<std::future<SubModParseResult>> legacyParseResultFutures;
std::chrono::milliseconds waitCacheDuration{ 0 };
};

[[nodiscard]] std::unique_ptr<Conditions::ConditionSet> ParseConditionsTxt(const std::filesystem::path& a_txtPath);
Expand Down Expand Up @@ -245,7 +246,6 @@ namespace Parsing
struct CachedModDirectory
{
std::filesystem::path path;
std::vector<CachedDirectoryEntry> entries;
std::vector<CachedSubModDirectory> subModDirectories;
};

Expand Down
72 changes: 56 additions & 16 deletions src/ReplacerMods.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,30 +92,52 @@ bool SubMod::AddReplacementAnimation(std::string_view a_animPath, uint16_t a_ori
{
WriteLocker locker(_dataLock);
_replacementAnimations.emplace_back(newReplacementAnimation.get());

// sort replacement animations by path
std::ranges::sort(_replacementAnimations, [](const auto& a_lhs, const auto& a_rhs) {
return a_lhs->_path < a_rhs->_path;
});
}

// load anim data
const auto animDataSearch = std::ranges::find_if(_replacementAnimDatas, [&](const ReplacementAnimData& a_replacementAnimData) {
return a_replacementAnimData.projectName == projectName && a_replacementAnimData.path == animFile.fullPath;
});

if (animDataSearch != _replacementAnimDatas.end()) {
newReplacementAnimation->LoadAnimData(*animDataSearch);
for (const auto& replacementAnimData : _replacementAnimDatas) {
if (replacementAnimData.projectName == projectName && Utils::ComparePaths(replacementAnimData.path, animFile.fullPath)) {
newReplacementAnimation->LoadAnimData(replacementAnimData);
}
}

InitializeReplacementAnimation(newReplacementAnimation.get());
a_replacerProjectData->AddReplacementAnimation(a_stringData, a_originalIndex, newReplacementAnimation);
AddReplacerProject(a_replacerProjectData);
}
}

return bAdded;
}

void SubMod::InitializeReplacementAnimation(ReplacementAnimation* a_replacementAnimation) const
{
if (_bKeepRandomResultsOnLoop_DEPRECATED || _bShareRandomResults_DEPRECATED) {
if (a_replacementAnimation->HasVariants()) {
auto& variants = a_replacementAnimation->GetVariants();
if (_bKeepRandomResultsOnLoop_DEPRECATED) {
variants.SetShouldResetRandomOnLoopOrEcho(false);
}
if (_bShareRandomResults_DEPRECATED) {
variants.SetVariantStateScope(Conditions::StateDataScope::kSubMod);
}
}
}

a_replacementAnimation->UpdateVariantCache();
if (_synchronizedConditionSet) {
a_replacementAnimation->SetSynchronizedConditionSet(_synchronizedConditionSet.get());
}
}

void SubMod::SortReplacementAnimationsByPath()
{
WriteLocker locker(_dataLock);

std::ranges::sort(_replacementAnimations, [](const auto& a_lhs, const auto& a_rhs) {
return a_lhs->_path < a_rhs->_path;
});
}

void SubMod::SetAnimationFiles(const std::vector<ReplacementAnimationFile>& a_animationFiles)
{
Parsing::ScopedTimer timer(Parsing::TimingBucket::kSetAnimationFiles);
Expand Down Expand Up @@ -1461,6 +1483,25 @@ void AnimationReplacements::MarkAsSynchronizedAnimation(bool a_bSynchronized)
}
}

ReplacerProjectData::ReplacerProjectData(RE::hkbCharacterStringData* a_stringData, RE::BShkbHkxDB::ProjectDBData* a_projectDBData) :
stringData(a_stringData),
projectDBData(a_projectDBData)
{
if (stringData) {
const auto animationCount = static_cast<size_t>(stringData->animationNames.size());
_animationPathToIndexMap.reserve(animationCount);
uint16_t pathIndex = 0;
for (const auto& animationName : stringData->animationNames) {
if (pathIndex == std::numeric_limits<uint16_t>::max()) {
break;
}

_animationPathToIndexMap.try_emplace(animationName.data(), pathIndex);
++pathIndex;
}
}
}

ReplacementAnimation* ReplacerProjectData::EvaluateConditionsAndGetReplacementAnimation(RE::hkbClipGenerator* a_clipGenerator, uint16_t a_originalIndex, RE::TESObjectREFR* a_refr) const
{
if (const auto replacementAnimations = GetAnimationReplacements(a_originalIndex)) {
Expand Down Expand Up @@ -1502,10 +1543,8 @@ uint16_t ReplacerProjectData::TryAddAnimationToAnimationBundleNames(std::string_
}

// Check if the animation is already in the list and return the index if it is
for (uint16_t i = 0; i < stringData->animationNames.size(); i++) {
if (stringData->animationNames[i].data() == a_path) {
return i;
}
if (const auto search = _animationPathToIndexMap.find(a_path); search != _animationPathToIndexMap.end()) {
return search->second;
}

// Check if the animation can be added to the list
Expand All @@ -1522,6 +1561,7 @@ uint16_t ReplacerProjectData::TryAddAnimationToAnimationBundleNames(std::string_
if (Settings::bFilterOutDuplicateAnimations && hash) {
_fileHashToIndexMap[*hash].emplace_back(a_path, newIndex);
}
_animationPathToIndexMap.try_emplace(std::string(a_path), newIndex);

return newIndex;
}
Expand Down
Loading
Loading