Load all mod descriptors and check dependencies

Author: BrickPi

src/headless/main.cpp

@@ -133,19 +133,16 @@ static bool run_headless(fs::path const& root, memory::vector<memory::string>& m
 
 	Logger::info("Commit hash: ", GameManager::get_commit_hash());
 
+	Logger::info("===== Setting base path... =====");
+	ret &= game_manager.set_base_path(roots);
+
 	Logger::info("===== Loading mod descriptors... =====");
-	ret &= game_manager.load_mod_descriptors(mods);
-	for (auto const& mod : game_manager.get_mod_manager().get_mods()) {
-		roots.emplace_back(root / mod.get_dataloader_root_path());
-		for (std::string_view path : mod.get_replace_paths()) {
-			if (std::find(replace_paths.begin(), replace_paths.end(), path) == replace_paths.end()) {
-				replace_paths.emplace_back(path);
-			} 
-		}
-	}
+	ret &= game_manager.load_mod_descriptors();
+
+	Logger::info("===== Loading mods... =====");
+	ret &= game_manager.load_mods(roots, replace_paths, mods);
 
 	Logger::info("===== Loading definitions... =====");
-	ret &= game_manager.set_roots(roots, replace_paths);
 	ret &= game_manager.load_definitions(
 		[](std::string_view key, Dataloader::locale_t locale, std::string_view localisation) -> bool {
 			return true;

src/openvic-simulation/GameManager.cpp

@@ -1,5 +1,12 @@
 #include "GameManager.hpp"
 
+#include <cstddef>
+#include <string_view>
+#include <vector>
+
+#include "openvic-simulation/dataloader/Dataloader.hpp"
+#include "openvic-simulation/utility/Logger.hpp"
+
 using namespace OpenVic;
 
 GameManager::GameManager(
@@ -8,25 +15,120 @@ GameManager::GameManager(
 		new_gamestate_updated_callback ? std::move(new_gamestate_updated_callback) : []() {}
 	}, definitions_loaded { false }, mod_descriptors_loaded { false } {}
 
-bool GameManager::load_mod_descriptors(std::span<const memory::string> descriptors) {
+bool GameManager::load_mod_descriptors() {
 	if (mod_descriptors_loaded) {
 		Logger::error("Cannot load mod descriptors - already loaded!");
 		return false;
 	}
 
-	if (!dataloader.load_mod_descriptors(descriptors, mod_manager)) {
+	if (!dataloader.load_mod_descriptors(mod_manager)) {
 		Logger::error("Failed to load mod descriptors!");
 		return false;
 	}
 	return true;
 }
 
-bool GameManager::set_roots(Dataloader::path_span_t roots, Dataloader::path_span_t replace_paths) {
-	if (!dataloader.set_roots(roots, replace_paths)) {
+bool GameManager::_get_mod_dependencies(Mod const* mod, std::vector<Mod const*>& dep_list) {
+	static constexpr size_t MAX_RECURSE = 16;
+	size_t current_recurse = 0;
+
+	static auto dep_cycle = [this, &current_recurse](auto self, Mod const* mod, std::vector<Mod const*>& dep_list) -> bool {
+		bool ret = true;
+		for (std::string_view dep_identifier : mod->get_dependencies()) {
+			if (!mod_manager.has_mod_identifier(dep_identifier)) {
+				Logger::error("Mod \"", mod->get_identifier(), "\" has unmet dependency \"", dep_identifier, "\" and cannot be loaded!");
+				return false;
+			}
+			Mod const* dep = mod_manager.get_mod_by_identifier(dep_identifier);
+			/* The poor man's cycle checking (cycles should be very rare and hard to accomplish with vic2 modding, this is a failsafe) */
+			if (current_recurse == MAX_RECURSE) {
+				Logger::error("Mod \"", mod->get_identifier(), "\" has cyclical or broken dependency chain and cannot be loaded!");
+				return false;
+			} else {
+				current_recurse++;
+				ret &= self(self, dep, dep_list); /* recursively search for mod dependencies */
+			}
+			if (std::find(dep_list.begin(), dep_list.end(), dep) == dep_list.end()) {
+				dep_list.emplace_back(dep);
+			}
+		}
+		return ret;
+	};
+	return dep_cycle(dep_cycle, mod, dep_list);
+}
+
+bool GameManager::load_mods(Dataloader::path_vector_t& roots, Dataloader::path_vector_t& replace_paths, std::span<const memory::string> requested_mods) {
+	if (requested_mods.empty()) {
+		return true;
+	}
+
+	bool ret = true;
+
+	std::vector<Mod const*> load_list;
+
+	/* Check loaded mod descriptors for requested mods, using either full name or user directory name
+	 * (Historical Project Mod 0.4.6 or HPM both valid, for example), and load them plus their dependencies.
+	 */
+	for (std::string_view requested_mod : requested_mods) {
+		auto it = std::find_if(
+			mod_manager.get_mods().begin(),
+			mod_manager.get_mods().end(),
+			[&requested_mod](Mod const& mod) -> bool {
+				return mod.get_identifier() == requested_mod || mod.get_user_dir() == requested_mod;
+			}
+		);
+
+		if (it == mod_manager.get_mods().end()) {
+			Logger::error("Requested mod \"", requested_mod, "\" does not exist!");
+			ret = false;
+			continue;
+		}
+
+		Mod const* mod_ptr = &*it;
+		std::vector<Mod const*> dependencies;
+		if(!_get_mod_dependencies(mod_ptr, dependencies)) {
+			ret = false;
+			continue;
+		}
+
+		/* Add mod plus dependencies to load_list in proper order. */
+		load_list.reserve(1 + dependencies.size());
+		for (Mod const* dep : dependencies) {
+			if (ret && std::find(load_list.begin(), load_list.end(), dep) == load_list.end()) {
+				load_list.emplace_back(dep);
+			}
+		}
+		if (ret && std::find(load_list.begin(), load_list.end(), mod_ptr) == load_list.end()) {
+			load_list.emplace_back(mod_ptr);
+		}
+	}
+
+	/* Actually registers all roots and replace paths to be loaded by the game. */
+	for (Mod const* mod : load_list) {
+		roots.emplace_back(roots[0] / mod->get_dataloader_root_path());
+		for (std::string_view path : mod->get_replace_paths()) {
+			if (std::find(replace_paths.begin(), replace_paths.end(), path) == replace_paths.end()) {
+				replace_paths.emplace_back(path);
+			}
+		}
+	}
+
+	/* Load only vanilla and push an error if mod loading failed. */
+	if (ret) {
+		mod_manager.set_loaded_mods(std::move(load_list));
+	} else {
+		mod_manager.set_loaded_mods({});
+		replace_paths.clear();
+		roots.erase(roots.begin()+1, roots.end());
+		Logger::error("Mod loading failed, loading base only!");
+	}
+
+	if (!dataloader.set_roots(roots, replace_paths, false)) {
 		Logger::error("Failed to set dataloader roots!");
-		return false;
+		ret = false;
 	}
-	return true;
+
+	return ret;
 }
 
 bool GameManager::load_definitions(Dataloader::localisation_callback_t localisation_callback) {

src/openvic-simulation/GameManager.hpp

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <optional>
-#include <span>
+#include <string_view>
 
 #include "openvic-simulation/DefinitionManager.hpp"
 #include "openvic-simulation/InstanceManager.hpp"
@@ -23,6 +23,8 @@ namespace OpenVic {
 		bool PROPERTY_CUSTOM_PREFIX(definitions_loaded, are);
 		bool PROPERTY_CUSTOM_PREFIX(mod_descriptors_loaded, are);
 
+		bool _get_mod_dependencies(Mod const* mod, std::vector<Mod const*>& load_list);
+
 	public:
 		GameManager(
 			InstanceManager::gamestate_updated_func_t new_gamestate_updated_callback
@@ -36,9 +38,15 @@ namespace OpenVic {
 			return instance_manager ? &*instance_manager : nullptr;
 		}
 
-		bool set_roots(Dataloader::path_span_t roots, Dataloader::path_span_t replace_paths);
+		inline bool set_base_path(Dataloader::path_span_t base_path) {
+			OV_ERR_FAIL_COND_V_MSG(base_path.size() > 1, "more than one dataloader base path provided", false);
+			OV_ERR_FAIL_COND_V_MSG(!dataloader.set_roots(base_path, {}), "failed to set dataloader base path", false);
+			return true;
+		};
+
+		bool load_mod_descriptors();
 
-		bool load_mod_descriptors(std::span<const memory::string> descriptors);
+		bool load_mods(Dataloader::path_vector_t& roots, Dataloader::path_vector_t& replace_paths, std::span<const memory::string> requested_mods);
 
 		bool load_definitions(Dataloader::localisation_callback_t localisation_callback);
 

src/openvic-simulation/dataloader/Dataloader.cpp

@@ -1,5 +1,6 @@
 #include "Dataloader.hpp"
 
+#include <string_view>
 #include <system_error>
 
 #include <openvic-dataloader/csv/Parser.hpp>
@@ -36,9 +37,11 @@ static fs::path ensure_forward_slash_path(std::string_view path) {
 #endif
 }
 
-bool Dataloader::set_roots(path_span_t new_roots, path_span_t new_replace_paths) {
+bool Dataloader::set_roots(path_span_t new_roots, path_span_t new_replace_paths, bool warn_on_override) {
 	if (!roots.empty()) {
-		Logger::warning("Overriding existing dataloader roots!");
+		if (warn_on_override) {
+			Logger::warning("Overriding existing dataloader roots!");
+		}
 		roots.clear();
 		replace_paths.clear();
 	}
@@ -307,19 +310,23 @@ void Dataloader::free_cache() {
 	cached_parsers.clear();
 }
 
-bool Dataloader::load_mod_descriptors(std::span<const memory::string> descriptors, ModManager& mod_manager) {
-	bool ret = true;
+bool Dataloader::load_mod_descriptors(ModManager& mod_manager) const {
+	static constexpr std::string_view mod_directory = "mod";
+	static constexpr std::string_view mod_descriptor_extension = ".mod";
 
-	for (std::string_view descriptor_path : descriptors) {
-		if (!mod_manager.load_mod_file(parse_defines(ensure_forward_slash_path(descriptor_path)).get_file_node())) {
-			Logger::error("Failed to load ", descriptor_path);
-			ret = false;
+	apply_to_files(
+		lookup_files_in_dir(mod_directory, mod_descriptor_extension),
+		[&mod_manager](fs::path const& file) -> bool {
+			if (!mod_manager.load_mod_file(parse_defines(file).get_file_node())) {
+				Logger::warning("Invalid mod descriptor at path: ", file, " could not be loaded!");
+			}
+			return true;
 		}
-	}
+	);
 
 	mod_manager.lock_mods();
 
-	return ret;
+	return true;
 }
 
 bool Dataloader::_load_interface_files(UIManager& ui_manager) const {

src/openvic-simulation/dataloader/Dataloader.hpp

@@ -102,8 +102,9 @@ namespace OpenVic {
 		///
 		/// @param new_roots Dataloader roots in reverse-load order, so base defines first and final loaded mod last
 		/// @param new_replace_paths All base define paths that should be ignored entirely in favour of mods.
+		/// @param warn_on_override Whether or not to log a warning if roots are overridden.
 		/// @return True if successful, false if failed.
-		bool set_roots(path_span_t new_roots, path_span_t new_replace_paths);
+		bool set_roots(path_span_t new_roots, path_span_t new_replace_paths, bool warn_on_override = true);
 
 		/* REQUIREMENTS:
 		 * DAT-24
@@ -126,8 +127,8 @@ namespace OpenVic {
 
 		string_set_t lookup_dirs_in_dir(std::string_view path) const;
 
-		/* Load all mod descriptors passed by the user. Importantly, loads dependencies and replace_paths for us to check. */
-		bool load_mod_descriptors(std::span<const memory::string> descriptors, ModManager& mod_manager);
+		/* Load all mod descriptors present in the mod/ directory. Importantly, loads dependencies and replace_paths for us to check. */
+		bool load_mod_descriptors(ModManager& mod_manager) const;
 
 		/* Load and parse all of the text defines data, including parsing cached condition and effect scripts after all the
 		 * static data is loaded. Paths to the base and mod defines must have been supplied with set_roots.*/

src/openvic-simulation/dataloader/ModManager.cpp

@@ -5,6 +5,7 @@
 #include "openvic-simulation/dataloader/NodeTools.hpp"
 #include "openvic-simulation/types/HasIdentifier.hpp"
 #include "openvic-simulation/types/IdentifierRegistry.hpp"
+#include "openvic-simulation/utility/ErrorMacros.hpp"
 
 using namespace OpenVic;
 using namespace OpenVic::NodeTools;
@@ -21,27 +22,38 @@ bool ModManager::load_mod_file(ast::NodeCPtr root) {
 	memory::vector<memory::string> replace_paths;
 	memory::vector<memory::string> dependencies;
 
-	bool ret = NodeTools::expect_dictionary_keys(
+	bool ret = NodeTools::expect_dictionary_keys_and_default_map(
+		map_key_value_ignore_invalid_callback<template_key_map_t<StringMapCaseSensitive>>,
 		"name", ONE_EXACTLY, expect_string(assign_variable_callback(identifier)),
 		"path", ONE_EXACTLY, expect_string(assign_variable_callback(path)),
 		"user_dir", ZERO_OR_ONE, expect_string(assign_variable_callback_opt(user_dir)),
 		"replace_path", ZERO_OR_MORE, expect_string(vector_callback_string(replace_paths)),
 		"dependencies", ZERO_OR_ONE, expect_list_reserve_length(dependencies, expect_string(vector_callback_string(dependencies)))
 	)(root);
 
-	memory::vector<std::string_view> previous_mods = mods.get_item_identifiers();
-	for (std::string_view dependency : dependencies) {
-		if (std::find(previous_mods.begin(), previous_mods.end(), dependency) == previous_mods.end()) {
-			ret = false;
-			Logger::error("Mod ", identifier, " has unmet dependency ", dependency);
-		}
-	}
+	OV_ERR_FAIL_COND_V(!ret, true);
+
+	Logger::info("Loaded mod descriptor for \"", identifier, "\"");
+	mods.add_item(
+		{ identifier, path, user_dir, std::move(replace_paths), std::move(dependencies) }
+	);
+	return true;
+}
 
-	if (ret) {
-		ret &= mods.add_item(
-			{ identifier, path, user_dir, std::move(replace_paths), std::move(dependencies) }
-		);
+void ModManager::set_loaded_mods(std::vector<Mod const*>&& new_loaded_mods) {
+	OV_ERR_FAIL_COND_MSG(mods_loaded, "set_loaded_mods called twice");
+
+	loaded_mods = std::move(new_loaded_mods);
+	mods_loaded = true;
+	for (Mod const* mod : loaded_mods) {
+		Logger::info("Loading mod \"", mod->get_identifier(), "\" at path ", mod->get_dataloader_root_path());
 	}
+}
+
+std::vector<Mod const*> const& ModManager::get_loaded_mods() const {
+	return loaded_mods;
+}
 
-	return ret;
-}
+size_t ModManager::get_loaded_mod_count() const {
+	return loaded_mods.size();
+}

src/openvic-simulation/dataloader/ModManager.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstddef>
 #include <string_view>
 #include <vector>
 
@@ -25,13 +26,18 @@ namespace OpenVic {
 	};
 
 	struct ModManager {
-	
+
 	private:
 		IdentifierRegistry<Mod> IDENTIFIER_REGISTRY(mod);
+		std::vector<Mod const*> loaded_mods;
+		bool mods_loaded = false;
 
 	public:
 		ModManager();
 
 		bool load_mod_file(ast::NodeCPtr root);
+		void set_loaded_mods(std::vector<Mod const*>&& new_loaded_mods);
+		std::vector<Mod const*> const& get_loaded_mods() const;
+		size_t get_loaded_mod_count() const;
 	};
-} // namespace OpenVic
+} // namespace OpenVic

tests/benchmarks/src/dataloading/Dataloading.cpp

@@ -13,7 +13,7 @@ TEST_CASE("Dataloading benchmark", "[benchmarks][benchmark-dataloading]") {
 	ankerl::nanobench::Bench().epochs(10).run("Dataloading", [&] {
 		OpenVic::GameManager game_manager { []() {} };
 
-		game_manager.set_roots(roots, {});
+		game_manager.set_base_path(roots);
 		game_manager.load_definitions(
 			[](std::string_view key, Dataloader::locale_t locale, std::string_view localisation) -> bool {
 				return true;