From 7ffdf15f31516eed5032520956b49e68dc1b6ea8 Mon Sep 17 00:00:00 2001 From: tehtelev <50070668+tehtelev@users.noreply.github.com> Date: Fri, 2 Jan 2026 18:30:06 +0300 Subject: [PATCH 1/3] Optimize placeholder replacement method in BlockType Replaced placeholder replacement method with optimized version for performance improvement during block and item initialization. --- Loading/BlockType.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Loading/BlockType.cs b/Loading/BlockType.cs index c0561d9..0aa3fa6 100644 --- a/Loading/BlockType.cs +++ b/Loading/BlockType.cs @@ -816,7 +816,11 @@ public static string[] GetCreativeTabs(AssetLocation code, Dictionary Date: Fri, 2 Jan 2026 18:36:05 +0300 Subject: [PATCH 2/3] Refactor CreateResolvedType and related methods --- Loading/RegistryObjectType.cs | 241 +++++++++++++++++++++++----------- 1 file changed, 161 insertions(+), 80 deletions(-) diff --git a/Loading/RegistryObjectType.cs b/Loading/RegistryObjectType.cs index b24b28e..07c8d08 100644 --- a/Loading/RegistryObjectType.cs +++ b/Loading/RegistryObjectType.cs @@ -272,124 +272,205 @@ private void loadInherits(ICoreAPI api, ref JObject entityTypeObject, string ent } - internal virtual RegistryObjectType CreateAndPopulate(ICoreServerAPI api, AssetLocation fullcode, JObject jobject, JsonSerializer deserializer, API.Datastructures.OrderedDictionary variant) + internal virtual RegistryObjectType CreateAndPopulate(ICoreServerAPI api, AssetLocation fullcode, JObject jobject, JsonSerializer deserializer, API.Datastructures.OrderedDictionary variant) + { + return this; + } + + + /// + /// Create and populate the resolved type + /// + protected T CreateResolvedType(ICoreServerAPI api, AssetLocation fullcode, JObject jobject, JsonSerializer deserializer, OrderedDictionary variant) where T : RegistryObjectType, new() + { + T resolvedType = new T() { - return this; - } + Code = Code, + VariantGroups = VariantGroups, + Enabled = Enabled, + jsonObject = jobject, // This is now already resolved JSON + Variant = variant + }; - protected T CreateResolvedType(ICoreServerAPI api, AssetLocation fullcode, JObject jobject, JsonSerializer deserializer, API.Datastructures.OrderedDictionary variant) where T : RegistryObjectType, new() + // solvedbytype is no longer needed since we already resolved the JSON earlier + + try { - T resolvedType = new T() - { - Code = Code, - VariantGroups = VariantGroups, - Enabled = Enabled, - jsonObject = jobject, - Variant = variant - }; + JsonUtil.PopulateObject(resolvedType, jobject, deserializer); + } + catch (Exception e) + { + api.Server.Logger.Error("Exception thrown while trying to parse json data of the type with code {0}, variant {1}. Will ignore most of the attributes. Exception:", this.Code, fullcode); + api.Server.Logger.Error(e); + } - try + resolvedType.Code = fullcode; + resolvedType.jsonObject = null; + return resolvedType; + } + + /// + /// Checks if the token needs resolution (contains "byType" or placeholders "{...}"). + /// This is a static check, independent of codePath and variant. + /// + private static bool NeedsResolve(JToken token) + { + if (token == null) return false; + + // Check if the object has properties ending with "byType" + if (token is JObject obj) + { + foreach (var prop in obj.Properties()) { - solveByType(jobject, fullcode.Path, variant); + if (prop.Name.EndsWith("byType", StringComparison.OrdinalIgnoreCase)) return true; + if (NeedsResolve(prop.Value)) return true; } - catch (Exception e) + return false; + } + else if (token is JArray arr) // Check array elements + { + // If at least one element needs resolution, return true + foreach (var item in arr) { - api.Server.Logger.Error("Exception thrown while trying to resolve *byType properties of type {0}, variant {1}. Will ignore most of the attributes. Exception thrown:", this.Code, fullcode); - api.Server.Logger.Error(e); + if (NeedsResolve(item)) return true; } + return false; + } + else if (token is JValue val && val.Type == JTokenType.String) // Check string values for placeholders + { + string str = (string)val.Value; + return str != null && str.Contains("{"); + } - try + return false; + } + + /// + /// Lazily resolves the JSON token for the given codePath and variant + /// Creates new containers (JObject/JArray) only for parts requiring changes + /// Unchanged subtrees are shared + /// + public static JToken Resolve(JToken token, string codePath, OrderedDictionary searchReplace) + { + if (token == null) + return null; + + // Check if the token needs resolution + if (token is JObject obj) + { + // Check if this object needs resolution at all + bool hasByType = false; + bool needsChildResolve = false; + foreach (var prop in obj.Properties()) { - JsonUtil.PopulateObject(resolvedType, jobject, deserializer); + if (prop.Name.EndsWith("byType", StringComparison.OrdinalIgnoreCase)) + { + hasByType = true; + } + if (NeedsResolve(prop.Value)) + { + needsChildResolve = true; + } } - catch (Exception e) + + if (!hasByType && !needsChildResolve) { - api.Server.Logger.Error("Exception thrown while trying to parse json data of the type with code {0}, variant {1}. Will ignore most of the attributes. Exception:", this.Code, fullcode); - api.Server.Logger.Error(e); + return token; // Share the original object since nothing changes } - resolvedType.Code = fullcode; - resolvedType.jsonObject = null; - return resolvedType; - } + // Create a new JObject only if needed + var newObj = new JObject(); + Dictionary propertiesToAdd = null; - protected static void solveByType(JToken json, string codePath, API.Datastructures.OrderedDictionary searchReplace) - { - if (json is JObject jsonObj) + foreach (var prop in obj.Properties()) { - List propertiesToRemove = null; - Dictionary propertiesToAdd = null; - - foreach (var entry in jsonObj) + var key = prop.Name; + if (key.EndsWith("byType", StringComparison.OrdinalIgnoreCase)) { - if (entry.Key.EndsWith("byType", StringComparison.OrdinalIgnoreCase)) + string trueKey = key.Substring(0, key.Length - "byType".Length); + var byTypeObj = prop.Value as JObject; + if (byTypeObj == null) continue; + + JToken selected = null; + foreach (var byTypeProp in byTypeObj.Properties()) { - string trueKey = entry.Key.Substring(0, entry.Key.Length - "byType".Length); - var jobj = entry.Value as JObject; - if (jobj == null) - { - throw new FormatException("Invalid value at key: " + entry.Key); - } - foreach (var byTypeProperty in jobj) + if (WildcardUtil.Match(byTypeProp.Name, codePath)) { - if (WildcardUtil.Match(byTypeProperty.Key, codePath)) - { - JToken typedToken = byTypeProperty.Value; // Unnecessary to solveByType specifically on this new token's contents as we will be doing a solveByType on all the tokens in the jsonObj anyhow, after adding the propertiesToAdd - if (propertiesToAdd == null) propertiesToAdd = new Dictionary(); - propertiesToAdd.Add(trueKey, typedToken); - break; // Replaces for first matched key only - } + selected = byTypeProp.Value; + break; // First matched } - if (propertiesToRemove == null) propertiesToRemove = new List(); - propertiesToRemove.Add(entry.Key); } - } + if (selected != null) + { + if (propertiesToAdd == null) propertiesToAdd = new Dictionary(); + propertiesToAdd[trueKey] = selected; // Store JToken directly + } + } + else + { + newObj[key] = Resolve(prop.Value, codePath, searchReplace); + } + } - if (propertiesToRemove != null) + // Add the resolved "byType" properties + if (propertiesToAdd != null) + { + foreach (var add in propertiesToAdd) { - foreach (var property in propertiesToRemove) + string trueKey = add.Key; + JToken selected = add.Value; + JToken resolvedSelected = Resolve(selected, codePath, searchReplace); + + if (newObj[trueKey] is JObject existing) { - jsonObj.Remove(property); + existing.Merge(resolvedSelected); // No mergeSettings, to match original (default Concat for arrays) } - - if (propertiesToAdd != null) + else { - foreach (var property in propertiesToAdd) - { - if (jsonObj[property.Key] is JObject jobject) - { - jobject.Merge(property.Value); - } - else - { - jsonObj[property.Key] = property.Value; - } - } + newObj[trueKey] = resolvedSelected; } } + } - foreach (var entry in jsonObj) + return newObj; + } + else if (token is JArray arr) + { + // Check if the array needs resolution + bool needs = false; + foreach (var item in arr) + { + if (NeedsResolve(item)) { - solveByType(entry.Value, codePath, searchReplace); + needs = true; + break; } } - else if (json.Type == JTokenType.String) + + if (!needs) return token; // Share original + + var newArr = new JArray(); + foreach (var item in arr) { - string value = (string)(json as JValue).Value; - if (value.Contains("{")) - { - (json as JValue).Value = RegistryObject.FillPlaceHolder(value, searchReplace); - } + newArr.Add(Resolve(item, codePath, searchReplace)); } - else if (json is JArray jarray) + return newArr; + } + else if (token is JValue val && val.Type == JTokenType.String) // String value + { + string str = (string)val.Value; + if (str != null && str.Contains("{")) { - foreach (var child in jarray) - { - solveByType(child, codePath, searchReplace); - } + return new JValue(RegistryObject.FillPlaceHolderOptimized(str, searchReplace)); // Using optimized version of placeholder replacement method } + return token; // Share original + } + else + { + return token; // Primitives shared } + } #endregion } From f421e3d9ab15f188f6e54fc92768000f366eef58 Mon Sep 17 00:00:00 2001 From: tehtelev <50070668+tehtelev@users.noreply.github.com> Date: Fri, 2 Jan 2026 18:39:06 +0300 Subject: [PATCH 3/3] Update Registryobjecttypeloader --- Loading/RegistryObjectTypeLoader.cs | 282 +++++++++++++++++----------- 1 file changed, 170 insertions(+), 112 deletions(-) diff --git a/Loading/RegistryObjectTypeLoader.cs b/Loading/RegistryObjectTypeLoader.cs index 75d63c9..a9046c0 100644 --- a/Loading/RegistryObjectTypeLoader.cs +++ b/Loading/RegistryObjectTypeLoader.cs @@ -1,4 +1,4 @@ -using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Diagnostics; @@ -10,6 +10,8 @@ using Vintagestory.API.MathTools; using Vintagestory.API.Server; using Vintagestory.API.Util; +using System.Threading.Tasks; +using System.Collections.Concurrent; #nullable disable @@ -25,7 +27,7 @@ public class VariantEntry public class ResolvedVariant { - public API.Datastructures.OrderedDictionary CodeParts = new (); + public API.Datastructures.OrderedDictionary CodeParts = new(); public AssetLocation Code; @@ -50,7 +52,7 @@ public void AddCodePart(string key, string val) public class ModRegistryObjectTypeLoader : ModSystem { - // Dict Key is filename (with .json) + // Dictionary key is filename (with .json) public Dictionary worldProperties; public Dictionary worldPropertiesVariants; @@ -76,6 +78,13 @@ public override double ExecuteOrder() return 0.2; } + private int _threadsCount; // Number of threads for parallel processing + private volatile bool _gatheringCompleted = false; // Flag indicating completion of type gathering + + + /// + /// Start loading assets + /// public override void AssetsLoaded(ICoreAPI coreApi) { if (!(coreApi is ICoreServerAPI api)) return; @@ -83,18 +92,33 @@ public override void AssetsLoaded(ICoreAPI coreApi) api.Logger.VerboseDebug("Starting to gather blocktypes, itemtypes and entities"); LoadWorldProperties(); + + // Determine number of threads int maxThreads = api.Server.IsDedicated ? 3 : 8; - int threads = GameMath.Clamp(Environment.ProcessorCount / 2 - 2, 1, maxThreads); - if (api.Server.ReducedServerThreads) threads = 1; + _threadsCount = GameMath.Clamp(Environment.ProcessorCount / 2 - 2, 1, maxThreads); + if (api.Server.ReducedServerThreads) _threadsCount = 1; - itemTypes = new Dictionary(); - blockTypes = new Dictionary(); - entityTypes = new Dictionary(); + // Load base types + LoadBaseTypes(); + + // Start one asynchronous thread to gather all types + TyronThreadPool.QueueTask(GatherAllTypes_Async, "gatheralltypes"); + // Wait for gathering to complete and load results + WaitAndLoadResults(); + } + + + /// + /// Load base types from JSON + /// + private void LoadBaseTypes() + { + // Loading itemTypes + itemTypes = new Dictionary(); foreach (KeyValuePair entry in api.Assets.GetMany(api.Server.Logger, "itemtypes/")) { if (!entry.Key.Path.EndsWithOrdinal(".json")) continue; - try { ItemType et = new ItemType(); @@ -103,78 +127,177 @@ public override void AssetsLoaded(ICoreAPI coreApi) } catch (Exception e) { - api.World.Logger.Error("Item type {0} could not be loaded. Will ignore. Exception thrown:", entry.Key); + api.World.Logger.Error("Item type {0} could not be loaded. Will ignore. Exception:", entry.Key); api.World.Logger.Error(e); continue; } } itemVariants = new List[itemTypes.Count]; - api.Logger.VerboseDebug("Starting parsing ItemTypes in " + threads + " threads"); - PrepareForLoading(threads); - foreach (KeyValuePair entry in api.Assets.GetMany(api.Server.Logger, "entities/")) + // Loading blockTypes + blockTypes = new Dictionary(); + foreach (KeyValuePair entry in api.Assets.GetMany(api.Server.Logger, "blocktypes/")) { if (!entry.Key.Path.EndsWithOrdinal(".json")) continue; - try { - EntityType et = new EntityType(); + BlockType et = new BlockType(); et.CreateBasetype(api, entry.Key.Path, entry.Key.Domain, entry.Value); - entityTypes.Add(entry.Key, et); + blockTypes.Add(entry.Key, et); } catch (Exception e) { - api.World.Logger.Error("Entity type {0} could not be loaded. Will ignore. Exception thrown:", entry.Key); + api.World.Logger.Error("Block type {0} could not be loaded. Will ignore. Exception:", entry.Key); api.World.Logger.Error(e); continue; } } - entityVariants = new List[entityTypes.Count]; + blockVariants = new List[blockTypes.Count]; - foreach (KeyValuePair entry in api.Assets.GetMany(api.Server.Logger, "blocktypes/")) + // Loading entityTypes + entityTypes = new Dictionary(); + foreach (KeyValuePair entry in api.Assets.GetMany(api.Server.Logger, "entities/")) { if (!entry.Key.Path.EndsWithOrdinal(".json")) continue; - try { - BlockType et = new BlockType(); + EntityType et = new EntityType(); et.CreateBasetype(api, entry.Key.Path, entry.Key.Domain, entry.Value); - blockTypes.Add(entry.Key, et); + entityTypes.Add(entry.Key, et); } catch (Exception e) { - api.World.Logger.Error("Block type {0} could not be loaded. Will ignore. Exception thrown:", entry.Key); + api.World.Logger.Error("Entity type {0} could not be loaded. Will ignore. Exception:", entry.Key); api.World.Logger.Error(e); continue; } } - blockVariants = new List[blockTypes.Count]; + entityVariants = new List[entityTypes.Count]; + } + + private void GatherAllTypes_Async() + { + try + { + // Using Parallel.For for parallel processing of each data type + // Processing itemTypes + if (itemTypes != null && itemTypes.Count > 0) + { + api.Logger.VerboseDebug($"Starting parallel processing of {itemTypes.Count} item types with {_threadsCount} threads"); + ProcessTypesInParallel(itemTypes, itemVariants); + } + + // Processing blockTypes + if (blockTypes != null && blockTypes.Count > 0) + { + api.Logger.VerboseDebug($"Starting parallel processing of {blockTypes.Count} block types with {_threadsCount} threads"); + ProcessTypesInParallel(blockTypes, blockVariants); + } + + // Processing entityTypes + if (entityTypes != null && entityTypes.Count > 0) + { + api.Logger.VerboseDebug($"Starting parallel processing of {entityTypes.Count} entity types with {_threadsCount} threads"); + ProcessTypesInParallel(entityTypes, entityVariants); + } + } + finally + { + _gatheringCompleted = true; + } + } + + + /// + /// Process all types in parallel threads + /// + private void ProcessTypesInParallel( + Dictionary baseTypes, + List[] resolvedTypeLists) + { + var baseTypeArray = baseTypes.Values.ToArray(); + int count = baseTypeArray.Length; + + if (count == 0) + return; + + // Using Partitioner with dynamic load distribution + // The value _threadsCount * _threadsCount seems to be the most optimal, but changing it doesn't significantly affect performance + var partitioner = Partitioner.Create(0, count, Math.Max(1, count / (_threadsCount* _threadsCount))); + + + // Start parallel processing using Partitioner and threads according to _threadsCount + Parallel.ForEach(partitioner, new ParallelOptions + { + MaxDegreeOfParallelism = _threadsCount + }, (range, loopState) => + { + // Process each type in the given range + for (int i = range.Item1; i < range.Item2; i++) + { + var val = baseTypeArray[i]; + + // Skip disabled types + if (!val.Enabled) + { + resolvedTypeLists[i] = new List(); + continue; + } - TyronThreadPool.QueueTask(GatherAllTypes_Async, "gatheralltypes"); // Now we've loaded everything, let's add one more gathering thread :) + try + { + List resolvedTypes = new List(); + GatherVariantsAndPopulate(val, resolvedTypes); + resolvedTypeLists[i] = resolvedTypes; + } + catch (Exception e) + { + api.Server.Logger.Error($"Error processing type {val.Code}: {e.Message}"); + // Do not interrupt execution for other types + } + } + }); + } + + /// + /// Wait for gathering to complete and load results + /// + private void WaitAndLoadResults() + { + // Wait for gathering to complete + api.Logger.VerboseDebug("Waiting for type gathering to complete..."); + while (!_gatheringCompleted) + { + Thread.Sleep(10); + } api.Logger.StoryEvent(Lang.Get("It remembers...")); api.Logger.VerboseDebug("Gathered all types, starting to load items"); + // Load results for items LoadItems(itemVariants); api.Logger.VerboseDebug("Parsed and loaded items"); api.Logger.StoryEvent(Lang.Get("...all that came before")); + // Load results for blocks LoadBlocks(blockVariants); api.Logger.VerboseDebug("Parsed and loaded blocks"); + // Load results for entities LoadEntities(entityVariants); api.Logger.VerboseDebug("Parsed and loaded entities"); api.TagRegistry.LoadTagsFromAssets(api); - api.Server.LogNotification("BlockLoader: Entities, Blocks and Items loaded"); - FreeRam(); - api.TriggerOnAssetsFirstLoaded(); } + + /// + /// Load world properties + /// private void LoadWorldProperties() { worldProperties = new Dictionary(); @@ -184,7 +307,7 @@ private void LoadWorldProperties() AssetLocation loc = entry.Key.Clone(); loc.Path = loc.Path.Replace("worldproperties/", ""); loc.RemoveEnding(); - + entry.Value.Code.Domain = entry.Key.Domain; worldProperties.Add(loc, entry.Value); @@ -239,7 +362,7 @@ void LoadEntities(List[] variantLists) #region Items void LoadItems(List[] variantLists) { - // Step2: create all the items from the itemtypes, and register them: this has to be on the main thread as the registry is not thread-safe + // Step 2: create all the items from the itemtypes, and register them: this has to be on the main thread as the registry is not thread-safe LoadFromVariants(variantLists, "item", (variants) => { foreach (ItemType type in variants) @@ -284,86 +407,21 @@ void LoadBlocks(List[] variantLists) api.Server.Logger.Error(e); } } - }); + }); } #endregion #region generic - void PrepareForLoading(int threadsCount) - { - // JSON parsing is slooowww, so let's multithread the **** out of it :) - for (int i = 0; i < threadsCount; i++) TyronThreadPool.QueueTask(GatherAllTypes_Async, "gatheralltypes" + i); - } - private void GatherAllTypes_Async() - { - GatherTypes_Async(itemVariants, itemTypes); - int timeOut = 1000; - bool logged = false; - while (blockVariants == null) - { - if (--timeOut == 0) return; - if (!logged) - { - api.Logger.VerboseDebug("Waiting for entityTypes to be gathered"); - logged = true; - } - Thread.Sleep(10); - } - if (logged) api.Logger.VerboseDebug("EntityTypes now all gathered"); - GatherTypes_Async(blockVariants, blockTypes); - timeOut = 1000; - logged = false; - while (entityVariants == null) - { - if (--timeOut == 0) return; - if (!logged) - { - api.Logger.VerboseDebug("Waiting for blockTypes to be gathered"); - logged = true; - } - Thread.Sleep(10); - } - if (logged) api.Logger.VerboseDebug("BlockTypes now all gathered"); - - GatherTypes_Async(entityVariants, entityTypes); - } /// - /// Each thread attempts to resolve and parse the whole list of types, using the parseStarted field for load-sharing + /// Gather variants and populate types /// - private void GatherTypes_Async(List[] resolvedTypeLists, Dictionary baseTypes) - { - int i = 0; - foreach (RegistryObjectType val in baseTypes.Values) - { - if (AsyncHelper.CanProceedOnThisThread(ref val.parseStarted)) // In each thread, only do work on RegistryObjectTypes which no other thread has yet worked on - { - List resolvedTypes = new List(); - try - { - if (val.Enabled) GatherVariantsAndPopulate(val, resolvedTypes); - } - finally - { - resolvedTypeLists[i] = resolvedTypes; - } - } - i++; - } - } - - - /// - /// This does the actual gathering and population, through GatherVariants calls - numerous calls if a base type has many variants - /// - /// - /// void GatherVariantsAndPopulate(RegistryObjectType baseType, List typesResolved) { List variants = null; @@ -386,20 +444,19 @@ void GatherVariantsAndPopulate(RegistryObjectType baseType, List()); + RegistryObjectType resolvedType = baseType.CreateAndPopulate(api, baseType.Code.Clone(), resolvedJson, deserializer, new OrderedDictionary()); typesResolved.Add(resolvedType); } else { - // Multiple variants - int count = 1; + // Multiple variants: No DeepClone! Resolve lazily for each foreach (ResolvedVariant variant in variants) { - JObject jobject = count++ == variants.Count ? baseType.jsonObject : baseType.jsonObject.DeepClone() as JObject; - // This DeepClone() is expensive, can we find a better way one day? - - RegistryObjectType resolvedType = baseType.CreateAndPopulate(api, variant.Code, jobject, deserializer, variant.CodeParts); - + string codePath = variant.Code.Path; + JObject resolvedJson = (JObject)RegistryObjectType.Resolve(baseType.jsonObject, codePath, variant.CodeParts); + RegistryObjectType resolvedType = baseType.CreateAndPopulate(api, variant.Code, resolvedJson, deserializer, variant.CodeParts); typesResolved.Add(resolvedType); } } @@ -444,7 +501,7 @@ List GatherVariants(AssetLocation baseCode, RegistryObjectVaria { List variantsFinal = new List(); - API.Datastructures.OrderedDictionary variantsMul = new (); + API.Datastructures.OrderedDictionary variantsMul = new(); // 1. Collect all types @@ -499,19 +556,20 @@ List GatherVariants(AssetLocation baseCode, RegistryObjectVaria var.ResolveCode(baseCode); } - + if (skipVariants != null) { List filteredVariants = new List(); HashSet skipVariantsHash = new HashSet(); List skipVariantsWildCards = new List(); - foreach(var val in skipVariants) + foreach (var val in skipVariants) { if (val.IsWildCard) { skipVariantsWildCards.Add(val); - } else + } + else { skipVariantsHash.Add(val); } @@ -557,7 +615,7 @@ List GatherVariants(AssetLocation baseCode, RegistryObjectVaria variantsFinal = filteredVariants; } - + return variantsFinal; } @@ -607,7 +665,7 @@ private void CollectFromStateList(RegistryObjectVariantGroup variantGroup, Regis VariantEntry old = stateList[k]; if (cvg.Code != old.Code) continue; - + stateList.RemoveAt(k); for (int j = 0; j < cvg.States.Length; j++) @@ -748,4 +806,4 @@ private void FreeRam() worldPropertiesVariants = null; } } -} \ No newline at end of file +}