XmlBuilderWorker.cpp 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777
  1. /*
  2. * Copyright (c) Contributors to the Open 3D Engine Project.
  3. * For complete copyright and license terms please see the LICENSE at the root of this distribution.
  4. *
  5. * SPDX-License-Identifier: Apache-2.0 OR MIT
  6. *
  7. */
  8. #include "XmlBuilderWorker.h"
  9. #include <AssetBuilderSDK/SerializationDependencies.h>
  10. #include <AzCore/Component/ComponentApplication.h>
  11. #include <AzCore/IO/SystemFile.h>
  12. #include <AzCore/std/string/wildcard.h>
  13. #include <AzCore/Dependency/Dependency.h>
  14. #include <AzFramework/FileFunc/FileFunc.h>
  15. #include <AzFramework/IO/LocalFileIO.h>
  16. #include <AzFramework/StringFunc/StringFunc.h>
  17. #include "Builders/CopyDependencyBuilder/SchemaBuilderWorker/SchemaUtils.h"
  18. namespace CopyDependencyBuilder
  19. {
  20. namespace Internal
  21. {
  22. bool AddFileExtention(const AZStd::string expectedExtension, AZStd::string& fileName, bool isOptional)
  23. {
  24. if (!AzFramework::StringFunc::Path::HasExtension(fileName.c_str()))
  25. {
  26. // Open 3D Engine makes use of some files without extensions, only replace the extension if there is an expected extension.
  27. if (!expectedExtension.empty())
  28. {
  29. AzFramework::StringFunc::Path::ReplaceExtension(fileName, expectedExtension.c_str());
  30. }
  31. }
  32. else if (!expectedExtension.empty())
  33. {
  34. AZStd::string existingExtension;
  35. AzFramework::StringFunc::Path::GetExtension(fileName.c_str(), existingExtension, false);
  36. if (expectedExtension.front() == '.')
  37. {
  38. existingExtension = '.' + existingExtension;
  39. }
  40. if (existingExtension != expectedExtension)
  41. {
  42. if (!isOptional)
  43. {
  44. AZ_Warning("XmlBuilderWorker", false, "Dependency %s already has an extension %s and the expected extension %s is different."
  45. "The original extension is not replaced.", fileName.c_str(), existingExtension.c_str(), expectedExtension.c_str());
  46. }
  47. return false;
  48. }
  49. }
  50. return true;
  51. }
  52. AZ::Data::AssetId TextToAssetId(const char* text)
  53. {
  54. // Parse the asset id
  55. // Asset data could look like "id={00000000-0000-0000-0000-000000000000}:0,type={00000000-0000-0000-0000-000000000000},hint={asset_path}"
  56. // SubId is 16 based according to AssetSerializer
  57. AZ::Data::AssetId assetId;
  58. if (!text)
  59. {
  60. AZ_Error("XmlBuilderWorker", false, "Null string given to TextToAssetId");
  61. return assetId;
  62. }
  63. const char* idGuidStart = strchr(text, '{');
  64. if (!idGuidStart)
  65. {
  66. AZ_Error("XmlBuilderWorker", false, "Invalid asset guid data! %s", text);
  67. return assetId;
  68. }
  69. const char* idGuidEnd = strchr(idGuidStart, ':');
  70. AZ_Error("XmlBuilderWorker", idGuidEnd, "Invalid asset guid data! %s", idGuidStart);
  71. const char* idSubIdStart = idGuidEnd + 1;
  72. assetId.m_guid = AZ::Uuid::CreateString(idGuidStart, idGuidEnd - idGuidStart);
  73. assetId.m_subId = static_cast<AZ::u32>(strtoul(idSubIdStart, nullptr, 16));
  74. return assetId;
  75. }
  76. bool ParseAttributeNode(
  77. const AzFramework::XmlSchemaAttribute& schemaAttribute,
  78. const AZ::rapidxml::xml_node<char>* xmlFileNode,
  79. AZStd::vector<AssetBuilderSDK::ProductDependency>& productDependencies,
  80. AssetBuilderSDK::ProductPathDependencySet& pathDependencies,
  81. const AZStd::string& sourceAssetFolder,
  82. const AZStd::string& nodeContent = "",
  83. const AZStd::string& watchFolder = "")
  84. {
  85. // Attribute nodes of the XML schema specify the attributes which are used to store product dependency info in the actual XML nodes
  86. AZStd::string schemaAttributeName = schemaAttribute.GetName();
  87. // The attribute name could be empty only if the XML element content specifies the product dependency
  88. // e.g. <entry>example.dds</entry>
  89. if (schemaAttributeName.empty() && nodeContent.empty())
  90. {
  91. return schemaAttribute.IsOptional();
  92. }
  93. AZ::rapidxml::xml_attribute<char>* xmlNodeNameAttr = xmlFileNode->first_attribute(schemaAttributeName.c_str(), 0, false);
  94. // Return early if the attribute specified in the schema doesn't exist
  95. if (!schemaAttributeName.empty() && !xmlNodeNameAttr)
  96. {
  97. return schemaAttribute.IsOptional();
  98. }
  99. AZStd::string dependencyValue = !schemaAttributeName.empty() ? xmlNodeNameAttr->value() : nodeContent;
  100. switch (schemaAttribute.GetType())
  101. {
  102. case AzFramework::XmlSchemaAttribute::AttributeType::RelativePath:
  103. {
  104. AssetBuilderSDK::ProductPathDependencyType productPathDependencyType =
  105. static_cast<AZ::u32>(schemaAttribute.GetPathDependencyType()) == static_cast<AZ::u32>(AssetBuilderSDK::ProductPathDependencyType::SourceFile)
  106. ? AssetBuilderSDK::ProductPathDependencyType::SourceFile
  107. : AssetBuilderSDK::ProductPathDependencyType::ProductFile;
  108. if (dependencyValue.empty())
  109. {
  110. return true;
  111. }
  112. // Reject values that don't pass the match pattern
  113. if(!schemaAttribute.GetMatchPattern().empty())
  114. {
  115. AZStd::regex match(schemaAttribute.GetMatchPattern(), AZStd::regex_constants::ECMAScript | AZStd::regex_constants::icase);
  116. if(!AZStd::regex_search(dependencyValue, match))
  117. {
  118. return true;
  119. }
  120. }
  121. if(!schemaAttribute.GetFindPattern().empty())
  122. {
  123. AZStd::regex find(schemaAttribute.GetFindPattern(), AZStd::regex_constants::ECMAScript | AZStd::regex_constants::icase);
  124. dependencyValue = AZStd::regex_replace(dependencyValue, find, schemaAttribute.GetReplacePattern());
  125. }
  126. if (AddFileExtention(schemaAttribute.GetExpectedExtension(), dependencyValue, schemaAttribute.IsOptional()))
  127. {
  128. if (schemaAttribute.IsRelativeToSourceAssetFolder())
  129. {
  130. AzFramework::StringFunc::AssetDatabasePath::Join(sourceAssetFolder.c_str(), dependencyValue.c_str(), dependencyValue);
  131. }
  132. else if (schemaAttribute.CacheRelativePath())
  133. {
  134. AZStd::string_view depFolder{ sourceAssetFolder.c_str() };
  135. if (watchFolder.length() && sourceAssetFolder.starts_with(watchFolder))
  136. {
  137. depFolder = &sourceAssetFolder[watchFolder.length()];
  138. if (depFolder.front() == '/')
  139. {
  140. depFolder = &sourceAssetFolder[watchFolder.length() + 1];
  141. }
  142. }
  143. AzFramework::StringFunc::AssetDatabasePath::Join(depFolder.data(), dependencyValue.c_str(), dependencyValue);
  144. }
  145. pathDependencies.emplace(dependencyValue, productPathDependencyType);
  146. }
  147. break;
  148. }
  149. case AzFramework::XmlSchemaAttribute::AttributeType::Asset:
  150. {
  151. productDependencies.emplace_back(AssetBuilderSDK::ProductDependency(TextToAssetId(dependencyValue.c_str()), {}));
  152. break;
  153. }
  154. default:
  155. AZ_Error("XmlBuilderWorker", false, "Unsupported schema attribute type. Choose from RelativePath and Asset.");
  156. break;
  157. }
  158. return true;
  159. }
  160. bool ParseElementNode(
  161. const AzFramework::XmlSchemaElement& xmlSchemaElement,
  162. const AZ::rapidxml::xml_node<char>* xmlFileNode,
  163. AZStd::vector<AssetBuilderSDK::ProductDependency>& productDependencies,
  164. AssetBuilderSDK::ProductPathDependencySet& pathDependencies,
  165. const AZStd::string& sourceAssetFolder,
  166. const AZStd::string& watchFolderPath)
  167. {
  168. // Check whether the XML node matches the schema
  169. AZStd::string schemaElementName = xmlSchemaElement.GetName();
  170. if (strcmp(schemaElementName.c_str(), xmlFileNode->name()) != 0 && schemaElementName != "*")
  171. {
  172. return false;
  173. }
  174. AZStd::vector<AssetBuilderSDK::ProductDependency> localProductDependencies;
  175. AssetBuilderSDK::ProductPathDependencySet localPathDependencies;
  176. // Continue parsing the source XML using the child element and attribute nodes
  177. for (const AzFramework::XmlSchemaElement& childSchemaElement : xmlSchemaElement.GetChildElements())
  178. {
  179. bool findMatchedChildNode = false;
  180. for (AZ::rapidxml::xml_node<char>* xmlFileChildNode = xmlFileNode->first_node(); xmlFileChildNode; xmlFileChildNode = xmlFileChildNode->next_sibling())
  181. {
  182. findMatchedChildNode = ParseElementNode(childSchemaElement, xmlFileChildNode, localProductDependencies, localPathDependencies, sourceAssetFolder, watchFolderPath) || findMatchedChildNode;
  183. }
  184. if (!findMatchedChildNode && !childSchemaElement.IsOptional())
  185. {
  186. return false;
  187. }
  188. }
  189. for (const AzFramework::XmlSchemaAttribute& schemaAttribute : xmlSchemaElement.GetAttributes())
  190. {
  191. AZStd::string xmlNodeValue = schemaAttribute.GetName().empty() ? xmlFileNode->value() : "";
  192. if (!ParseAttributeNode(schemaAttribute, xmlFileNode, localProductDependencies, localPathDependencies, sourceAssetFolder, xmlNodeValue, watchFolderPath))
  193. {
  194. return false;
  195. }
  196. }
  197. // Only merge the dependencies if the attributes parsed cleanly. If a required dependency was missing, then don't add anything that was found.
  198. productDependencies.insert(productDependencies.end(), localProductDependencies.begin(), localProductDependencies.end());
  199. pathDependencies.insert(localPathDependencies.begin(), localPathDependencies.end());
  200. return true;
  201. }
  202. AZ::Outcome <AZ::rapidxml::xml_node<char>*, AZStd::string> GetSourceFileRootNode(
  203. const AZStd::string& filePath,
  204. AZStd::vector<char>& charBuffer,
  205. AZ::rapidxml::xml_document<char>& xmlDoc)
  206. {
  207. AZ::IO::FileIOStream fileStream;
  208. if (!fileStream.Open(filePath.c_str(), AZ::IO::OpenMode::ModeRead | AZ::IO::OpenMode::ModeBinary))
  209. {
  210. return AZ::Failure(AZStd::string::format("Failed to open source file %s.", filePath.c_str()));
  211. }
  212. AZ::IO::SizeType length = fileStream.GetLength();
  213. if (length == 0)
  214. {
  215. return AZ::Failure(AZStd::string("Failed to get the file stream length."));
  216. }
  217. charBuffer.resize_no_construct(length + 1);
  218. fileStream.Read(length, charBuffer.data());
  219. charBuffer.back() = 0;
  220. if (!xmlDoc.parse<AZ::rapidxml::parse_default>(charBuffer.data()))
  221. {
  222. return AZ::Failure(AZStd::string::format("Failed to parse the source file %s.", filePath.c_str()));
  223. }
  224. AZ::rapidxml::xml_node<char>* xmlRootNode = xmlDoc.first_node();
  225. if (!xmlRootNode)
  226. {
  227. return AZ::Failure(AZStd::string::format("Failed to get the root node of the source file %s.", filePath.c_str()));
  228. }
  229. return AZ::Success(AZStd::move(xmlRootNode));
  230. }
  231. AZStd::string GetSourceFileVersion(
  232. const AZ::rapidxml::xml_node<char>* xmlFileRootNode,
  233. const AZStd::string& rootNodeAttributeName)
  234. {
  235. if (!rootNodeAttributeName.empty())
  236. {
  237. AZ::rapidxml::xml_attribute<char>* xmlNodeNameAttr = xmlFileRootNode->first_attribute(rootNodeAttributeName.c_str(), 0, false);
  238. if (xmlNodeNameAttr)
  239. {
  240. return xmlNodeNameAttr->value();
  241. }
  242. }
  243. AZStd::string result = "0";
  244. for (size_t count = 1; count < MaxVersionPartsCount; ++count)
  245. {
  246. result += ".0";
  247. }
  248. return result;
  249. }
  250. bool MatchesVersionConstraints(const AZ::Version<MaxVersionPartsCount>& version, const AZStd::vector<AZStd::string>& versionConstraints)
  251. {
  252. if (versionConstraints.size() == 0)
  253. {
  254. return true;
  255. }
  256. AZ::Dependency<MaxVersionPartsCount> dependency;
  257. AZ::Outcome<void, AZStd::string> parseOutcome = dependency.ParseVersions(versionConstraints);
  258. if (!parseOutcome.IsSuccess())
  259. {
  260. AZ_Error("XmlBuilderWorker", false, parseOutcome.TakeError().c_str());
  261. return false;
  262. }
  263. return dependency.IsFullfilledBy(AZ::Specifier<MaxVersionPartsCount>(AZ::Uuid::CreateNull(), version));
  264. }
  265. AZ::Outcome<AZ::Version<MaxVersionPartsCount>, AZStd::string> ParseFromString(const AZStd::string& versionStr)
  266. {
  267. AZ::Version<MaxVersionPartsCount> result;
  268. AZStd::vector<AZStd::string> versionParts;
  269. AzFramework::StringFunc::Tokenize(versionStr.c_str(), versionParts, VERSION_SEPARATOR_CHAR);
  270. size_t versionPartsCount = versionParts.size();
  271. if (versionPartsCount > MaxVersionPartsCount)
  272. {
  273. return AZ::Failure(AZStd::string::format(
  274. "Failed to parse invalid version string \"%s\". "
  275. "Only version number with at most %zu parts is supported. "
  276. , versionStr.c_str(), MaxVersionPartsCount));
  277. }
  278. for (int index = 0; index < MaxVersionPartsCount; ++index)
  279. {
  280. if (index >= versionPartsCount)
  281. {
  282. result.m_parts[index] = 0;
  283. }
  284. else
  285. {
  286. AZStd::string versionPart = versionParts[index];
  287. if (!AZStd::all_of(versionPart.begin(), versionPart.end(), isdigit))
  288. {
  289. return AZ::Failure(AZStd::string::format(
  290. "Failed to parse invalid version string \"%s\". "
  291. "Unexpected separator character encountered. "
  292. "Expected: \"%d\""
  293. , versionStr.c_str(), VERSION_SEPARATOR_CHAR));
  294. }
  295. result.m_parts[index] = AZStd::stoi(versionParts[index]);
  296. }
  297. }
  298. return AZ::Success(result);
  299. }
  300. }
  301. // m_skipServer (3rd Param) should be false - we want to process xml files on the server as it's a generic data format which could
  302. // have meaningful data for a server
  303. XmlBuilderWorker::XmlBuilderWorker()
  304. : CopyDependencyBuilderWorker("xml", true, false)
  305. {
  306. }
  307. void XmlBuilderWorker::RegisterBuilderWorker()
  308. {
  309. AssetBuilderSDK::AssetBuilderDesc xmlSchemaBuilderDescriptor;
  310. xmlSchemaBuilderDescriptor.m_name = "XmlBuilderWorker";
  311. xmlSchemaBuilderDescriptor.m_patterns.push_back(AssetBuilderSDK::AssetBuilderPattern("(?!.*libs\\/gameaudio\\/).*\\.xml", AssetBuilderSDK::AssetBuilderPattern::PatternType::Regex));
  312. xmlSchemaBuilderDescriptor.m_patterns.push_back(AssetBuilderSDK::AssetBuilderPattern("*.vegdescriptorlist", AssetBuilderSDK::AssetBuilderPattern::PatternType::Wildcard));
  313. xmlSchemaBuilderDescriptor.m_busId = azrtti_typeid<XmlBuilderWorker>();
  314. xmlSchemaBuilderDescriptor.m_version = 10;
  315. xmlSchemaBuilderDescriptor.m_createJobFunction =
  316. AZStd::bind(&XmlBuilderWorker::CreateJobs, this, AZStd::placeholders::_1, AZStd::placeholders::_2);
  317. xmlSchemaBuilderDescriptor.m_processJobFunction =
  318. AZStd::bind(&XmlBuilderWorker::ProcessJob, this, AZStd::placeholders::_1, AZStd::placeholders::_2);
  319. BusConnect(xmlSchemaBuilderDescriptor.m_busId);
  320. AssetBuilderSDK::AssetBuilderBus::Broadcast(&AssetBuilderSDK::AssetBuilderBusTraits::RegisterBuilderInformation, xmlSchemaBuilderDescriptor);
  321. bool success;
  322. AZStd::vector<AZStd::string> assetSafeFolders;
  323. AzToolsFramework::AssetSystemRequestBus::BroadcastResult(success, &AzToolsFramework::AssetSystemRequestBus::Events::GetAssetSafeFolders, assetSafeFolders);
  324. for (const AZStd::string& assetSafeFolder : assetSafeFolders)
  325. {
  326. AZStd::string schemaFolder = "Schema";
  327. AzFramework::StringFunc::AssetDatabasePath::Join(assetSafeFolder.c_str(), schemaFolder.c_str(), schemaFolder);
  328. AddSchemaFileDirectory(schemaFolder);
  329. }
  330. }
  331. void XmlBuilderWorker::AddSchemaFileDirectory(const AZStd::string& schemaFileDirectory)
  332. {
  333. if (!AZ::IO::FileIOBase::GetInstance()->Exists(schemaFileDirectory.c_str()))
  334. {
  335. return;
  336. }
  337. char resolvedPath[AZ_MAX_PATH_LEN];
  338. AZ::IO::FileIOBase::GetInstance()->ResolvePath(schemaFileDirectory.c_str(), resolvedPath, AZ_MAX_PATH_LEN);
  339. m_schemaFileDirectories.emplace_back(resolvedPath);
  340. }
  341. void XmlBuilderWorker::UnregisterBuilderWorker()
  342. {
  343. BusDisconnect();
  344. }
  345. AZ::Outcome<AZStd::vector<AssetBuilderSDK::SourceFileDependency>, AZStd::string> XmlBuilderWorker::GetSourceDependencies(
  346. const AssetBuilderSDK::CreateJobsRequest& request) const
  347. {
  348. AZStd::vector<AssetBuilderSDK::SourceFileDependency> sourceDependencies;
  349. AZStd::vector<AZStd::string> matchedSchemas;
  350. AZStd::string fullPath;
  351. AzFramework::StringFunc::AssetDatabasePath::Join(request.m_watchFolder.c_str(), request.m_sourceFile.c_str(), fullPath);
  352. // Iterate through each schema file and check whether the source XML matches its file path pattern
  353. for (const AZStd::string& schemaFileDirectory : m_schemaFileDirectories)
  354. {
  355. AZ::Outcome<AZStd::list<AZStd::string>, AZStd::string> findFilesResult = AzFramework::FileFunc::FindFilesInPath(schemaFileDirectory, SchemaNamePattern, true);
  356. if (!findFilesResult.IsSuccess())
  357. {
  358. return AZ::Failure(AZStd::string::format("Failed to find schema files in directory %s.", schemaFileDirectory.c_str()));
  359. }
  360. for (const AZStd::string& schemaPath : findFilesResult.GetValue())
  361. {
  362. AzFramework::XmlSchemaAsset schemaAsset;
  363. AZ::ObjectStream::FilterDescriptor loadFilter = AZ::ObjectStream::FilterDescriptor(&AZ::Data::AssetFilterNoAssetLoading, AZ::ObjectStream::FILTERFLAG_IGNORE_UNKNOWN_CLASSES);
  364. if (!AZ::Utils::LoadObjectFromFileInPlace(schemaPath, schemaAsset, nullptr, loadFilter))
  365. {
  366. return AZ::Failure(AZStd::string::format("Failed to load schema file: %s.", schemaPath.c_str()));
  367. }
  368. if (SourceFileDependsOnSchema(schemaAsset, fullPath.c_str()))
  369. {
  370. matchedSchemas.emplace_back(schemaPath);
  371. }
  372. }
  373. }
  374. // If we have matched any schemas, then add both the schemas as well as the path dependencies as source dependencies.
  375. if (matchedSchemas.size() > 0)
  376. {
  377. for (const AZStd::string& schemaPath : matchedSchemas)
  378. {
  379. AssetBuilderSDK::SourceFileDependency sourceFileDependency;
  380. sourceFileDependency.m_sourceFileDependencyPath = schemaPath;
  381. sourceDependencies.emplace_back(sourceFileDependency);
  382. }
  383. AZStd::vector<AssetBuilderSDK::ProductDependency> productDependencies;
  384. AssetBuilderSDK::ProductPathDependencySet pathDependencies;
  385. if (MatchExistingSchema(fullPath, matchedSchemas, productDependencies, pathDependencies, request.m_watchFolder) != SchemaMatchResult::Error)
  386. {
  387. // Product dependencies with wildcards are treated as source dependencies
  388. for (const auto& pathDependency : pathDependencies)
  389. {
  390. if (!pathDependency.m_dependencyPath.contains('*') && !pathDependency.m_dependencyPath.contains('?'))
  391. continue;
  392. AssetBuilderSDK::SourceFileDependency sourceFileDependency;
  393. sourceFileDependency.m_sourceFileDependencyPath = pathDependency.m_dependencyPath;
  394. sourceFileDependency.m_sourceDependencyType = AssetBuilderSDK::SourceFileDependency::SourceFileDependencyType::Wildcards;
  395. sourceDependencies.emplace_back(sourceFileDependency);
  396. }
  397. }
  398. }
  399. return AZ::Success(sourceDependencies);
  400. }
  401. bool XmlBuilderWorker::ParseProductDependencies(
  402. const AssetBuilderSDK::ProcessJobRequest& request,
  403. AZStd::vector<AssetBuilderSDK::ProductDependency>& productDependencies,
  404. AssetBuilderSDK::ProductPathDependencySet& pathDependencies)
  405. {
  406. AZStd::vector<AZStd::string> matchedSchemas;
  407. // We've already iterate through all the schemas and found source dependencies in CreateJobs
  408. // Retrieve the matched schemas from the job parameters in ProcessJob to avoid redundant work
  409. const auto& paramMap = request.m_jobDescription.m_jobParameters;
  410. auto startIter = paramMap.find(AZ_CRC_CE("sourceDependencyStartPoint"));
  411. auto sourceNumIter = paramMap.find(AZ_CRC_CE("sourceDependenciesNum"));
  412. if (startIter != paramMap.end() && sourceNumIter != paramMap.end())
  413. {
  414. int startPoint = AzFramework::StringFunc::ToInt(startIter->second.c_str());
  415. int sourceDependenciesNum = AzFramework::StringFunc::ToInt(sourceNumIter->second.c_str());
  416. for (int index = 0; index < sourceDependenciesNum; ++index)
  417. {
  418. auto thisSchemaIter = paramMap.find(startPoint + index);
  419. if (thisSchemaIter != paramMap.end())
  420. {
  421. matchedSchemas.emplace_back(thisSchemaIter->second);
  422. }
  423. }
  424. }
  425. // If a schema is found or not found, the result is valid. Return false if there was an error.
  426. return MatchExistingSchema(request.m_fullPath, matchedSchemas, productDependencies, pathDependencies, request.m_watchFolder) != SchemaMatchResult::Error;
  427. }
  428. XmlBuilderWorker::SchemaMatchResult XmlBuilderWorker::MatchLastUsedSchema(
  429. [[maybe_unused]] const AZStd::string& sourceFilePath,
  430. [[maybe_unused]] AZStd::vector<AssetBuilderSDK::ProductDependency>& productDependencies,
  431. [[maybe_unused]] AssetBuilderSDK::ProductPathDependencySet& pathDependencies,
  432. [[maybe_unused]] const AZStd::string& watchFolderPath) const
  433. {
  434. // Check the existing schema info stored in the asset database
  435. // LY-99056
  436. return SchemaMatchResult::NoMatchFound;
  437. }
  438. XmlBuilderWorker::SchemaMatchResult XmlBuilderWorker::MatchExistingSchema(
  439. const AZStd::string& sourceFilePath,
  440. AZStd::vector<AZStd::string>& sourceDependencyPaths,
  441. AZStd::vector<AssetBuilderSDK::ProductDependency>& productDependencies,
  442. AssetBuilderSDK::ProductPathDependencySet& pathDependencies,
  443. const AZStd::string& watchFolderPath) const
  444. {
  445. if (m_printDebug)
  446. {
  447. printf("Searching %zu source dependency paths\n", sourceDependencyPaths.size());
  448. }
  449. if (sourceDependencyPaths.empty())
  450. {
  451. // Iterate through all the schema files if no source dependencies are detected in CreateJobs
  452. for (const AZStd::string& schemaFileDirectory : m_schemaFileDirectories)
  453. {
  454. if (m_printDebug)
  455. {
  456. printf("Finding files in %s\n", schemaFileDirectory.c_str());
  457. }
  458. AZ::Outcome<AZStd::list<AZStd::string>, AZStd::string> searchResult = AzFramework::FileFunc::FindFilesInPath(schemaFileDirectory, SchemaNamePattern, true);
  459. if (searchResult.IsSuccess())
  460. {
  461. AZStd::list<AZStd::string> newSchemaFiles = searchResult.GetValue();
  462. if (m_printDebug)
  463. {
  464. printf("Found %zu files\n", newSchemaFiles.size());
  465. }
  466. for (const AZStd::string& newSchemaFile : newSchemaFiles)
  467. {
  468. if (m_printDebug)
  469. {
  470. printf("Adding %s\n", newSchemaFile.c_str());
  471. }
  472. sourceDependencyPaths.emplace_back(newSchemaFile);
  473. }
  474. }
  475. }
  476. }
  477. for (const AZStd::string& schemaFilePath : sourceDependencyPaths)
  478. {
  479. SchemaMatchResult matchResult = ParseXmlFile(schemaFilePath, sourceFilePath, productDependencies, pathDependencies, watchFolderPath);
  480. if (m_printDebug)
  481. {
  482. printf("Match on %s returns %d\n", schemaFilePath.c_str(), (int)matchResult);
  483. }
  484. switch (matchResult)
  485. {
  486. case SchemaMatchResult::MatchFound:
  487. // Update the LastUsedSchema info stored in the asset database
  488. // LY-99056
  489. AZ_Printf("XmlBuilderWorker", "Schema file %s found for source %s.", schemaFilePath.c_str(), sourceFilePath.c_str());
  490. return matchResult;
  491. case SchemaMatchResult::NoMatchFound:
  492. // Continue searching through schemas if this one didn't match.
  493. break;
  494. case SchemaMatchResult::Error:
  495. default:
  496. return matchResult;
  497. }
  498. }
  499. return SchemaMatchResult::NoMatchFound;
  500. }
  501. XmlBuilderWorker::SchemaMatchResult XmlBuilderWorker::ParseXmlFile(
  502. const AZStd::string& schemaFilePath,
  503. const AZStd::string& sourceFilePath,
  504. AZStd::vector<AssetBuilderSDK::ProductDependency>& productDependencies,
  505. AssetBuilderSDK::ProductPathDependencySet& pathDependencies,
  506. const AZStd::string& watchFolderPath) const
  507. {
  508. if (schemaFilePath.empty())
  509. {
  510. return SchemaMatchResult::NoMatchFound;
  511. }
  512. AzFramework::XmlSchemaAsset schemaAsset;
  513. AZ::ObjectStream::FilterDescriptor loadFilter = AZ::ObjectStream::FilterDescriptor(&AZ::Data::AssetFilterNoAssetLoading, AZ::ObjectStream::FILTERFLAG_IGNORE_UNKNOWN_CLASSES);
  514. if (!AZ::Utils::LoadObjectFromFileInPlace(schemaFilePath, schemaAsset, nullptr, loadFilter))
  515. {
  516. AZ_Error("XmlBuilderWorker", false, "Failed to load schema file: %s.", schemaFilePath.c_str());
  517. // This isn't a blocking error, the error was on this schema, so try checking the next schema for a match.
  518. return SchemaMatchResult::NoMatchFound;
  519. }
  520. // Get the source file root node and version info
  521. AZStd::vector<char> xmlFileBuffer;
  522. AZ::rapidxml::xml_document<char> xmlFileDoc;
  523. AZ::Outcome <AZ::rapidxml::xml_node<char>*, AZStd::string> rootNodeOutcome = Internal::GetSourceFileRootNode(sourceFilePath, xmlFileBuffer, xmlFileDoc);
  524. if (!rootNodeOutcome.IsSuccess())
  525. {
  526. AZ_Error("XmlBuilderWorker", false, rootNodeOutcome.TakeError().c_str());
  527. // The XML file couldn't be loaded.
  528. // We can't know whether this is intentionally an empty file any more than if it were an empty xml with a root node that that were incorrect
  529. // So we leave it as "nothing will match this" and emit the above error
  530. return SchemaMatchResult::NoMatchFound;
  531. }
  532. AZ::rapidxml::xml_node<char>* xmlFileRootNode = rootNodeOutcome.GetValue();
  533. AZStd::string sourceFileVersionStr = Internal::GetSourceFileVersion(xmlFileRootNode, schemaAsset.GetVersionSearchRule().GetRootNodeAttributeName());
  534. AZ::Outcome <AZ::Version<MaxVersionPartsCount>, AZStd::string> SourceFileVersionOutcome = Internal::ParseFromString(sourceFileVersionStr);
  535. if (!SourceFileVersionOutcome.IsSuccess())
  536. {
  537. AZ_Warning("XmlBuilderWorker", false, SourceFileVersionOutcome.TakeError().c_str());
  538. // This isn't a blocking error, the error was on this schema, so try checking the next schema for a match.
  539. return SchemaMatchResult::NoMatchFound;
  540. }
  541. AZ::Version<MaxVersionPartsCount> version = SourceFileVersionOutcome.GetValue();
  542. AZ::Outcome <void, bool> matchingRuleOutcome = SearchForMatchingRule(sourceFilePath, schemaFilePath, version, schemaAsset.GetMatchingRules(), watchFolderPath);
  543. if (!matchingRuleOutcome.IsSuccess())
  544. {
  545. // This isn't a blocking error, the error was on this schema, so try checking the next schema for a match.
  546. return SchemaMatchResult::NoMatchFound;
  547. }
  548. bool dependencySearchRuleResult = false;
  549. if (schemaAsset.UseAZSerialization())
  550. {
  551. AZ::SerializeContext* context = nullptr;
  552. AZ::ComponentApplicationBus::BroadcastResult(context, &AZ::ComponentApplicationBus::Events::GetSerializeContext);
  553. dependencySearchRuleResult = AssetBuilderSDK::GatherProductDependenciesForFile(
  554. *context,
  555. sourceFilePath,
  556. productDependencies,
  557. pathDependencies);
  558. }
  559. else
  560. {
  561. AZStd::string sourceAssetFolder = sourceFilePath;
  562. AzFramework::StringFunc::Path::GetFullPath(sourceAssetFolder.c_str(), sourceAssetFolder);
  563. dependencySearchRuleResult = SearchForDependencySearchRule(xmlFileRootNode, schemaFilePath, version,
  564. schemaAsset.GetDependencySearchRules(), productDependencies, pathDependencies, sourceAssetFolder, watchFolderPath);
  565. if (!dependencySearchRuleResult)
  566. {
  567. AZ_Warning("XmlBuilderWorker", false, "File %s matches schema %s's maching rules defined for version %s,"
  568. "but has no matching dependency search rules. No dependencies will be emitted for this file."
  569. "To resolve this warning, add a new dependency search rule that matches this version and leave it empty if no dependencies need to be emitted.",
  570. sourceFilePath.c_str(), schemaFilePath.c_str(), sourceFileVersionStr.c_str());
  571. }
  572. }
  573. // The schema matched, so return either a match was found or there was an error.
  574. return dependencySearchRuleResult ? SchemaMatchResult::MatchFound : SchemaMatchResult::Error;
  575. }
  576. AZ::Outcome <void, bool> XmlBuilderWorker::SearchForMatchingRule(
  577. const AZStd::string& sourceFilePath,
  578. [[maybe_unused]] const AZStd::string& schemaFilePath,
  579. const AZ::Version<MaxVersionPartsCount>& version,
  580. const AZStd::vector<AzFramework::MatchingRule>& matchingRules,
  581. [[maybe_unused]] const AZStd::string& watchFolderPath) const
  582. {
  583. if (matchingRules.size() == 0)
  584. {
  585. AZ_Error("XmlBuilderWorker", false, "Matching rules are missing.");
  586. return AZ::Failure(false);
  587. }
  588. // Check each matching rule
  589. for (const AzFramework::MatchingRule& matchingRule : matchingRules)
  590. {
  591. if (!matchingRule.Valid())
  592. {
  593. AZ_Error("XmlBuilderWorker", false, "Matching rules defined in schema file %s are invalid.", schemaFilePath.c_str());
  594. return AZ::Failure(false);
  595. }
  596. AZStd::string filePathPattern = matchingRule.GetFilePathPattern();
  597. AZStd::string excludedFilePathPattern = matchingRule.GetExcludedFilePathPattern();
  598. if (!Internal::MatchesVersionConstraints(version, matchingRule.GetVersionConstraints()) ||
  599. !AZStd::wildcard_match(filePathPattern.c_str(), sourceFilePath.c_str()) ||
  600. (!excludedFilePathPattern.empty() && AZStd::wildcard_match(excludedFilePathPattern.c_str(), sourceFilePath.c_str())))
  601. {
  602. continue;
  603. }
  604. else
  605. {
  606. return AZ::Success();
  607. }
  608. }
  609. return AZ::Failure(true);
  610. }
  611. bool XmlBuilderWorker::SearchForDependencySearchRule(
  612. AZ::rapidxml::xml_node<char>* xmlFileRootNode,
  613. [[maybe_unused]] const AZStd::string& schemaFilePath,
  614. const AZ::Version<MaxVersionPartsCount>& version,
  615. const AZStd::vector<AzFramework::DependencySearchRule>& dependencySearchRules,
  616. AZStd::vector<AssetBuilderSDK::ProductDependency>& productDependencies,
  617. AssetBuilderSDK::ProductPathDependencySet& pathDependencies,
  618. const AZStd::string& sourceAssetFolder,
  619. const AZStd::string& watchFolderPath) const
  620. {
  621. if (dependencySearchRules.size() == 0)
  622. {
  623. AZ_Error("XmlBuilderWorker", false, "Dependency search rules are missing.");
  624. return false;
  625. }
  626. for (const AzFramework::DependencySearchRule& dependencySearchRule : dependencySearchRules)
  627. {
  628. if (!Internal::MatchesVersionConstraints(version, dependencySearchRule.GetVersionConstraints()))
  629. {
  630. continue;
  631. }
  632. // Pre-calculate the list of all the XML nodes and mappings from node names to the corresponding nodes
  633. // This could help to reduce the number of traversals when we need to find a match which could appear multiple times in the source file
  634. AZStd::unordered_map<AZStd::string, AZStd::vector<const AZ::rapidxml::xml_node<char>*>> xmlNodeMappings;
  635. AZStd::vector<const AZ::rapidxml::xml_node<char>*> xmlNodeList;
  636. TraverseSourceFile(xmlFileRootNode, xmlNodeMappings, xmlNodeList);
  637. AZStd::vector<AzFramework::SearchRuleDefinition> searchRuleDefinitions = dependencySearchRule.GetSearchRules();
  638. for (const AzFramework::SearchRuleDefinition& searchRuleDefinition : searchRuleDefinitions)
  639. {
  640. AzFramework::XmlSchemaElement searchRuleRootNode = searchRuleDefinition.GetSearchRuleStructure();
  641. AZStd::vector<const AZ::rapidxml::xml_node<char>*> validNodes;
  642. if (searchRuleRootNode.GetName() == "*")
  643. {
  644. // If the schema element node name is "*", it could match any node in the source XML file
  645. // We can use this to specify an attribute which contains product dependency info and could exist in any XML node
  646. validNodes = xmlNodeList;
  647. }
  648. else
  649. {
  650. if (searchRuleDefinition.IsRelativeToXmlRoot())
  651. {
  652. // If the dependency search rule is relative to the root, we will only care about the match at the root level
  653. validNodes = { xmlFileRootNode };
  654. }
  655. else
  656. {
  657. // Otherwise we need to check for any match that appears in the XML file structure
  658. validNodes = xmlNodeMappings[searchRuleRootNode.GetName()];
  659. }
  660. }
  661. for (const AZ::rapidxml::xml_node<char>* validNode : validNodes)
  662. {
  663. Internal::ParseElementNode(searchRuleRootNode, validNode, productDependencies, pathDependencies, sourceAssetFolder, watchFolderPath);
  664. }
  665. }
  666. return true;
  667. }
  668. return false;
  669. }
  670. void XmlBuilderWorker::TraverseSourceFile(
  671. const AZ::rapidxml::xml_node<char>* currentNode,
  672. AZStd::unordered_map<AZStd::string, AZStd::vector<const AZ::rapidxml::xml_node<char>*>>& xmlNodeMappings,
  673. AZStd::vector<const AZ::rapidxml::xml_node<char>*>& xmlNodeList) const
  674. {
  675. if (!currentNode)
  676. {
  677. return;
  678. }
  679. xmlNodeMappings[currentNode->name()].emplace_back(currentNode);
  680. xmlNodeList.emplace_back(currentNode);
  681. for (AZ::rapidxml::xml_node<char>* childNode = currentNode->first_node(); childNode; childNode = childNode->next_sibling())
  682. {
  683. TraverseSourceFile(childNode, xmlNodeMappings, xmlNodeList);
  684. }
  685. }
  686. }