AssetBuilderComponent.cpp 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194
  1. /*
  2. * Copyright (c) Contributors to the Open 3D Engine Project.
  3. * For complete copyright and license terms please see the LICENSE at the root of this distribution.
  4. *
  5. * SPDX-License-Identifier: Apache-2.0 OR MIT
  6. *
  7. */
  8. #include <AssetBuilderApplication.h>
  9. #include <AzCore/Asset/AssetManager.h>
  10. #include <AzCore/Component/TickBus.h>
  11. #include <AzCore/Component/ComponentApplicationBus.h>
  12. #include <AzCore/Component/ComponentApplication.h>
  13. #include <AzCore/IO/IStreamer.h>
  14. #include <AzCore/IO/Path/Path.h>
  15. #include <AzCore/IO/Streamer/FileRequest.h>
  16. #include <AzCore/RTTI/RTTI.h>
  17. #include <AzCore/Serialization/Utils.h>
  18. #include <AzCore/Settings/SettingsRegistryMergeUtils.h>
  19. #include <AzCore/StringFunc/StringFunc.h>
  20. #include <AzCore/std/algorithm.h>
  21. #include <AzCore/Utils/Utils.h>
  22. #include <AzFramework/Asset/AssetProcessorMessages.h>
  23. #include <AzFramework/Asset/AssetSystemBus.h>
  24. #include <AzFramework/IO/LocalFileIO.h>
  25. #include <AzFramework/Network/AssetProcessorConnection.h>
  26. #include <AzFramework/Platform/PlatformDefaults.h>
  27. #include <AzToolsFramework/API/EditorAssetSystemAPI.h>
  28. #include <AzToolsFramework/Debug/TraceContext.h>
  29. #include <AssetBuilderSDK/AssetBuilderSDK.h>
  30. #include <AssetBuilderComponent.h>
  31. #include <AssetBuilderInfo.h>
  32. #include <AzCore/Memory/AllocatorManager.h>
  33. #include <AssetBuilderSDK/AssetBuilderBusses.h>
  34. #include <AzCore/Interface/Interface.h>
  35. #include <AzFramework/Asset/AssetSystemComponent.h>
  36. #include <ToolsComponents/ToolsAssetCatalogComponent.h>
  37. #include <AssetBuilderStatic.h>
  38. // Command-line parameter options:
  39. static const char* const s_paramHelp = "help"; // Print help information.
  40. static const char* const s_paramTask = "task"; // Task to run.
  41. static const char* const s_paramProjectName = "project-name"; // Name of the current project.
  42. static const char* const s_paramProjectCacheRoot = "project-cache-path"; // Full path to the project cache folder.
  43. static const char* const s_paramModule = "module"; // For resident mode, the path to the builder dll folder, otherwise the full path to a single builder dll to use.
  44. static const char* const s_paramPort = "port"; // Optional, port number to use to connect to the AP.
  45. static const char* const s_paramIp = "remoteip"; // optional, IP address to use to connect to the AP
  46. static const char* const s_paramId = "id"; // UUID string that identifies the builder. Only used for resident mode when the AP directly starts up the AssetBuilder.
  47. static const char* const s_paramInput = "input"; // For non-resident mode, full path to the file containing the serialized job request.
  48. static const char* const s_paramOutput = "output"; // For non-resident mode, full path to the file to write the job response to.
  49. static const char* const s_paramDebug = "debug"; // Debug mode for the create and process job of the specified file.
  50. static const char* const s_paramDebugCreate = "debug_create"; // Debug mode for the create job of the specified file.
  51. static const char* const s_paramDebugProcess = "debug_process"; // Debug mode for the process job of the specified file.
  52. static const char* const s_paramPlatformTags = "tags"; // Additional list of tags to add platform tag list.
  53. static const char* const s_paramPlatform = "platform"; // Platform to use
  54. static const char* const s_paramRegisterBuilders = "register"; // Indicates the AP is starting up and requesting a list of registered builders
  55. // Task modes:
  56. static const char* const s_taskResident = "resident"; // stays up and running indefinitely, accepting jobs via network connection
  57. static const char* const s_taskCreateJob = "create"; // runs a builders createJobs function
  58. static const char* const s_taskProcessJob = "process"; // runs processJob function
  59. static const char* const s_taskDebug = "debug"; // runs a one shot job in a fake environment for a specified file.
  60. static const char* const s_taskDebugCreate = "debug_create"; // runs a one shot job in a fake environment for a specified file.
  61. static const char* const s_taskDebugProcess = "debug_process"; // runs a one shot job in a fake environment for a specified file.
  62. //! Scoped Setters for the SettingsRegistry to its previous value on destruction
  63. struct ScopedSettingsRegistrySetter
  64. {
  65. using SettingsRegistrySetterTypes = AZStd::variant<bool, AZ::s64, AZ::u64, double, AZStd::string_view>;
  66. using SettingsRegistryGetterTypes = AZStd::variant<bool, AZ::s64, AZ::u64, double, AZStd::string>;
  67. ScopedSettingsRegistrySetter(AZ::SettingsRegistryInterface& settingsRegistry, AZStd::string_view jsonPointer,
  68. SettingsRegistrySetterTypes newValue)
  69. : m_settingsRegistry(settingsRegistry)
  70. , m_jsonPointer(jsonPointer)
  71. {
  72. AZStd::string oldValue;
  73. if (m_settingsRegistry.Get(oldValue, jsonPointer))
  74. {
  75. m_oldValue = AZStd::move(oldValue);
  76. }
  77. AZStd::visit([this](auto&& value) {m_settingsRegistry.Set(m_jsonPointer, AZStd::move(value)); }, AZStd::move(newValue));
  78. }
  79. ~ScopedSettingsRegistrySetter()
  80. {
  81. // Reset the old value within the Settings Registry if it was set
  82. // Or remove it if not
  83. if (m_oldValue)
  84. {
  85. AZStd::visit([this](auto&& value) {m_settingsRegistry.Set(m_jsonPointer, AZStd::move(value)); }, AZStd::move(*m_oldValue));
  86. }
  87. else
  88. {
  89. m_settingsRegistry.Remove(m_jsonPointer);
  90. }
  91. }
  92. AZ::SettingsRegistryInterface& m_settingsRegistry;
  93. AZStd::string_view m_jsonPointer;
  94. AZStd::optional<SettingsRegistryGetterTypes> m_oldValue;
  95. };
  96. //! FileIO classes which resets the set key to its previous value on destruction
  97. struct ScopedAliasSetter
  98. {
  99. ScopedAliasSetter(AZ::IO::FileIOBase& fileIoBase, const char* alias,
  100. const char* newValue)
  101. : m_fileIoBase(fileIoBase)
  102. , m_alias(alias)
  103. {
  104. if (const char* oldValue = m_fileIoBase.GetAlias(m_alias); oldValue != nullptr)
  105. {
  106. m_oldValue = oldValue;
  107. }
  108. m_fileIoBase.SetAlias(alias, newValue);
  109. }
  110. ~ScopedAliasSetter()
  111. {
  112. // Reset the old alias if it was set or clear it if not
  113. if (m_oldValue)
  114. {
  115. m_fileIoBase.SetAlias(m_alias, m_oldValue->c_str());
  116. }
  117. else
  118. {
  119. m_fileIoBase.ClearAlias(m_alias);
  120. }
  121. }
  122. AZ::IO::FileIOBase& m_fileIoBase;
  123. const char* m_alias;
  124. AZStd::optional<AZStd::string> m_oldValue;
  125. };
  126. //////////////////////////////////////////////////////////////////////////
  127. void AssetBuilderComponent::PrintHelp()
  128. {
  129. AZ_TracePrintf("Help", "\nAssetBuilder is part of the Asset Processor so tasks are run in an isolated environment.\n");
  130. AZ_TracePrintf("Help", "The following command line options are available for the AssetBuilder.\n");
  131. AZ_TracePrintf("Help", "%s - Print help information.\n", s_paramHelp);
  132. AZ_TracePrintf("Help", "%s - Task to run.\n", s_paramTask);
  133. AZ_TracePrintf("Help", "%s - Name of the current project.\n", s_paramProjectName);
  134. AZ_TracePrintf("Help", "%s - Full path to the project cache folder.\n", s_paramProjectCacheRoot);
  135. AZ_TracePrintf("Help", "%s - For resident mode, the path to the builder dll folder, otherwise the full path to a single builder dll to use.\n", s_paramModule);
  136. AZ_TracePrintf("Help", "%s - Optional, port number to use to connect to the AP.\n", s_paramPort);
  137. AZ_TracePrintf("Help", "%s - UUID string that identifies the builder. Only used for resident mode when the AP directly starts up the AssetBuilder.\n", s_paramId);
  138. AZ_TracePrintf("Help", "%s - For non-resident mode, full path to the file containing the serialized job request.\n", s_paramInput);
  139. AZ_TracePrintf("Help", "%s - For non-resident mode, full path to the file to write the job response to.\n", s_paramOutput);
  140. AZ_TracePrintf("Help", "%s - Debug mode for the create and process job of the specified file.\n", s_paramDebug);
  141. AZ_TracePrintf("Help", " Debug mode optionally uses -%s, -%s, -%s, -%s and -gameroot.\n", s_paramInput, s_paramOutput, s_paramModule, s_paramPort);
  142. AZ_TracePrintf("Help", " Example: -%s Objects\\Tutorials\\shapes.fbx\n", s_paramDebug);
  143. AZ_TracePrintf("Help", "%s - Debug mode for the create job of the specified file.\n", s_paramDebugCreate);
  144. AZ_TracePrintf("Help", "%s - Debug mode for the process job of the specified file.\n", s_paramDebugProcess);
  145. AZ_TracePrintf("Help", "%s - Additional tags to add to the debug platform for job processing. One tag can be supplied per option\n", s_paramPlatformTags);
  146. AZ_TracePrintf("Help", "%s - Platform to use for debugging. ex: pc\n", s_paramPlatform);
  147. }
  148. bool AssetBuilderComponent::IsInDebugMode(const AzFramework::CommandLine& commandLine)
  149. {
  150. if (commandLine.HasSwitch(s_paramDebug) || commandLine.HasSwitch(s_paramDebugCreate) || commandLine.HasSwitch(s_paramDebugProcess))
  151. {
  152. return true;
  153. }
  154. if (commandLine.HasSwitch(s_paramTask))
  155. {
  156. const AZStd::string& task = commandLine.GetSwitchValue(s_paramTask, 0);
  157. if (task == s_taskDebug || task == s_taskDebugCreate || task == s_taskDebugProcess)
  158. {
  159. return true;
  160. }
  161. }
  162. return false;
  163. }
  164. void AssetBuilderComponent::Activate()
  165. {
  166. BuilderBus::Handler::BusConnect();
  167. AssetBuilderSDK::AssetBuilderBus::Handler::BusConnect();
  168. AzToolsFramework::AssetDatabase::AssetDatabaseRequestsBus::Handler::BusConnect();
  169. // the asset builder app never writes source files, only assets, so there is no need to do any kind of asset upgrading
  170. AZ::Data::AssetManager::Instance().SetAssetInfoUpgradingEnabled(false);
  171. }
  172. void AssetBuilderComponent::Deactivate()
  173. {
  174. BuilderBus::Handler::BusDisconnect();
  175. AssetBuilderSDK::AssetBuilderBus::Handler::BusDisconnect();
  176. AzFramework::EngineConnectionEvents::Bus::Handler::BusDisconnect();
  177. AzToolsFramework::AssetDatabase::AssetDatabaseRequestsBus::Handler::BusDisconnect();
  178. }
  179. void AssetBuilderComponent::Reflect(AZ::ReflectContext* context)
  180. {
  181. if (AZ::SerializeContext* serializeContext = azrtti_cast<AZ::SerializeContext*>(context))
  182. {
  183. serializeContext->Class<AssetBuilderComponent, AZ::Component>()
  184. ->Version(1);
  185. }
  186. }
  187. bool AssetBuilderComponent::DoHelloPing()
  188. {
  189. using namespace AssetBuilder;
  190. BuilderHelloRequest request;
  191. BuilderHelloResponse response;
  192. AZStd::string id;
  193. if (!GetParameter(s_paramId, id))
  194. {
  195. return false;
  196. }
  197. request.m_uuid = AZ::Uuid::CreateString(id.c_str());
  198. AZ_TracePrintf(
  199. "AssetBuilderComponent", "RunInResidentMode: Pinging asset processor with the builder UUID %s\n",
  200. request.m_uuid.ToString<AZStd::string>().c_str());
  201. bool result = AzFramework::AssetSystem::SendRequest(request, response);
  202. AZ_Error("AssetBuilder", result, "Failed to send hello request to Asset Processor");
  203. // This error is only shown if we successfully got a response AND the response explicitly indicates the AP rejected the builder
  204. AZ_Error("AssetBuilder", !result || response.m_accepted, "Asset Processor rejected connection request");
  205. if (result)
  206. {
  207. AZ_TracePrintf("AssetBuilder", "Builder ID: %s\n", response.m_uuid.ToString<AZStd::string>().c_str());
  208. }
  209. return result;
  210. }
  211. bool AssetBuilderComponent::Run()
  212. {
  213. AZ_TracePrintf("AssetBuilderComponent", "Run: Parsing command line.\n");
  214. const AzFramework::CommandLine* commandLine = nullptr;
  215. AzFramework::ApplicationRequests::Bus::BroadcastResult(commandLine, &AzFramework::ApplicationRequests::GetCommandLine);
  216. if (commandLine->HasSwitch(s_paramHelp))
  217. {
  218. PrintHelp();
  219. UnloadBuilders();
  220. return true;
  221. }
  222. AZStd::string task;
  223. AZStd::string debugFile;
  224. if (GetParameter(s_paramDebug, debugFile, false))
  225. {
  226. task = s_taskDebug;
  227. }
  228. else if (GetParameter(s_paramDebugCreate, debugFile, false))
  229. {
  230. task = s_taskDebugCreate;
  231. }
  232. else if (GetParameter(s_paramDebugProcess, debugFile, false))
  233. {
  234. task = s_taskDebugProcess;
  235. }
  236. else if (!GetParameter(s_paramTask, task))
  237. {
  238. AZ_Error("AssetBuilder", false, "No task specified. Use -help for options.");
  239. UnloadBuilders();
  240. return false;
  241. }
  242. bool isDebugTask = (task == s_taskDebug || task == s_taskDebugCreate || task == s_taskDebugProcess);
  243. if (!GetParameter(s_paramProjectName, m_gameName, !isDebugTask))
  244. {
  245. m_gameName = AZ::Utils::GetProjectName();
  246. }
  247. if (!GetParameter(s_paramProjectCacheRoot, m_gameCache, !isDebugTask))
  248. {
  249. if (!isDebugTask)
  250. {
  251. UnloadBuilders();
  252. return false;
  253. }
  254. }
  255. AZ_TracePrintf("AssetBuilderComponent", "Run: Connecting back to Asset Processor...\n");
  256. bool connectedToAssetProcessor = ConnectToAssetProcessor();
  257. //AP connection is required to access the asset catalog
  258. AZ_Error("AssetBuilder", connectedToAssetProcessor, "Failed to establish a network connection to the AssetProcessor. Use -help for options.");
  259. bool registerBuilders = commandLine->GetNumSwitchValues(s_paramRegisterBuilders) > 0;
  260. IBuilderApplication* builderApplication = AZ::Interface<IBuilderApplication>::Get();
  261. if (!builderApplication)
  262. {
  263. AZ_Error("AssetBuilder", false, "Failed to retreive IBuilderApplication interface");
  264. return false;
  265. }
  266. builderApplication->InitializeBuilderComponents();
  267. bool result = false;
  268. if (connectedToAssetProcessor)
  269. {
  270. if (task == s_taskResident)
  271. {
  272. result = RunInResidentMode(registerBuilders);
  273. }
  274. else if (task == s_taskDebug)
  275. {
  276. static const bool runCreateJobs = true;
  277. static const bool runProcessJob = true;
  278. result = RunDebugTask(AZStd::move(debugFile), runCreateJobs, runProcessJob);
  279. }
  280. else if (task == s_taskDebugCreate)
  281. {
  282. static const bool runCreateJobs = true;
  283. static const bool runProcessJob = false;
  284. result = RunDebugTask(AZStd::move(debugFile), runCreateJobs, runProcessJob);
  285. }
  286. else if (task == s_taskDebugProcess)
  287. {
  288. static const bool runCreateJobs = false;
  289. static const bool runProcessJob = true;
  290. result = RunDebugTask(AZStd::move(debugFile), runCreateJobs, runProcessJob);
  291. }
  292. else
  293. {
  294. result = RunOneShotTask(task);
  295. }
  296. }
  297. // note that we destroy (unload) the builder dlls soon after this (see UnloadBuilders() below),
  298. // so we must tick here before that occurs.
  299. // ticking here causes assets that have a 0 refcount (and are thus in the destroy list) to actually be destroyed.
  300. AZ::SystemTickBus::Broadcast(&AZ::SystemTickBus::Events::OnSystemTick);
  301. AZ_Error("AssetBuilder", result, "Failed to handle `%s` request", task.c_str());
  302. AzFramework::AssetSystemRequestBus::Broadcast(&AzFramework::AssetSystem::AssetSystemRequests::StartDisconnectingAssetProcessor);
  303. UnloadBuilders();
  304. return result;
  305. }
  306. bool AssetBuilderComponent::ConnectToAssetProcessor()
  307. {
  308. //get the asset processor connection params from the bootstrap
  309. AzFramework::AssetSystem::ConnectionSettings connectionSettings;
  310. bool succeeded = AzFramework::AssetSystem::ReadConnectionSettingsFromSettingsRegistry(connectionSettings);
  311. if (!succeeded)
  312. {
  313. AZ_Error("Asset Builder", false, "Getting bootstrap params failed");
  314. return false;
  315. }
  316. //override bootstrap params
  317. //the asset builder may have been given an optional ip to use
  318. AZStd::string overrideIp;
  319. if (GetParameter(s_paramIp, overrideIp, false))
  320. {
  321. connectionSettings.m_assetProcessorIp = overrideIp;
  322. }
  323. //the asset builder may have been given an optional port to use
  324. AZStd::string overridePort;
  325. if (GetParameter(s_paramPort, overridePort, false))
  326. {
  327. connectionSettings.m_assetProcessorPort = static_cast<AZ::u16>(AZStd::stoi(overridePort));
  328. }
  329. //the asset builder may have been given an optional asset platform to use
  330. AZStd::string overrideAssetPlatform;
  331. if (GetParameter(s_paramPlatform, overrideAssetPlatform, false))
  332. {
  333. connectionSettings.m_assetPlatform = overrideAssetPlatform;
  334. }
  335. //the asset builder may have been given an optional project name to use
  336. AZStd::string overrideProjectName;
  337. if (GetParameter(s_paramProjectName, overrideProjectName, false))
  338. {
  339. connectionSettings.m_projectName = overrideProjectName;
  340. }
  341. connectionSettings.m_connectionIdentifier = "Asset Builder";
  342. connectionSettings.m_connectionDirection = AzFramework::AssetSystem::ConnectionSettings::ConnectionDirection::ConnectToAssetProcessor;
  343. connectionSettings.m_launchAssetProcessorOnFailedConnection = false; // builders shouldn't launch the AssetProcessor
  344. connectionSettings.m_waitUntilAssetProcessorIsReady = false; // builders are what make the AssetProcessor ready, so the cannot wait until the AssetProcessor is ready
  345. connectionSettings.m_waitForConnect = true; // application is a builder so it needs to wait for a connection
  346. //connect to Asset Processor.
  347. bool connectedToAssetProcessor = false;
  348. AzFramework::AssetSystemRequestBus::BroadcastResult(connectedToAssetProcessor,
  349. &AzFramework::AssetSystemRequestBus::Events::EstablishAssetProcessorConnection, connectionSettings);
  350. return connectedToAssetProcessor;
  351. }
  352. //////////////////////////////////////////////////////////////////////////
  353. bool AssetBuilderComponent::SendRegisteredBuildersToAp()
  354. {
  355. AssetBuilder::BuilderRegistrationRequest registrationRequest;
  356. for (const auto& [uuid, desc] : m_assetBuilderDescMap)
  357. {
  358. AssetBuilder::BuilderRegistration registration;
  359. registration.m_name = desc->m_name;
  360. registration.m_analysisFingerprint = desc->m_analysisFingerprint;
  361. registration.m_flags = desc->m_flags;
  362. registration.m_flagsByJobKey = desc->m_flagsByJobKey;
  363. registration.m_version = desc->m_version;
  364. registration.m_busId = desc->m_busId;
  365. registration.m_patterns = desc->m_patterns;
  366. registration.m_productsToKeepOnFailure = desc->m_productsToKeepOnFailure;
  367. registrationRequest.m_builders.push_back(AZStd::move(registration));
  368. }
  369. bool result = SendRequest(registrationRequest);
  370. AZ_Error("AssetBuilder", result, "Failed to send builder registration request to Asset Processor");
  371. return result;
  372. }
  373. bool AssetBuilderComponent::RunInResidentMode(bool sendRegistration)
  374. {
  375. using namespace AssetBuilder;
  376. using namespace AZStd::placeholders;
  377. AZ_TracePrintf("AssetBuilderComponent", "RunInResidentMode: Starting resident mode (waiting for commands to arrive)\n");
  378. AzFramework::SocketConnection::GetInstance()->AddMessageHandler(CreateJobsNetRequest::MessageType(), AZStd::bind(&AssetBuilderComponent::CreateJobsResidentHandler, this, _1, _2, _3, _4));
  379. AzFramework::SocketConnection::GetInstance()->AddMessageHandler(ProcessJobNetRequest::MessageType(), AZStd::bind(&AssetBuilderComponent::ProcessJobResidentHandler, this, _1, _2, _3, _4));
  380. bool result = DoHelloPing() && ((sendRegistration && SendRegisteredBuildersToAp()) || !sendRegistration);
  381. if (result)
  382. {
  383. m_running = true;
  384. m_jobThreadDesc.m_name = "Builder Job Thread";
  385. m_jobThread = AZStd::thread(m_jobThreadDesc, AZStd::bind(&AssetBuilderComponent::JobThread, this));
  386. AzFramework::EngineConnectionEvents::Bus::Handler::BusConnect(); // Listen for disconnects
  387. AZ_TracePrintf("AssetBuilder", "Resident mode ready\n");
  388. m_mainEvent.acquire();
  389. AZ_TracePrintf("AssetBuilder", "Shutting down\n");
  390. m_running = false;
  391. }
  392. if (m_jobThread.joinable())
  393. {
  394. m_jobEvent.release();
  395. m_jobThread.join();
  396. }
  397. return result;
  398. }
  399. bool AssetBuilderComponent::RunDebugTask(AZStd::string&& debugFile, bool runCreateJobs, bool runProcessJob)
  400. {
  401. AZ_TracePrintf("AssetBuilderComponent", "RunDebugTask - running debug task on file : %s\n", debugFile.c_str());
  402. AZ_TracePrintf("AssetBuilderComponent", "RunDebugTask - CreateJobs: %s\n", runCreateJobs ? "True" : "False");
  403. AZ_TracePrintf("AssetBuilderComponent", "RunDebugTask - ProcessJob: %s\n", runProcessJob ? "True" : "False");
  404. if (debugFile.empty())
  405. {
  406. if (!GetParameter(s_paramInput, debugFile))
  407. {
  408. AZ_Error("AssetBuilder", false, "No input file was specified. Use -help for options.");
  409. return false;
  410. }
  411. }
  412. AZ::StringFunc::Path::Normalize(debugFile);
  413. if (!GetParameter(s_paramProjectCacheRoot, m_gameCache, false))
  414. {
  415. if (m_gameCache.empty())
  416. {
  417. // Query the project cache root path from the Settings Registry
  418. auto settingsRegistry = AZ::SettingsRegistry::Get();
  419. if (!settingsRegistry || !settingsRegistry->Get(m_gameCache, AZ::SettingsRegistryMergeUtils::FilePathKey_CacheProjectRootFolder))
  420. {
  421. m_gameCache = ".";
  422. }
  423. }
  424. }
  425. bool result = false;
  426. AZ::Data::AssetInfo info;
  427. AZStd::string watchFolder;
  428. AzToolsFramework::AssetSystemRequestBus::BroadcastResult(result, &AzToolsFramework::AssetSystemRequestBus::Events::GetSourceInfoBySourcePath, debugFile.c_str(), info, watchFolder);
  429. if (!result)
  430. {
  431. AZ_Error("AssetBuilder", false, "Failed to locate asset info for '%s'.", debugFile.c_str());
  432. return false;
  433. }
  434. AZStd::string binDir;
  435. AZStd::string module;
  436. if (GetParameter(s_paramModule, module, false))
  437. {
  438. AZ::StringFunc::Path::GetFullPath(module.c_str(), binDir);
  439. if (!LoadBuilder(module))
  440. {
  441. AZ_Error("AssetBuilder", false, "Failed to load module '%s'.", module.c_str());
  442. return false;
  443. }
  444. }
  445. else
  446. {
  447. const char* executableFolder = nullptr;
  448. AZ::ComponentApplicationBus::BroadcastResult(executableFolder, &AZ::ComponentApplicationBus::Events::GetExecutableFolder);
  449. if (!executableFolder)
  450. {
  451. AZ_Error("AssetBuilder", false, "Unable to determine application root.");
  452. return false;
  453. }
  454. AZ::StringFunc::Path::Join(executableFolder, "Builders", binDir);
  455. if (!LoadBuilders(binDir))
  456. {
  457. AZ_Error("AssetBuilder", false, "Failed to load one or more builders from '%s'.", binDir.c_str());
  458. return false;
  459. }
  460. }
  461. AZStd::string baseTempDirPath;
  462. if (!GetParameter(s_paramOutput, baseTempDirPath, false))
  463. {
  464. AZStd::string fileName;
  465. AZ::StringFunc::Path::GetFullFileName(debugFile.c_str(), fileName);
  466. AZStd::replace(fileName.begin(), fileName.end(), '.', '_');
  467. AZ::StringFunc::Path::Join(binDir.c_str(), "Debug", baseTempDirPath);
  468. AZ::StringFunc::Path::Join(baseTempDirPath.c_str(), fileName.c_str(), baseTempDirPath);
  469. }
  470. // Default tags for the debug task are "tools" and "debug"
  471. // Additional tags are parsed from command line parameters
  472. AZStd::unordered_set<AZStd::string> platformTags{"tools", "debug"};
  473. {
  474. const AzFramework::CommandLine* commandLine = nullptr;
  475. AzFramework::ApplicationRequests::Bus::BroadcastResult(commandLine, &AzFramework::ApplicationRequests::GetCommandLine);
  476. if (commandLine)
  477. {
  478. size_t tagSwitchSize = commandLine->GetNumSwitchValues(s_paramPlatformTags);
  479. for (int tagIndex = 0; tagIndex < tagSwitchSize; ++tagIndex)
  480. {
  481. platformTags.emplace(commandLine->GetSwitchValue(s_paramPlatformTags, tagIndex));
  482. }
  483. }
  484. }
  485. AZStd::string platform;
  486. if(!GetParameter(s_paramPlatform, platform, false))
  487. {
  488. platform = "debug platform";
  489. }
  490. auto* fileIO = AZ::IO::FileIOBase::GetInstance();
  491. for (auto& it : m_assetBuilderDescMap)
  492. {
  493. AZStd::unique_ptr<AssetBuilderSDK::AssetBuilderDesc>& builder = it.second;
  494. AZ_Assert(builder, "Invalid description for builder registered.");
  495. if (!IsBuilderForFile(info.m_relativePath, *builder))
  496. {
  497. AZ_TracePrintf(AssetBuilderSDK::InfoWindow, "Skipping '%s'.\n", builder->m_name.c_str());
  498. continue;
  499. }
  500. AZ_TracePrintf(AssetBuilderSDK::InfoWindow, "Debugging builder '%s'.\n", builder->m_name.c_str());
  501. AZStd::string tempDirPath;
  502. AZ::StringFunc::Path::Join(baseTempDirPath.c_str(), builder->m_name.c_str(), tempDirPath);
  503. AZStd::vector<AssetBuilderSDK::PlatformInfo> enabledDebugPlatformInfos =
  504. {
  505. {
  506. platform.c_str(), platformTags
  507. }
  508. };
  509. AZStd::vector<AssetBuilderSDK::JobDescriptor> jobDescriptions;
  510. if (runCreateJobs)
  511. {
  512. AZStd::string createJobsTempDirPath;
  513. AZ::StringFunc::Path::Join(tempDirPath.c_str(), "CreateJobs", createJobsTempDirPath);
  514. AZ::IO::Result fileResult = fileIO->CreatePath(createJobsTempDirPath.c_str());
  515. if (!fileResult)
  516. {
  517. AZ_Error("AssetBuilder", false, "Unable to create or clear debug folder '%s'.", createJobsTempDirPath.c_str());
  518. return false;
  519. }
  520. AssetBuilderSDK::CreateJobsRequest createRequest(builder->m_busId, info.m_relativePath, watchFolder,
  521. enabledDebugPlatformInfos, info.m_assetId.m_guid);
  522. AZ_TraceContext("Source", debugFile);
  523. AZ_TraceContext("Platforms", AssetBuilderSDK::PlatformInfo::PlatformVectorAsString(createRequest.m_enabledPlatforms));
  524. AssetBuilderSDK::CreateJobsResponse createResponse;
  525. builder->m_createJobFunction(createRequest, createResponse);
  526. AZStd::string responseFile;
  527. AZ::StringFunc::Path::Join(createJobsTempDirPath.c_str(), "CreateJobsResponse.xml", responseFile);
  528. if (!AZ::Utils::SaveObjectToFile(responseFile, AZ::DataStream::ST_XML, &createResponse))
  529. {
  530. AZ_Error("AssetBuilder", false, "Failed to serialize response to file: %s", responseFile.c_str());
  531. return false;
  532. }
  533. if (runProcessJob)
  534. {
  535. jobDescriptions = AZStd::move(createResponse.m_createJobOutputs);
  536. }
  537. }
  538. AZ::SystemTickBus::Broadcast(&AZ::SystemTickBus::Events::OnSystemTick); // flush assets in case any are present with 0 refcount.
  539. if (runProcessJob)
  540. {
  541. AZStd::string processJobTempDirPath;
  542. AZ::StringFunc::Path::Join(tempDirPath.c_str(), "ProcessJobs", processJobTempDirPath);
  543. AZ::IO::Result fileResult = fileIO->CreatePath(processJobTempDirPath.c_str());
  544. if (!fileResult)
  545. {
  546. AZ_Error("AssetBuilder", false, "Unable to create debug or clear folder '%s'.", processJobTempDirPath.c_str());
  547. return false;
  548. }
  549. AssetBuilderSDK::PlatformInfo enabledDebugPlatformInfo = {
  550. platform.c_str(), { "tools", "debug" }
  551. };
  552. AssetBuilderSDK::ProcessJobRequest processRequest;
  553. processRequest.m_watchFolder = watchFolder;
  554. processRequest.m_sourceFile = info.m_relativePath;
  555. processRequest.m_platformInfo = enabledDebugPlatformInfo;
  556. processRequest.m_sourceFileUUID = info.m_assetId.m_guid;
  557. AZ::StringFunc::AssetDatabasePath::Join(processRequest.m_watchFolder.c_str(), processRequest.m_sourceFile.c_str(), processRequest.m_fullPath);
  558. processRequest.m_tempDirPath = processJobTempDirPath;
  559. processRequest.m_jobId = 0;
  560. processRequest.m_builderGuid = builder->m_busId;
  561. processRequest.m_platformInfo.m_tags = platformTags;
  562. AZ_TraceContext("Source", debugFile);
  563. if (jobDescriptions.empty())
  564. {
  565. for (const auto& platformInfo : enabledDebugPlatformInfos)
  566. {
  567. AssetBuilderSDK::JobDescriptor placeholder;
  568. placeholder.SetPlatformIdentifier(platformInfo.m_identifier.c_str());
  569. placeholder.m_jobKey = AZStd::string::format("%s_DEBUG", builder->m_name.c_str());
  570. placeholder.m_jobParameters[AZ_CRC_CE("Debug")] = "true";
  571. jobDescriptions.emplace_back(AZStd::move(placeholder));
  572. }
  573. }
  574. for (size_t i = 0; i < jobDescriptions.size(); ++i)
  575. {
  576. AssetBuilderSDK::AssetBuilderTraceBus::Broadcast(&AssetBuilderSDK::AssetBuilderTraceBus::Events::ResetErrorCount);
  577. AssetBuilderSDK::AssetBuilderTraceBus::Broadcast(&AssetBuilderSDK::AssetBuilderTraceBus::Events::ResetWarningCount);
  578. processRequest.m_jobDescription = jobDescriptions[i];
  579. AssetBuilderSDK::ProcessJobResponse processResponse;
  580. ProcessJob(builder->m_processJobFunction, processRequest, processResponse);
  581. AZStd::string responseFile;
  582. AZ::StringFunc::Path::Join(processJobTempDirPath.c_str(),
  583. AZStd::string::format("%zu_%s", i, AssetBuilderSDK::s_processJobResponseFileName).c_str(), responseFile);
  584. if (!AZ::Utils::SaveObjectToFile(responseFile, AZ::DataStream::ST_XML, &processResponse))
  585. {
  586. AZ_Error("AssetBuilder", false, "Failed to serialize response to file: %s", responseFile.c_str());
  587. return false;
  588. }
  589. }
  590. }
  591. }
  592. return true;
  593. }
  594. void AssetBuilderComponent::FlushFileStreamerCache()
  595. {
  596. // Force a file streamer flush to ensure that file handles don't remain used or locked between jobs.
  597. auto streamer = AZ::Interface<AZ::IO::IStreamer>::Get();
  598. AZStd::binary_semaphore wait;
  599. AZ::IO::FileRequestPtr flushRequest = streamer->FlushCaches();
  600. streamer->SetRequestCompleteCallback(flushRequest, [&wait]([[maybe_unused]] AZ::IO::FileRequestHandle request)
  601. {
  602. wait.release();
  603. });
  604. streamer->QueueRequest(flushRequest);
  605. wait.acquire();
  606. }
  607. void AssetBuilderComponent::ProcessJob(const AssetBuilderSDK::ProcessJobFunction& job, const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& outResponse)
  608. {
  609. // Setup the alias' as appropriate to the job in question.
  610. auto ioBase = AZ::IO::FileIOBase::GetInstance();
  611. AZ_Assert(ioBase != nullptr, "AZ::IO::FileIOBase must be ready for use.");
  612. auto settingsRegistry = AZ::SettingsRegistry::Get();
  613. AZ_Assert(settingsRegistry != nullptr, "SettingsRegistry must be ready for use in the AssetBuilder.");
  614. // The root path is the cache plus the platform name.
  615. AZ::IO::FixedMaxPath newProjectCache(m_gameCache);
  616. // Check if the platform identifier is a valid "asset platform"
  617. // If so, use it, other wise use the OS default platform as a fail safe
  618. // This is to make sure the "debug platform" isn't added as a path segment
  619. // the Cache ProjectCache folder
  620. if (AzFramework::PlatformHelper::GetPlatformIdFromName(request.m_platformInfo.m_identifier) != AzFramework::PlatformId::Invalid)
  621. {
  622. newProjectCache /= request.m_platformInfo.m_identifier;
  623. }
  624. else
  625. {
  626. newProjectCache /= AzFramework::OSPlatformToDefaultAssetPlatform(AZ_TRAIT_OS_PLATFORM_CODENAME);
  627. }
  628. // Now set the paths and run the job.
  629. {
  630. // Save out the prior paths.
  631. ScopedAliasSetter projectPlatformCacheAliasScope(*ioBase, "@products@", newProjectCache.c_str());
  632. ScopedSettingsRegistrySetter cacheRootFolderScope(*settingsRegistry,
  633. AZ::SettingsRegistryMergeUtils::FilePathKey_CacheRootFolder, newProjectCache.Native());
  634. // Invoke the Process Job function
  635. job(request, outResponse);
  636. }
  637. // The asset building ProcessJob method might read any number of source files while processing the asset.
  638. // Ensure that any exclusive file handle locks caused by this are cleared so that other AssetBuilder processes
  639. // running in parallel have the ability to read those files as well.
  640. // This needs to occur after the ProcessJob call, but before the file aliases get cleared.
  641. FlushFileStreamerCache();
  642. UpdateResultCode(request, outResponse);
  643. }
  644. bool AssetBuilderComponent::RunOneShotTask(const AZStd::string& task)
  645. {
  646. AZ_TracePrintf("AssetBuilderComponent", "RunOneShotTask - running one-shot task [%s]\n", task.c_str());
  647. // Load the requested module. This is not a required param for the task, since the builders can be in gems.
  648. AZStd::string modulePath;
  649. if (GetParameter(s_paramModule, modulePath) && !LoadBuilder(modulePath))
  650. {
  651. return false;
  652. }
  653. AZStd::string inputFilePath, outputFilePath;
  654. if (!GetParameter(s_paramInput, inputFilePath)
  655. || !GetParameter(s_paramOutput, outputFilePath))
  656. {
  657. return false;
  658. }
  659. AZ::StringFunc::Path::Normalize(inputFilePath);
  660. AZ::StringFunc::Path::Normalize(outputFilePath);
  661. if (task == s_taskCreateJob)
  662. {
  663. auto func = [this](const AssetBuilderSDK::CreateJobsRequest& request, AssetBuilderSDK::CreateJobsResponse& response)
  664. {
  665. AZ_TraceContext("Source", request.m_sourceFile);
  666. AZ_TraceContext("Platforms", AssetBuilderSDK::PlatformInfo::PlatformVectorAsString(request.m_enabledPlatforms));
  667. auto assetBuilderDescIt = m_assetBuilderDescMap.find(request.m_builderid);
  668. if (assetBuilderDescIt != m_assetBuilderDescMap.end())
  669. {
  670. assetBuilderDescIt->second->m_createJobFunction(request, response);
  671. }
  672. else
  673. {
  674. AZ_Error("AssetBuilder", false, "Builder UUID [%s] does not exist in the AssetBuilderDescMap for source file %s",
  675. request.m_builderid.ToString<AZStd::fixed_string<64>>().c_str(), request.m_sourceFile.c_str());
  676. }
  677. // The asset building CreateJob method might read any number of source files to gather a dependency list.
  678. // Ensure that any exclusive file handle locks caused by this are cleared so that other AssetBuilder processes
  679. // running in parallel have the ability to read those files as well.
  680. FlushFileStreamerCache();
  681. };
  682. return HandleTask<AssetBuilderSDK::CreateJobsRequest, AssetBuilderSDK::CreateJobsResponse>(inputFilePath, outputFilePath, func);
  683. }
  684. else if (task == s_taskProcessJob)
  685. {
  686. auto func = [this](const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response)
  687. {
  688. AZ_TraceContext("Source", request.m_fullPath);
  689. AZ_TraceContext("Platform", request.m_platformInfo.m_identifier);
  690. auto assetBuilderDescIt = m_assetBuilderDescMap.find(request.m_builderGuid);
  691. if (assetBuilderDescIt != m_assetBuilderDescMap.end())
  692. {
  693. ProcessJob(assetBuilderDescIt->second->m_processJobFunction, request, response);
  694. }
  695. else
  696. {
  697. AZ_Error("AssetBuilder", false, "Builder UUID [%s] does not exist in the AssetBuilderDescMap for source file %s",
  698. request.m_builderGuid.ToString<AZStd::fixed_string<64>>().c_str(), request.m_sourceFile.c_str());
  699. }
  700. };
  701. return HandleTask<AssetBuilderSDK::ProcessJobRequest, AssetBuilderSDK::ProcessJobResponse>(inputFilePath, outputFilePath, func);
  702. }
  703. else
  704. {
  705. AZ_Error("AssetBuilder", false, "Unknown task");
  706. return false;
  707. }
  708. }
  709. void AssetBuilderComponent::Disconnected(AzFramework::SocketConnection* /*connection*/)
  710. {
  711. // If we lose connection to the AP, print out an error and shut down.
  712. // This prevents builders from running indefinitely if the AP crashes
  713. AZ_Error("AssetBuilder", false, "Lost connection to Asset Processor, shutting down");
  714. m_mainEvent.release();
  715. }
  716. bool AssetBuilderComponent::GetAssetDatabaseLocation(AZStd::string& location)
  717. {
  718. AZ_Error("AssetBuilder", false,
  719. "Accessing the database directly from a builder is not supported. Many queries behave unexpectedly from builders as the Asset"
  720. "Processor continuously updates tables as well as risking dead locks. Please use the AssetSystemRequestBus or similar buses "
  721. "to safely query information from the database.");
  722. location = "<Unsupported>";
  723. return false;
  724. }
  725. template<typename TNetRequest, typename TNetResponse>
  726. void AssetBuilderComponent::ResidentJobHandler(AZ::u32 serial, const void* data, AZ::u32 dataLength, JobType jobType)
  727. {
  728. auto job = AZStd::make_unique<Job>();
  729. job->m_netResponse = AZStd::make_unique<TNetResponse>();
  730. job->m_requestSerial = serial;
  731. job->m_jobType = jobType;
  732. auto* request = AZ::Utils::LoadObjectFromBuffer<TNetRequest>(data, dataLength);
  733. if (!request)
  734. {
  735. AZ_Error("AssetBuilder", false, "Problem deserializing net request");
  736. AzFramework::AssetSystem::SendResponse(*(job->m_netResponse), serial);
  737. return;
  738. }
  739. job->m_netRequest = AZStd::unique_ptr<TNetRequest>(request);
  740. // Queue up the job for the worker thread
  741. {
  742. AZStd::lock_guard<AZStd::mutex> lock(m_jobMutex);
  743. if (!m_queuedJob)
  744. {
  745. m_queuedJob.swap(job);
  746. }
  747. else
  748. {
  749. AZ_Error("AssetBuilder", false, "Builder already has a job queued");
  750. AzFramework::AssetSystem::SendResponse(*(job->m_netResponse), serial);
  751. return;
  752. }
  753. }
  754. // Wake up the job thread
  755. m_jobEvent.release();
  756. }
  757. bool AssetBuilderComponent::IsBuilderForFile(const AZStd::string& filePath, const AssetBuilderSDK::AssetBuilderDesc& builderDescription) const
  758. {
  759. for (const AssetBuilderSDK::AssetBuilderPattern& pattern : builderDescription.m_patterns)
  760. {
  761. AssetBuilderSDK::FilePatternMatcher matcher(pattern);
  762. if (matcher.MatchesPath(filePath))
  763. {
  764. return true;
  765. }
  766. }
  767. return false;
  768. }
  769. void AssetBuilderComponent::JobThread()
  770. {
  771. while (m_running)
  772. {
  773. m_jobEvent.acquire();
  774. AZStd::unique_ptr<Job> job;
  775. {
  776. AZStd::lock_guard<AZStd::mutex> lock(m_jobMutex);
  777. job.swap(m_queuedJob);
  778. }
  779. if (!job)
  780. {
  781. if (m_running)
  782. {
  783. AZ_TracePrintf("AssetBuilder", "JobThread woke up, but there was no queued job\n");
  784. }
  785. continue;
  786. }
  787. AssetBuilderSDK::AssetBuilderTraceBus::Broadcast(&AssetBuilderSDK::AssetBuilderTraceBus::Events::ResetErrorCount);
  788. AssetBuilderSDK::AssetBuilderTraceBus::Broadcast(&AssetBuilderSDK::AssetBuilderTraceBus::Events::ResetWarningCount);
  789. size_t allocatedBytesBefore = 0;
  790. size_t capacityBytesBefore = 0;
  791. AZ::AllocatorManager::Instance().GarbageCollect();
  792. AZ::AllocatorManager::Instance().GetAllocatorStats(allocatedBytesBefore, capacityBytesBefore);
  793. AZ_TracePrintf("AssetBuilder", "AllocatorManager before: allocatedBytes = %zu capacityBytes = %zu\n", allocatedBytesBefore, capacityBytesBefore);
  794. switch (job->m_jobType)
  795. {
  796. case JobType::Create:
  797. {
  798. using namespace AssetBuilder;
  799. auto* netRequest = azrtti_cast<CreateJobsNetRequest*>(job->m_netRequest.get());
  800. auto* netResponse = azrtti_cast<CreateJobsNetResponse*>(job->m_netResponse.get());
  801. AZ_Assert(netRequest && netResponse, "Request or response is null");
  802. AZ::IO::FixedMaxPath fullPath(netRequest->m_request.m_watchFolder);
  803. fullPath /= netRequest->m_request.m_sourceFile;
  804. AZ_TracePrintf("AssetBuilder", "Source = %s\n", fullPath.c_str());
  805. AZ_TracePrintf("AssetBuilder", "Platforms = %s\n", AssetBuilderSDK::PlatformInfo::PlatformVectorAsString(netRequest->m_request.m_enabledPlatforms).c_str());
  806. auto assetBuilderDescIt = m_assetBuilderDescMap.find(netRequest->m_request.m_builderid);
  807. if (assetBuilderDescIt != m_assetBuilderDescMap.end())
  808. {
  809. assetBuilderDescIt->second->m_createJobFunction(netRequest->m_request, netResponse->m_response);
  810. }
  811. else
  812. {
  813. AZ_Error("AssetBuilder", false, "Builder UUID [%s] does not exist in the AssetBuilderDescMap for source file %s",
  814. netRequest->m_request.m_builderid.ToString<AZStd::fixed_string<64>>().c_str(), netRequest->m_request.m_sourceFile.c_str());
  815. }
  816. break;
  817. }
  818. case JobType::Process:
  819. {
  820. using namespace AssetBuilder;
  821. AZ_TracePrintf("AssetBuilder", "Running processJob task\n");
  822. auto* netRequest = azrtti_cast<ProcessJobNetRequest*>(job->m_netRequest.get());
  823. auto* netResponse = azrtti_cast<ProcessJobNetResponse*>(job->m_netResponse.get());
  824. AZ_Assert(netRequest && netResponse, "Request or response is null");
  825. AZ_TracePrintf("AssetBuilder", "Source = %s\n", netRequest->m_request.m_fullPath.c_str());
  826. AZ_TracePrintf("AssetBuilder", "Platform = %s\n", netRequest->m_request.m_jobDescription.GetPlatformIdentifier().c_str());
  827. auto assetBuilderDescIt = m_assetBuilderDescMap.find(netRequest->m_request.m_builderGuid);
  828. if (assetBuilderDescIt != m_assetBuilderDescMap.end())
  829. {
  830. auto* toolsCatalog = AZ::Interface<AssetProcessor::IToolsAssetCatalog>::Get();
  831. if (toolsCatalog)
  832. {
  833. toolsCatalog->SetActivePlatform(netRequest->m_request.m_jobDescription.GetPlatformIdentifier());
  834. }
  835. else
  836. {
  837. AZ_Warning("AssetBuilder", false, "Failed to retrieve IToolsAssetCatalog interface, cannot set current platform");
  838. }
  839. ProcessJob(assetBuilderDescIt->second->m_processJobFunction, netRequest->m_request, netResponse->m_response);
  840. }
  841. else
  842. {
  843. AZ_Error("AssetBuilder", false, "Builder UUID [%s] does not exist in the AssetBuilderDescMap for source file %s",
  844. netRequest->m_request.m_builderGuid.ToString<AZStd::fixed_string<64>>().c_str(), netRequest->m_request.m_sourceFile.c_str());
  845. }
  846. break;
  847. }
  848. default:
  849. AZ_Error("AssetBuilder", false, "Unhandled job request type");
  850. continue;
  851. }
  852. size_t allocatedBytesAfter = 0;
  853. size_t capacityBytesAfter = 0;
  854. AZ::AllocatorManager::Instance().GarbageCollect();
  855. AZ_MALLOC_TRIM(0);
  856. AZ::AllocatorManager::Instance().GetAllocatorStats(allocatedBytesAfter, capacityBytesAfter);
  857. AZ_TracePrintf("AssetBuilder", "AllocatorManager after: allocatedBytes = %zu capacityBytes = %zu; allocated change = %zd\n",
  858. allocatedBytesAfter, capacityBytesAfter, allocatedBytesAfter - allocatedBytesBefore);
  859. AZ::u32 warningCount, errorCount;
  860. AssetBuilderSDK::AssetBuilderTraceBus::BroadcastResult(warningCount, &AssetBuilderSDK::AssetBuilderTraceBus::Events::GetWarningCount);
  861. AssetBuilderSDK::AssetBuilderTraceBus::BroadcastResult(errorCount, &AssetBuilderSDK::AssetBuilderTraceBus::Events::GetErrorCount);
  862. AZ_TracePrintf("S", "%d errors, %d warnings\n", errorCount, warningCount);
  863. //Flush our output so the AP can properly associate all output with the current job
  864. std::fflush(stdout);
  865. std::fflush(stderr);
  866. AZ::SystemTickBus::Broadcast(&AZ::SystemTickBus::Events::OnSystemTick);
  867. AZ::TickBus::Broadcast(&AZ::TickEvents::OnTick, 0.00f, AZ::ScriptTimePoint(AZStd::chrono::steady_clock::now()));
  868. AZ::AllocatorManager::Instance().GarbageCollect();
  869. AzFramework::AssetSystem::SendResponse(*(job->m_netResponse), job->m_requestSerial);
  870. }
  871. }
  872. void AssetBuilderComponent::CreateJobsResidentHandler(AZ::u32 /*typeId*/, AZ::u32 serial, const void* data, AZ::u32 dataLength)
  873. {
  874. using namespace AssetBuilder;
  875. ResidentJobHandler<CreateJobsNetRequest, CreateJobsNetResponse>(serial, data, dataLength, JobType::Create);
  876. }
  877. void AssetBuilderComponent::ProcessJobResidentHandler(AZ::u32 /*typeId*/, AZ::u32 serial, const void* data, AZ::u32 dataLength)
  878. {
  879. using namespace AssetBuilder;
  880. ResidentJobHandler<ProcessJobNetRequest, ProcessJobNetResponse>(serial, data, dataLength, JobType::Process);
  881. }
  882. //////////////////////////////////////////////////////////////////////////
  883. template<typename TRequest, typename TResponse>
  884. bool AssetBuilderComponent::HandleTask(const AZStd::string& inputFilePath, const AZStd::string& outputFilePath, const AZStd::function<void(const TRequest& request, TResponse& response)>& assetBuilderFunc)
  885. {
  886. TRequest request;
  887. TResponse response;
  888. if (!AZ::Utils::LoadObjectFromFileInPlace(inputFilePath, request))
  889. {
  890. AZ_Error("AssetBuilder", false, "Failed to deserialize request from file: %s", inputFilePath.c_str());
  891. return false;
  892. }
  893. assetBuilderFunc(request, response);
  894. if (!AZ::Utils::SaveObjectToFile(outputFilePath, AZ::DataStream::ST_XML, &response))
  895. {
  896. AZ_Error("AssetBuilder", false, "Failed to serialize response to file: %s", outputFilePath.c_str());
  897. return false;
  898. }
  899. return true;
  900. }
  901. void AssetBuilderComponent::UpdateResultCode(const AssetBuilderSDK::ProcessJobRequest& request, AssetBuilderSDK::ProcessJobResponse& response) const
  902. {
  903. if (request.m_jobDescription.m_failOnError)
  904. {
  905. AZ::u32 errorCount = 0;
  906. AssetBuilderSDK::AssetBuilderTraceBus::BroadcastResult(errorCount, &AssetBuilderSDK::AssetBuilderTraceBus::Events::GetErrorCount);
  907. if (errorCount > 0 && response.m_resultCode == AssetBuilderSDK::ProcessJobResult_Success)
  908. {
  909. response.m_resultCode = AssetBuilderSDK::ProcessJobResult_Failed;
  910. }
  911. }
  912. }
  913. bool AssetBuilderComponent::GetParameter(const char* paramName, AZStd::string& outValue, [[maybe_unused]] bool required /*= true*/) const
  914. {
  915. const AzFramework::CommandLine* commandLine = nullptr;
  916. AzFramework::ApplicationRequests::Bus::BroadcastResult(commandLine, &AzFramework::ApplicationRequests::GetCommandLine);
  917. size_t optionCount = commandLine->GetNumSwitchValues(paramName);
  918. if (optionCount > 0)
  919. {
  920. outValue = commandLine->GetSwitchValue(paramName, optionCount - 1);
  921. }
  922. if (outValue.empty())
  923. {
  924. AZ_Error("AssetBuilder", !required, "Missing required parameter `%s`. Use -help for options.", paramName);
  925. return false;
  926. }
  927. return true;
  928. }
  929. const char* AssetBuilderComponent::GetLibraryExtension()
  930. {
  931. return "*" AZ_TRAIT_OS_DYNAMIC_LIBRARY_EXTENSION;
  932. }
  933. bool AssetBuilderComponent::LoadBuilders([[maybe_unused]] const AZStd::string& builderFolder)
  934. {
  935. // LoadBuilders by folder has been removed. Builders should all live within gems
  936. AZ_TracePrintf("AssetBuilderComponent", "LoadBuilders - Called LoadBuilders for [%s] - SKIPPING\n", builderFolder.c_str());
  937. return true;
  938. }
  939. bool AssetBuilderComponent::LoadBuilder(const AZStd::string& filePath)
  940. {
  941. using AssetBuilder::AssetBuilderType;
  942. auto assetBuilderInfo = AZStd::make_unique<AssetBuilder::ExternalModuleAssetBuilderInfo>(QString::fromUtf8(filePath.c_str()));
  943. if (assetBuilderInfo->GetAssetBuilderType() == AssetBuilderType::Valid)
  944. {
  945. if (!assetBuilderInfo->IsLoaded())
  946. {
  947. AZ_Warning("AssetBuilder", false, "AssetBuilder was not able to load the library: %s\n", filePath.c_str());
  948. return false;
  949. }
  950. }
  951. AssetBuilderType builderType = assetBuilderInfo->GetAssetBuilderType();
  952. if (builderType == AssetBuilderType::Valid)
  953. {
  954. AZ_TracePrintf("AssetBuilder", "LoadBuilder - Initializing and registering builder [%s]\n", assetBuilderInfo->GetName().toUtf8().constData());
  955. m_currentAssetBuilder = assetBuilderInfo.get();
  956. m_currentAssetBuilder->Initialize();
  957. m_currentAssetBuilder = nullptr;
  958. m_assetBuilderInfoList.push_back(AZStd::move(assetBuilderInfo));
  959. return true;
  960. }
  961. if (builderType == AssetBuilderType::Invalid)
  962. {
  963. return false;
  964. }
  965. return true;
  966. }
  967. void AssetBuilderComponent::UnloadBuilders()
  968. {
  969. m_assetBuilderDescMap.clear();
  970. for (auto& assetBuilderInfo : m_assetBuilderInfoList)
  971. {
  972. AZ_TracePrintf("AssetBuilderComponent", "UnloadBuilders - unloading builder [%s]\n", assetBuilderInfo->GetName().toUtf8().constData());
  973. assetBuilderInfo->UnInitialize();
  974. }
  975. m_assetBuilderInfoList.clear();
  976. }
  977. bool AssetBuilderComponent::FindBuilderInformation(const AZ::Uuid& builderGuid, AssetBuilderSDK::AssetBuilderDesc& descriptionOut)
  978. {
  979. auto iter = m_assetBuilderDescMap.find(builderGuid);
  980. if (iter != m_assetBuilderDescMap.end())
  981. {
  982. descriptionOut = *iter->second;
  983. return true;
  984. }
  985. else
  986. {
  987. return false;
  988. }
  989. }
  990. void AssetBuilderComponent::RegisterBuilderInformation(const AssetBuilderSDK::AssetBuilderDesc& builderDesc)
  991. {
  992. m_assetBuilderDescMap.insert({ builderDesc.m_busId, AZStd::make_unique<AssetBuilderSDK::AssetBuilderDesc>(builderDesc) });
  993. if (m_currentAssetBuilder)
  994. {
  995. m_currentAssetBuilder->RegisterBuilderDesc(builderDesc.m_busId);
  996. }
  997. }
  998. void AssetBuilderComponent::RegisterComponentDescriptor(AZ::ComponentDescriptor* descriptor)
  999. {
  1000. if (m_currentAssetBuilder)
  1001. {
  1002. m_currentAssetBuilder->RegisterComponentDesc(descriptor);
  1003. }
  1004. }