BaseAssetManagerTest.cpp 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411
  1. /*
  2. * Copyright (c) Contributors to the Open 3D Engine Project.
  3. * For complete copyright and license terms please see the LICENSE at the root of this distribution.
  4. *
  5. * SPDX-License-Identifier: Apache-2.0 OR MIT
  6. *
  7. */
  8. #include <Tests/Asset/BaseAssetManagerTest.h>
  9. #include <AzCore/Asset/AssetSerializer.h>
  10. #include <AzCore/Console/IConsole.h>
  11. #include <AzCore/Interface/Interface.h>
  12. #include <AzCore/IO/SystemFile.h>
  13. #include <AzCore/IO/Streamer/Streamer.h>
  14. #include <AzCore/IO/FileIO.h>
  15. #include <AzCore/IO/GenericStreams.h>
  16. #include <AzCore/Math/Crc.h>
  17. #include <AzCore/Jobs/JobManager.h>
  18. #include <AzCore/Jobs/JobContext.h>
  19. #include <AzCore/Outcome/Outcome.h>
  20. #include <AzCore/Serialization/SerializeContext.h>
  21. #include <AzCore/Serialization/ObjectStream.h>
  22. #include <AzCore/Serialization/Utils.h>
  23. #include <AzCore/std/parallel/thread.h>
  24. #include <AzCore/std/functional.h>
  25. #include <AzCore/std/parallel/condition_variable.h>
  26. #include <AzCore/UnitTest/TestTypes.h>
  27. #include <AZTestShared/Utils/Utils.h>
  28. #include <Streamer/IStreamerMock.h>
  29. namespace UnitTest
  30. {
  31. /**
  32. * Find the current status of the reload
  33. */
  34. AZ::Data::AssetData::AssetStatus TestAssetManager::GetReloadStatus(const AssetId& assetId)
  35. {
  36. AZStd::lock_guard<AZStd::recursive_mutex> assetLock(m_assetMutex);
  37. auto reloadInfo = m_reloads.find(assetId);
  38. if (reloadInfo != m_reloads.end())
  39. {
  40. return reloadInfo->second.GetStatus();
  41. }
  42. return AZ::Data::AssetData::AssetStatus::NotLoaded;
  43. }
  44. const AZ::Data::AssetManager::OwnedAssetContainerMap& TestAssetManager::GetAssetContainers() const
  45. {
  46. return m_ownedAssetContainers;
  47. }
  48. const AssetManager::AssetMap& TestAssetManager::GetAssets() const
  49. {
  50. return m_assets;
  51. }
  52. void BaseAssetManagerTest::SetUp()
  53. {
  54. SerializeContextFixture::SetUp();
  55. SuppressTraceOutput(false);
  56. AZ::JobManagerDesc jobDesc;
  57. AZ::JobManagerThreadDesc threadDesc;
  58. for (size_t threadCount = 0; threadCount < GetNumJobManagerThreads(); threadCount++)
  59. {
  60. jobDesc.m_workerThreads.push_back(threadDesc);
  61. }
  62. m_jobManager = aznew AZ::JobManager(jobDesc);
  63. m_jobContext = aznew AZ::JobContext(*m_jobManager);
  64. AZ::JobContext::SetGlobalContext(m_jobContext);
  65. m_prevFileIO = IO::FileIOBase::GetInstance();
  66. IO::FileIOBase::SetInstance(&m_fileIO);
  67. m_streamer = CreateStreamer();
  68. if (m_streamer)
  69. {
  70. Interface<IO::IStreamer>::Register(m_streamer);
  71. }
  72. }
  73. void BaseAssetManagerTest::TearDown()
  74. {
  75. if (m_streamer)
  76. {
  77. Interface<IO::IStreamer>::Unregister(m_streamer);
  78. }
  79. DestroyStreamer(m_streamer);
  80. // Clean up any temporary asset files created during the test.
  81. for (auto& assetName : m_assetsWritten)
  82. {
  83. DeleteAssetFromDisk(assetName);
  84. }
  85. // Make sure to clear the memory from the name storage before shutting down the allocator.
  86. m_assetsWritten.clear();
  87. m_assetsWritten.shrink_to_fit();
  88. IO::FileIOBase::SetInstance(m_prevFileIO);
  89. AZ::JobContext::SetGlobalContext(nullptr);
  90. delete m_jobContext;
  91. delete m_jobManager;
  92. // Reset back to default suppression settings to avoid affecting other tests
  93. SuppressTraceOutput(true);
  94. SerializeContextFixture::TearDown();
  95. }
  96. void BaseAssetManagerTest::SuppressTraceOutput(bool suppress)
  97. {
  98. UnitTest::TestRunner::Instance().m_suppressAsserts = suppress;
  99. UnitTest::TestRunner::Instance().m_suppressErrors = suppress;
  100. UnitTest::TestRunner::Instance().m_suppressWarnings = suppress;
  101. UnitTest::TestRunner::Instance().m_suppressPrintf = suppress;
  102. UnitTest::TestRunner::Instance().m_suppressOutput = suppress;
  103. }
  104. void BaseAssetManagerTest::WriteAssetToDisk(const AZStd::string& assetName, [[maybe_unused]] const AZStd::string& assetIdGuid)
  105. {
  106. AZ::IO::Path assetFileName = GetTestFolderPath() / assetName;
  107. AssetWithCustomData asset;
  108. EXPECT_TRUE(AZ::Utils::SaveObjectToFile(assetFileName.Native(), AZ::DataStream::ST_XML, &asset, m_serializeContext));
  109. // Keep track of every asset written so that we can remove it on teardown
  110. m_assetsWritten.emplace_back(AZStd::move(assetFileName).Native());
  111. }
  112. void BaseAssetManagerTest::DeleteAssetFromDisk(const AZStd::string& assetName)
  113. {
  114. AZ::IO::FileIOBase* fileIO = AZ::IO::FileIOBase::GetInstance();
  115. if (fileIO->Exists(assetName.c_str()))
  116. {
  117. fileIO->Remove(assetName.c_str());
  118. }
  119. }
  120. void BaseAssetManagerTest::BlockUntilAssetJobsAreComplete()
  121. {
  122. auto maxTimeout = AZStd::chrono::steady_clock::now() + DefaultTimeoutSeconds;
  123. while (AssetManager::Instance().HasActiveJobsOrStreamerRequests())
  124. {
  125. if (AZStd::chrono::steady_clock::now() > maxTimeout)
  126. {
  127. break;
  128. }
  129. AZStd::this_thread::yield();
  130. }
  131. EXPECT_FALSE(AssetManager::Instance().HasActiveJobsOrStreamerRequests());
  132. }
  133. MemoryStreamerWrapper::MemoryStreamerWrapper()
  134. {
  135. using ::testing::_;
  136. using ::testing::NiceMock;
  137. using ::testing::Return;
  138. ON_CALL(m_mockStreamer, SuspendProcessing()).WillByDefault([this]()
  139. {
  140. m_suspended = true;
  141. });
  142. ON_CALL(m_mockStreamer, ResumeProcessing()).WillByDefault([this]()
  143. {
  144. AZStd::unique_lock lock(m_mutex);
  145. m_suspended = false;
  146. while (!m_processingQueue.empty())
  147. {
  148. FileRequestHandle requestHandle = m_processingQueue.front();
  149. m_processingQueue.pop();
  150. const auto& onCompleteCallback = GetReadRequest(requestHandle)->m_callback;
  151. if (onCompleteCallback)
  152. {
  153. onCompleteCallback(requestHandle);
  154. }
  155. }
  156. });
  157. ON_CALL(m_mockStreamer, Read(_, ::testing::An<IStreamerTypes::RequestMemoryAllocator&>(), _, _, _, _))
  158. .WillByDefault(
  159. [this](
  160. [[maybe_unused]] AZStd::string_view relativePath, IStreamerTypes::RequestMemoryAllocator& allocator, size_t size,
  161. AZStd::chrono::microseconds deadline, IStreamerTypes::Priority priority, [[maybe_unused]] size_t offset)
  162. {
  163. AZStd::unique_lock lock(m_mutex);
  164. ReadRequest request;
  165. // Save off the requested deadline and priority
  166. request.m_deadline = deadline;
  167. request.m_priority = priority;
  168. request.m_data = allocator.Allocate(size, size, 8);
  169. const auto* virtualFile = FindFile(relativePath);
  170. AZ_Assert(
  171. virtualFile->size() == size, "Streamer read request size did not match size of saved file: %d vs %d (%.*s)",
  172. virtualFile->size(), size,
  173. relativePath.size(), relativePath.data());
  174. AZ_Assert(size > 0, "Size is zero %.*s", relativePath.size(), relativePath.data());
  175. memcpy(request.m_data.m_address, virtualFile->data(), size);
  176. // Create a real file request result and return it
  177. request.m_request = m_context.GetNewExternalRequest();
  178. m_readRequests.push_back(request);
  179. return request.m_request;
  180. });
  181. ON_CALL(m_mockStreamer, SetRequestCompleteCallback(_, _))
  182. .WillByDefault([this](FileRequestPtr& request, AZ::IO::IStreamer::OnCompleteCallback callback) -> FileRequestPtr&
  183. {
  184. // Save off the callback just so that we can call it when the request is "done"
  185. AZStd::unique_lock lock(m_mutex);
  186. ReadRequest* readRequest = GetReadRequest(request);
  187. readRequest->m_callback = callback;
  188. return request;
  189. });
  190. ON_CALL(m_mockStreamer, QueueRequest(_))
  191. .WillByDefault([this](const auto& fileRequest)
  192. {
  193. if (!m_suspended)
  194. {
  195. decltype(ReadRequest::m_callback) onCompleteCallback;
  196. AZStd::unique_lock lock(m_mutex);
  197. ReadRequest* readRequest = GetReadRequest(fileRequest);
  198. onCompleteCallback = readRequest->m_callback;
  199. if (onCompleteCallback)
  200. {
  201. onCompleteCallback(fileRequest);
  202. m_readRequests.erase(readRequest);
  203. }
  204. }
  205. else
  206. {
  207. AZStd::unique_lock lock(m_mutex);
  208. m_processingQueue.push(fileRequest);
  209. }
  210. });
  211. ON_CALL(m_mockStreamer, GetRequestStatus(_))
  212. .WillByDefault([]([[maybe_unused]] FileRequestHandle request)
  213. {
  214. // Return whatever request status has been set in this class
  215. return IO::IStreamerTypes::RequestStatus::Completed;
  216. });
  217. ON_CALL(m_mockStreamer, GetReadRequestResult(_, _, _, _))
  218. .WillByDefault([this](
  219. [[maybe_unused]] FileRequestHandle request, void*& buffer, AZ::u64& numBytesRead,
  220. IStreamerTypes::ClaimMemory claimMemory)
  221. {
  222. // Make sure the requestor plans to free the data buffer we allocated.
  223. EXPECT_EQ(claimMemory, IStreamerTypes::ClaimMemory::Yes);
  224. AZStd::unique_lock lock(m_mutex);
  225. ReadRequest* readRequest = GetReadRequest(request);
  226. // Provide valid data buffer results.
  227. numBytesRead = readRequest->m_data.m_size;
  228. buffer = readRequest->m_data.m_address;
  229. return true;
  230. });
  231. ON_CALL(m_mockStreamer, RescheduleRequest(_, _, _))
  232. .WillByDefault([this](IO::FileRequestPtr target, AZStd::chrono::microseconds newDeadline, IO::IStreamerTypes::Priority newPriority)
  233. {
  234. AZStd::unique_lock lock(m_mutex);
  235. ReadRequest* readRequest = GetReadRequest(target);
  236. readRequest->m_deadline = newDeadline;
  237. readRequest->m_priority = newPriority;
  238. return target;
  239. });
  240. }
  241. ReadRequest* MemoryStreamerWrapper::GetReadRequest(FileRequestHandle request)
  242. {
  243. auto itr = AZStd::find_if(
  244. m_readRequests.begin(), m_readRequests.end(),
  245. [request](const ReadRequest& searchItem) -> bool
  246. {
  247. return (searchItem.m_request == request);
  248. });
  249. return itr;
  250. }
  251. AZStd::vector<char>* MemoryStreamerWrapper::FindFile(AZStd::string_view path)
  252. {
  253. auto itr = m_virtualFiles.find(path);
  254. if (itr == m_virtualFiles.end())
  255. {
  256. // Path didn't work as-is, does it have the test folder prefixed? If so try removing it
  257. AZ::IO::Path testFolderPath = GetTestFolderPath();
  258. if (AZ::IO::PathView(path).IsRelativeTo(testFolderPath))
  259. {
  260. AZ::IO::Path pathWithoutFolder = AZ::IO::PathView(path).LexicallyProximate(testFolderPath).String();
  261. itr = m_virtualFiles.find(pathWithoutFolder);
  262. }
  263. else // Path isn't prefixed, so try adding it
  264. {
  265. itr = m_virtualFiles.find(GetTestFolderPath() / path);
  266. }
  267. }
  268. if (itr != m_virtualFiles.end())
  269. {
  270. return &itr->second;
  271. }
  272. // Currently no test expects a file not to exist so we assert to make it easy to quickly find where something went wrong
  273. // If we ever need to test for a non-existent file this assert should just be conditionally disabled for that specific test
  274. AZ_Assert(false, "Failed to find virtual file %.*s", AZ_STRING_ARG(path))
  275. return nullptr;
  276. }
  277. void DisklessAssetManagerBase::SetUp()
  278. {
  279. using ::testing::_;
  280. using ::testing::NiceMock;
  281. using ::testing::Return;
  282. BaseAssetManagerTest::SetUp();
  283. ON_CALL(m_fileIO, Size(::testing::Matcher<const char*>(::testing::_), _))
  284. .WillByDefault(
  285. [this](const char* path, u64& size)
  286. {
  287. AZStd::scoped_lock lock(m_streamerWrapper->m_mutex);
  288. const auto* file = m_streamerWrapper->FindFile(path);
  289. if (file)
  290. {
  291. size = file->size();
  292. return ResultCode::Success;
  293. }
  294. AZ_Error("DisklessAssetManagerBase", false, "Failed to find virtual file %.*s", path);
  295. return ResultCode::Error;
  296. });
  297. m_prevFileIO = IO::FileIOBase::GetInstance();
  298. IO::FileIOBase::SetInstance(nullptr);
  299. IO::FileIOBase::SetInstance(&m_fileIO);
  300. }
  301. void DisklessAssetManagerBase::TearDown()
  302. {
  303. IO::FileIOBase::SetInstance(nullptr);
  304. IO::FileIOBase::SetInstance(m_prevFileIO);
  305. BaseAssetManagerTest::TearDown();
  306. }
  307. IO::IStreamer* DisklessAssetManagerBase::CreateStreamer()
  308. {
  309. m_streamerWrapper = AZStd::make_unique<MemoryStreamerWrapper>();
  310. return &(m_streamerWrapper->m_mockStreamer);
  311. }
  312. void DisklessAssetManagerBase::DestroyStreamer(IO::IStreamer*)
  313. {
  314. m_streamerWrapper = nullptr;
  315. }
  316. void DisklessAssetManagerBase::WriteAssetToDisk(const AZStd::string& assetName, const AZStd::string&)
  317. {
  318. AZ::IO::Path assetFileName = GetTestFolderPath() / assetName;
  319. AssetWithCustomData asset;
  320. EXPECT_TRUE(m_streamerWrapper->WriteMemoryFile(assetFileName.Native(), &asset, m_serializeContext));
  321. }
  322. void DisklessAssetManagerBase::DeleteAssetFromDisk(const AZStd::string&)
  323. {
  324. }
  325. }