DOOM-3-BFG/neo/framework/File_Resource.cpp

/*
===========================================================================

Doom 3 BFG Edition GPL Source Code
Copyright (C) 1993-2012 id Software LLC, a ZeniMax Media company. 

This file is part of the Doom 3 BFG Edition GPL Source Code ("Doom 3 BFG Edition Source Code").  

Doom 3 BFG Edition Source Code is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

Doom 3 BFG Edition Source Code is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Doom 3 BFG Edition Source Code.  If not, see <http://www.gnu.org/licenses/>.

In addition, the Doom 3 BFG Edition Source Code is also subject to certain additional terms. You should have received a copy of these additional terms immediately following the terms and conditions of the GNU General Public License which accompanied the Doom 3 BFG Edition Source Code.  If not, please request a copy in writing from id Software at the address below.

If you have questions concerning this license or the applicable additional terms, you may contact in writing id Software LLC, c/o ZeniMax Media Inc., Suite 120, Rockville, Maryland 20850 USA.

===========================================================================
*/

#include "../idlib/precompiled.h"
#pragma hdrstop

/*
================================================================================================

idResourceContainer

================================================================================================
*/

/*
========================
idResourceContainer::ReOpen 
========================
*/ 
void idResourceContainer::ReOpen() {
	delete resourceFile;
	resourceFile = fileSystem->OpenFileRead( fileName );
}

/*
========================
idResourceContainer::Init 
========================
*/ 
bool idResourceContainer::Init( const char *_fileName, uint8 containerIndex ) {

	if ( idStr::Icmp( _fileName, "_ordered.resources" ) == 0 ) {
		resourceFile = fileSystem->OpenFileReadMemory( _fileName );
	} else {
		resourceFile = fileSystem->OpenFileRead( _fileName );
	}

	if ( resourceFile == NULL ) {
		idLib::Warning( "Unable to open resource file %s", _fileName );
		return false;
	}

	resourceFile->ReadBig( resourceMagic );
	if ( resourceMagic != RESOURCE_FILE_MAGIC ) {
		idLib::FatalError( "resourceFileMagic != RESOURCE_FILE_MAGIC" );
	}

	fileName = _fileName;

	resourceFile->ReadBig( tableOffset );
	resourceFile->ReadBig( tableLength );
	// read this into a memory buffer with a single read
	char * const buf = (char *)Mem_Alloc( tableLength, TAG_RESOURCE );
	resourceFile->Seek( tableOffset, FS_SEEK_SET );
	resourceFile->Read( buf, tableLength );
	idFile_Memory memFile( "resourceHeader", (const char *)buf, tableLength );

	// Parse the resourceFile header, which includes every resource used
	// by the game.
	memFile.ReadBig( numFileResources );

	cacheTable.SetNum( numFileResources );

	for ( int i = 0; i < numFileResources; i++ ) {
		idResourceCacheEntry &rt = cacheTable[ i ];
		rt.Read( &memFile );
		rt.filename.BackSlashesToSlashes();
		rt.filename.ToLower();
		rt.containerIndex = containerIndex;

		const int key = cacheHash.GenerateKey( rt.filename, false );
		bool found = false;
		//for ( int index = cacheHash.GetFirst( key ); index != idHashIndex::NULL_INDEX; index = cacheHash.GetNext( index ) ) {
		//	idResourceCacheEntry & rtc = cacheTable[ index ];
		//	if ( idStr::Icmp( rtc.filename, rt.filename ) == 0 ) {
		//		found = true;
		//		break;
		//	}
		//}
		if ( !found ) {
			//idLib::Printf( "rez file name: %s\n", rt.filename.c_str() );
			cacheHash.Add( key, i );
		}
	}
	Mem_Free( buf );

	return true;
}


/*
========================
idResourceContainer::WriteManifestFile 
========================
*/ 
void idResourceContainer::WriteManifestFile( const char *name, const idStrList &list ) {
	idStr filename( name );
	filename.SetFileExtension( "manifest" );
	filename.Insert( "maps/", 0 );
	idFile *outFile = fileSystem->OpenFileWrite( filename );
	if ( outFile != NULL ) {
		int num = list.Num();
		outFile->WriteBig( num );
		for ( int i = 0; i < num; i++ ) {
			outFile->WriteString( list[ i ] );
		}
		delete outFile;
	}
}

/*
========================
idResourceContainer::ReadManifestFile 
========================
*/ 
int idResourceContainer::ReadManifestFile( const char *name, idStrList &list ) {
	idFile *inFile = fileSystem->OpenFileRead( name );
	if ( inFile != NULL ) {
		list.SetGranularity( 16384 );
		idStr str;
		int num;
		list.Clear();
		inFile->ReadBig( num );
		for ( int i = 0; i < num; i++ ) {
			inFile->ReadString( str );
			list.Append( str );
		}
		delete inFile;
	}
	return list.Num();
}


/*
========================
idResourceContainer::UpdateResourceFile 
========================
*/ 
void idResourceContainer::UpdateResourceFile( const char *_filename, const idStrList &_filesToUpdate ) {
	idFile *outFile = fileSystem->OpenFileWrite( va( "%s.new", _filename ) );
	if ( outFile == NULL ) {
		idLib::Warning( "Unable to open resource file %s or new output file", _filename );
		return;
	}

	uint32 magic = 0;
	int _tableOffset = 0;
	int _tableLength = 0;
	idList< idResourceCacheEntry > entries;
	idStrList filesToUpdate = _filesToUpdate;

	idFile *inFile = fileSystem->OpenFileRead( _filename );
	if ( inFile == NULL ) {
		magic = RESOURCE_FILE_MAGIC;

		outFile->WriteBig( magic );
		outFile->WriteBig( _tableOffset );
		outFile->WriteBig( _tableLength );

	} else {
		inFile->ReadBig( magic );
		if ( magic != RESOURCE_FILE_MAGIC ) {
			delete inFile;
			return;
		}

		inFile->ReadBig( _tableOffset );
		inFile->ReadBig( _tableLength );
		// read this into a memory buffer with a single read
		char * const buf = (char *)Mem_Alloc( _tableLength, TAG_RESOURCE );
		inFile->Seek( _tableOffset, FS_SEEK_SET );
		inFile->Read( buf, _tableLength );
		idFile_Memory memFile( "resourceHeader", (const char *)buf, _tableLength );

		int _numFileResources = 0;
		memFile.ReadBig( _numFileResources );

		outFile->WriteBig( magic );
		outFile->WriteBig( _tableOffset );
		outFile->WriteBig( _tableLength );

		entries.SetNum( _numFileResources );

		for ( int i = 0; i < _numFileResources; i++ ) {
			entries[ i ].Read( &memFile );


			idLib::Printf( "examining %s\n", entries[ i ].filename.c_str() );
			byte * fileData = NULL;

			for ( int j = filesToUpdate.Num() - 1; j >= 0; j-- ) {
				if ( filesToUpdate[ j ].Icmp( entries[ i ].filename ) == 0 ) {
					idFile *newFile = fileSystem->OpenFileReadMemory( filesToUpdate[ j ] );
					if ( newFile != NULL ) {
						idLib::Printf( "Updating %s\n", filesToUpdate[ j ].c_str() );
						entries[ i ].length = newFile->Length();
						fileData = (byte *)Mem_Alloc( entries[ i ].length, TAG_TEMP );
						newFile->Read( fileData, newFile->Length() );
						delete newFile;
					}
					filesToUpdate.RemoveIndex( j );
				}
			}

			if ( fileData == NULL ) {
				inFile->Seek( entries[ i ].offset, FS_SEEK_SET );
				fileData = (byte *)Mem_Alloc( entries[ i ].length, TAG_TEMP );
				inFile->Read( fileData, entries[ i ].length );
			}

			entries[ i ].offset = outFile->Tell();
			outFile->Write( ( void* )fileData, entries[ i ].length );

			Mem_Free( fileData );
		}

		Mem_Free( buf );
	}

	while ( filesToUpdate.Num() > 0 ) {
		idFile *newFile = fileSystem->OpenFileReadMemory( filesToUpdate[ 0 ] );
		if ( newFile != NULL ) {
			idLib::Printf( "Appending %s\n", filesToUpdate[ 0 ].c_str() );
			idResourceCacheEntry rt;
			rt.filename = filesToUpdate[ 0 ];
			rt.length = newFile->Length();
			byte * fileData = (byte *)Mem_Alloc( rt.length, TAG_TEMP );
			newFile->Read( fileData, rt.length );
			int idx = entries.Append( rt );
			if ( idx >= 0 ) {
				entries[ idx ].offset = outFile->Tell();
				outFile->Write( ( void* )fileData, entries[ idx ].length );
			}
			delete newFile;
			Mem_Free( fileData );
		}
		filesToUpdate.RemoveIndex( 0 );
	}

	_tableOffset = outFile->Tell();
	outFile->WriteBig( entries.Num() );

	// write the individual resource entries
	for ( int i = 0; i < entries.Num(); i++ ) {
		entries[ i ].Write( outFile );
	}

	// go back and write the header offsets again, now that we have file offsets and lengths
	_tableLength = outFile->Tell() - _tableOffset;
	outFile->Seek( 0, FS_SEEK_SET );
	outFile->WriteBig( magic );
	outFile->WriteBig( _tableOffset );
	outFile->WriteBig( _tableLength );

	delete outFile;
	delete inFile;
}


/*
========================
idResourceContainer::ExtractResourceFile 
========================
*/ 
void idResourceContainer::SetContainerIndex( const int & _idx ) {
	for ( int i = 0; i < cacheTable.Num(); i++ ) {
		cacheTable[ i ].containerIndex = _idx;
	}
}

/*
========================
idResourceContainer::ExtractResourceFile 
========================
*/ 
void idResourceContainer::ExtractResourceFile ( const char * _fileName, const char * _outPath, bool _copyWavs ) {
	idFile *inFile = fileSystem->OpenFileRead( _fileName );

	if ( inFile == NULL ) {
		idLib::Warning( "Unable to open resource file %s", _fileName );
		return;
	}

	uint32 magic;
	inFile->ReadBig( magic );
	if ( magic != RESOURCE_FILE_MAGIC ) {
		delete inFile;
		return;
	}

	int _tableOffset;
	int _tableLength;
	inFile->ReadBig( _tableOffset );
	inFile->ReadBig( _tableLength );
	// read this into a memory buffer with a single read
	char * const buf = (char *)Mem_Alloc( _tableLength, TAG_RESOURCE );
	inFile->Seek( _tableOffset, FS_SEEK_SET );
	inFile->Read( buf, _tableLength );
	idFile_Memory memFile( "resourceHeader", (const char *)buf, _tableLength );

	int _numFileResources;
	memFile.ReadBig( _numFileResources );

	for ( int i = 0; i < _numFileResources; i++ ) {
		idResourceCacheEntry rt;
		rt.Read( &memFile );
		rt.filename.BackSlashesToSlashes();
		rt.filename.ToLower();
		byte *fbuf = NULL;
		if ( _copyWavs && ( rt.filename.Find( ".idwav" ) >= 0 ||  rt.filename.Find( ".idxma" ) >= 0 ||  rt.filename.Find( ".idmsf" ) >= 0 ) ) {
			rt.filename.SetFileExtension( "wav" );
			rt.filename.Replace( "generated/", "" );
			int len = fileSystem->GetFileLength( rt.filename );
			fbuf =  (byte *)Mem_Alloc( len, TAG_RESOURCE );
			fileSystem->ReadFile( rt.filename, (void**)&fbuf, NULL );
		} else {
			inFile->Seek( rt.offset, FS_SEEK_SET );
			fbuf =  (byte *)Mem_Alloc( rt.length, TAG_RESOURCE );
			inFile->Read( fbuf, rt.length );
		}
		idStr outName = _outPath;
		outName.AppendPath( rt.filename );
		idFile *outFile = fileSystem->OpenExplicitFileWrite( outName );
		if ( outFile != NULL ) {
			outFile->Write( ( byte* )fbuf, rt.length );
			delete outFile;
		}
		Mem_Free( fbuf );
	}
	delete inFile;
	Mem_Free( buf );
}



/*
========================
idResourceContainer::Open 
========================
*/ 
void idResourceContainer::WriteResourceFile( const char *manifestName, const idStrList &manifest, const bool &_writeManifest ) {

	if ( manifest.Num() == 0 ) {
		return;
	}

	idLib::Printf( "Writing resource file %s\n", manifestName );

	// build multiple output files at 1GB each
	idList < idStrList > outPutFiles;

	idFileManifest outManifest;
	int64 size = 0;
	idStrList flist;
	flist.SetGranularity( 16384 );
	for ( int i = 0; i < manifest.Num(); i++ ) {
		flist.Append( manifest[ i ] );
		size += fileSystem->GetFileLength( manifest[ i ] );
		if ( size > 1024 * 1024 * 1024 ) {
			outPutFiles.Append( flist );
			size = 0;
			flist.Clear();
		}
		outManifest.AddFile( manifest[ i ] );
	}

	outPutFiles.Append( flist );

	if ( _writeManifest ) {
		idStr temp = manifestName;
		temp.Replace( "maps/", "manifests/" );
		temp.StripFileExtension();
		temp.SetFileExtension( "manifest" );
		outManifest.WriteManifestFile( temp );
	}

	for ( int idx = 0; idx < outPutFiles.Num(); idx++ ) {

		idStrList &fileList = outPutFiles[ idx ];
		if ( fileList.Num() == 0 ) {
			continue;
		}

		idStr fileName = manifestName;
		if ( idx > 0 ) {
			fileName = va( "%s_%02d", manifestName, idx );
		} 
		fileName.SetFileExtension( "resources" );

		idFile *resFile = fileSystem->OpenFileWrite( fileName );

		if ( resFile == NULL ) {
			idLib::Warning( "Cannot open %s for writing.\n", fileName.c_str() );
			return;
		}

		idLib::Printf( "Writing resource file %s\n", fileName.c_str() );

		int	tableOffset = 0;
		int	tableLength = 0;
		int	tableNewLength = 0;
		uint32	resourceFileMagic = RESOURCE_FILE_MAGIC;

		resFile->WriteBig( resourceFileMagic );
		resFile->WriteBig( tableOffset );
		resFile->WriteBig( tableLength );

		idList< idResourceCacheEntry > entries;

		entries.Resize( fileList.Num() );

		for ( int i = 0; i < fileList.Num(); i++ ) {
			idResourceCacheEntry ent;

			ent.filename = fileList[ i ];
			ent.length = 0;
			ent.offset = 0;

			idFile *file = fileSystem->OpenFileReadMemory( ent.filename, false );
			idFile_Memory *fm = dynamic_cast< idFile_Memory* >( file );
			if ( fm == NULL ) {
				continue;
			}
			// if the entry is uncompressed, align the file pointer to a 16 byte boundary
			// so it will be usable if memory mapped
			ent.length = fm->Length();

			// always get the offset, even if the file will have zero length
			ent.offset = resFile->Tell();

			entries.Append( ent );

			if ( ent.length == 0 ) {
				ent.filename = "";
				delete fm;
				continue;
			}

			resFile->Write( fm->GetDataPtr(), ent.length );

			delete fm;

			// pacifier every ten megs
			if ( ( ent.offset + ent.length ) / 10000000 != ent.offset / 10000000 ) {
				idLib::Printf( "." );
			}
		}

		idLib::Printf( "\n" );

		// write the table out now that we have all the files
		tableOffset = resFile->Tell();

		// count how many we are going to write for this platform
		int	numFileResources = entries.Num();

		resFile->WriteBig( numFileResources );

		// write the individual resource entries
		for ( int i = 0; i < entries.Num(); i++ ) {
			entries[ i ].Write( resFile );
			if ( i + 1 == numFileResources ) {
				// we just wrote out the last new entry
				tableNewLength = resFile->Tell() - tableOffset;
			}
		}

		// go back and write the header offsets again, now that we have file offsets and lengths
		tableLength = resFile->Tell() - tableOffset;
		resFile->Seek( 0, FS_SEEK_SET );
		resFile->WriteBig( resourceFileMagic );
		resFile->WriteBig( tableOffset );
		resFile->WriteBig( tableLength );
		delete resFile;
	}
}