DOOM-3-BFG/neo/idlib/LangDict.cpp

/*
===========================================================================

Doom 3 BFG Edition GPL Source Code
Copyright (C) 1993-2012 id Software LLC, a ZeniMax Media company. 

This file is part of the Doom 3 BFG Edition GPL Source Code ("Doom 3 BFG Edition Source Code").  

Doom 3 BFG Edition Source Code is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

Doom 3 BFG Edition Source Code is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Doom 3 BFG Edition Source Code.  If not, see <http://www.gnu.org/licenses/>.

In addition, the Doom 3 BFG Edition Source Code is also subject to certain additional terms. You should have received a copy of these additional terms immediately following the terms and conditions of the GNU General Public License which accompanied the Doom 3 BFG Edition Source Code.  If not, please request a copy in writing from id Software at the address below.

If you have questions concerning this license or the applicable additional terms, you may contact in writing id Software LLC, c/o ZeniMax Media Inc., Suite 120, Rockville, Maryland 20850 USA.

===========================================================================
*/
#pragma hdrstop
#include "precompiled.h"

// This is the default language dict that the entire system uses, but you can instantiate your own idLangDict classes to manipulate a language dictionary in a tool
idLangDict	idLocalization::languageDict;

idCVar lang_maskLocalizedStrings( "lang_maskLocalizedStrings", "0", CVAR_BOOL, "Masks all localized strings to help debugging.  When set will replace strings with an equal length of W's and ending in an X.  Note: The masking occurs at string table load time." );

/*
========================
idLocalization::ClearDictionary
========================
*/
void idLocalization::ClearDictionary() {
	languageDict.Clear();
}

/*
========================
idLocalization::LoadDictionary
========================
*/
bool idLocalization::LoadDictionary( const byte * data, int dataLen, const char * fileName ) {
	return languageDict.Load( data, dataLen, fileName );
}

/*
========================
idLocalization::GetString 
========================
*/
const char * idLocalization::GetString( const char * inString ) {
	return languageDict.GetString( inString );
}

/*
========================
idLocalization::FindString 
========================
*/
const char * idLocalization::FindString( const char * inString ) {
	return languageDict.FindString( inString );
}

/*
========================
idLocalization::VerifyUTF8
========================
*/
utf8Encoding_t idLocalization::VerifyUTF8( const uint8 * buffer, const int bufferLen, const char * name ) {
	utf8Encoding_t encoding;
	idStr::IsValidUTF8( buffer, bufferLen, encoding );
	if ( encoding == UTF8_INVALID ) {
		idLib::FatalError( "Language file %s is not valid UTF-8 or plain ASCII.", name );
	} else if ( encoding == UTF8_INVALID_BOM ) {
		idLib::FatalError( "Language file %s is marked as UTF-8 but has invalid encoding.", name );
	} else if ( encoding == UTF8_ENCODED_NO_BOM ) {
		idLib::FatalError( "Language file %s has no byte order marker. Fix this or roll back to a version that has the marker.", name );
	} else if ( encoding != UTF8_ENCODED_BOM && encoding != UTF8_PURE_ASCII ) {
		idLib::FatalError( "Language file %s has unknown utf8Encoding_t.", name );
	}
	return encoding;
}

// string entries can refer to other string entries, 
// recursing up to this many times before we decided someone did something stupid
const char * idLangDict::KEY_PREFIX = "#str_";	// all keys should be prefixed with this for redirection to work
const int idLangDict::KEY_PREFIX_LEN = idStr::Length( KEY_PREFIX );

/*
========================
idLangDict::idLangDict
========================
*/
idLangDict::idLangDict() : keyIndex( 4096, 4096 ) {
}

/*
========================
idLangDict::~idLangDict
========================
*/
idLangDict::~idLangDict() {
	Clear();
}

/*
========================
idLangDict::Clear
========================
*/
void idLangDict::Clear() {
	//mem.PushHeap();
	for ( int i = 0; i < keyVals.Num(); i++ ) {
		if ( keyVals[i].value == NULL ) {
			continue;
		}
		blockAlloc.Free( keyVals[i].value );
		keyVals[i].value = NULL;
	}
	//mem.PopHeap();
}

/*
========================
idLangDict::Load
========================
*/
bool idLangDict::Load( const byte * buffer, const int bufferLen, const char *name ) {

	if ( buffer == NULL || bufferLen <= 0 ) {
		// let whoever called us deal with the failure (so sys_lang can be reset)
		return false;
	}

	idLib::Printf( "Reading %s", name );

	bool utf8 = false;

	// in all but retail builds, ensure that the byte-order mark is NOT MISSING so that
	// we can avoid debugging UTF-8 code
#ifndef ID_RETAIL
	utf8Encoding_t encoding = idLocalization::VerifyUTF8( buffer, bufferLen, name );
	if ( encoding == UTF8_ENCODED_BOM ) {
		utf8 = true;
	} else if ( encoding == UTF8_PURE_ASCII ) {
		utf8 = false;
	} else {
		assert( false );	// this should have been handled in VerifyUTF8 with a FatalError
		return false;
	}
#else
	// in release we just check the BOM so we're not scanning the lang file twice on startup
	if ( bufferLen > 3 && buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF ) {
		utf8 = true;
	}
#endif

	if ( utf8 ) {
		idLib::Printf( " as UTF-8\n" );
	} else {
		idLib::Printf( " as ASCII\n" );
	}

	idStr tempKey;
	idStr tempVal;

	int line = 0;
	int numStrings = 0;

	int i = 0;
	while ( i < bufferLen ) {
		uint32 c = buffer[i++];
		if ( c == '/' ) { // comment, read until new line
			while ( i < bufferLen ) {
				c = buffer[i++];
				if ( c == '\n' ) {
					line++;
					break;
				}
			}
		} else if ( c == '}' ) {
			break;
		} else if ( c == '\n' ) {
			line++;
		} else if ( c == '\"' ) {
			int keyStart = i;
			int keyEnd = -1;
			while ( i < bufferLen ) {
				c = buffer[i++];
				if ( c == '\"' ) {
					keyEnd = i - 1;
					break;
				}
			}
			if ( keyEnd < keyStart ) {
				idLib::FatalError( "%s File ended while reading key at line %d", name, line );
			}
			tempKey.CopyRange( (char *)buffer, keyStart, keyEnd );

			int valStart = -1;
			while ( i < bufferLen ) {
				c = buffer[i++];
				if ( c == '\"' ) {
					valStart = i;
					break;
				}
			}
			if ( valStart < 0 ) {
				idLib::FatalError( "%s File ended while reading value at line %d", name, line );
			}
			int valEnd = -1;
			tempVal.CapLength( 0 );
			while ( i < bufferLen ) {
				c = utf8 ? idStr::UTF8Char( buffer, i ) : buffer[i++];
				if ( !utf8 && c >= 0x80 ) {
					// this is a serious error and we must check this to avoid accidentally shipping a file where someone squased UTF-8 encodings
					idLib::FatalError( "Language file %s is supposed to be plain ASCII, but has byte values > 127!", name );
				}
				if ( c == '\"' ) {
					valEnd = i - 1;
					continue;
				}
				if ( c == '\n' ) {
					line++;
					break;
				}
				if ( c == '\r' ) {
					continue;
				}
				if ( c == '\\' ) {
					c = utf8 ? idStr::UTF8Char( buffer, i ) : buffer[i++];
					if ( c == 'n' ) {
						c = '\n';
					} else if ( c == 't' ) {
						c = '\t';
					} else if ( c == '\"' ) {
						c = '\"';
					} else if ( c == '\\' ) {
						c = '\\';
					} else {
						idLib::Warning( "Unknown escape sequence %x at line %d", c, line );
					}
				}
				tempVal.AppendUTF8Char( c );
			}
			if ( valEnd < valStart ) {
				idLib::FatalError( "%s File ended while reading value at line %d", name, line );
			}
			if ( lang_maskLocalizedStrings.GetBool() && tempVal.Length() > 0 && tempKey.Find( "#font_" ) == -1 ) {
				int len = tempVal.Length();
				if ( len > 0 ) {
					tempVal.Fill( 'W', len - 1 );
				} else {
					tempVal.Empty();
				}
				tempVal.Append( 'X' );
			}
			AddKeyVal( tempKey, tempVal );
			numStrings++;
		}
	}

	idLib::Printf( "%i strings read\n", numStrings );
	
	// get rid of any waste due to geometric list growth
	//mem.PushHeap();
	keyVals.Condense();
	//mem.PopHeap();

	return true;
}

/*
========================
idLangDict::Save
========================
*/
bool idLangDict::Save( const char * fileName ) {
	idFile * outFile = fileSystem->OpenFileWrite( fileName );
	if ( outFile == NULL ) {
		idLib::Warning( "Error saving: %s", fileName );
		return false;
	}
	byte bof[3] = { 0xEF, 0xBB, 0xBF };
	outFile->Write( bof, 3 );
	outFile->WriteFloatString( "// string table\n//\n\n{\n" );
	for ( int j = 0; j < keyVals.Num(); j++ ) {
		const idLangKeyValue & kvp = keyVals[j];
		if ( kvp.value == NULL ) {
			continue;
		}
		outFile->WriteFloatString( "\t\"%s\"\t\"", kvp.key );
		for ( int k = 0; kvp.value[k] != 0; k++ ) {
			char ch = kvp.value[k];
			if ( ch == '\t' ) {
				outFile->Write( "\\t", 2 );
			} else if ( ch == '\n' || ch == '\r' ) {
				outFile->Write( "\\n", 2 );
			} else if ( ch == '"' ) {
				outFile->Write( "\\\"", 2 );
			} else if ( ch == '\\' ) {
				outFile->Write( "\\\\", 2 );
			} else {
				outFile->Write( &ch, 1 );
			}
		}
		outFile->WriteFloatString( "\"\n" );
	}
	outFile->WriteFloatString( "\n}\n" );
	delete outFile;
	return true;
}

/*
========================
idLangDict::GetString
========================
*/
const char * idLangDict::GetString( const char * str ) const {
	const char * localized = FindString( str );
	if ( localized == NULL ) {
		return str;
	}
	return localized;
}

/*
========================
idLangDict::FindStringIndex
========================
*/
int idLangDict::FindStringIndex( const char * str ) const {
	if ( str == NULL ) {
		return -1;
	}
	int hash = idStr::IHash( str );
	for ( int i = keyIndex.GetFirst( hash ); i >= 0; i = keyIndex.GetNext( i ) ) {
		if ( idStr::Icmp( str, keyVals[i].key ) == 0 ) {
			return i;
		}
	}
	return -1;
}

/*
========================
idLangDict::FindString_r
========================
*/
const char * idLangDict::FindString_r( const char * str, int & depth ) const {
	depth++;
	if ( depth > MAX_REDIRECTION_DEPTH ) {
		// This isn't an error because we assume the error will be obvious somewhere in a GUI or something,
		// and the whole point of tracking the depth is to avoid a crash.
		idLib::Warning( "String '%s', indirection depth > %d", str, MAX_REDIRECTION_DEPTH );
		return NULL;
	}

	if ( str == NULL || str[0] == '\0' ) {
		return NULL;
	}

	int index = FindStringIndex( str );
	if ( index < 0 ) {
		return NULL;
	}
	const char * value = keyVals[index].value;
	if ( value == NULL ) {
		return NULL;
	}
	if ( IsStringId( value ) ) {
		// this string is re-directed to another entry
		return FindString_r( value, depth );
	}
	return value;
}

/*
========================
idLangDict::FindString
========================
*/
const char * idLangDict::FindString( const char * str ) const {
	int depth = 0;
	return FindString_r( str, depth );
}

/*
========================
idLangDict::DeleteString
========================
*/
bool idLangDict::DeleteString( const char * key ) {
	return DeleteString( FindStringIndex( key ) );
}

/*
========================
idLangDict::DeleteString
========================
*/
bool idLangDict::DeleteString( const int idx ) {
	if ( idx < 0 || idx >= keyVals.Num() ) {
		return false;
	}
	
	//mem.PushHeap();
	blockAlloc.Free( keyVals[idx].value );
	keyVals[idx].value = NULL;
	//mem.PopHeap();

	return true;
}

/*
========================
idLangDict::RenameStringKey
========================
*/
bool idLangDict::RenameStringKey( const char * oldKey, const char * newKey ) {
	int index = FindStringIndex( oldKey );
	if ( index < 0 ) {
		return false;
	}
	//mem.PushHeap();
	blockAlloc.Free( keyVals[index].key );
	int newKeyLen = idStr::Length( newKey );
	keyVals[index].key = blockAlloc.Alloc( newKeyLen + 1 );
	idStr::Copynz( keyVals[index].key, newKey, newKeyLen + 1 );
	int oldHash = idStr::IHash( oldKey );
	int newHash = idStr::IHash( newKey );
	if ( oldHash != newHash ) {
		keyIndex.Remove( oldHash, index );
		keyIndex.Add( newHash, index );
	}
	//mem.PopHeap();

	return true;
}

/*
========================
idLangDict::SetString
========================
*/
bool idLangDict::SetString( const char * key, const char * val ) {
	int index = FindStringIndex( key );
	if ( index < 0 ) {
		return false;
	}
	//mem.PushHeap();
	if ( keyVals[index].value != NULL ) {
		blockAlloc.Free( keyVals[index].value );
	}
	int valLen = idStr::Length( val );
	keyVals[index].value = blockAlloc.Alloc( valLen + 1 );
	idStr::Copynz( keyVals[index].value, val, valLen + 1 );
	//mem.PopHeap();
	return true;
}

/*
========================
idLangDict::AddKeyVal
========================
*/
void idLangDict::AddKeyVal( const char * key, const char * val ) {
	if ( SetString( key, val ) ) {
		return;
	}
	//mem.PushHeap();
	int keyLen = idStr::Length( key );
	char * k = blockAlloc.Alloc( keyLen + 1 );
	idStr::Copynz( k, key, keyLen + 1 );
	char * v = NULL;
	if ( val != NULL ) {
		int valLen = idStr::Length( val );
		v = blockAlloc.Alloc( valLen + 1 );
		idStr::Copynz( v, val, valLen + 1 );
	}
	int index = keyVals.Append( idLangKeyValue( k, v ) );
	int hash = idStr::IHash( key );
	keyIndex.Add( hash, index );
	//mem.PopHeap();
}

/*
========================
idLangDict::AddString
========================
*/
const char * idLangDict::AddString( const char * val ) {
	int i = Sys_Milliseconds();
	idStr key;
	sprintf( key, "#str_%06d", ( i++ % 1000000 ) );
	while ( FindStringIndex( key ) > 0 ) {
		sprintf( key, "#str_%06d", ( i++ % 1000000 ) );
	}
	AddKeyVal( key, val );
	int index = FindStringIndex( key );
	return keyVals[index].key;
}

/*
========================
idLangDict::GetNumKeyVals
========================
*/
int idLangDict::GetNumKeyVals() const {
	return keyVals.Num();
}

/*
========================
idLangDict::GetKeyVal
========================
*/
const idLangKeyValue * idLangDict::GetKeyVal( int i ) const {
	return &keyVals[i];
}

/*
========================
idLangDict::IsStringId
========================
*/
bool idLangDict::IsStringId( const char * str ) {
	return idStr::Icmpn( str, KEY_PREFIX, KEY_PREFIX_LEN ) == 0;
}

/*
========================
idLangDict::GetLocalizedString
========================
*/
const char * idLangDict::GetLocalizedString( const idStrId & strId ) const {
	if ( strId.GetIndex() >= 0 && strId.GetIndex() < keyVals.Num() ) {
		if ( keyVals[ strId.GetIndex() ].value == NULL ) {
			return keyVals[ strId.GetIndex() ].key;
		} else {
			return keyVals[ strId.GetIndex() ].value;
		}
	}
	return "";
}

/*
================================================================================================
idStrId
================================================================================================
*/

/*
========================
idStrId::Set
========================
*/
void idStrId::Set( const char * key ) {
	if ( key == NULL || key[0] == 0 ) {
		index = -1;
	} else {
		index = idLocalization::languageDict.FindStringIndex( key );
		if ( index < 0 ) {
			// don't allow setting of string ID's to an unknown ID... this should only be allowed from 
			// the string table tool because additions from anywhere else are not guaranteed to be 
			// saved to the .lang file.
			idLib::Warning( "Attempted to set unknown string ID '%s'", key );
		}
	}
}

/*
========================
idStrId::GetKey
========================
*/
const char * idStrId::GetKey() const {
	if ( index >= 0 && index < idLocalization::languageDict.keyVals.Num() ) {
		return idLocalization::languageDict.keyVals[index].key;
	}
	return "";
}

/*
========================
idStrId::GetLocalizedString
========================
*/
const char * idStrId::GetLocalizedString() const {
	return idLocalization::languageDict.GetLocalizedString( *this );
}