/////////////////////////////////////////////////////////////////////////////////////// // // pip.cpp KSH 6 Feb 2002 // // Pre-in-Place stuff // /////////////////////////////////////////////////////////////////////////////////////// // start autoduck documentation // @DOC pip // @module pip | None // @subindex Scripting Database // @index script | pip // TODO // Use Shrink to shrink the memory block to get rid of the 2048 padding at the end. // (This will only be effective when the bottom-up heap is being used though) // TODO but non-essential: // Make Unload work if passed the pointer. // Make GetFileSize work if passed the pointer. // 11Dec02 JCB - Andre wanted me to use tolower() instead of strncmpi() #include #include #include #include #include #include #include namespace Pip { #define IN_PLACE_DECOMPRESSION_MARGIN 3072 enum EQuadAligned { NOT_QUAD_WORD_ALIGNED=0, QUAD_WORD_ALIGNED }; #define CURRENT_PRE_VERSION 0xabcd0003 // as of 3/14/2001 (and as of 2/12/2002) struct SPreHeader { int mSize; int mVersion; int mNumFiles; }; struct SPreContained { uint32 mDataSize; uint32 mCompressedSize; uint16 mNameSize; // In makepre.cpp, mNameSize is stored in 4 bytes. When the pre is in memory though, I'm // borrowing the two high bytes to use as a usage indicator to indicate whether this // contained file is 'open', ie has had Load called on it. The count value is the number // of times the file has been opened using Load. Gets decremented when Unload is called. // (The LoadPre function checks that they are all zero to start with when a new pre is loaded) uint16 mUsage; // Mick - added space for a checksum of mpName // as otherwise we spend over five seconds at boot up in re-calculating checksums n^2 times uint32 mChecksum; // Keep mpName the last member of this struct, cos I calculate the space used by the other // members by subtracting &p->mDataSize from p->mpName. // mpName will probably not move anyway, cos this strcture is part of the pre file format. char mpName[1]; }; static SPreContained *sSkipToNextPreContained(SPreContained *p_preContained, EQuadAligned quadWordAlignedData=QUAD_WORD_ALIGNED); static SPreHeader *sSkipOverPreName(const char *p_pre_name); static SPreContained *sSeeIfFileIsInAnyPre(uint32 fileNameCRC); #ifdef __NOPT_ASSERT__ static const char *sGetPreName(SPreContained *p_contained_file); #endif #define MAX_PRE_FILES 100 // char* because each pre is prefixed with the name of the pre. static char *spp_pre_files[MAX_PRE_FILES]; struct SUnPreedFile { uint8 *mpFileData; uint32 mFileNameChecksum; uint32 mFileSize; uint32 mUsage; }; #define MAX_UNPREED_FILES 200 static SUnPreedFile sp_unpreed_files[MAX_UNPREED_FILES]; // This class only exists so that I can declare an instance of it and hence // use its constructor to initialise the above arrays. // Saves me having to call an init function from somewhere early on in the code, // which might get moved around later and cause problems. // This way, the arrays are guaranteed to have got initialised even before main() is // called. I think ... class CInit { public: CInit() { int i; for (i=0; imDataSize; if (p_preContained->mCompressedSize) { total_data_size=p_preContained->mCompressedSize; } total_data_size=(total_data_size+3)&~3; // printf ("p_preContained = %p\n, total_data = %d, name = (%p) %s",p_preContained,total_data_size,p_preContained->mpName,p_preContained->mpName); uint32 p_next=(uint32)p_preContained->mpName; p_next+=p_preContained->mNameSize; if (quadWordAlignedData) { p_next=(p_next+15)&~15; } p_next+=total_data_size; return (SPreContained*)p_next; } static SPreHeader *sSkipOverPreName(const char *p_pre_name) { Dbg_MsgAssert(p_pre_name,("NULL p_pre_name")); int len=strlen(p_pre_name)+1; // +1 for terminator len=(len+15)&~15; // Round up to a multiple of 16 return (SPreHeader*)(p_pre_name+len); } // Loads a pre file into memory. The name must have no path since the pre is // assumed to be in the data\pre directory. // Ie, a valid name would be "alc.pre" void LoadPre(const char *p_preFileName) { Dbg_MsgAssert(p_preFileName,("NULL p_preFileName")); // Check to see if the pre is already loaded, and return without doing anything if it is. for (int i=0; ioriginal_file_size ?")); #else File::Read(p_old_file_data+name_size, 1, original_file_size, p_file); #endif File::Close(p_file); } // Calculate the new buffer size required. // Note that even if no decompression is required, the new buffer will probably need to // be bigger than the old, because the contained files need to be moved so that they all start // on 16 byte boundaries. This is required by the collision code for example. uint32 new_pre_buffer_size=name_size; new_pre_buffer_size+=sizeof(SPreHeader); SPreHeader *p_pre_header=sSkipOverPreName(p_old_file_data); uint32 num_files=p_pre_header->mNumFiles; SPreContained *p_contained=(SPreContained*)(p_pre_header+1); for (uint32 f=0; fmUsage==0,("The file %s in %s has mUsage=%d ??",p_contained->mpName,p_preFileName,p_contained->mUsage)); new_pre_buffer_size+=(uint32)p_contained->mpName-(uint32)p_contained; new_pre_buffer_size+=p_contained->mNameSize; new_pre_buffer_size=(new_pre_buffer_size+15)&~15; new_pre_buffer_size+=(p_contained->mDataSize+3)&~3; p_contained=sSkipToNextPreContained(p_contained,NOT_QUAD_WORD_ALIGNED); } // Need to add a small margin to prevent decompressed data overtaking the compressed data. new_pre_buffer_size+=IN_PLACE_DECOMPRESSION_MARGIN; // At this point we have: // // original_file_size The exact size of the original pre file. Will be a multiple of 4. // // name_size The size of the pre name, rounded up to a multiple of 4 // // old_pre_buffer_size The size of the memory buffer pointed to by p_old_file_data // Equals name_size + (original_file_size, rounded up to a multiple of 2048) // // new_pre_buffer_size The required size of the new buffer, to contain the decompressed pre, // with all the files moved to be at 16 byte boundaries. // #ifdef __NOPT_ASSERT__ printf("Decompressing and rearranging pre file ...\n"); #endif // Note: It does not matter if new_pre_buffer_size is not a multiple of 2048 any more. // old_pre_buffer_size was only a multiple of 2048 to ensure that the file loading // did not overrun the end of the buffer. I think file loading may only load a whole // number of sectors. // Reallocate the buffer. char *p_new_file_data=NULL; if (Mem::Manager::sHandle().GetContextDirection()==Mem::Allocator::vTOP_DOWN) { // If using the top-down heap expand the buffer downwards ... // p_new_file_data will become a pointer lower down in memory than p_old_file_data. // The old data pointed to by p_old_file_data will still be there. // The memory between p_new_file_data and p_old_file_data is all free to use. Hoorah! p_new_file_data=(char*)Mem::ReallocateDown(new_pre_buffer_size,p_old_file_data); } else { // If using the top-down heap expand the buffer upwards ... p_new_file_data=(char*)Mem::ReallocateUp(new_pre_buffer_size,p_old_file_data); Dbg_MsgAssert(p_new_file_data,("ReallocateUp failed!")); // Now need to move the data up so that it can be decompressed into the gap below. uint32 *p_source=(uint32*)(p_old_file_data+old_pre_buffer_size); uint32 *p_dest=(uint32*)(p_new_file_data+new_pre_buffer_size); // Loading backwards cos the destination overlaps the source. // Note: Did some timing tests to see if this was was slow, but it's not really. // To load all the pre files in the game, one after the other, takes an average of // 32.406 seconds when the top down heap is used. When using the bottom up, which // necessitates doing this copy, it takes 33.518 seconds. // So per pre file that isn't much. (There's about 60 pre files) Dbg_MsgAssert((old_pre_buffer_size&3)==0,("old_pre_buffer_size not a multiple of 4 ?")); uint32 num_longs=old_pre_buffer_size/4; for (uint32 i=0; imNumFiles=p_source_header->mNumFiles; p_dest_header->mVersion=p_source_header->mVersion; // Copy down each of the contained files, decompressing those that need it. SPreContained *p_source_contained=(SPreContained*)(p_source_header+1); SPreContained *p_dest_contained=(SPreContained*)(p_dest_header+1); for (uint32 f=0; fmDataSize=p_source_contained->mDataSize; p_dest_contained->mChecksum=p_source_contained->mChecksum; p_dest_contained->mCompressedSize=0; // The new file will not be compressed. p_dest_contained->mUsage=0; p_dest_contained->mNameSize=p_source_contained->mNameSize; for (int i=0; imNameSize; ++i) { p_dest_contained->mpName[i]=p_source_contained->mpName[i]; } // Pre-calculate p_next_source_contained because decompression may (and often will) // cause the new data to overwrite the contents of p_source_contained. SPreContained *p_next_source_contained=sSkipToNextPreContained(p_source_contained,NOT_QUAD_WORD_ALIGNED); uint8 *p_source=(uint8*)(p_source_contained->mpName+p_source_contained->mNameSize); uint8 *p_dest=(uint8*)(p_dest_contained->mpName+p_dest_contained->mNameSize); p_dest=(uint8*)( ((uint32)p_dest+15)&~15 ); if (p_source_contained->mCompressedSize) { uint32 num_bytes_decompressed=p_dest_contained->mDataSize; uint8 *p_end=DecodeLZSS(p_source,p_dest,p_source_contained->mCompressedSize); Dbg_MsgAssert(p_end==p_dest+num_bytes_decompressed,("Eh? DecodeLZSS wrote %d bytes, expected it to write %d",p_end-p_dest,num_bytes_decompressed)); // For neatness, write zero's into the pad bytes at the end, otherwise they'll // be uninitialised data. while (num_bytes_decompressed & 3) { *p_end++=0; ++num_bytes_decompressed; } } else { // Uncompressed, so just copy the data down. uint32 *p_source_long=(uint32*)p_source; uint32 *p_dest_long=(uint32*)p_dest; // mDataSize is not necessarily a multiple of 4, but the actual data will be // padded at the end so that it does occupy a whole number of long words. // So the +3 is to ensure that n is rounded up to the next whole number of longs. uint32 n=(p_source_contained->mDataSize+3)/4; for (uint32 i=0; imSize=(uint32)p_dest_contained-(uint32)p_dest_header; //printf("Wasted space = %d\n",new_pre_buffer_size-((uint32)p_dest_contained-(uint32)p_new_file_data)); spp_pre_files[spare_index]=p_new_file_data; #ifdef __NOPT_ASSERT__ printf("Done\n"); #endif } // Removes the specified pre from memory. The name must have no path since the pre is // assumed to be in the data\pre directory. // Ie, a valid name would be "alc.pre" // // Won't do anything if the pre is gone already. // Asserts if any of the files in the pre are still 'open' in that they haven't had Unload called. bool UnloadPre(const char *p_preFileName) { bool success = false; Dbg_MsgAssert(p_preFileName,("NULL p_preFileName")); for (int i=0; imNumFiles; SPreContained *p_contained=(SPreContained*)(p_pre_header+1); for (int f=0; fmUsage,("Tried to unload %s when the file %s contained within it was still open! (mUsage=%d)",spp_pre_files[i],p_contained->mpName,p_contained->mUsage)); p_contained=sSkipToNextPreContained(p_contained); } #endif // Delete it. Mem::Free(spp_pre_files[i]); spp_pre_files[i]=NULL; // we've successfully unloaded a pre success = true; } } } // Not found, so nothing to do. return success; } // Searches each of the loaded pre files for the passed contained file. // Returns NULL if not found. static SPreContained *sSeeIfFileIsInAnyPre(uint32 fileNameCRC) { for (int i=0; imNumFiles; SPreContained *p_contained=(SPreContained*)(p_pre_header+1); for (int f=0; fmpName) == p_contained->mChecksum, //("Checksum for %s (%x) not %x",p_contained->mpName,Crc::GenerateCRCFromString(p_contained->mpName),p_contained->mChecksum)); //if ( Crc::GenerateCRCFromString(p_contained->mpName) == fileNameCRC ) if ( p_contained->mChecksum == fileNameCRC ) { return p_contained; } p_contained=sSkipToNextPreContained(p_contained); } } } return NULL; } #ifdef __NOPT_ASSERT__ // Finds which pre the passed file is contained in, and returns a pointer to the pre name. // Returns "Unknown" if not found anywhere. static const char *sGetPreName(SPreContained *p_contained_file) { for (int i=0; imNumFiles; SPreContained *p_contained=(SPreContained*)(p_pre_header+1); for (int f=0; fmUsage; Dbg_MsgAssert(p_contained_file->mCompressedSize==0,("The file '%s' is stored compressed in %s !",p_fileName,sGetPreName(p_contained_file))); uint32 p_data=(uint32)p_contained_file->mpName+p_contained_file->mNameSize; p_data=(p_data+15)&~15; return (void*)p_data; } // Next, see if it is one of the unpreed files that is already loaded. for (int i=0; impFileData=p_file_data; p_new_unpreed_file->mFileSize=file_size; p_new_unpreed_file->mFileNameChecksum=filename_checksum; p_new_unpreed_file->mUsage=1; return p_file_data; } void Unload(uint32 fileNameCRC) { // See if it is one of the unpreed files. #ifdef __NOPT_ASSERT__ bool is_an_unpreed_file=false; #endif for (int i=0; impName,sGetPreName(p_contained_file))); // Decrement the usage. if (p_contained_file->mUsage) { --p_contained_file->mUsage; } } } // If the file is unpreed, this will decrement the usage and free the memory specially allocated // for it if the usage has reached zero. // If the file is in a loaded pre, it will decrement that files usage in the pre. // If the file is both of the above, it will assert. // If it is neither, it won't do anything. void Unload(const char *p_fileName) { Dbg_MsgAssert(p_fileName,("NULL p_fileName")); Unload( Crc::GenerateCRCFromString(p_fileName) ); } uint32 GetFileSize(uint32 fileNameCRC) { // See if it is one of the unpreed files. #ifdef __NOPT_ASSERT__ bool is_an_unpreed_file=false; #endif for (int i=0; impName,sGetPreName(p_contained_file))); Dbg_MsgAssert(p_contained_file->mDataSize,("Zero mDataSize ??")); return p_contained_file->mDataSize; } // Dbg_MsgAssert(0,("File '%s' not found loaded anywhere",p_fileName)); return 0; } uint32 GetFileSize(const char *p_fileName) { Dbg_MsgAssert(p_fileName,("NULL p_fileName")); uint32 file_size = GetFileSize( Crc::GenerateCRCFromString(p_fileName) ); if ( file_size == 0 ) { Dbg_MsgAssert(0,("File '%s' not found loaded anywhere",p_fileName)); } return file_size; } const char *GetNextLoadedPre(const char *p_pre_name) { if (p_pre_name==NULL) { for (int i=0; imNumFiles; SPreContained *p_contained=(SPreContained*)(p_pre_header+1); for (int f=0; fmUsage) { return true; } p_contained=sSkipToNextPreContained(p_contained); } } } } return false; } // @script | LoadPipPre | // @uparm "string" | filename // @parmopt name | heap | 0 (checksum value) | Which heap to use. // Possible values are TopDown or BottomUp // If no heap is specified it will use whatever the current heap is. bool ScriptLoadPipPre(Script::CStruct *pParams, Script::CScript *pScript) { const char *p_filename; pParams->GetString(NONAME, &p_filename, Script::ASSERT); uint32 chosen_heap=0; pParams->GetChecksum("Heap",&chosen_heap); switch (chosen_heap) { case 0: chosen_heap = 0; // Use whatever the current heap is. break; case 0x477fc6de: // TopDown Mem::Manager::sHandle().PushContext(Mem::Manager::sHandle().TopDownHeap()); break; case 0xc80bf12d: // BottomUp Mem::Manager::sHandle().PushContext(Mem::Manager::sHandle().BottomUpHeap()); break; default: Dbg_Warning("Heap '%s' not supported by LoadPipPre, using current heap instead.",Script::FindChecksumName(chosen_heap)); chosen_heap=0; break; } LoadPre(p_filename); if (chosen_heap) { Mem::Manager::sHandle().PopContext(); } return true; } // @script | UnLoadPipPre | // @uparm "string" | filename bool ScriptUnloadPipPre(Script::CStruct *pParams, Script::CScript *pScript) { const char *p_filename; pParams->GetString(NONAME, &p_filename, Script::ASSERT); return UnloadPre(p_filename); } // @script | DumpPipPreStatus | Prints the status of the currently loaded pre files. // @flag ShowPreedFiles | Lists all the files contained within each pre, together with their usage value. // @flag ShowUnPreedFiles | Lists all the files currently open which were not in a pre, // so were individually opened. // @flag ShowOnlyOpenFiles | When listing the files in each pre, only display those whose usage is // greater than zero. bool ScriptDumpPipPreStatus(Script::CStruct *pParams, Script::CScript *pScript) { bool show_preed_files=pParams->ContainsFlag("ShowPreedFiles"); bool show_unpreed_files=pParams->ContainsFlag("ShowUnPreedFiles"); bool show_only_open_files=pParams->ContainsFlag("ShowOnlyOpenFiles"); printf("Currently loaded pre files:\n"); for (int i=0; imNumFiles; SPreContained *p_contained=(SPreContained*)(p_pre_header+1); for (int f=0; fmUsage) { printf("Usage:%d File: %s\n",p_contained->mUsage,p_contained->mpName); // Delay so that printf does not break when there are lots of files. for (volatile int pp=0; pp<100000; ++pp); } p_contained=sSkipToNextPreContained(p_contained); } } } } if (show_unpreed_files) { printf("Currently loaded un-preed files:\n"); for (int i=0; iLoadFile(p_fileName,&file_size, p_dest); if (!p_file_data) { // nope, so try loading "Quickly" (basically if we are on a CD) file_size = File::CanFileBeLoadedQuickly( p_fileName ); if (file_size) { // It can be loaded quickly. if (!p_dest) { p_file_data=(uint8*)Mem::Malloc((file_size+2047)&~2047); } else { p_file_data = (uint8*)p_dest; } bool fileLoaded = File::LoadFileQuicklyPlease( p_fileName, p_file_data ); if ( !fileLoaded ) { Dbg_MsgAssert( 0,( "File %s failed to load quickly.\n", p_fileName)); if (!p_dest) { Mem::Free(p_file_data); } p_file_data=NULL; } } else { // can't load quickly, so probably loading from the PC // Open the file & get its file size void *p_file = File::Open(p_fileName, "rb"); if (!p_file) { Dbg_MsgAssert(0,("Could not open file '%s'",p_fileName)); } file_size=File::GetFileSize(p_file); if (!file_size) { Dbg_MsgAssert(0,("Zero file size for file %s",p_fileName)); } if (!p_dest) { // Allocate memory. // Just to be safe, allocate a buffer of size file_size rounded up to the // next multiple of 2048, cos maybe loading a file off CD will always load // whole numbers of sectors. // Haven't checked that though. p_file_data=(uint8*)Mem::Malloc((file_size+2047)&~2047); Dbg_MsgAssert(p_file_data,("Could not allocate memory for file %s",p_fileName)); } else { p_file_data = (uint8*)p_dest; } // Load the file into memory then close the file. #ifdef __NOPT_ASSERT__ long bytes_read=File::Read(p_file_data, 1, file_size, p_file); Dbg_MsgAssert(bytes_read<=file_size,("bytes_read>file_size ?")); #else File::Read(p_file_data, 1, file_size, p_file); #endif File::Close(p_file); } // Shrink memory back down to accurate usage - saves 43K total in the school!!! if (!p_dest && p_file_data) { Mem::ReallocateShrink(file_size,p_file_data); } } if (p_filesize) { *p_filesize = file_size; } // If we specified a destination, then make sure we did not overflow it if (p_dest) { Dbg_MsgAssert(((file_size + 2047)&~2047) < maxSize,("file size (%d) overflows buffer (%d) for %s\n",file_size,maxSize,p_fileName)); } return (void *)p_file_data; } }