feat(filesystem): Implement access to shadowed files inside archives (#1483)

Author: xezon

Core/GameEngine/Include/Common/ArchiveFileSystem.h

@@ -84,26 +84,19 @@ class ArchivedDirectoryInfo;
 class DetailedArchivedDirectoryInfo;
 class ArchivedFileInfo;
 
-typedef std::map<AsciiString, DetailedArchivedDirectoryInfo> DetailedArchivedDirectoryInfoMap;
-typedef std::map<AsciiString, ArchivedDirectoryInfo> ArchivedDirectoryInfoMap;
-typedef std::map<AsciiString, ArchivedFileInfo> ArchivedFileInfoMap;
-typedef std::map<AsciiString, ArchiveFile *> ArchiveFileMap;
-typedef std::map<AsciiString, AsciiString> ArchivedFileLocationMap; // first string is the file name, second one is the archive filename.
+typedef std::map<AsciiString, DetailedArchivedDirectoryInfo> DetailedArchivedDirectoryInfoMap; // Archived directory name to detailed archived directory info
+typedef std::map<AsciiString, ArchivedDirectoryInfo> ArchivedDirectoryInfoMap; // Archived directory name to archived directory info
+typedef std::map<AsciiString, ArchivedFileInfo> ArchivedFileInfoMap; // Archived file name to archived file info
+typedef std::map<AsciiString, ArchiveFile *> ArchiveFileMap; // Archive file name to archive data
+typedef std::multimap<AsciiString, ArchiveFile *> ArchivedFileLocationMap; // Archived file name to archive data
 
 class ArchivedDirectoryInfo
 {
 public:
-	AsciiString								m_directoryName;
-	ArchivedDirectoryInfoMap	m_directories;
-	ArchivedFileLocationMap		m_files;
-
-	void clear()
-	{
-		m_directoryName.clear();
-		m_directories.clear();
-		m_files.clear();
-	}
-
+	AsciiString								m_path; // The full path to this directory
+	AsciiString								m_directoryName; // The current directory
+	ArchivedDirectoryInfoMap	m_directories; // Contained leaf directories
+	ArchivedFileLocationMap		m_files; // Contained files
 };
 
 class DetailedArchivedDirectoryInfo
@@ -112,13 +105,6 @@ class DetailedArchivedDirectoryInfo
 	AsciiString												m_directoryName;
 	DetailedArchivedDirectoryInfoMap	m_directories;
 	ArchivedFileInfoMap								m_files;
-
-	void clear()
-	{
-		m_directoryName.clear();
-		m_directories.clear();
-		m_files.clear();
-	}
 };
 
 class ArchivedFileInfo
@@ -130,23 +116,16 @@ class ArchivedFileInfo
 	UnsignedInt m_size;
 
 	ArchivedFileInfo()
+		: m_offset(0)
+		, m_size(0)
 	{
-		clear();
-	}
-
-	void clear()
-	{
-		m_filename.clear();
-		m_archiveFilename.clear();
-		m_offset = 0;
-		m_size = 0;
 	}
 };
 
 
 class ArchiveFileSystem : public SubsystemInterface
 {
-	public:
+public:
 	ArchiveFileSystem();
 	virtual ~ArchiveFileSystem();
 
@@ -158,24 +137,38 @@ class ArchiveFileSystem : public SubsystemInterface
 	// ArchiveFile operations
 	virtual ArchiveFile*	openArchiveFile( const Char *filename ) = 0;		///< Create new or return existing Archive file from file name
 	virtual void					closeArchiveFile( const Char *filename ) = 0;		///< Close the one specified big file.
-	virtual void					closeAllArchiveFiles( void ) = 0;								///< Close all Archivefiles currently open
+	virtual void					closeAllArchiveFiles( void ) = 0;								///< Close all Archive files currently open
 
 	// File operations
-	virtual File*					openFile( const Char *filename, Int access = 0);	///< Search Archive files for specified file name and open it if found
-	virtual void					closeAllFiles( void ) = 0;									///< Close all files associated with ArchiveFiles
-	virtual Bool					doesFileExist(const Char *filename) const;		///< return true if that file exists in an archive file somewhere.
+	virtual File*					openFile( const Char *filename, Int access = 0, FileInstance instance = 0);	///< Search Archive files for specified file name and open it if found
+	virtual void					closeAllFiles( void ) = 0;									///< Close all files associated with Archive files
+	virtual Bool					doesFileExist(const Char *filename, FileInstance instance = 0) const;	///< return true if that file exists in an archive file somewhere.
 
 	void					getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; ///< search the given directory for files matching the searchName (egs. *.ini, *.rep).  Possibly search subdirectories.  Scans each Archive file.
-	Bool					getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const; ///< see FileSystem.h
+	Bool					getFileInfo(const AsciiString& filename, FileInfo *fileInfo, FileInstance instance = 0) const; ///< see FileSystem.h
 
 	virtual Bool	loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0;
 
 	// Unprotected this for copy-protection routines
-	AsciiString						getArchiveFilenameForFile(const AsciiString& filename) const;
+	ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const;
+
 	void loadMods( void );
 
+	ArchivedDirectoryInfo* friend_getArchivedDirectoryInfo(const Char* directory);
+
 protected:
-	virtual void					loadIntoDirectoryTree(const ArchiveFile *archiveFile, const AsciiString& archiveFilename, Bool overwrite = FALSE );	///< load the archive file's header information and apply it to the global archive directory tree.
+	struct ArchivedDirectoryInfoResult
+	{
+		ArchivedDirectoryInfoResult() : dirInfo(NULL) {}
+		Bool valid() const { return dirInfo != NULL; }
+
+		ArchivedDirectoryInfo* dirInfo;
+		AsciiString lastToken; ///< Synonymous for file name if the search directory was a file path
+	};
+
+	ArchivedDirectoryInfoResult getArchivedDirectoryInfo(const Char* directory);
+
+	virtual void loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool overwrite = FALSE);	///< load the archive file's header information and apply it to the global archive directory tree.
 
 	ArchiveFileMap m_archiveFileMap;
 	ArchivedDirectoryInfo m_rootDirectory;

Core/GameEngine/Include/Common/FileSystem.h

@@ -66,6 +66,7 @@
 
 typedef std::set<AsciiString, rts::less_than_nocase<AsciiString> > FilenameList;
 typedef FilenameList::iterator FilenameListIter;
+typedef UnsignedByte FileInstance;
 
 //----------------------------------------------------------------------------
 //           Type Defines
@@ -80,7 +81,7 @@ typedef FilenameList::iterator FilenameListIter;
 #define USER_W3D_DIR_PATH "%sW3D/"					///< .w3d files live here
 #define USER_TGA_DIR_PATH "%sTextures/"		///< User .tga texture files live here
 
-// the following defines are only to be used while maintaining legacy compatability
+// the following defines are only to be used while maintaining legacy compatibility
 // with old files until they are completely gone and in the regular art set
 #ifdef MAINTAIN_LEGACY_FILES
 #define LEGACY_W3D_DIR_PATH "../LegacyArt/W3D/"				///< .w3d files live here
@@ -102,7 +103,16 @@ typedef FilenameList::iterator FilenameListIter;
 	#define TEST_TGA_DIR_PATH "../TestArt/"		///< .tga texture files live here
 #endif
 
+#ifndef ENABLE_FILESYSTEM_LOGGING
+#define ENABLE_FILESYSTEM_LOGGING (0)
+#endif
+
+
 struct FileInfo {
+
+	Int64 size() const { return (Int64)sizeHigh << 32 | sizeLow; }
+	Int64 timestamp() const { return (Int64)timestampHigh << 32 | timestampLow; }
+
 	Int sizeHigh;
 	Int sizeLow;
 	Int timestampHigh;
@@ -115,11 +125,13 @@ struct FileInfo {
 /**
   * FileSystem is an interface class for creating specific FileSystem objects.
   *
-	* A FileSystem object's implemenation decides what derivative of File object needs to be
+	* A FileSystem object's implementation decides what derivative of File object needs to be
 	* created when FileSystem::Open() gets called.
 	*/
+// TheSuperHackers @feature xezon 23/08/2025 Implements file instance access.
+// Can be used to access different versions of files in different archives under the same name.
+// Instance 0 refers to the top file that shadows all other files under the same name.
 //===============================
-
 class FileSystem : public SubsystemInterface
 {
   FileSystem(const FileSystem&);
@@ -133,23 +145,30 @@ class FileSystem : public SubsystemInterface
 	void reset();
 	void update();
 
-	File* openFile( const Char *filename, Int access = File::NONE, size_t bufferSize = File::BUFFERSIZE );		///< opens a File interface to the specified file
-	Bool doesFileExist(const Char *filename) const;								///< returns TRUE if the file exists.  filename should have no directory.
+	File* openFile( const Char *filename, Int access = File::NONE, size_t bufferSize = File::BUFFERSIZE, FileInstance instance = 0 ); ///< opens a File interface to the specified file
+	Bool doesFileExist(const Char *filename, FileInstance instance = 0) const; ///< returns TRUE if the file exists.  filename should have no directory.
 	void getFileListInDirectory(const AsciiString& directory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; ///< search the given directory for files matching the searchName (egs. *.ini, *.rep).  Possibly search subdirectories.
-	Bool getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const; ///< fills in the FileInfo struct for the file given. returns TRUE if successful.
+	Bool getFileInfo(const AsciiString& filename, FileInfo *fileInfo, FileInstance instance = 0) const; ///< fills in the FileInfo struct for the file given. returns TRUE if successful.
 
 	Bool createDirectory(AsciiString directory); ///< create a directory of the given name.
 
 	Bool areMusicFilesOnCD();
 	void loadMusicFilesFromCD();
 	void unloadMusicFilesFromCD();
-	AsciiString normalizePath(const AsciiString& path) const;	///< normalizes a file path. The path can refer to a directory. File path must be absolute, but does not need to exist. Returns an empty string on failure.
+
+	static AsciiString normalizePath(const AsciiString& path);	///< normalizes a file path. The path can refer to a directory. File path must be absolute, but does not need to exist. Returns an empty string on failure.
 	static Bool isPathInDirectory(const AsciiString& testPath, const AsciiString& basePath);	///< determines if a file path is within a base path. Both paths must be absolute, but do not need to exist.
 
 protected:
 #if ENABLE_FILESYSTEM_EXISTENCE_CACHE
+	struct FileExistData
+	{
+		FileExistData() : instanceExists(0), instanceDoesNotExist(~FileInstance(0)) {}
+		FileInstance instanceExists;
+		FileInstance instanceDoesNotExist;
+	};
 	typedef std::hash_map<
-		rts::string_key<AsciiString>, bool,
+		rts::string_key<AsciiString>, FileExistData,
 		rts::string_key_hash<AsciiString>,
 		rts::string_key_equal<AsciiString> > FileExistMap;
 	mutable FileExistMap m_fileExist;

Core/GameEngine/Source/Common/System/ArchiveFile.cpp

@@ -90,50 +90,46 @@ ArchiveFile::~ArchiveFile()
 }
 
 ArchiveFile::ArchiveFile()
+	: m_file(NULL)
 {
-	m_rootDirectory.clear();
 }
 
 void ArchiveFile::addFile(const AsciiString& path, const ArchivedFileInfo *fileInfo)
 {
-	AsciiString temp;
-	temp = path;
-	temp.toLower();
-	AsciiString token;
-	AsciiString debugpath;
-
 	DetailedArchivedDirectoryInfo *dirInfo = &m_rootDirectory;
 
-	temp.nextToken(&token, "\\/");
+	AsciiString token;
+	AsciiString tokenizer = path;
+	tokenizer.toLower();
+	tokenizer.nextToken(&token, "\\/");
 
-	while (token.getLength() > 0) {
-		if (dirInfo->m_directories.find(token) == dirInfo->m_directories.end())
+	while (token.getLength() > 0)
+	{
+		DetailedArchivedDirectoryInfoMap::iterator tempiter = dirInfo->m_directories.find(token);
+		if (tempiter == dirInfo->m_directories.end())
+		{
+			dirInfo = &(dirInfo->m_directories[token]);
+			dirInfo->m_directoryName = token;
+		}
+		else
 		{
-			dirInfo->m_directories[token].clear();
-			dirInfo->m_directories[token].m_directoryName = token;
+			dirInfo = &tempiter->second;
 		}
 
-		debugpath.concat(token);
-		debugpath.concat('\\');
-		dirInfo = &(dirInfo->m_directories[token]);
-		temp.nextToken(&token, "\\/");
+		tokenizer.nextToken(&token, "\\/");
 	}
 
 	dirInfo->m_files[fileInfo->m_filename] = *fileInfo;
-	//path.concat(fileInfo->m_filename);
 }
 
 void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const
 {
-
-	AsciiString searchDir;
 	const DetailedArchivedDirectoryInfo *dirInfo = &m_rootDirectory;
 
-	searchDir = originalDirectory;
-	searchDir.toLower();
 	AsciiString token;
-
-	searchDir.nextToken(&token, "\\/");
+	AsciiString tokenizer = originalDirectory;
+	tokenizer.toLower();
+	tokenizer.nextToken(&token, "\\/");
 
 	while (token.getLength() > 0) {
 
@@ -148,7 +144,7 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co
 			return;
 		}
 
-		searchDir.nextToken(&token, "\\/");
+		tokenizer.nextToken(&token, "\\/");
 	}
 
 	getFileListInDirectory(dirInfo, originalDirectory, searchName, filenameList, searchSubdirectories);
@@ -198,17 +194,15 @@ void ArchiveFile::attachFile(File *file)
 
 const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& filename) const
 {
-	AsciiString path;
-	path = filename;
-	path.toLower();
-	AsciiString token;
-
 	const DetailedArchivedDirectoryInfo *dirInfo = &m_rootDirectory;
 
-	path.nextToken(&token, "\\/");
-
-	while ((token.find('.') == NULL) || (path.find('.') != NULL)) {
+	AsciiString token;
+	AsciiString tokenizer = filename;
+	tokenizer.toLower();
+	tokenizer.nextToken(&token, "\\/");
 
+	while (!token.find('.') || tokenizer.find('.'))
+	{
 		DetailedArchivedDirectoryInfoMap::const_iterator it = dirInfo->m_directories.find(token);
 		if (it != dirInfo->m_directories.end())
 		{
@@ -219,7 +213,7 @@ const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& fil
 			return NULL;
 		}
 
-		path.nextToken(&token, "\\/");
+		tokenizer.nextToken(&token, "\\/");
 	}
 
 	ArchivedFileInfoMap::const_iterator it = dirInfo->m_files.find(token);