// Ryzom - MMORPG Framework // Copyright (C) 2010 Winch Gate Property Limited // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as // published by the Free Software Foundation, either version 3 of the // License, or (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . //----------------------------------------------------------------------------- // includes //----------------------------------------------------------------------------- // nel #include "nel/misc/variable.h" #include "nel/misc/file.h" // game share #include "game_share/utils.h" // local #include "file_manager.h" //------------------------------------------------------------------------------------------------- // namespaces //------------------------------------------------------------------------------------------------- using namespace std; using namespace NLMISC; //using namespace NLNET; using namespace PATCHMAN; //----------------------------------------------------------------------------- // some NLMISC Variable //----------------------------------------------------------------------------- NLMISC::CVariable FileCacheSize("patchman","FileCacheSize","Minimum size for re file cache",100*1024*1024,0,true); namespace PATCHMAN { //----------------------------------------------------------------------------- // handy utils //----------------------------------------------------------------------------- static NLMISC::CSString getTempFileName(const NLMISC::CSString& fileName) { return fileName+".download.tmp"; } static NLMISC::CSString getIndexFileName(const NLMISC::CSString& rootDirectory) { return rootDirectory+".patchman.file_index"; } //----------------------------------------------------------------------------- // methods SFileInfo //----------------------------------------------------------------------------- bool SFileInfo::updateFileInfo(const NLMISC::CSString& fileName,const NLMISC::CSString& fullFileName, SFileInfo::TUpdateMethod updateMethod, IFileInfoUpdateListener* updateListener) { // if file doesn't exist then just drop out if (!NLMISC::CFile::fileExists(fullFileName)) { clear(); return false; } // give ourselves 5 attempts in case the file is being accessed... for(uint32 i=0;i<5;++i) { try { uint32 newFileSize= NLMISC::CFile::getFileSize(fullFileName); uint32 newFileTime= NLMISC::CFile::getFileModificationDate(fullFileName); // work out whether the record has changed (based on size and time stamp) bool changed= ( (FileTime!=newFileTime) || (FileSize!=newFileSize) ); // note: it is possible to hit an exception in the checksum calculation if the file is being accessed // by someone else if (updateMethod==FORCE_RECALCULATE || (updateMethod==RECALCULATE_IF_CHANGED && (newFileSize!=FileSize || newFileTime!=FileTime))) { // call the updateListener object's callback so that they can write a log, etc if (updateListener!=NULL) { updateListener->cbFileInfoRescanning(fileName,newFileSize); } // workout the new checksum and update the 'changed' flag accordingly CHashKeyMD5 newChecksum= NLMISC::getMD5(fullFileName); changed|= (Checksum!=newChecksum); Checksum= newChecksum; } // update the fields in our new record FileName= fileName; FileSize= newFileSize; FileTime= newFileTime; // if there's an update listener object, then let them know about the file update if (changed && updateListener!=NULL) { updateListener->cbFileInfoUpdate(*this); } // return true if we are unchanged, otherwise false return !changed; } catch(...) { nlwarning("Exception thrown in getMD5(\"%s\") ... will try again in a few seconds",fullFileName.c_str()); nlSleep(5); } } nlwarning("Failed to get info on file: %s",fullFileName.c_str()); clear(); return false; } //----------------------------------------------------------------------------- // methods CFileSpec //----------------------------------------------------------------------------- CFileSpec::CFileSpec() { } CFileSpec::CFileSpec(const NLMISC::CSString& fileSpec) { _NameSpec= NLMISC::CFile::getFilename(fileSpec); _PathSpec= NLMISC::CFile::getPath(fileSpec); _NameIsWild= _NameSpec.contains('*') || _NameSpec.contains('?'); _PathIsWild= _PathSpec.contains('*') || _PathSpec.contains('?'); _AcceptAllNames= (_NameSpec=="*"); _AcceptAllPaths= (_PathSpec=="*/"); } bool CFileSpec::matches(const NLMISC::CSString& fullFileName) const { CSString name= NLMISC::CFile::getFilename(fullFileName); CSString path= NLMISC::CFile::getPath(fullFileName); return (_AcceptAllPaths || pathMatches(path)) && (_AcceptAllNames || nameMatches(name)); } bool CFileSpec::nameMatches(const NLMISC::CSString& fileName) const { return _AcceptAllNames || ( _NameIsWild ? testWildCard(fileName,_NameSpec) : (fileName==_NameSpec) ); } bool CFileSpec::pathMatches(const NLMISC::CSString& path) const { // treat the special case where the file that we're set to accept all paths if (_AcceptAllPaths) return true; // treat the special case where the file that we're testing has no path and we // are looking for files in the root directory only if (path.empty() && (_PathSpec=="./") ) return true; return _PathIsWild ? testWildCard(path,_PathSpec) : (path==_PathSpec); } NLMISC::CSString CFileSpec::toString() const { return _PathSpec+_NameSpec; } const CSString& CFileSpec::nameSpec() const { return _NameSpec; } const CSString& CFileSpec::pathSpec() const { return _PathSpec; } bool CFileSpec::isWild() const { return _NameIsWild || _PathIsWild; } bool CFileSpec::nameIsWild() const { return _NameIsWild; } bool CFileSpec::pathIsWild() const { return _PathIsWild; } //----------------------------------------------------------------------------- // methods CRepositoryDirectory //----------------------------------------------------------------------------- CRepositoryDirectory::CRepositoryDirectory(const NLMISC::CSString& path) { _Root= NLMISC::CPath::getFullPath(path); _IndexFileIsUpToDate= false; } void CRepositoryDirectory::clear() { // clear out our directories map _DirectoryTree.clear(); } void CRepositoryDirectory::rescanFull(IFileInfoUpdateListener* updateListener) { // rescan our directories recursively, starting at the root _rescanDirectory("",true,updateListener); // update the index file as required if (!_IndexFileIsUpToDate) writeIndex(); } void CRepositoryDirectory::rescanPartial(IFileInfoUpdateListener* updateListener) { // if the directory tree is empty then start with the rooot directory... if (_DirectoryTree.empty()) { _rescanDirectory("",false,updateListener); return; } // try to get hold of a ref to the last directory that we rescanned TDirectoryTree::iterator it= _DirectoryTree.find(_LastRescan); if (it==_DirectoryTree.end()) { // we failed to get a ref to the last directory so wrap back round to the start it= _DirectoryTree.begin(); } else { // increement our iterator to get hold of a ref to the next directory in the map ++it; // if we reach end of map then wrap back to the start if (it==_DirectoryTree.end()) { it= _DirectoryTree.begin(); } } // scan the next directory in our map (this is not recursive _rescanDirectory(it->first,false,updateListener); _LastRescan= it->first; // update the index file as required if (!_IndexFileIsUpToDate) writeIndex(); } void CRepositoryDirectory::updateFile(const NLMISC::CSString& fileName,SFileInfo::TUpdateMethod updateMethod, IFileInfoUpdateListener* updateListener) { // if the file doesn't exist then give up if (!NLMISC::CFile::fileExists(_Root+fileName)) return; // split the file name into path and fileName const NLMISC::CSString path= NLMISC::CFile::getPath(fileName); // get hold of the directory object for this file (or create a new one if need be) TFileInfoMap& directory= _DirectoryTree[path]; // get hold of the file info object for this (or create a new one if need be) SFileInfo& fileInfo= directory[fileName]; // update the file info for the given file _IndexFileIsUpToDate&= fileInfo.updateFileInfo(fileName,_Root+fileName,updateMethod,updateListener); } // query methods void CRepositoryDirectory::getFileInfo(const NLMISC::CSString& fileSpec,TFileInfoVector& result,IFileRequestValidator* validator,const NLNET::IModuleProxy *sender) const { // split into directory and file name const CFileSpec spec(fileSpec); // setup a set of paths to scan std::vector paths; // if the path has wirlcards in it then do a search for wildcard matches otherwise just do a lookup if (spec.pathIsWild()) { for (TDirectoryTree::const_iterator it=_DirectoryTree.begin(); it!=_DirectoryTree.end(); ++it) { if (spec.pathMatches(it->first)) paths.push_back(it); } } else { TDirectoryTree::const_iterator it= _DirectoryTree.find(spec.pathSpec()); if (it != _DirectoryTree.end()) paths.push_back(it); } // run through the selected paths looking for files for (uint32 i=0;ifirst+spec.nameSpec(); // if the filename doesn't have wildcards then just do a straight lookup... if (!spec.nameIsWild()) { // see whether we have a match... TFileInfoMap::const_iterator fit= paths[i]->second.find(fullFilePattern); if (fit!=paths[i]->second.end()) { // call the overloadable validation callback before adding the file info record to the result container if (validator==NULL || validator->cbValidateFileInfoRequest(sender,fit->second.FileName)) { result.push_back(fit->second); } } continue; } // run through the files in the given path for (TFileInfoMap::const_iterator fit= paths[i]->second.begin(); fit!= paths[i]->second.end(); ++fit) { // get hold of the file name for this map entry const NLMISC::CSString& name=fit->second.FileName; // do either a wildcard compare or a quick and dirty string compare if (testWildCard(name,fullFilePattern)) { // call the overloadable validation callback before adding the file info record to the result container if (validator==NULL || validator->cbValidateFileInfoRequest(sender,name)) { result.push_back(fit->second); } } } } } void CRepositoryDirectory::getFile(const NLMISC::CSString& fileName,NLMISC::CSString& resultData,IFileRequestValidator* validator,const NLNET::IModuleProxy *sender) const { // start by clearing out the result container... resultData.clear(); // allow the overloadable validation callback a chance to prohibit read if (validator!=NULL && !validator->cbValidateDownloadRequest(sender,fileName)) return; // if the file exists then go ahead and read it NLMISC::CSString fullFileName= _Root+fileName; if (NLMISC::CFile::fileExists(fullFileName)) { resultData.readFromFile(fullFileName); } } const NLMISC::CSString &CRepositoryDirectory::getRootDirectory() const { return _Root; } bool CRepositoryDirectory::readIndex() { // start by clearing out our containers clear(); // make sure the file exists (return false if not found) NLMISC::CSString indexFileName= getIndexFileName(_Root); if (!NLMISC::CFile::fileExists(indexFileName)) return false; // read the file contents NLMISC::CSString index; index.readFromFile(indexFileName); DROP_IF(index.empty(),"Failed to read data from index file: "+indexFileName,return false); nlinfo("CRepositoryDirectory_Reading index file: %s",indexFileName.c_str()); NLMISC::CVectorSString lines; index.splitLines(lines); for (uint32 i=0;isecond; for (TFileInfoMap::const_iterator fit= theDirectory.begin(); fit!=theDirectory.end(); ++fit) { const SFileInfo& theInfo= fit->second; outputText+= NLMISC::toString("%10u,%12u, %s, %s\n",theInfo.FileSize,theInfo.FileTime,theInfo.Checksum.toString().c_str(),theInfo.FileName.c_str()); } } // write the resulting buffer to disk _IndexFileIsUpToDate= outputText.writeToFile(indexFileName); return _IndexFileIsUpToDate; } void CRepositoryDirectory::_rescanDirectory(const NLMISC::CSString& directoryName, bool recurse, IFileInfoUpdateListener* updateListener) { // nldebug("VERBOSE_Scanning directory: root=%s directory=%s",_Root.c_str(),directoryName.c_str()); // make sure we exist in the '_DirectoryTree' map (and get a handle to it) TFileInfoMap& theDirectory= _DirectoryTree[directoryName]; // first scan for directories std::vector pathContents; NLMISC::CPath::getPathContent(_Root+directoryName,false,true,false,pathContents); // run through the directories we found... for (uint32 i=pathContents.size();i--;) { NLMISC::CSString childDirectoryName= NLMISC::CSString(pathContents[i]).leftCrop(_Root.size()); // make sure they exist in the '_DirectoryTree' map _DirectoryTree[childDirectoryName]; // if we're recursing then go for it if (recurse) { _rescanDirectory(childDirectoryName,recurse,updateListener); } } // run through all of the files in our map flagging them as 'not updated' for (TFileInfoMap::iterator fit= theDirectory.begin(); fit!= theDirectory.end(); ++fit) { fit->second.FileName.clear(); } // now scan for files pathContents.clear(); NLMISC::CPath::getPathContent(_Root+directoryName,false,false,true,pathContents); // run through the files adding them to ourself for (uint32 i=pathContents.size();i--;) { // if the file is system file then skip it if (pathContents[i].find("/.")!=std::string::npos) continue; // construct the file name NLMISC::CSString fileName= NLMISC::CSString(pathContents[i]).leftCrop(_Root.size()); // get hold of the directory entry for this file (or create a new one if not exist) and update it _IndexFileIsUpToDate&= _DirectoryTree[directoryName][fileName].updateFileInfo(fileName,pathContents[i],SFileInfo::RECALCULATE_IF_CHANGED,updateListener); } // run through all of the files in our map looking for files that are not updated and that need erasing TFileInfoMap::iterator fit= theDirectory.begin(); while (fit!=theDirectory.end()) { TFileInfoMap::iterator thisIt= fit; ++fit; if (thisIt->second.FileName.empty()) { // if there's an update listener object, then let them know about the file update if (updateListener!=NULL) { updateListener->cbFileInfoErased(thisIt->first); } // erase the entry in our files map and flag the index file as out of date theDirectory.erase(thisIt); _IndexFileIsUpToDate= false; } } } //----------------------------------------------------------------------------- // methods CFileManager //----------------------------------------------------------------------------- bool CFileManager::load(const NLMISC::CSString& fileName, uint32 startOffset, uint32 numBytes, NLMISC::CSString& result) { // clear out the return value before we begin result.clear(); // make sure the file exists if (!NLMISC::CFile::fileExists(fileName)) return false; // nldebug("Loading file data for: %s",fileName.c_str()); // get the file's vital statistics from disk uint32 fileSize= NLMISC::CFile::getFileSize(fileName); uint32 fileTime= NLMISC::CFile::getFileModificationDate(fileName); // run through the files to see if the one we're after is here TCacheFiles::iterator it= _CacheFiles.begin(); for (; it!=_CacheFiles.end();++it) { // if we've found the file we're after then break out here if (it->FileName==fileName && fileSize==it->FileSize && fileTime==it->FileTime) { // nldebug("- Found data in Ram @ offset: %d",it->StartOffset); break; } } // if we didn't find the file then we have to load it if (it==_CacheFiles.end()) { // nldebug("- Found data NOT already in Ram"); // setup a data block for this file SCacheFileEntry newFileEntry; newFileEntry.FileName= fileName; newFileEntry.FileSize= fileSize; newFileEntry.FileTime= fileTime; newFileEntry.StartOffset= ~0u; // if the buffer is too small to load this file then reallocate it and clear out the _CacheFiles vector if (_CacheBuffer.size()_CacheBuffer.size()) { // clear out all remaining files between us and the end of buffer while (!_CacheFiles.empty() && _CacheFiles.front().StartOffset>=newFileEntry.StartOffset) { // nldebug("- Ditching cache entry: %s",_CacheFiles.front().FileName.c_str()); _CacheFiles.pop_front(); nlassert(!_CacheFiles.empty()); } // not enough space at end of file so we'll need to spin round to the start newFileEntry.StartOffset= 0; requiredEndOffset= fileSize; } // our start offset is now OK, so make a bit of room for our data as required while (!_CacheFiles.empty() && _CacheFiles.front().StartOffset>=newFileEntry.StartOffset && _CacheFiles.front().StartOffsetfileSize,"Ignoring request for data where end offset > file size: "+fileName,return false); // nldebug("- Retrieving data from buffer @offset: %d (%d bytes)",it->StartOffset+startOffset,numBytes); // copy out the data chunk that we're after result.resize(numBytes); memcpy(&result[0],&_CacheBuffer[it->StartOffset+startOffset],numBytes); // we succeeded so return true return true; } TRepositoryDirectoryPtr CFileManager::getRepositoryDirectory(const NLMISC::CSString& path) { // get hold of a ref to the map entry pointing at the directory we want (create a new map entry if need be) TRepositoryDirectoryRefPtr& thePtr=_RepositoryDirectories[path]; // if the map entry is null then create a new one if (thePtr==NULL) { thePtr= new CRepositoryDirectory(path); thePtr->readIndex(); } return &*thePtr; } bool CFileManager::save(const NLMISC::CSString& fileName, const NLMISC::CMemStream& data) { NLMISC::CSString tmpFileName= fileName+"__patchman__.sav"; // make sure the destination file is deleted before we begin if (NLMISC::CFile::fileExists(fileName)) { NLMISC::CFile::deleteFile(fileName); } // try to write the tmp file try { // make sure that the directory structure exists foe the file NLMISC::CSString path= NLMISC::CFile::getPath(fileName); if (!path.empty()) { NLMISC::CFile::createDirectoryTree(path); } // go ahead and write the data to disk... COFile outputFile(tmpFileName); outputFile.serialBuffer(const_cast(data.buffer()),data.size()); } catch(...) { } // make sure that the file write succeeded if (NLMISC::CFile::getFileSize(tmpFileName)!=data.size()) { nlwarning("Failed to save file '%s' because failed to save tmp file: %s",fileName.c_str(),tmpFileName.c_str()); NLMISC::CFile::deleteFile(tmpFileName); return false; } // write succeeded so rename the tmp file to the correct file name bool ok= NLMISC::CFile::moveFile(fileName.c_str(),tmpFileName.c_str()); DROP_IF(!ok,"Failed to save file '"+fileName+"' because failed to rename tmp file: '"+tmpFileName+"'",return false); return true; } uint32 CFileManager::getFileSize(const NLMISC::CSString& fileName) { try { return NLMISC::CFile::getFileSize(fileName); } catch(...) { return 0; } } } // end of namespace //----------------------------------------------------------------------------- // NLMISC_COMMANDS for testing the singleton interface //----------------------------------------------------------------------------- NLMISC_CATEGORISED_COMMAND(patchman,fileManagerLoad,"Load a file segment via the file manager"," ") { if (args.size()!=3) return false; CSString fileName= args[0]; CSString startOffset=args[1]; CSString numBytes=args[2]; CSString data; bool ok= CFileManager::getInstance().load(fileName,startOffset.atoui(),numBytes.atoui(),data); if (ok) { log.displayNL("Loaded %d bytes from file: %s[%d] (requested %d) starting: %s",data.size(),fileName.c_str(),startOffset.atoui(),numBytes.atoui(),data.left(20).quote().c_str()); } else { log.displayNL("Load failed for file: %s (from offset %d to %d)",fileName.c_str(),startOffset.atoui(),startOffset.atoui()+numBytes.atoui()-1); } return true; } NLMISC_CATEGORISED_COMMAND(patchman,fileManagerSave,"Save a file via the file manager"," ") { if (args.size()!=2) return false; CSString fileName= args[0]; CMemStream data; data.serialBuffer((uint8*)(&args[1][0]),args[1].size()); CFileManager::getInstance().save(fileName,data); return true; } NLMISC_CATEGORISED_COMMAND(patchman,testCFileSpec,"test the CFileSpec class","") { if (args.size()!=0) return false; { nlinfo("test a"); CFileSpec fsa("foo/bar"); nlassert(fsa.matches("foo/bar")); nlassert(!fsa.matches("foo/bard")); nlassert(!fsa.matches("food/bar")); nlassert(!fsa.matches("foo/d/bar")); nlassert(!fsa.nameIsWild()); nlassert(!fsa.pathIsWild()); nlassert(fsa.nameMatches("bar")); nlassert(fsa.pathMatches("foo/")); nlassert(!fsa.nameMatches("bard")); nlassert(!fsa.pathMatches("food/")); nlassert(!fsa.pathMatches("foo/d/")); nlassert(fsa.nameSpec()=="bar"); nlassert(fsa.pathSpec()=="foo/"); nlassert(fsa.toString()=="foo/bar"); } { nlinfo("test b"); CFileSpec fsb("foo/bar*"); nlassert(fsb.matches("foo/bar")); nlassert(fsb.matches("foo/bard")); nlassert(!fsb.matches("food/bar")); nlassert(!fsb.matches("foo/d/bar")); nlassert(fsb.nameIsWild()); nlassert(!fsb.pathIsWild()); nlassert(fsb.nameMatches("bard")); nlassert(fsb.pathMatches("foo/")); nlassert(fsb.nameMatches("bard")); nlassert(!fsb.pathMatches("food/")); nlassert(!fsb.pathMatches("foo/d/")); nlassert(fsb.nameSpec()=="bar*"); nlassert(fsb.pathSpec()=="foo/"); nlassert(fsb.toString()=="foo/bar*"); } { nlinfo("test c"); CFileSpec fsc("foo*/bar"); nlassert(fsc.matches("foo/bar")); nlassert(!fsc.matches("foo/bard")); nlassert(fsc.matches("food/bar")); nlassert(fsc.matches("foo/d/bar")); nlassert(!fsc.nameIsWild()); nlassert(fsc.pathIsWild()); nlassert(fsc.nameMatches("bar")); nlassert(fsc.pathMatches("foo/")); nlassert(!fsc.nameMatches("bard")); nlassert(fsc.pathMatches("food/")); nlassert(fsc.pathMatches("foo/d/")); nlassert(fsc.nameSpec()=="bar"); nlassert(fsc.pathSpec()=="foo*/"); nlassert(fsc.toString()=="foo*/bar"); } { nlinfo("test d"); CFileSpec fsd("foo*/bar*"); nlassert(fsd.matches("foo/bar")); nlassert(fsd.matches("foo/bard")); nlassert(fsd.matches("food/bar")); nlassert(fsd.matches("foo/d/bar")); nlassert(fsd.nameIsWild()); nlassert(fsd.pathIsWild()); nlassert(fsd.nameMatches("bar")); nlassert(fsd.pathMatches("foo/")); nlassert(fsd.nameMatches("bard")); nlassert(fsd.pathMatches("food/")); nlassert(fsd.pathMatches("foo/d/")); nlassert(fsd.nameSpec()=="bar*"); nlassert(fsd.pathSpec()=="foo*/"); nlassert(fsd.toString()=="foo*/bar*"); } return true; } class CFileSpec { public: // ctors CFileSpec(); CFileSpec(const NLMISC::CSString& fileSpec); // test a complete match (filename and path) bool matches(const NLMISC::CSString& fullFileName) const; // test whether the given file name matches the filename part of the filespec // note - the supplied filename should already have been stripped of its path bool nameMatches(const NLMISC::CSString& fileName) const; // test whether the given path matches the path part of the filespec // note - the supplied path should not have an attached file name bool pathMatches(const NLMISC::CSString& path) const; // retrieve the filespec as a single string (for serialising, etc) NLMISC::CSString toString() const; // accessors const NLMISC::CSString& nameSpec() const; const NLMISC::CSString& pathSpec() const; bool nameIsWild() const; bool pathIsWild() const; private: NLMISC::CSString _NameSpec; NLMISC::CSString _PathSpec; bool _NameIsWild; // true if _NameSpec contains wildcards ('*' or '?') bool _PathIsWild; // true if _PathSpec contains wildcards ('*' or '?') };