// NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/> // Copyright (C) 2010 Winch Gate Property Limited // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as // published by the Free Software Foundation, either version 3 of the // License, or (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see <http://www.gnu.org/licenses/>. #include "stdmisc.h" #ifndef NL_OS_WINDOWS #include <errno.h> #endif #include "nel/misc/file.h" #include "nel/misc/debug.h" #include "nel/misc/big_file.h" #include "nel/misc/path.h" #include "nel/misc/command.h" #include "nel/misc/sstring.h" #include "nel/misc/xml_pack.h" using namespace std; #define NLMISC_DONE_FILE_OPENED 40 namespace NLMISC { typedef std::list<uint64> TFileAccessTimes; // list of times at which a given file is opened for reading typedef CHashMap<std::string,TFileAccessTimes> TFileAccessLog; // map from file name to read access times typedef NLMISC::CSynchronized<TFileAccessLog> TSynchronizedFileAccessLog; static TSynchronizedFileAccessLog IFileAccessLog("IFileAccessLog"); static bool IFileAccessLoggingEnabled= false; static uint64 IFileAccessLogStartTime= 0; uint32 CIFile::_NbBytesSerialized = 0; uint32 CIFile::_NbBytesLoaded = 0; uint32 CIFile::_ReadFromFile = 0; uint32 CIFile::_ReadingFromFile = 0; uint32 CIFile::_FileOpened = 0; uint32 CIFile::_FileRead = 0; CSynchronized<std::deque<std::string> > CIFile::_OpenedFiles(""); // ====================================================================================================== CIFile::CIFile() : IStream(true) { _F = NULL; _Cache = NULL; _ReadPos = 0; _FileSize = 0; _BigFileOffset = 0; _IsInBigFile = false; _IsInXMLPackFile = false; _CacheFileOnOpen = false; _IsAsyncLoading = false; _AllowBNPCacheFileOnOpen= true; } // ====================================================================================================== CIFile::CIFile(const std::string &path, bool text) : IStream(true) { _F=NULL; _Cache = NULL; _ReadPos = 0; _FileSize = 0; _BigFileOffset = 0; _IsInBigFile = false; _IsInXMLPackFile = false; _CacheFileOnOpen = false; _IsAsyncLoading = false; _AllowBNPCacheFileOnOpen= true; open(path, text); } // ====================================================================================================== CIFile::~CIFile() { close(); } // ====================================================================================================== void CIFile::loadIntoCache() { const uint32 READPACKETSIZE = 64 * 1024; const uint32 INTERPACKETSLEEP = 5; _Cache = new uint8[_FileSize]; if(!_IsAsyncLoading) { _ReadingFromFile += _FileSize; int read = (int)fread (_Cache, _FileSize, 1, _F); _FileRead++; _ReadingFromFile -= _FileSize; _ReadFromFile += read * _FileSize; } else { uint index= 0; while(index<_FileSize) { if( _NbBytesLoaded + (_FileSize-index) > READPACKETSIZE ) { sint n= READPACKETSIZE-_NbBytesLoaded; n= max(n, 1); _ReadingFromFile += n; int read = (int)fread (_Cache+index, n, 1, _F); _FileRead++; _ReadingFromFile -= n; _ReadFromFile += read * n; index+= n; nlSleep (INTERPACKETSLEEP); _NbBytesLoaded= 0; } else { uint n= _FileSize-index; _ReadingFromFile += n; int read = (int)fread (_Cache+index, n, 1, _F); _FileRead++; _ReadingFromFile -= n; _ReadFromFile += read * n; _NbBytesLoaded+= n; index+= n; } } } } // ====================================================================================================== bool CIFile::open(const std::string &path, bool text) { // Log opened files { CSynchronized<deque<string> >::CAccessor fileOpened(&_OpenedFiles); fileOpened.value().push_front (path); if (fileOpened.value().size () > NLMISC_DONE_FILE_OPENED) fileOpened.value().resize (NLMISC_DONE_FILE_OPENED); _FileOpened++; } close(); // can't open empty filename if(path.empty ()) return false; // IFile Access Log management if (IFileAccessLoggingEnabled) { // get the current time uint64 timeNow = NLMISC::CTime::getPerformanceTime(); // get a handle for the container TSynchronizedFileAccessLog::CAccessor synchronizedFileAccesLog(&IFileAccessLog); TFileAccessLog& fileAccessLog= synchronizedFileAccesLog.value(); // add the current time to the container entry for the given path (creating a new container entry if required) fileAccessLog[path].push_back(timeNow); } char mode[3]; mode[0] = 'r'; mode[1] = 'b'; // No more reading in text mode mode[2] = '\0'; _FileName = path; _ReadPos = 0; // Bigfile or xml pack access requested ? string::size_type pos; if ((pos = path.find('@')) != string::npos) { // check for a double @ to identify XML pack file if (pos+1 < path.size() && path[pos+1] == '@') { // xml pack file _IsInXMLPackFile = true; if(_AllowBNPCacheFileOnOpen) { _F = CXMLPack::getInstance().getFile(path, _FileSize, _BigFileOffset, _CacheFileOnOpen, _AlwaysOpened); } else { bool dummy; _F = CXMLPack::getInstance().getFile (path, _FileSize, _BigFileOffset, dummy, _AlwaysOpened); } } else { // bnp file _IsInBigFile = true; if(_AllowBNPCacheFileOnOpen) { _F = CBigFile::getInstance().getFile (path, _FileSize, _BigFileOffset, _CacheFileOnOpen, _AlwaysOpened); } else { bool dummy; _F = CBigFile::getInstance().getFile (path, _FileSize, _BigFileOffset, dummy, _AlwaysOpened); } } if(_F != NULL) { // Start to load the bigfile or xml file at the file offset. nlfseek64 (_F, _BigFileOffset, SEEK_SET); // Load into cache ? if (_CacheFileOnOpen) { // load file in the cache loadIntoCache(); if (!_AlwaysOpened) { fclose (_F); _F = NULL; } return (_Cache != NULL); } } } else { _IsInBigFile = false; _IsInXMLPackFile = false; _BigFileOffset = 0; _AlwaysOpened = false; _F = fopen (path.c_str(), mode); if (_F != NULL) { /* THIS CODE REPLACED BY SADGE BECAUSE SOMETIMES ftell() RETRUNS 0 FOR NO GOOD REASON - LEADING TO CLIENT CRASH nlfseek64 (_F, 0, SEEK_END); _FileSize = ftell(_F); nlfseek64 (_F, 0, SEEK_SET); nlassert(_FileSize==filelength(fileno(_F))); THE FOLLOWING WORKS BUT IS NOT PORTABLE _FileSize=filelength(fileno(_F)); */ _FileSize=CFile::getFileSize (_F); if (_FileSize == 0) { nlwarning ("FILE: Size of file '%s' is 0", path.c_str()); fclose (_F); _F = NULL; } } else { nlwarning("Failed to open file '%s', error %u : %s", path.c_str(), errno, strerror(errno)); _FileSize = 0; } if ((_CacheFileOnOpen) && (_F != NULL)) { // load file in the cache loadIntoCache(); fclose (_F); _F = NULL; return (_Cache != NULL); } } return (_F != NULL); } // ====================================================================================================== void CIFile::setCacheFileOnOpen (bool newState) { _CacheFileOnOpen = newState; } // ====================================================================================================== void CIFile::setAsyncLoading (bool newState) { _IsAsyncLoading = true; } // ====================================================================================================== void CIFile::close() { if (_CacheFileOnOpen) { if (_Cache) { delete[] _Cache; _Cache = NULL; } } else { if (_IsInBigFile || _IsInXMLPackFile) { if (!_AlwaysOpened) { if (_F) { fclose (_F); _F = NULL; } } } else { if (_F) { fclose (_F); _F = NULL; } } } nlassert(_Cache == NULL); resetPtrTable(); } // ====================================================================================================== void CIFile::flush() { if (_CacheFileOnOpen) { } else { if (_F) { fflush (_F); } } } // ====================================================================================================== void CIFile::getline (char *buffer, uint32 bufferSize) { if (bufferSize == 0) return; uint read = 0; for(;;) { if (read == bufferSize - 1) { *buffer = '\0'; return; } try { // read one byte serialBuffer ((uint8 *)buffer, 1); } catch (EFile &) { *buffer = '\0'; return; } if (*buffer == '\n') { *buffer = '\0'; return; } // skip '\r' char if (*buffer != '\r') { buffer++; read++; } } } // ====================================================================================================== bool CIFile::eof () { return _ReadPos >= (sint32)_FileSize; } // ====================================================================================================== void CIFile::serialBuffer(uint8 *buf, uint len) throw(EReadError) { if (len == 0) return; // Check the read pos if ((_ReadPos < 0) || ((_ReadPos+len) > _FileSize)) throw EReadError (_FileName); if ((_CacheFileOnOpen) && (_Cache == NULL)) throw EFileNotOpened (_FileName); if ((!_CacheFileOnOpen) && (_F == NULL)) throw EFileNotOpened (_FileName); if (_IsAsyncLoading) { _NbBytesSerialized += len; if (_NbBytesSerialized > 64 * 1024) { // give up time slice nlSleep (0); _NbBytesSerialized = 0; } } if (_CacheFileOnOpen) { memcpy (buf, _Cache + _ReadPos, len); _ReadPos += len; } else { int read; _ReadingFromFile += len; read=(int)fread(buf, len, 1, _F); _FileRead++; _ReadingFromFile -= len; _ReadFromFile += /*read **/ len; if (read != 1 /*< (int)len*/) throw EReadError(_FileName); _ReadPos += len; } } // ====================================================================================================== void CIFile::serialBit(bool &bit) throw(EReadError) { // Simple for now. uint8 v=bit; serialBuffer(&v, 1); bit=(v!=0); } // ====================================================================================================== bool CIFile::seek (sint32 offset, IStream::TSeekOrigin origin) const throw(EStream) { if ((_CacheFileOnOpen) && (_Cache == NULL)) return false; if ((!_CacheFileOnOpen) && (_F == NULL)) return false; switch (origin) { case IStream::begin: _ReadPos = offset; break; case IStream::current: _ReadPos = _ReadPos + offset; break; case IStream::end: _ReadPos = _FileSize + offset; break; default: nlstop; } if (_CacheFileOnOpen) return true; // seek in the file. NB: if not in bigfile, _BigFileOffset==0. if (nlfseek64(_F, _BigFileOffset+_ReadPos, SEEK_SET) != 0) return false; return true; } // ====================================================================================================== sint32 CIFile::getPos () const throw(EStream) { return _ReadPos; } // ====================================================================================================== std::string CIFile::getStreamName() const { return _FileName; } // ====================================================================================================== void CIFile::allowBNPCacheFileOnOpen(bool newState) { _AllowBNPCacheFileOnOpen= newState; } // ====================================================================================================== void CIFile::dump (std::vector<std::string> &result) { CSynchronized<deque<string> >::CAccessor acces(&_OpenedFiles); const deque<string> &openedFile = acces.value(); // Resize the destination array result.clear (); result.reserve (openedFile.size ()); // Add the waiting strings deque<string>::const_reverse_iterator ite = openedFile.rbegin (); while (ite != openedFile.rend ()) { result.push_back (*ite); // Next task ite++; } } // ====================================================================================================== void CIFile::clearDump () { CSynchronized<deque<string> >::CAccessor acces(&_OpenedFiles); acces.value().clear(); } // ====================================================================================================== uint CIFile::getDbgStreamSize() const { return getFileSize(); } // ====================================================================================================== // ====================================================================================================== // ====================================================================================================== // ====================================================================================================== COFile::COFile() : IStream(false) { _F=NULL; _FileName = ""; } // ====================================================================================================== COFile::COFile(const std::string &path, bool append, bool text, bool useTempFile) : IStream(false) { _F=NULL; open(path, append, text, useTempFile); } // ====================================================================================================== COFile::~COFile() { internalClose(false); } // ====================================================================================================== bool COFile::open(const std::string &path, bool append, bool text, bool useTempFile) { close(); // can't open empty filename if(path.empty ()) return false; _FileName = path; _TempFileName.clear(); char mode[3]; mode[0] = (append)?'a':'w'; // ACE: NEVER SAVE IN TEXT MODE!!! mode[1] = (text)?'\0':'b'; mode[1] = 'b'; mode[2] = '\0'; string fileToOpen = path; if (useTempFile) { CFile::getTemporaryOutputFilename (path, _TempFileName); fileToOpen = _TempFileName; } // if appending to file and using a temporary file, copycat temporary file from original... if (append && useTempFile && CFile::fileExists(_FileName)) { // open fails if can't copy original content if (!CFile::copyFile(_TempFileName, _FileName)) return false; } _F=fopen(fileToOpen.c_str(), mode); return _F!=NULL; } // ====================================================================================================== void COFile::close() { internalClose(true); } // ====================================================================================================== void COFile::internalClose(bool success) { if(_F) { fclose(_F); // Temporary filename ? if (!_TempFileName.empty()) { // Delete old if (success) { // Bug under windows, sometimes the file is not deleted uint retry = 1000; while (--retry) { if (CFile::fileExists(_FileName)) CFile::deleteFile (_FileName); if (CFile::moveFile (_FileName.c_str(), _TempFileName.c_str())) break; nlSleep (0); } if (!retry) throw ERenameError (_FileName, _TempFileName); } else CFile::deleteFile (_TempFileName); } _F=NULL; } resetPtrTable(); } // ====================================================================================================== void COFile::flush() { if(_F) { fflush(_F); } } // ====================================================================================================== void COFile::serialBuffer(uint8 *buf, uint len) throw(EWriteError) { if(!_F) throw EFileNotOpened(_FileName); if(fwrite(buf, len, 1, _F) != 1) // if(fwrite(buf, 1, len, _F) != len) { if (ferror(_F) && errno == 28 /*ENOSPC*/) { throw EDiskFullError(_FileName); } throw EWriteError(_FileName); } } // ====================================================================================================== void COFile::serialBit(bool &bit) throw(EWriteError) { // Simple for now. uint8 v=bit; serialBuffer(&v, 1); } // ====================================================================================================== bool COFile::seek (sint32 offset, IStream::TSeekOrigin origin) const throw(EStream) { if (_F) { int origin_c = SEEK_SET; switch (origin) { case IStream::begin: origin_c=SEEK_SET; break; case IStream::current: origin_c=SEEK_CUR; break; case IStream::end: origin_c=SEEK_END; break; default: nlstop; } if (nlfseek64 (_F, offset, origin_c)!=0) return false; return true; } return false; } // ====================================================================================================== sint32 COFile::getPos () const throw(EStream) { if (_F) { return ftell (_F); } return 0; } // ====================================================================================================== std::string COFile::getStreamName() const { return _FileName; } // ====================================================================================================== // ====================================================================================================== // ====================================================================================================== // ====================================================================================================== NLMISC_CATEGORISED_COMMAND(nel, iFileAccessLogStart, "Start file access logging", "") { if (!args.empty()) return false; IFileAccessLoggingEnabled= true; if (IFileAccessLogStartTime==0) { uint64 timeNow = NLMISC::CTime::getPerformanceTime(); IFileAccessLogStartTime= timeNow; } return true; } // ====================================================================================================== NLMISC_CATEGORISED_COMMAND(nel, iFileAccessLogStop, "Stop file access logging", "") { if (!args.empty()) return false; IFileAccessLoggingEnabled= false; return true; } // ====================================================================================================== NLMISC_CATEGORISED_COMMAND(nel, iFileAccessLogClear, "Clear file access logs", "") { if (!args.empty()) return false; TSynchronizedFileAccessLog::CAccessor(&IFileAccessLog).value().clear(); return true; } // ====================================================================================================== NLMISC_CATEGORISED_COMMAND(nel, iFileAccessLogDisplay, "Display file access logs", "") { if (!args.empty()) return false; log.displayNL("-- FILE ACCESS LOG BEGIN --"); TSynchronizedFileAccessLog::CAccessor fileAccesLog(&IFileAccessLog); TFileAccessLog::const_iterator it= fileAccesLog.value().begin(); TFileAccessLog::const_iterator itEnd= fileAccesLog.value().end(); uint32 count=0; while (it!=itEnd) { uint32 numTimes= (uint32)it->second.size(); CSString fileName= it->first; if (fileName.contains("@")) { log.display("%d,%s,%s,",numTimes,fileName.splitTo("@").c_str(),fileName.splitFrom("@").c_str()); } else { log.display("%d,,%s,",numTimes,fileName.c_str()); } TFileAccessTimes::const_iterator atIt= it->second.begin(); TFileAccessTimes::const_iterator atItEnd=it->second.end(); while (atIt!=atItEnd) { uint64 delta= (*atIt-IFileAccessLogStartTime); log.display("%"NL_I64"u,",delta); ++atIt; } log.displayNL(""); ++count; ++it; } log.displayNL("-- FILE ACCESS LOG END (%d Unique Files Accessed) --",count); return true; } }