diff --git a/code/nel/include/nel/misc/big_file.h b/code/nel/include/nel/misc/big_file.h index 4c11b01b0..14851d69e 100644 --- a/code/nel/include/nel/misc/big_file.h +++ b/code/nel/include/nel/misc/big_file.h @@ -20,7 +20,7 @@ #include "types_nl.h" #include "tds.h" #include "singleton.h" - +#include "callback.h" namespace NLMISC { @@ -86,6 +86,66 @@ public: // Used for CPath only for the moment ! char *getFileNamePtr(const std::string &sFileName, const std::string &sBigFileName); + typedef CCallback TUnpackProgressCallback; + + // Unpack all files in sBigFileName to sDestDir and send progress notifications to optional callback + static bool unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback = NULL); + + // A BNPFile header (filename is a char* pointing on FileNames and is always lowercase) + struct BNPFile + { + BNPFile() : Name(NULL), Size(0), Pos(0) { } + char* Name; + uint32 Size; + uint32 Pos; + }; + + // A SBNPFile header (filename is a std::string and keeps the original case) + struct SBNPFile + { + SBNPFile() : Size(0), Pos(0) { } + std::string Name; + uint32 Size; + uint32 Pos; + }; + + // A BNP structure + struct BNP + { + BNP() : FileNames(NULL), ThreadFileId(0), CacheFileOnOpen(false), AlwaysOpened(false), InternalUse(false), OffsetFromBeginning(0) { } + + // FileName of the BNP. important to open it in getFile() (for other threads or if not always opened). + std::string BigFileName; + // map of files in the BNP. + char *FileNames; + std::vector Files; + std::vector SFiles; + + // Since many seek may be done on a FILE*, each thread should have its own FILE opened. + uint32 ThreadFileId; + bool CacheFileOnOpen; + bool AlwaysOpened; + bool InternalUse; + + // Offset written in BNP header + uint32 OffsetFromBeginning; + + // Read BNP header from FILE* and init member variables + bool readHeader(FILE* file); + + // Read BNP header from BigFileName and init member variables + bool readHeader(); + + // Append BNP header to the big file BigFileName (to use after appendFile calls) + bool appendHeader(); + + // Append a file to BigFileName + bool appendFile(const std::string &filename); + + // Unpack BigFileName to sDestDir and send progress notifications to optional callback + bool unpack(const std::string &sDestDir, TUnpackProgressCallback *callback = NULL); + }; + // *************** private: class CThreadFileArray; @@ -118,38 +178,6 @@ private: uint32 _CurrentId; }; - // A BNPFile header - struct BNPFile - { - BNPFile() : Name(NULL), Size(0), Pos(0) { } - char *Name; - uint32 Size; - uint32 Pos; - }; - - struct CBNPFileComp - { - bool operator()(const BNPFile &f, const BNPFile &s ) - { - return strcmp( f.Name, s.Name ) < 0; - } - }; - - // A BNP structure - struct BNP - { - BNP() : FileNames(NULL) { } - - // FileName of the BNP. important to open it in getFile() (for other threads or if not always opened). - std::string BigFileName; - // map of files in the BNP. - char *FileNames; - std::vector Files; - // Since many seek may be done on a FILE*, each thread should have its own FILE opened. - uint32 ThreadFileId; - bool CacheFileOnOpen; - bool AlwaysOpened; - }; private: // CBigFile(); // Singleton mode -> access it with the getInstance function diff --git a/code/nel/include/nel/misc/path.h b/code/nel/include/nel/misc/path.h index 20ea9b2a9..9c3e2e21b 100644 --- a/code/nel/include/nel/misc/path.h +++ b/code/nel/include/nel/misc/path.h @@ -510,9 +510,10 @@ public: /** Make path absolute * \param relativePath - The relative path * \param directory - the directory to which the path is relative to + * \param simplify - if we should simplify or not the path (convert . and .. in path) * returns the absolute path, or empty if something went wrong. */ - static std::string makePathAbsolute (const std::string &relativePath, const std::string &directory ); + static std::string makePathAbsolute (const std::string &relativePath, const std::string &directory, bool simplify = false ); /** Return if a path is absolute or not. * \param path - The path diff --git a/code/nel/src/misc/big_file.cpp b/code/nel/src/misc/big_file.cpp index f5ee067f4..1de1c2b73 100644 --- a/code/nel/src/misc/big_file.cpp +++ b/code/nel/src/misc/big_file.cpp @@ -16,6 +16,7 @@ #include "stdmisc.h" +#include "nel/misc/file.h" #include "nel/misc/big_file.h" #include "nel/misc/path.h" @@ -128,151 +129,27 @@ bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions) bnp.BigFileName= sBigFileName; - // Allocate a new ThreadSafe FileId for this bnp. bnp.ThreadFileId= _ThreadFileArray.allocate(); // Get a ThreadSafe handle on the file CHandleFile &handle= _ThreadFileArray.get(bnp.ThreadFileId); + // Open the big file. handle.File = fopen (sBigFileName.c_str(), "rb"); if (handle.File == NULL) return false; - uint32 nFileSize=CFile::getFileSize (handle.File); - //nlfseek64 (handle.File, 0, SEEK_END); - //uint32 nFileSize = ftell (handle.File); - // Result - if (nlfseek64 (handle.File, nFileSize-4, SEEK_SET) != 0) + // Used internally by CBigFile, use optimizations and lower case of filenames + bnp.InternalUse = true; + + // read BNP header + if (!bnp.readHeader(handle.File)) { fclose (handle.File); handle.File = NULL; return false; } - - uint32 nOffsetFromBeginning; - if (fread (&nOffsetFromBeginning, sizeof(uint32), 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - -#ifdef NL_BIG_ENDIAN - NLMISC_BSWAP32(nOffsetFromBeginning); -#endif - - if (nlfseek64 (handle.File, nOffsetFromBeginning, SEEK_SET) != 0) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - - // Read the file count - uint32 nNbFile; - if (fread (&nNbFile, sizeof(uint32), 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - -#ifdef NL_BIG_ENDIAN - NLMISC_BSWAP32(nNbFile); -#endif - - map tempMap; - for (uint32 i = 0; i < nNbFile; ++i) - { - char FileName[256]; - uint8 nStringSize; - if (fread (&nStringSize, 1, 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - - if (fread (FileName, nStringSize, 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - - FileName[nStringSize] = 0; - uint32 nFileSize2; - if (fread (&nFileSize2, sizeof(uint32), 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - -#ifdef NL_BIG_ENDIAN - NLMISC_BSWAP32(nFileSize2); -#endif - - uint32 nFilePos; - if (fread (&nFilePos, sizeof(uint32), 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - -#ifdef NL_BIG_ENDIAN - NLMISC_BSWAP32(nFilePos); -#endif - - BNPFile bnpfTmp; - bnpfTmp.Pos = nFilePos; - bnpfTmp.Size = nFileSize2; - tempMap.insert (make_pair(toLower(string(FileName)), bnpfTmp)); - } - - if (nlfseek64 (handle.File, 0, SEEK_SET) != 0) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - - // Convert temp map - if (nNbFile > 0) - { - uint nSize = 0, nNb = 0; - map::iterator it = tempMap.begin(); - while (it != tempMap.end()) - { - nSize += (uint)it->first.size() + 1; - nNb++; - it++; - } - - bnp.FileNames = new char[nSize]; - memset(bnp.FileNames, 0, nSize); - bnp.Files.resize(nNb); - - it = tempMap.begin(); - nSize = 0; - nNb = 0; - while (it != tempMap.end()) - { - strcpy(bnp.FileNames+nSize, it->first.c_str()); - - bnp.Files[nNb].Name = bnp.FileNames+nSize; - bnp.Files[nNb].Size = it->second.Size; - bnp.Files[nNb].Pos = it->second.Pos; - - nSize += (uint)it->first.size() + 1; - nNb++; - it++; - } - } - // End of temp map conversion - if (nOptions&BF_CACHE_FILE_ON_OPEN) bnp.CacheFileOnOpen = true; else @@ -314,6 +191,369 @@ void CBigFile::remove (const std::string &sBigFileName) } } +//// *************************************************************************** +bool CBigFile::BNP::readHeader() +{ + // Only external use + if (InternalUse || BigFileName.empty()) return false; + + FILE *f = fopen (BigFileName.c_str(), "rb"); + if (f == NULL) return false; + + bool res = readHeader(f); + fclose (f); + + return res; +} + +//// *************************************************************************** +bool CBigFile::BNP::readHeader(FILE *file) +{ + if (file == NULL) return false; + + uint32 nFileSize=CFile::getFileSize (file); + + // Result + if (nlfseek64 (file, nFileSize-4, SEEK_SET) != 0) + { + return false; + } + + if (fread (&OffsetFromBeginning, sizeof(uint32), 1, file) != 1) + { + return false; + } + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(OffsetFromBeginning); +#endif + + if (nlfseek64 (file, OffsetFromBeginning, SEEK_SET) != 0) + { + return false; + } + + // Read the file count + uint32 nNbFile; + if (fread (&nNbFile, sizeof(uint32), 1, file) != 1) + { + return false; + } + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nNbFile); +#endif + + map tempMap; + + if (!InternalUse) SFiles.clear(); + + for (uint32 i = 0; i < nNbFile; ++i) + { + uint8 nStringSize; + if (fread (&nStringSize, 1, 1, file) != 1) + { + return false; + } + + char sFileName[256]; + if (fread (sFileName, 1, nStringSize, file) != nStringSize) + { + return false; + } + + sFileName[nStringSize] = 0; + + uint32 nFileSize2; + if (fread (&nFileSize2, sizeof(uint32), 1, file) != 1) + { + return false; + } + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nFileSize2); +#endif + + uint32 nFilePos; + if (fread (&nFilePos, sizeof(uint32), 1, file) != 1) + { + return false; + } + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nFilePos); +#endif + + if (InternalUse) + { + BNPFile bnpfTmp; + bnpfTmp.Pos = nFilePos; + bnpfTmp.Size = nFileSize2; + tempMap.insert (make_pair(toLower(string(sFileName)), bnpfTmp)); + } + else + { + SBNPFile bnpfTmp; + bnpfTmp.Name = sFileName; + bnpfTmp.Pos = nFilePos; + bnpfTmp.Size = nFileSize2; + SFiles.push_back(bnpfTmp); + } + } + + if (nlfseek64 (file, 0, SEEK_SET) != 0) + { + return false; + } + + // Convert temp map + if (InternalUse && nNbFile > 0) + { + uint nSize = 0, nNb = 0; + map::iterator it = tempMap.begin(); + while (it != tempMap.end()) + { + nSize += (uint)it->first.size() + 1; + nNb++; + it++; + } + + FileNames = new char[nSize]; + memset(FileNames, 0, nSize); + Files.resize(nNb); + + it = tempMap.begin(); + nSize = 0; + nNb = 0; + while (it != tempMap.end()) + { + strcpy(FileNames+nSize, it->first.c_str()); + + Files[nNb].Name = FileNames+nSize; + Files[nNb].Size = it->second.Size; + Files[nNb].Pos = it->second.Pos; + + nSize += (uint)it->first.size() + 1; + nNb++; + it++; + } + } + // End of temp map conversion + + return true; +} + +bool CBigFile::BNP::appendHeader() +{ + // Only external use + if (InternalUse || BigFileName.empty()) return false; + + FILE *f = fopen (BigFileName.c_str(), "ab"); + if (f == NULL) return false; + + uint32 nNbFile = (uint32)SFiles.size(); + + // value to be serialized + uint32 nNbFile2 = nNbFile; + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nNbFile2); +#endif + + if (fwrite (&nNbFile2, sizeof(uint32), 1, f) != 1) + { + fclose(f); + return false; + } + + for (uint32 i = 0; i < nNbFile; ++i) + { + uint8 nStringSize = (uint8)SFiles[i].Name.length(); + if (fwrite (&nStringSize, 1, 1, f) != 1) + { + fclose(f); + return false; + } + + if (fwrite (SFiles[i].Name.c_str(), 1, nStringSize, f) != nStringSize) + { + fclose(f); + return false; + } + + uint32 nFileSize = SFiles[i].Size; + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nFileSize); +#endif + + if (fwrite (&nFileSize, sizeof(uint32), 1, f) != 1) + { + fclose(f); + return false; + } + + uint32 nFilePos = SFiles[i].Pos; + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nFilePos); +#endif + + if (fwrite (&nFilePos, sizeof(uint32), 1, f) != 1) + { + fclose(f); + return false; + } + } + + uint32 nOffsetFromBeginning = OffsetFromBeginning; + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nOffsetFromBeginning); +#endif + + if (fwrite (&nOffsetFromBeginning, sizeof(uint32), 1, f) != 1) + { + fclose(f); + return false; + } + + fclose (f); + return true; +} + +// *************************************************************************** +bool CBigFile::BNP::appendFile(const std::string &filename) +{ + // Only external use + if (InternalUse || BigFileName.empty()) return false; + + // Check if we can read the source file + if (!CFile::fileExists(filename)) return false; + + SBNPFile ftmp; + ftmp.Name = CFile::getFilename(filename); + ftmp.Size = CFile::getFileSize(filename); + ftmp.Pos = OffsetFromBeginning; + SFiles.push_back(ftmp); + OffsetFromBeginning += ftmp.Size; + + FILE *f1 = fopen(BigFileName.c_str(), "ab"); + if (f1 == NULL) return false; + + FILE *f2 = fopen(filename.c_str(), "rb"); + if (f2 == NULL) + { + fclose(f1); + return false; + } + + uint8 *ptr = new uint8[ftmp.Size]; + + if (fread (ptr, ftmp.Size, 1, f2) != 1) + { + nlwarning("%s read error", filename.c_str()); + } + else if (fwrite (ptr, ftmp.Size, 1, f1) != 1) + { + nlwarning("%s write error", BigFileName.c_str()); + } + + delete [] ptr; + + fclose(f2); + fclose(f1); + + return true; +} + +// *************************************************************************** +bool CBigFile::BNP::unpack(const std::string &sDestDir, TUnpackProgressCallback *callback) +{ + // Only external use + if (InternalUse || BigFileName.empty()) return false; + + FILE *bnp = fopen (BigFileName.c_str(), "rb"); + if (bnp == NULL) + return false; + + // only read header is not already read + if (SFiles.empty() && !readHeader(bnp)) + { + fclose (bnp); + return false; + } + + CFile::createDirectory(sDestDir); + + uint32 totalUncompressed = 0, total = 0; + + for (uint32 i = 0; i < SFiles.size(); ++i) + { + total += SFiles[i].Size; + } + + FILE *out = NULL; + + for (uint32 i = 0; i < SFiles.size(); ++i) + { + const SBNPFile &rBNPFile = SFiles[i]; + string filename = CPath::standardizePath(sDestDir) + rBNPFile.Name; + + if (callback && !(*callback)(filename, totalUncompressed, total)) + { + fclose (bnp); + return false; + } + + out = fopen (filename.c_str(), "wb"); + if (out != NULL) + { + nlfseek64 (bnp, rBNPFile.Pos, SEEK_SET); + uint8 *ptr = new uint8[rBNPFile.Size]; + bool readError = fread (ptr, rBNPFile.Size, 1, bnp) != 1; + if (readError) + { + nlwarning("%s read error errno = %d: %s", filename.c_str(), errno, strerror(errno)); + } + bool writeError = fwrite (ptr, rBNPFile.Size, 1, out) != 1; + if (writeError) + { + nlwarning("%s write error errno = %d: %s", filename.c_str(), errno, strerror(errno)); + } + bool diskFull = ferror(out) && errno == 28 /* ENOSPC*/; + fclose (out); + delete [] ptr; + if (diskFull) + { + fclose (bnp); + throw NLMISC::EDiskFullError(filename); + } + if (writeError) + { + fclose (bnp); + throw NLMISC::EWriteError(filename); + } + if (readError) + { + fclose (bnp); + throw NLMISC::EReadError(filename); + } + } + + totalUncompressed += rBNPFile.Size; + + if (callback && !(*callback)(filename, totalUncompressed, total)) + { + fclose (bnp); + return false; + } + } + + fclose (bnp); + return true; +} + // *************************************************************************** bool CBigFile::isBigFileAdded(const std::string &sBigFileName) const { @@ -359,6 +599,14 @@ void CBigFile::removeAll () } } +struct CBNPFileComp +{ + bool operator()(const CBigFile::BNPFile &f, const CBigFile::BNPFile &s ) + { + return strcmp( f.Name, s.Name ) < 0; + } +}; + // *************************************************************************** bool CBigFile::getFileInternal (const std::string &sFileName, BNP *&zeBnp, BNPFile *&zeBnpFile) { @@ -504,5 +752,12 @@ void CBigFile::getBigFilePaths(std::vector &bigFilePaths) } } +// *************************************************************************** +bool CBigFile::unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback) +{ + BNP bnpFile; + bnpFile.BigFileName = sBigFileName; + return bnpFile.unpack(sDestDir, callback); +} } // namespace NLMISC diff --git a/code/nel/src/misc/path.cpp b/code/nel/src/misc/path.cpp index 7ec61fdd7..58370eb2b 100644 --- a/code/nel/src/misc/path.cpp +++ b/code/nel/src/misc/path.cpp @@ -260,7 +260,7 @@ void CFileContainer::getFileListByPath(const std::string &extension, const std:: { string ext = SSMext.get(first->idExt); string p = SSMpath.get(first->idPath); - + if (strstr(p.c_str(), path.c_str()) != NULL && (ext == extension || extension.empty())) { filenames.push_back(first->Name); @@ -2546,55 +2546,112 @@ bool CPath::makePathRelative (const char *basePath, std::string &relativePath) return false; } -std::string CPath::makePathAbsolute( const std::string &relativePath, const std::string &directory ) +std::string CPath::makePathAbsolute( const std::string &relativePath, const std::string &directory, bool simplify ) { if( relativePath.empty() ) return ""; if( directory.empty() ) return ""; + std::string absolutePath; + #ifdef NL_OS_WINDOWS // Windows network address. Eg.: \\someshare\path - if( ( relativePath[ 0 ] == '\\' ) && ( relativePath[ 1 ] == '\\' ) ) - return relativePath; + if ((relativePath[0] == '\\') && (relativePath[1] == '\\')) + { + absolutePath = relativePath; + } // Normal Windows absolute path. Eg.: C:\something // - if( isalpha( relativePath[ 0 ] ) && ( relativePath[ 1 ] == ':' ) && ( ( relativePath[ 2 ] == '\\' ) || ( relativePath[ 2 ] == '/' ) ) ) - return relativePath; + else if (isalpha(relativePath[0]) && (relativePath[1] == ':') && ((relativePath[2] == '\\') || (relativePath[2] == '/'))) + { + absolutePath = relativePath; + } #else // Unix filesystem absolute path - if( relativePath[ 0 ] == '/' ) - return relativePath; - + if (relativePath[0] == '/') + { + absolutePath = relativePath; + } #endif - - // Add a slash to the directory if necessary. - // If the relative path starts with dots we need a slash. - // If the relative path starts with a slash we don't. - // If it starts with neither, we need a slash. - bool needSlash = true; - char c = relativePath[ 0 ]; - if( ( c == '\\' ) || ( c == '/' ) ) - needSlash = false; - - bool hasSlash = false; - std::string npath = directory; - c = npath[ npath.size() - 1 ]; - if( ( c == '\\' ) || ( c == '/' ) ) - hasSlash = true; - - if( needSlash && !hasSlash ) - npath += '/'; else - if( hasSlash && !needSlash ) - npath.resize( npath.size() - 1 ); - - // Now build the new absolute path - npath += relativePath; - npath = standardizePath( npath, false ); + { + // Add a slash to the directory if necessary. + // If the relative path starts with dots we need a slash. + // If the relative path starts with a slash we don't. + // If it starts with neither, we need a slash. + bool needSlash = true; + char c = relativePath[0]; + if ((c == '\\') || (c == '/')) + needSlash = false; - return npath; + bool hasSlash = false; + absolutePath = directory; + c = absolutePath[absolutePath.size() - 1]; + if ((c == '\\') || (c == '/')) + hasSlash = true; + + if (needSlash && !hasSlash) + absolutePath += '/'; + else + if (hasSlash && !needSlash) + absolutePath.resize(absolutePath.size() - 1); + + // Now build the new absolute path + absolutePath += relativePath; + absolutePath = standardizePath(absolutePath, true); + } + + if (simplify) + { + // split all components path to manage parent directories + std::vector tokens; + explode(absolutePath, std::string("/"), tokens, true); + + std::vector directoryParts; + + // process all components + for(uint i = 0, len = tokens.size(); i < len; ++i) + { + std::string token = tokens[i]; + + // current directory + if (token != ".") + { + // parent directory + if (token == "..") + { + // remove last directory + directoryParts.pop_back(); + } + else + { + // append directory + directoryParts.push_back(token); + } + } + } + + if (!directoryParts.empty()) + { + absolutePath = directoryParts[0]; + + // rebuild the whole absolute path + for(uint i = 1, len = directoryParts.size(); i < len; ++i) + absolutePath += "/" + directoryParts[i]; + + // add trailing slash + absolutePath += "/"; + } + else + { + // invalid path + absolutePath.clear(); + } + } + + return absolutePath; } bool CPath::isAbsolutePath(const std::string &path)