From 9d319a568821a484691e06add41189737b7cf679 Mon Sep 17 00:00:00 2001 From: kervala Date: Sat, 23 Jan 2016 10:18:09 +0100 Subject: [PATCH] Changed: Centralize BNP unpacking and packing methods, to be reused in other tools/programs --- code/nel/include/nel/misc/big_file.h | 94 +++-- code/nel/src/misc/big_file.cpp | 513 ++++++++++++++++++++------- 2 files changed, 445 insertions(+), 162 deletions(-) diff --git a/code/nel/include/nel/misc/big_file.h b/code/nel/include/nel/misc/big_file.h index 4c11b01b0..14851d69e 100644 --- a/code/nel/include/nel/misc/big_file.h +++ b/code/nel/include/nel/misc/big_file.h @@ -20,7 +20,7 @@ #include "types_nl.h" #include "tds.h" #include "singleton.h" - +#include "callback.h" namespace NLMISC { @@ -86,6 +86,66 @@ public: // Used for CPath only for the moment ! char *getFileNamePtr(const std::string &sFileName, const std::string &sBigFileName); + typedef CCallback TUnpackProgressCallback; + + // Unpack all files in sBigFileName to sDestDir and send progress notifications to optional callback + static bool unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback = NULL); + + // A BNPFile header (filename is a char* pointing on FileNames and is always lowercase) + struct BNPFile + { + BNPFile() : Name(NULL), Size(0), Pos(0) { } + char* Name; + uint32 Size; + uint32 Pos; + }; + + // A SBNPFile header (filename is a std::string and keeps the original case) + struct SBNPFile + { + SBNPFile() : Size(0), Pos(0) { } + std::string Name; + uint32 Size; + uint32 Pos; + }; + + // A BNP structure + struct BNP + { + BNP() : FileNames(NULL), ThreadFileId(0), CacheFileOnOpen(false), AlwaysOpened(false), InternalUse(false), OffsetFromBeginning(0) { } + + // FileName of the BNP. important to open it in getFile() (for other threads or if not always opened). + std::string BigFileName; + // map of files in the BNP. + char *FileNames; + std::vector Files; + std::vector SFiles; + + // Since many seek may be done on a FILE*, each thread should have its own FILE opened. + uint32 ThreadFileId; + bool CacheFileOnOpen; + bool AlwaysOpened; + bool InternalUse; + + // Offset written in BNP header + uint32 OffsetFromBeginning; + + // Read BNP header from FILE* and init member variables + bool readHeader(FILE* file); + + // Read BNP header from BigFileName and init member variables + bool readHeader(); + + // Append BNP header to the big file BigFileName (to use after appendFile calls) + bool appendHeader(); + + // Append a file to BigFileName + bool appendFile(const std::string &filename); + + // Unpack BigFileName to sDestDir and send progress notifications to optional callback + bool unpack(const std::string &sDestDir, TUnpackProgressCallback *callback = NULL); + }; + // *************** private: class CThreadFileArray; @@ -118,38 +178,6 @@ private: uint32 _CurrentId; }; - // A BNPFile header - struct BNPFile - { - BNPFile() : Name(NULL), Size(0), Pos(0) { } - char *Name; - uint32 Size; - uint32 Pos; - }; - - struct CBNPFileComp - { - bool operator()(const BNPFile &f, const BNPFile &s ) - { - return strcmp( f.Name, s.Name ) < 0; - } - }; - - // A BNP structure - struct BNP - { - BNP() : FileNames(NULL) { } - - // FileName of the BNP. important to open it in getFile() (for other threads or if not always opened). - std::string BigFileName; - // map of files in the BNP. - char *FileNames; - std::vector Files; - // Since many seek may be done on a FILE*, each thread should have its own FILE opened. - uint32 ThreadFileId; - bool CacheFileOnOpen; - bool AlwaysOpened; - }; private: // CBigFile(); // Singleton mode -> access it with the getInstance function diff --git a/code/nel/src/misc/big_file.cpp b/code/nel/src/misc/big_file.cpp index f5ee067f4..9eaa73bdf 100644 --- a/code/nel/src/misc/big_file.cpp +++ b/code/nel/src/misc/big_file.cpp @@ -16,6 +16,7 @@ #include "stdmisc.h" +#include "nel/misc/file.h" #include "nel/misc/big_file.h" #include "nel/misc/path.h" @@ -138,141 +139,17 @@ bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions) handle.File = fopen (sBigFileName.c_str(), "rb"); if (handle.File == NULL) return false; - uint32 nFileSize=CFile::getFileSize (handle.File); - //nlfseek64 (handle.File, 0, SEEK_END); - //uint32 nFileSize = ftell (handle.File); - // Result - if (nlfseek64 (handle.File, nFileSize-4, SEEK_SET) != 0) + // Used internally by CBigFile, use optimizations and lower case of filenames + bnp.InternalUse = true; + + // read BNP header + if (!bnp.readHeader(handle.File)) { fclose (handle.File); handle.File = NULL; return false; } - - uint32 nOffsetFromBeginning; - if (fread (&nOffsetFromBeginning, sizeof(uint32), 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - -#ifdef NL_BIG_ENDIAN - NLMISC_BSWAP32(nOffsetFromBeginning); -#endif - - if (nlfseek64 (handle.File, nOffsetFromBeginning, SEEK_SET) != 0) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - - // Read the file count - uint32 nNbFile; - if (fread (&nNbFile, sizeof(uint32), 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - -#ifdef NL_BIG_ENDIAN - NLMISC_BSWAP32(nNbFile); -#endif - - map tempMap; - for (uint32 i = 0; i < nNbFile; ++i) - { - char FileName[256]; - uint8 nStringSize; - if (fread (&nStringSize, 1, 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - - if (fread (FileName, nStringSize, 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - - FileName[nStringSize] = 0; - uint32 nFileSize2; - if (fread (&nFileSize2, sizeof(uint32), 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - -#ifdef NL_BIG_ENDIAN - NLMISC_BSWAP32(nFileSize2); -#endif - - uint32 nFilePos; - if (fread (&nFilePos, sizeof(uint32), 1, handle.File) != 1) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - -#ifdef NL_BIG_ENDIAN - NLMISC_BSWAP32(nFilePos); -#endif - - BNPFile bnpfTmp; - bnpfTmp.Pos = nFilePos; - bnpfTmp.Size = nFileSize2; - tempMap.insert (make_pair(toLower(string(FileName)), bnpfTmp)); - } - - if (nlfseek64 (handle.File, 0, SEEK_SET) != 0) - { - fclose (handle.File); - handle.File = NULL; - return false; - } - - // Convert temp map - if (nNbFile > 0) - { - uint nSize = 0, nNb = 0; - map::iterator it = tempMap.begin(); - while (it != tempMap.end()) - { - nSize += (uint)it->first.size() + 1; - nNb++; - it++; - } - - bnp.FileNames = new char[nSize]; - memset(bnp.FileNames, 0, nSize); - bnp.Files.resize(nNb); - - it = tempMap.begin(); - nSize = 0; - nNb = 0; - while (it != tempMap.end()) - { - strcpy(bnp.FileNames+nSize, it->first.c_str()); - - bnp.Files[nNb].Name = bnp.FileNames+nSize; - bnp.Files[nNb].Size = it->second.Size; - bnp.Files[nNb].Pos = it->second.Pos; - - nSize += (uint)it->first.size() + 1; - nNb++; - it++; - } - } - // End of temp map conversion - if (nOptions&BF_CACHE_FILE_ON_OPEN) bnp.CacheFileOnOpen = true; else @@ -314,6 +191,369 @@ void CBigFile::remove (const std::string &sBigFileName) } } +//// *************************************************************************** +bool CBigFile::BNP::readHeader() +{ + // Only external use + if (InternalUse || BigFileName.empty()) return false; + + FILE *f = fopen (BigFileName.c_str(), "rb"); + if (f == NULL) return false; + + bool res = readHeader(f); + fclose (f); + + return res; +} + +//// *************************************************************************** +bool CBigFile::BNP::readHeader(FILE *file) +{ + if (file == NULL) return false; + + uint32 nFileSize=CFile::getFileSize (file); + + // Result + if (nlfseek64 (file, nFileSize-4, SEEK_SET) != 0) + { + return false; + } + + if (fread (&OffsetFromBeginning, sizeof(uint32), 1, file) != 1) + { + return false; + } + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(OffsetFromBeginning); +#endif + + if (nlfseek64 (file, OffsetFromBeginning, SEEK_SET) != 0) + { + return false; + } + + // Read the file count + uint32 nNbFile; + if (fread (&nNbFile, sizeof(uint32), 1, file) != 1) + { + return false; + } + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nNbFile); +#endif + + map tempMap; + + if (!InternalUse) SFiles.clear(); + + for (uint32 i = 0; i < nNbFile; ++i) + { + uint8 nStringSize; + if (fread (&nStringSize, 1, 1, file) != 1) + { + return false; + } + + char sFileName[256]; + if (fread (sFileName, 1, nStringSize, file) != nStringSize) + { + return false; + } + + sFileName[nStringSize] = 0; + + uint32 nFileSize2; + if (fread (&nFileSize2, sizeof(uint32), 1, file) != 1) + { + return false; + } + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nFileSize2); +#endif + + uint32 nFilePos; + if (fread (&nFilePos, sizeof(uint32), 1, file) != 1) + { + return false; + } + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nFilePos); +#endif + + if (InternalUse) + { + BNPFile bnpfTmp; + bnpfTmp.Pos = nFilePos; + bnpfTmp.Size = nFileSize2; + tempMap.insert (make_pair(toLower(string(sFileName)), bnpfTmp)); + } + else + { + SBNPFile bnpfTmp; + bnpfTmp.Name = sFileName; + bnpfTmp.Pos = nFilePos; + bnpfTmp.Size = nFileSize2; + SFiles.push_back(bnpfTmp); + } + } + + if (nlfseek64 (file, 0, SEEK_SET) != 0) + { + return false; + } + + // Convert temp map + if (InternalUse && nNbFile > 0) + { + uint nSize = 0, nNb = 0; + map::iterator it = tempMap.begin(); + while (it != tempMap.end()) + { + nSize += (uint)it->first.size() + 1; + nNb++; + it++; + } + + FileNames = new char[nSize]; + memset(FileNames, 0, nSize); + Files.resize(nNb); + + it = tempMap.begin(); + nSize = 0; + nNb = 0; + while (it != tempMap.end()) + { + strcpy(FileNames+nSize, it->first.c_str()); + + Files[nNb].Name = FileNames+nSize; + Files[nNb].Size = it->second.Size; + Files[nNb].Pos = it->second.Pos; + + nSize += (uint)it->first.size() + 1; + nNb++; + it++; + } + } + // End of temp map conversion + + return true; +} + +bool CBigFile::BNP::appendHeader() +{ + // Only external use + if (InternalUse || BigFileName.empty()) return false; + + FILE *f = fopen (BigFileName.c_str(), "ab"); + if (f == NULL) return false; + + uint32 nNbFile = (uint32)SFiles.size(); + + // value to be serialized + uint32 nNbFile2 = nNbFile; + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nNbFile2); +#endif + + if (fwrite (&nNbFile2, sizeof(uint32), 1, f) != 1) + { + fclose(f); + return false; + } + + for (uint32 i = 0; i < nNbFile; ++i) + { + uint8 nStringSize = (uint8)SFiles[i].Name.length(); + if (fwrite (&nStringSize, 1, 1, f) != 1) + { + fclose(f); + return false; + } + + if (fwrite (SFiles[i].Name.c_str(), 1, nStringSize, f) != nStringSize) + { + fclose(f); + return false; + } + + uint32 nFileSize = SFiles[i].Size; + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nFileSize); +#endif + + if (fwrite (&nFileSize, sizeof(uint32), 1, f) != 1) + { + fclose(f); + return false; + } + + uint32 nFilePos = SFiles[i].Pos; + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nFilePos); +#endif + + if (fwrite (&nFilePos, sizeof(uint32), 1, f) != 1) + { + fclose(f); + return false; + } + } + + uint32 nOffsetFromBeginning = OffsetFromBeginning; + +#ifdef NL_BIG_ENDIAN + NLMISC_BSWAP32(nOffsetFromBeginning); +#endif + + if (fwrite (&nOffsetFromBeginning, sizeof(uint32), 1, f) != 1) + { + fclose(f); + return false; + } + + fclose (f); + return true; +} + +// *************************************************************************** +bool CBigFile::BNP::appendFile(const std::string &filename) +{ + // Only external use + if (InternalUse || BigFileName.empty()) return false; + + // Check if we can read the source file + if (!CFile::fileExists(filename)) return false; + + SBNPFile ftmp; + ftmp.Name = CFile::getFilename(filename); + ftmp.Size = CFile::getFileSize(filename); + ftmp.Pos = OffsetFromBeginning; + SFiles.push_back(ftmp); + OffsetFromBeginning += ftmp.Size; + + FILE *f1 = fopen(BigFileName.c_str(), "ab"); + if (f1 == NULL) return false; + + FILE *f2 = fopen(filename.c_str(), "rb"); + if (f2 == NULL) + { + fclose(f1); + return false; + } + + uint8 *ptr = new uint8[ftmp.Size]; + + if (fread (ptr, ftmp.Size, 1, f2) != 1) + { + nlwarning("%s read error", filename.c_str()); + } + else if (fwrite (ptr, ftmp.Size, 1, f1) != 1) + { + nlwarning("%s write error", BigFileName.c_str()); + } + + delete [] ptr; + + fclose(f2); + fclose(f1); + + return true; +} + +// *************************************************************************** +bool CBigFile::BNP::unpack(const std::string &sDestDir, TUnpackProgressCallback *callback) +{ + // Only external use + if (InternalUse || BigFileName.empty()) return false; + + FILE *bnp = fopen (BigFileName.c_str(), "rb"); + if (bnp == NULL) + return false; + + // only read header is not already read + if (SFiles.empty() && !readHeader(bnp)) + { + fclose (bnp); + return false; + } + + CFile::createDirectory(sDestDir); + + uint32 totalUncompressed = 0, total = 0; + + for (uint32 i = 0; i < SFiles.size(); ++i) + { + total += SFiles[i].Size; + } + + FILE *out = NULL; + + for (uint32 i = 0; i < SFiles.size(); ++i) + { + const SBNPFile &rBNPFile = SFiles[i]; + string filename = CPath::standardizePath(sDestDir) + rBNPFile.Name; + + if (callback && !(*callback)(filename, totalUncompressed, total)) + { + fclose (bnp); + return false; + } + + out = fopen (filename.c_str(), "wb"); + if (out != NULL) + { + nlfseek64 (bnp, rBNPFile.Pos, SEEK_SET); + uint8 *ptr = new uint8[rBNPFile.Size]; + bool readError = fread (ptr, rBNPFile.Size, 1, bnp) != 1; + if (readError) + { + nlwarning("%s read error errno = %d: %s", filename.c_str(), errno, strerror(errno)); + } + bool writeError = fwrite (ptr, rBNPFile.Size, 1, out) != 1; + if (writeError) + { + nlwarning("%s write error errno = %d: %s", filename.c_str(), errno, strerror(errno)); + } + bool diskFull = ferror(out) && errno == 28 /* ENOSPC*/; + fclose (out); + delete [] ptr; + if (diskFull) + { + fclose (bnp); + throw NLMISC::EDiskFullError(filename); + } + if (writeError) + { + fclose (bnp); + throw NLMISC::EWriteError(filename); + } + if (readError) + { + fclose (bnp); + throw NLMISC::EReadError(filename); + } + } + + totalUncompressed += rBNPFile.Size; + + if (callback && !(*callback)(filename, totalUncompressed, total)) + { + fclose (bnp); + return false; + } + } + + fclose (bnp); + return true; +} + // *************************************************************************** bool CBigFile::isBigFileAdded(const std::string &sBigFileName) const { @@ -359,6 +599,14 @@ void CBigFile::removeAll () } } +struct CBNPFileComp +{ + bool operator()(const CBigFile::BNPFile &f, const CBigFile::BNPFile &s ) + { + return strcmp( f.Name, s.Name ) < 0; + } +}; + // *************************************************************************** bool CBigFile::getFileInternal (const std::string &sFileName, BNP *&zeBnp, BNPFile *&zeBnpFile) { @@ -504,5 +752,12 @@ void CBigFile::getBigFilePaths(std::vector &bigFilePaths) } } +// *************************************************************************** +bool CBigFile::unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback) +{ + BNP bnpFile; + bnpFile.BigFileName = sBigFileName; + return bnpFile.unpack(sDestDir, callback); +} } // namespace NLMISC