aleajactaest/khanat-opennel-code
1
0
Fork 0
mirror of https://port.numenaute.org/aleajactaest/khanat-opennel-code.git synced 2024-11-10 09:19:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge with develop

Browse source
This commit is contained in:
kervala 2016-01-23 10:19:01 +01:00
parent b8225c4fef
commit 39e8135a82
4 changed files with 539 additions and 198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

94
code/nel/include/nel/misc/big_file.h
View file

@ -20,7 +20,7 @@
#include "types_nl.h" #include "types_nl.h"
#include "tds.h" #include "tds.h"
#include "singleton.h" #include "singleton.h"
#include "callback.h"
namespace NLMISC { namespace NLMISC {
@ -86,6 +86,66 @@ public:
// Used for CPath only for the moment ! // Used for CPath only for the moment !
char *getFileNamePtr(const std::string &sFileName, const std::string &sBigFileName); char *getFileNamePtr(const std::string &sFileName, const std::string &sBigFileName);
typedef CCallback<bool /* continue */, const std::string &/* filename */, uint32 /* currentSize */, uint32 /* totalSize */> TUnpackProgressCallback;
// Unpack all files in sBigFileName to sDestDir and send progress notifications to optional callback
static bool unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback = NULL);
// A BNPFile header (filename is a char* pointing on FileNames and is always lowercase)
struct BNPFile
{
BNPFile() : Name(NULL), Size(0), Pos(0) { }
char* Name;
uint32 Size;
uint32 Pos;
};
// A SBNPFile header (filename is a std::string and keeps the original case)
struct SBNPFile
{
SBNPFile() : Size(0), Pos(0) { }
std::string Name;
uint32 Size;
uint32 Pos;
};
// A BNP structure
struct BNP
{
BNP() : FileNames(NULL), ThreadFileId(0), CacheFileOnOpen(false), AlwaysOpened(false), InternalUse(false), OffsetFromBeginning(0) { }
// FileName of the BNP. important to open it in getFile() (for other threads or if not always opened).
std::string BigFileName;
// map of files in the BNP.
char *FileNames;
std::vector<BNPFile> Files;
std::vector<SBNPFile> SFiles;
// Since many seek may be done on a FILE*, each thread should have its own FILE opened.
uint32 ThreadFileId;
bool CacheFileOnOpen;
bool AlwaysOpened;
bool InternalUse;
// Offset written in BNP header
uint32 OffsetFromBeginning;
// Read BNP header from FILE* and init member variables
bool readHeader(FILE* file);
// Read BNP header from BigFileName and init member variables
bool readHeader();
// Append BNP header to the big file BigFileName (to use after appendFile calls)
bool appendHeader();
// Append a file to BigFileName
bool appendFile(const std::string &filename);
// Unpack BigFileName to sDestDir and send progress notifications to optional callback
bool unpack(const std::string &sDestDir, TUnpackProgressCallback *callback = NULL);
};
// *************** // ***************
private: private:
class CThreadFileArray; class CThreadFileArray;
@ -118,38 +178,6 @@ private:
uint32 _CurrentId; uint32 _CurrentId;
}; };
// A BNPFile header
struct BNPFile
{
BNPFile() : Name(NULL), Size(0), Pos(0) { }
char *Name;
uint32 Size;
uint32 Pos;
};
struct CBNPFileComp
{
bool operator()(const BNPFile &f, const BNPFile &s )
{
return strcmp( f.Name, s.Name ) < 0;
}
};
// A BNP structure
struct BNP
{
BNP() : FileNames(NULL) { }
// FileName of the BNP. important to open it in getFile() (for other threads or if not always opened).
std::string BigFileName;
// map of files in the BNP.
char *FileNames;
std::vector<BNPFile> Files;
// Since many seek may be done on a FILE*, each thread should have its own FILE opened.
uint32 ThreadFileId;
bool CacheFileOnOpen;
bool AlwaysOpened;
};
private: private:
// CBigFile(); // Singleton mode -> access it with the getInstance function // CBigFile(); // Singleton mode -> access it with the getInstance function

3
code/nel/include/nel/misc/path.h
View file

@ -510,9 +510,10 @@ public:
/** Make path absolute /** Make path absolute
* \param relativePath - The relative path * \param relativePath - The relative path
* \param directory - the directory to which the path is relative to * \param directory - the directory to which the path is relative to
* \param simplify - if we should simplify or not the path (convert . and .. in path)
* returns the absolute path, or empty if something went wrong. * returns the absolute path, or empty if something went wrong.
*/ */
static std::string makePathAbsolute (const std::string &relativePath, const std::string &directory ); static std::string makePathAbsolute (const std::string &relativePath, const std::string &directory, bool simplify = false );
/** Return if a path is absolute or not. /** Return if a path is absolute or not.
* \param path - The path * \param path - The path

515
code/nel/src/misc/big_file.cpp
View file

@ -16,6 +16,7 @@
#include "stdmisc.h" #include "stdmisc.h"
#include "nel/misc/file.h"
#include "nel/misc/big_file.h" #include "nel/misc/big_file.h"
#include "nel/misc/path.h" #include "nel/misc/path.h"
@ -128,151 +129,27 @@ bool CBigFile::add (const std::string &sBigFileName, uint32 nOptions)
bnp.BigFileName= sBigFileName; bnp.BigFileName= sBigFileName;
// Allocate a new ThreadSafe FileId for this bnp. // Allocate a new ThreadSafe FileId for this bnp.
bnp.ThreadFileId= _ThreadFileArray.allocate(); bnp.ThreadFileId= _ThreadFileArray.allocate();
// Get a ThreadSafe handle on the file // Get a ThreadSafe handle on the file
CHandleFile &handle= _ThreadFileArray.get(bnp.ThreadFileId); CHandleFile &handle= _ThreadFileArray.get(bnp.ThreadFileId);
// Open the big file. // Open the big file.
handle.File = fopen (sBigFileName.c_str(), "rb"); handle.File = fopen (sBigFileName.c_str(), "rb");
if (handle.File == NULL) if (handle.File == NULL)
return false; return false;
uint32 nFileSize=CFile::getFileSize (handle.File);
//nlfseek64 (handle.File, 0, SEEK_END);
//uint32 nFileSize = ftell (handle.File);
// Result // Used internally by CBigFile, use optimizations and lower case of filenames
if (nlfseek64 (handle.File, nFileSize-4, SEEK_SET) != 0) bnp.InternalUse = true;
// read BNP header
if (!bnp.readHeader(handle.File))
{ {
fclose (handle.File); fclose (handle.File);
handle.File = NULL; handle.File = NULL;
return false; return false;
} }
uint32 nOffsetFromBeginning;
if (fread (&nOffsetFromBeginning, sizeof(uint32), 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nOffsetFromBeginning);
#endif
if (nlfseek64 (handle.File, nOffsetFromBeginning, SEEK_SET) != 0)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
// Read the file count
uint32 nNbFile;
if (fread (&nNbFile, sizeof(uint32), 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nNbFile);
#endif
map<string,BNPFile> tempMap;
for (uint32 i = 0; i < nNbFile; ++i)
{
char FileName[256];
uint8 nStringSize;
if (fread (&nStringSize, 1, 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
if (fread (FileName, nStringSize, 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
FileName[nStringSize] = 0;
uint32 nFileSize2;
if (fread (&nFileSize2, sizeof(uint32), 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFileSize2);
#endif
uint32 nFilePos;
if (fread (&nFilePos, sizeof(uint32), 1, handle.File) != 1)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFilePos);
#endif
BNPFile bnpfTmp;
bnpfTmp.Pos = nFilePos;
bnpfTmp.Size = nFileSize2;
tempMap.insert (make_pair(toLower(string(FileName)), bnpfTmp));
}
if (nlfseek64 (handle.File, 0, SEEK_SET) != 0)
{
fclose (handle.File);
handle.File = NULL;
return false;
}
// Convert temp map
if (nNbFile > 0)
{
uint nSize = 0, nNb = 0;
map<string,BNPFile>::iterator it = tempMap.begin();
while (it != tempMap.end())
{
nSize += (uint)it->first.size() + 1;
nNb++;
it++;
}
bnp.FileNames = new char[nSize];
memset(bnp.FileNames, 0, nSize);
bnp.Files.resize(nNb);
it = tempMap.begin();
nSize = 0;
nNb = 0;
while (it != tempMap.end())
{
strcpy(bnp.FileNames+nSize, it->first.c_str());
bnp.Files[nNb].Name = bnp.FileNames+nSize;
bnp.Files[nNb].Size = it->second.Size;
bnp.Files[nNb].Pos = it->second.Pos;
nSize += (uint)it->first.size() + 1;
nNb++;
it++;
}
}
// End of temp map conversion
if (nOptions&BF_CACHE_FILE_ON_OPEN) if (nOptions&BF_CACHE_FILE_ON_OPEN)
bnp.CacheFileOnOpen = true; bnp.CacheFileOnOpen = true;
else else
@ -314,6 +191,369 @@ void CBigFile::remove (const std::string &sBigFileName)
} }
} }
//// ***************************************************************************
bool CBigFile::BNP::readHeader()
{
// Only external use
if (InternalUse || BigFileName.empty()) return false;
FILE *f = fopen (BigFileName.c_str(), "rb");
if (f == NULL) return false;
bool res = readHeader(f);
fclose (f);
return res;
}
//// ***************************************************************************
bool CBigFile::BNP::readHeader(FILE *file)
{
if (file == NULL) return false;
uint32 nFileSize=CFile::getFileSize (file);
// Result
if (nlfseek64 (file, nFileSize-4, SEEK_SET) != 0)
{
return false;
}
if (fread (&OffsetFromBeginning, sizeof(uint32), 1, file) != 1)
{
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(OffsetFromBeginning);
#endif
if (nlfseek64 (file, OffsetFromBeginning, SEEK_SET) != 0)
{
return false;
}
// Read the file count
uint32 nNbFile;
if (fread (&nNbFile, sizeof(uint32), 1, file) != 1)
{
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nNbFile);
#endif
map<string, BNPFile> tempMap;
if (!InternalUse) SFiles.clear();
for (uint32 i = 0; i < nNbFile; ++i)
{
uint8 nStringSize;
if (fread (&nStringSize, 1, 1, file) != 1)
{
return false;
}
char sFileName[256];
if (fread (sFileName, 1, nStringSize, file) != nStringSize)
{
return false;
}
sFileName[nStringSize] = 0;
uint32 nFileSize2;
if (fread (&nFileSize2, sizeof(uint32), 1, file) != 1)
{
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFileSize2);
#endif
uint32 nFilePos;
if (fread (&nFilePos, sizeof(uint32), 1, file) != 1)
{
return false;
}
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFilePos);
#endif
if (InternalUse)
{
BNPFile bnpfTmp;
bnpfTmp.Pos = nFilePos;
bnpfTmp.Size = nFileSize2;
tempMap.insert (make_pair(toLower(string(sFileName)), bnpfTmp));
}
else
{
SBNPFile bnpfTmp;
bnpfTmp.Name = sFileName;
bnpfTmp.Pos = nFilePos;
bnpfTmp.Size = nFileSize2;
SFiles.push_back(bnpfTmp);
}
}
if (nlfseek64 (file, 0, SEEK_SET) != 0)
{
return false;
}
// Convert temp map
if (InternalUse && nNbFile > 0)
{
uint nSize = 0, nNb = 0;
map<string,BNPFile>::iterator it = tempMap.begin();
while (it != tempMap.end())
{
nSize += (uint)it->first.size() + 1;
nNb++;
it++;
}
FileNames = new char[nSize];
memset(FileNames, 0, nSize);
Files.resize(nNb);
it = tempMap.begin();
nSize = 0;
nNb = 0;
while (it != tempMap.end())
{
strcpy(FileNames+nSize, it->first.c_str());
Files[nNb].Name = FileNames+nSize;
Files[nNb].Size = it->second.Size;
Files[nNb].Pos = it->second.Pos;
nSize += (uint)it->first.size() + 1;
nNb++;
it++;
}
}
// End of temp map conversion
return true;
}
bool CBigFile::BNP::appendHeader()
{
// Only external use
if (InternalUse || BigFileName.empty()) return false;
FILE *f = fopen (BigFileName.c_str(), "ab");
if (f == NULL) return false;
uint32 nNbFile = (uint32)SFiles.size();
// value to be serialized
uint32 nNbFile2 = nNbFile;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nNbFile2);
#endif
if (fwrite (&nNbFile2, sizeof(uint32), 1, f) != 1)
{
fclose(f);
return false;
}
for (uint32 i = 0; i < nNbFile; ++i)
{
uint8 nStringSize = (uint8)SFiles[i].Name.length();
if (fwrite (&nStringSize, 1, 1, f) != 1)
{
fclose(f);
return false;
}
if (fwrite (SFiles[i].Name.c_str(), 1, nStringSize, f) != nStringSize)
{
fclose(f);
return false;
}
uint32 nFileSize = SFiles[i].Size;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFileSize);
#endif
if (fwrite (&nFileSize, sizeof(uint32), 1, f) != 1)
{
fclose(f);
return false;
}
uint32 nFilePos = SFiles[i].Pos;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nFilePos);
#endif
if (fwrite (&nFilePos, sizeof(uint32), 1, f) != 1)
{
fclose(f);
return false;
}
}
uint32 nOffsetFromBeginning = OffsetFromBeginning;
#ifdef NL_BIG_ENDIAN
NLMISC_BSWAP32(nOffsetFromBeginning);
#endif
if (fwrite (&nOffsetFromBeginning, sizeof(uint32), 1, f) != 1)
{
fclose(f);
return false;
}
fclose (f);
return true;
}
// ***************************************************************************
bool CBigFile::BNP::appendFile(const std::string &filename)
{
// Only external use
if (InternalUse || BigFileName.empty()) return false;
// Check if we can read the source file
if (!CFile::fileExists(filename)) return false;
SBNPFile ftmp;
ftmp.Name = CFile::getFilename(filename);
ftmp.Size = CFile::getFileSize(filename);
ftmp.Pos = OffsetFromBeginning;
SFiles.push_back(ftmp);
OffsetFromBeginning += ftmp.Size;
FILE *f1 = fopen(BigFileName.c_str(), "ab");
if (f1 == NULL) return false;
FILE *f2 = fopen(filename.c_str(), "rb");
if (f2 == NULL)
{
fclose(f1);
return false;
}
uint8 *ptr = new uint8[ftmp.Size];
if (fread (ptr, ftmp.Size, 1, f2) != 1)
{
nlwarning("%s read error", filename.c_str());
}
else if (fwrite (ptr, ftmp.Size, 1, f1) != 1)
{
nlwarning("%s write error", BigFileName.c_str());
}
delete [] ptr;
fclose(f2);
fclose(f1);
return true;
}
// ***************************************************************************
bool CBigFile::BNP::unpack(const std::string &sDestDir, TUnpackProgressCallback *callback)
{
// Only external use
if (InternalUse || BigFileName.empty()) return false;
FILE *bnp = fopen (BigFileName.c_str(), "rb");
if (bnp == NULL)
return false;
// only read header is not already read
if (SFiles.empty() && !readHeader(bnp))
{
fclose (bnp);
return false;
}
CFile::createDirectory(sDestDir);
uint32 totalUncompressed = 0, total = 0;
for (uint32 i = 0; i < SFiles.size(); ++i)
{
total += SFiles[i].Size;
}
FILE *out = NULL;
for (uint32 i = 0; i < SFiles.size(); ++i)
{
const SBNPFile &rBNPFile = SFiles[i];
string filename = CPath::standardizePath(sDestDir) + rBNPFile.Name;
if (callback && !(*callback)(filename, totalUncompressed, total))
{
fclose (bnp);
return false;
}
out = fopen (filename.c_str(), "wb");
if (out != NULL)
{
nlfseek64 (bnp, rBNPFile.Pos, SEEK_SET);
uint8 *ptr = new uint8[rBNPFile.Size];
bool readError = fread (ptr, rBNPFile.Size, 1, bnp) != 1;
if (readError)
{
nlwarning("%s read error errno = %d: %s", filename.c_str(), errno, strerror(errno));
}
bool writeError = fwrite (ptr, rBNPFile.Size, 1, out) != 1;
if (writeError)
{
nlwarning("%s write error errno = %d: %s", filename.c_str(), errno, strerror(errno));
}
bool diskFull = ferror(out) && errno == 28 /* ENOSPC*/;
fclose (out);
delete [] ptr;
if (diskFull)
{
fclose (bnp);
throw NLMISC::EDiskFullError(filename);
}
if (writeError)
{
fclose (bnp);
throw NLMISC::EWriteError(filename);
}
if (readError)
{
fclose (bnp);
throw NLMISC::EReadError(filename);
}
}
totalUncompressed += rBNPFile.Size;
if (callback && !(*callback)(filename, totalUncompressed, total))
{
fclose (bnp);
return false;
}
}
fclose (bnp);
return true;
}
// *************************************************************************** // ***************************************************************************
bool CBigFile::isBigFileAdded(const std::string &sBigFileName) const bool CBigFile::isBigFileAdded(const std::string &sBigFileName) const
{ {
@ -359,6 +599,14 @@ void CBigFile::removeAll ()
} }
} }
struct CBNPFileComp
{
bool operator()(const CBigFile::BNPFile &f, const CBigFile::BNPFile &s )
{
return strcmp( f.Name, s.Name ) < 0;
}
};
// *************************************************************************** // ***************************************************************************
bool CBigFile::getFileInternal (const std::string &sFileName, BNP *&zeBnp, BNPFile *&zeBnpFile) bool CBigFile::getFileInternal (const std::string &sFileName, BNP *&zeBnp, BNPFile *&zeBnpFile)
{ {
@ -504,5 +752,12 @@ void CBigFile::getBigFilePaths(std::vector<std::string> &bigFilePaths)
} }
} }
// ***************************************************************************
bool CBigFile::unpack(const std::string &sBigFileName, const std::string &sDestDir, TUnpackProgressCallback *callback)
{
BNP bnpFile;
bnpFile.BigFileName = sBigFileName;
return bnpFile.unpack(sDestDir, callback);
}
} // namespace NLMISC } // namespace NLMISC

121
code/nel/src/misc/path.cpp
View file

@ -2546,55 +2546,112 @@ bool CPath::makePathRelative (const char *basePath, std::string &relativePath)
return false; return false;
} }
std::string CPath::makePathAbsolute( const std::string &relativePath, const std::string &directory ) std::string CPath::makePathAbsolute( const std::string &relativePath, const std::string &directory, bool simplify )
{ {
if( relativePath.empty() ) if( relativePath.empty() )
return ""; return "";
if( directory.empty() ) if( directory.empty() )
return ""; return "";
std::string absolutePath;
#ifdef NL_OS_WINDOWS #ifdef NL_OS_WINDOWS
// Windows network address. Eg.: \\someshare\path // Windows network address. Eg.: \\someshare\path
if( ( relativePath[ 0 ] == '\\' ) && ( relativePath[ 1 ] == '\\' ) ) if ((relativePath[0] == '\\') && (relativePath[1] == '\\'))
return relativePath; {
absolutePath = relativePath;
}
// Normal Windows absolute path. Eg.: C:\something // Normal Windows absolute path. Eg.: C:\something
// //
if( isalpha( relativePath[ 0 ] ) && ( relativePath[ 1 ] == ':' ) && ( ( relativePath[ 2 ] == '\\' ) || ( relativePath[ 2 ] == '/' ) ) ) else if (isalpha(relativePath[0]) && (relativePath[1] == ':') && ((relativePath[2] == '\\') || (relativePath[2] == '/')))
return relativePath; {
absolutePath = relativePath;
}
#else #else
// Unix filesystem absolute path // Unix filesystem absolute path
if( relativePath[ 0 ] == '/' ) if (relativePath[0] == '/')
return relativePath; {
absolutePath = relativePath;
}
#endif #endif
// Add a slash to the directory if necessary.
// If the relative path starts with dots we need a slash.
// If the relative path starts with a slash we don't.
// If it starts with neither, we need a slash.
bool needSlash = true;
char c = relativePath[ 0 ];
if( ( c == '\\' ) || ( c == '/' ) )
needSlash = false;
bool hasSlash = false;
std::string npath = directory;
c = npath[ npath.size() - 1 ];
if( ( c == '\\' ) || ( c == '/' ) )
hasSlash = true;
if( needSlash && !hasSlash )
npath += '/';
else else
if( hasSlash && !needSlash ) {
npath.resize( npath.size() - 1 ); // Add a slash to the directory if necessary.
// If the relative path starts with dots we need a slash.
// If the relative path starts with a slash we don't.
// If it starts with neither, we need a slash.
bool needSlash = true;
char c = relativePath[0];
if ((c == '\\') || (c == '/'))
needSlash = false;
// Now build the new absolute path bool hasSlash = false;
npath += relativePath; absolutePath = directory;
npath = standardizePath( npath, false ); c = absolutePath[absolutePath.size() - 1];
if ((c == '\\') || (c == '/'))
hasSlash = true;
return npath; if (needSlash && !hasSlash)
absolutePath += '/';
else
if (hasSlash && !needSlash)
absolutePath.resize(absolutePath.size() - 1);
// Now build the new absolute path
absolutePath += relativePath;
absolutePath = standardizePath(absolutePath, true);
}
if (simplify)
{
// split all components path to manage parent directories
std::vector<std::string> tokens;
explode(absolutePath, std::string("/"), tokens, true);
std::vector<std::string> directoryParts;
// process all components
for(uint i = 0, len = tokens.size(); i < len; ++i)
{
std::string token = tokens[i];
// current directory
if (token != ".")
{
// parent directory
if (token == "..")
{
// remove last directory
directoryParts.pop_back();
}
else
{
// append directory
directoryParts.push_back(token);
}
}
}
if (!directoryParts.empty())
{
absolutePath = directoryParts[0];
// rebuild the whole absolute path
for(uint i = 1, len = directoryParts.size(); i < len; ++i)
absolutePath += "/" + directoryParts[i];
// add trailing slash
absolutePath += "/";
}
else
{
// invalid path
absolutePath.clear();
}
}
return absolutePath;
} }
bool CPath::isAbsolutePath(const std::string &path) bool CPath::isAbsolutePath(const std::string &path)