1/* (c) Magnus Auvinen. See licence.txt in the root of the distribution for more information. */
2/* If you are missing that file, acquire a complete release at teeworlds.com. */
3
4#include "datafile.h"
5
6#include "uuid_manager.h"
7
8#include <base/bytes.h>
9#include <base/dbg.h>
10#include <base/fs.h>
11#include <base/hash_ctxt.h>
12#include <base/io.h>
13#include <base/log.h>
14#include <base/math.h>
15#include <base/mem.h>
16#include <base/str.h>
17
18#include <engine/storage.h>
19
20#include <zlib.h>
21
22#include <cstdlib>
23#include <limits>
24#include <unordered_set>
25
26static constexpr int MAX_ITEM_TYPE = 0xFFFF;
27static constexpr int MAX_ITEM_ID = 0xFFFF;
28static constexpr int OFFSET_UUID_TYPE = 0x8000;
29
30static inline void SwapEndianInPlace(void *pObj, size_t Size)
31{
32#if defined(CONF_ARCH_ENDIAN_BIG)
33 swap_endian(pObj, sizeof(int), Size / sizeof(int));
34#endif
35}
36
37template<typename T>
38static inline void SwapEndianInPlace(T *pObj)
39{
40 static_assert(sizeof(T) % sizeof(int) == 0);
41 SwapEndianInPlace(pObj, sizeof(T));
42}
43
44static inline int SwapEndianInt(int Number)
45{
46 SwapEndianInPlace(pObj: &Number);
47 return Number;
48}
49
50class CItemEx
51{
52public:
53 int m_aUuid[sizeof(CUuid) / sizeof(int32_t)];
54
55 static CItemEx FromUuid(CUuid Uuid)
56 {
57 CItemEx Result;
58 for(size_t i = 0; i < std::size(Result.m_aUuid); i++)
59 {
60 Result.m_aUuid[i] = bytes_be_to_uint(bytes: &Uuid.m_aData[i * sizeof(int32_t)]);
61 }
62 return Result;
63 }
64
65 CUuid ToUuid() const
66 {
67 CUuid Result;
68 for(size_t i = 0; i < std::size(m_aUuid); i++)
69 {
70 uint_to_bytes_be(bytes: &Result.m_aData[i * sizeof(int32_t)], value: m_aUuid[i]);
71 }
72 return Result;
73 }
74};
75
76class CDatafileItemType
77{
78public:
79 int m_Type;
80 int m_Start;
81 int m_Num;
82};
83
84class CDatafileItem
85{
86public:
87 unsigned m_TypeAndId;
88 int m_Size;
89
90 int Type() const
91 {
92 return (m_TypeAndId >> 16u) & MAX_ITEM_TYPE;
93 }
94
95 int Id() const
96 {
97 return m_TypeAndId & MAX_ITEM_ID;
98 }
99};
100
101class CDatafileHeader
102{
103public:
104 char m_aId[4];
105 int m_Version;
106 int m_Size;
107 int m_Swaplen;
108 int m_NumItemTypes;
109 int m_NumItems;
110 int m_NumRawData;
111 int m_ItemSize;
112 int m_DataSize;
113
114 constexpr size_t SizeOffset()
115 {
116 // The size of these members is not included in m_Size and m_Swaplen
117 return sizeof(m_aId) + sizeof(m_Version) + sizeof(m_Size) + sizeof(m_Swaplen);
118 }
119};
120
121class CDatafileInfo
122{
123public:
124 CDatafileItemType *m_pItemTypes;
125 int *m_pItemOffsets;
126 int *m_pDataOffsets;
127 int *m_pDataSizes;
128
129 char *m_pItemStart;
130 char *m_pDataStart;
131};
132
133class CDatafile
134{
135public:
136 IOHANDLE m_File;
137 char m_aFullName[IO_MAX_PATH_LENGTH];
138 const char *m_pBaseName;
139 char m_aPath[IO_MAX_PATH_LENGTH];
140 unsigned m_FileSize;
141 SHA256_DIGEST m_Sha256;
142 unsigned m_Crc;
143
144 CDatafileInfo m_Info;
145 CDatafileHeader m_Header;
146 int m_DataStartOffset;
147 void **m_ppDataPtrs;
148 int *m_pDataSizes;
149 char *m_pData;
150
151 int GetFileDataSize(int Index) const
152 {
153 dbg_assert(Index >= 0 && Index < m_Header.m_NumRawData, "Invalid Index: %d", Index);
154
155 if(Index == m_Header.m_NumRawData - 1)
156 {
157 return m_Header.m_DataSize - m_Info.m_pDataOffsets[Index];
158 }
159
160 return m_Info.m_pDataOffsets[Index + 1] - m_Info.m_pDataOffsets[Index];
161 }
162
163 int GetDataSize(int Index) const
164 {
165 // Invalid data indices may appear in map items
166 if(Index < 0 || Index >= m_Header.m_NumRawData)
167 {
168 return 0;
169 }
170
171 if(m_ppDataPtrs[Index] == nullptr)
172 {
173 if(m_Info.m_pDataSizes != nullptr)
174 {
175 return m_Info.m_pDataSizes[Index];
176 }
177 else
178 {
179 return GetFileDataSize(Index);
180 }
181 }
182
183 const int Size = m_pDataSizes[Index];
184 if(Size < 0)
185 {
186 return 0; // summarize all errors as zero size
187 }
188 return Size;
189 }
190
191 void *GetData(int Index, bool Swap) const
192 {
193 // Invalid data indices may appear in map items
194 if(Index < 0 || Index >= m_Header.m_NumRawData)
195 {
196 return nullptr;
197 }
198
199 // Data already loaded
200 if(m_ppDataPtrs[Index] != nullptr)
201 {
202 return m_ppDataPtrs[Index];
203 }
204
205 // Don't try to load the data again if it previously failed
206 if(m_pDataSizes[Index] < 0)
207 {
208 return nullptr;
209 }
210
211 const unsigned DataSize = GetFileDataSize(Index);
212 if(m_Info.m_pDataSizes != nullptr)
213 {
214 // v4 has compressed data
215 const unsigned OriginalUncompressedSize = m_Info.m_pDataSizes[Index];
216 log_trace("datafile", "loading data. index=%d size=%d uncompressed=%d", Index, DataSize, OriginalUncompressedSize);
217 if(OriginalUncompressedSize == 0)
218 {
219 log_error("datafile", "data size invalid. data will be ignored. index=%d size=%d uncompressed=%d", Index, DataSize, OriginalUncompressedSize);
220 m_ppDataPtrs[Index] = nullptr;
221 m_pDataSizes[Index] = -1;
222 return nullptr;
223 }
224
225 // read the compressed data
226 void *pCompressedData = malloc(size: DataSize);
227 if(pCompressedData == nullptr)
228 {
229 log_error("datafile", "out of memory. could not allocate memory for compressed data. index=%d size=%d", Index, DataSize);
230 m_ppDataPtrs[Index] = nullptr;
231 m_pDataSizes[Index] = -1;
232 return nullptr;
233 }
234 unsigned ActualDataSize = 0;
235 if(io_seek(io: m_File, offset: m_DataStartOffset + m_Info.m_pDataOffsets[Index], origin: IOSEEK_START) == 0)
236 {
237 ActualDataSize = io_read(io: m_File, buffer: pCompressedData, size: DataSize);
238 }
239 if(DataSize != ActualDataSize)
240 {
241 log_error("datafile", "truncation error. could not read all compressed data. index=%d wanted=%d got=%d", Index, DataSize, ActualDataSize);
242 free(ptr: pCompressedData);
243 m_ppDataPtrs[Index] = nullptr;
244 m_pDataSizes[Index] = -1;
245 return nullptr;
246 }
247
248 // decompress the data
249 m_ppDataPtrs[Index] = static_cast<char *>(malloc(size: OriginalUncompressedSize));
250 if(m_ppDataPtrs[Index] == nullptr)
251 {
252 free(ptr: pCompressedData);
253 log_error("datafile", "out of memory. could not allocate memory for uncompressed data. index=%d size=%d", Index, OriginalUncompressedSize);
254 m_pDataSizes[Index] = -1;
255 return nullptr;
256 }
257 unsigned long UncompressedSize = OriginalUncompressedSize;
258 const int Result = uncompress(dest: static_cast<Bytef *>(m_ppDataPtrs[Index]), destLen: &UncompressedSize, source: static_cast<Bytef *>(pCompressedData), sourceLen: DataSize);
259 free(ptr: pCompressedData);
260 if(Result != Z_OK || UncompressedSize != OriginalUncompressedSize)
261 {
262 log_error("datafile", "failed to uncompress data. index=%d result=%d wanted=%d got=%ld", Index, Result, OriginalUncompressedSize, UncompressedSize);
263 free(ptr: m_ppDataPtrs[Index]);
264 m_ppDataPtrs[Index] = nullptr;
265 m_pDataSizes[Index] = -1;
266 return nullptr;
267 }
268 m_pDataSizes[Index] = OriginalUncompressedSize;
269 }
270 else
271 {
272 log_trace("datafile", "loading data. index=%d size=%d", Index, DataSize);
273 m_ppDataPtrs[Index] = malloc(size: DataSize);
274 if(m_ppDataPtrs[Index] == nullptr)
275 {
276 log_error("datafile", "out of memory. could not allocate memory for uncompressed data. index=%d size=%d", Index, DataSize);
277 m_pDataSizes[Index] = -1;
278 return nullptr;
279 }
280 unsigned ActualDataSize = 0;
281 if(io_seek(io: m_File, offset: m_DataStartOffset + m_Info.m_pDataOffsets[Index], origin: IOSEEK_START) == 0)
282 {
283 ActualDataSize = io_read(io: m_File, buffer: m_ppDataPtrs[Index], size: DataSize);
284 }
285 if(DataSize != ActualDataSize)
286 {
287 log_error("datafile", "truncation error. could not read all uncompressed data. index=%d wanted=%d got=%d", Index, DataSize, ActualDataSize);
288 free(ptr: m_ppDataPtrs[Index]);
289 m_ppDataPtrs[Index] = nullptr;
290 m_pDataSizes[Index] = -1;
291 return nullptr;
292 }
293 m_pDataSizes[Index] = DataSize;
294 }
295 if(Swap)
296 {
297 SwapEndianInPlace(pObj: m_ppDataPtrs[Index], Size: m_pDataSizes[Index]);
298 }
299 return m_ppDataPtrs[Index];
300 }
301
302 int GetFileItemSize(int Index) const
303 {
304 dbg_assert(Index >= 0 && Index < m_Header.m_NumItems, "Invalid Index: %d", Index);
305
306 if(Index == m_Header.m_NumItems - 1)
307 {
308 return m_Header.m_ItemSize - m_Info.m_pItemOffsets[Index];
309 }
310
311 return m_Info.m_pItemOffsets[Index + 1] - m_Info.m_pItemOffsets[Index];
312 }
313
314 int GetItemSize(int Index) const
315 {
316 return GetFileItemSize(Index) - sizeof(CDatafileItem);
317 }
318
319 CDatafileItem *GetItem(int Index) const
320 {
321 dbg_assert(Index >= 0 && Index < m_Header.m_NumItems, "Invalid Index: %d", Index);
322
323 return static_cast<CDatafileItem *>(static_cast<void *>(m_Info.m_pItemStart + m_Info.m_pItemOffsets[Index]));
324 }
325
326 bool Validate() const
327 {
328#define Check(Test, ErrorMessage, ...) \
329 do \
330 { \
331 if(!(Test)) \
332 { \
333 log_error("datafile", "invalid file information: " ErrorMessage, ##__VA_ARGS__); \
334 return false; \
335 } \
336 } while(false)
337
338 // validate item types
339 int64_t CountedItems = 0;
340 std::unordered_set<int> UsedItemTypes;
341 for(int Index = 0; Index < m_Header.m_NumItemTypes; Index++)
342 {
343 const CDatafileItemType &ItemType = m_Info.m_pItemTypes[Index];
344 Check(ItemType.m_Type >= 0 && ItemType.m_Type <= MAX_ITEM_TYPE, "item type has invalid type. index=%d type=%d", Index, ItemType.m_Type);
345 const auto [_, Inserted] = UsedItemTypes.insert(x: ItemType.m_Type);
346 Check(Inserted, "item type has duplicate type. index=%d type=%d", Index, ItemType.m_Type);
347 Check(ItemType.m_Num > 0, "item type has invalid number of items. index=%d type=%d num=%d", Index, ItemType.m_Type, ItemType.m_Num);
348 Check(ItemType.m_Start == CountedItems, "item type has invalid start. index=%d type=%d start=%d", Index, ItemType.m_Type, ItemType.m_Start);
349 CountedItems += ItemType.m_Num;
350 if(CountedItems > m_Header.m_NumItems)
351 {
352 break;
353 }
354 }
355 Check(CountedItems == m_Header.m_NumItems, "mismatched number of items in item types. counted=%" PRId64 " header=%d", CountedItems, m_Header.m_NumItems);
356
357 // validate item offsets
358 int PrevItemOffset = -1;
359 for(int Index = 0; Index < m_Header.m_NumItems; Index++)
360 {
361 const int Offset = m_Info.m_pItemOffsets[Index];
362 if(Index == 0)
363 {
364 Check(Offset == 0, "first item offset is not zero. offset=%d", Offset);
365 }
366 else
367 {
368 Check(Offset > PrevItemOffset, "item offset not greater than previous. index=%d offset=%d previous=%d", Index, Offset, PrevItemOffset);
369 }
370 Check(Offset < m_Header.m_ItemSize, "item offset larger than total item size. index=%d offset=%d total=%d", Index, Offset, m_Header.m_ItemSize);
371 PrevItemOffset = Offset;
372 }
373
374 // validate item sizes, types and IDs
375 int64_t TotalItemSize = 0;
376 for(int TypeIndex = 0; TypeIndex < m_Header.m_NumItemTypes; TypeIndex++)
377 {
378 std::unordered_set<int> UsedItemIds;
379 const CDatafileItemType &ItemType = m_Info.m_pItemTypes[TypeIndex];
380 for(int ItemIndex = ItemType.m_Start; ItemIndex < ItemType.m_Start + ItemType.m_Num; ItemIndex++)
381 {
382 const int FileItemSize = GetFileItemSize(Index: ItemIndex);
383 Check(FileItemSize >= (int)sizeof(CDatafileItem), "map item too small for header. type_index=%d item_index=%d size=%d header=%d", TypeIndex, ItemIndex, FileItemSize, (int)sizeof(CDatafileItem));
384 const CDatafileItem *pItem = GetItem(Index: ItemIndex);
385 Check(pItem->Type() == ItemType.m_Type, "mismatched item type. type_index=%d item_index=%d type=%d expected=%d", TypeIndex, ItemIndex, pItem->Type(), ItemType.m_Type);
386 // Many old maps contain duplicate map items of type ITEMTYPE_EX due to a bug in DDNet tools.
387 if(pItem->Type() != ITEMTYPE_EX)
388 {
389 const auto [_, Inserted] = UsedItemIds.insert(x: pItem->Id());
390 Check(Inserted, "map item has duplicate ID. type_index=%d item_index=%d type=%d ID=%d", TypeIndex, ItemIndex, pItem->Type(), pItem->Id());
391 }
392 Check(pItem->m_Size >= 0, "map item size invalid. type_index=%d item_index=%d size=%d", TypeIndex, ItemIndex, pItem->m_Size);
393 Check(pItem->m_Size % sizeof(int) == 0, "map item size not integer aligned. type_index=%d item_index=%d size=%d", TypeIndex, ItemIndex, pItem->m_Size);
394 Check(pItem->m_Size == GetItemSize(ItemIndex), "map item size does not match file. type_index=%d item_index=%d size=%d file_size=%d", TypeIndex, ItemIndex, pItem->m_Size, GetItemSize(ItemIndex));
395 TotalItemSize += FileItemSize;
396 if(TotalItemSize > m_Header.m_ItemSize)
397 {
398 break;
399 }
400 }
401 }
402 Check(TotalItemSize == m_Header.m_ItemSize, "mismatched total item size. expected=%" PRId64 " header=%d", TotalItemSize, m_Header.m_ItemSize);
403
404 // validate data offsets
405 int PrevDataOffset = -1;
406 for(int Index = 0; Index < m_Header.m_NumRawData; Index++)
407 {
408 const int Offset = m_Info.m_pDataOffsets[Index];
409 if(Index == 0)
410 {
411 Check(Offset == 0, "first data offset must be zero. offset=%d", Offset);
412 }
413 else
414 {
415 Check(Offset > PrevDataOffset, "data offset not greater than previous. index=%d offset=%d previous=%d", Index, Offset, PrevDataOffset);
416 }
417 Check(Offset < m_Header.m_DataSize, "data offset larger than total data size. index=%d offset=%d total=%d", Index, Offset, m_Header.m_DataSize);
418 PrevDataOffset = Offset;
419 }
420
421 // validate data sizes
422 if(m_Info.m_pDataSizes != nullptr)
423 {
424 for(int Index = 0; Index < m_Header.m_NumRawData; Index++)
425 {
426 const int Size = m_Info.m_pDataSizes[Index];
427 Check(Size >= 0, "data size invalid. index=%d size=%d", Index, Size);
428 if(Size == 0)
429 {
430 // Data of size zero is not allowed, but due to existing maps with this quirk we instead allow
431 // the file to be loaded and fail loading the data in the GetData function if the size is zero.
432 log_warn("datafile", "invalid file information: data size invalid. index=%d size=%d", Index, Size);
433 }
434 }
435 }
436
437 return true;
438#undef Check
439 }
440};
441
442CDataFileReader::~CDataFileReader()
443{
444 Close();
445}
446
447CDataFileReader &CDataFileReader::operator=(CDataFileReader &&Other)
448{
449 m_pDataFile = Other.m_pDataFile;
450 Other.m_pDataFile = nullptr;
451 return *this;
452}
453
454bool CDataFileReader::Open(const char *pFullName, IStorage *pStorage, const char *pPath, int StorageType)
455{
456 dbg_assert(m_pDataFile == nullptr, "File already open");
457
458 log_trace("datafile", "loading '%s' from '%s'", pFullName, pPath);
459
460 IOHANDLE File = pStorage->OpenFile(pFilename: pPath, Flags: IOFLAG_READ, Type: StorageType);
461 if(!File)
462 {
463 log_error("datafile", "failed to open file '%s' for reading", pPath);
464 return false;
465 }
466
467 // determine size and hashes of the file and store them
468 int64_t FileSize = 0;
469 unsigned Crc = 0;
470 SHA256_DIGEST Sha256;
471 {
472 SHA256_CTX Sha256Ctxt;
473 sha256_init(ctxt: &Sha256Ctxt);
474 unsigned char aBuffer[64 * 1024];
475 while(true)
476 {
477 const unsigned Bytes = io_read(io: File, buffer: aBuffer, size: sizeof(aBuffer));
478 if(Bytes == 0)
479 break;
480 FileSize += Bytes;
481 Crc = crc32(crc: Crc, buf: aBuffer, len: Bytes);
482 sha256_update(ctxt: &Sha256Ctxt, data: aBuffer, data_len: Bytes);
483 }
484 Sha256 = sha256_finish(ctxt: &Sha256Ctxt);
485 if(io_seek(io: File, offset: 0, origin: IOSEEK_START) != 0)
486 {
487 io_close(io: File);
488 log_error("datafile", "could not seek to start after calculating hashes");
489 return false;
490 }
491 }
492
493 // read header
494 CDatafileHeader Header;
495 if(io_read(io: File, buffer: &Header, size: sizeof(Header)) != sizeof(Header))
496 {
497 io_close(io: File);
498 log_error("datafile", "could not read file header. file truncated or not a datafile.");
499 return false;
500 }
501
502 // check header magic
503 if((Header.m_aId[0] != 'A' || Header.m_aId[1] != 'T' || Header.m_aId[2] != 'A' || Header.m_aId[3] != 'D') &&
504 (Header.m_aId[0] != 'D' || Header.m_aId[1] != 'A' || Header.m_aId[2] != 'T' || Header.m_aId[3] != 'A'))
505 {
506 io_close(io: File);
507 log_error("datafile", "wrong header magic. magic=%x%x%x%x", Header.m_aId[0], Header.m_aId[1], Header.m_aId[2], Header.m_aId[3]);
508 return false;
509 }
510
511 SwapEndianInPlace(pObj: &Header);
512
513 // check header version
514 if(Header.m_Version != 3 && Header.m_Version != 4)
515 {
516 io_close(io: File);
517 log_error("datafile", "unsupported header version. version=%d", Header.m_Version);
518 return false;
519 }
520
521 // validate header information
522 if(Header.m_NumItemTypes < 0 ||
523 Header.m_NumItemTypes > MAX_ITEM_TYPE + 1 ||
524 Header.m_NumItems < 0 ||
525 Header.m_NumRawData < 0 ||
526 Header.m_ItemSize < 0 ||
527 Header.m_ItemSize % sizeof(int) != 0 ||
528 Header.m_DataSize < 0)
529 {
530 io_close(io: File);
531 log_error("datafile", "invalid header information. num_types=%d num_items=%d num_data=%d item_size=%d data_size=%d",
532 Header.m_NumItemTypes, Header.m_NumItems, Header.m_NumRawData, Header.m_ItemSize, Header.m_DataSize);
533 return false;
534 }
535
536 // calculate and validate sizes
537 int64_t Size = 0;
538 Size += (int64_t)Header.m_NumItemTypes * sizeof(CDatafileItemType);
539 Size += (int64_t)Header.m_NumItems * sizeof(int);
540 Size += (int64_t)Header.m_NumRawData * sizeof(int);
541 int64_t SizeFix = 0;
542 if(Header.m_Version == 4) // v4 has uncompressed data sizes as well
543 {
544 // The size of the uncompressed data sizes was not included in
545 // Header.m_Size and Header.m_Swaplen of version 4 maps prior
546 // to commit 3dd1ea0d8f6cb442ac41bd223279f41d1ed1b2bb. We also
547 // support loading maps created prior to this commit by fixing
548 // the sizes transparently when loading.
549 SizeFix = (int64_t)Header.m_NumRawData * sizeof(int);
550 Size += SizeFix;
551 }
552 Size += Header.m_ItemSize;
553
554 if((int64_t)sizeof(Header) + Size + (int64_t)Header.m_DataSize != FileSize)
555 {
556 io_close(io: File);
557 log_error("datafile", "invalid header data size or truncated file. data_size=%d file_size=%" PRId64, Header.m_DataSize, FileSize);
558 return false;
559 }
560
561 const int64_t HeaderFileSize = (int64_t)Header.m_Size + Header.SizeOffset();
562 if(HeaderFileSize != FileSize)
563 {
564 if(SizeFix != 0 && HeaderFileSize + SizeFix == FileSize)
565 {
566 log_warn("datafile", "fixing invalid header size. size=%d fix=+%" PRId64, Header.m_Size, SizeFix);
567 Header.m_Size += SizeFix;
568 }
569 else
570 {
571 io_close(io: File);
572 log_error("datafile", "invalid header size or truncated file. size=%" PRId64 " actual=%" PRId64, HeaderFileSize, FileSize);
573 return false;
574 }
575 }
576
577 const int64_t HeaderSwaplen = (int64_t)Header.m_Swaplen + Header.SizeOffset();
578 const int64_t FileSizeSwaplen = FileSize - Header.m_DataSize;
579 if(HeaderSwaplen != FileSizeSwaplen)
580 {
581 if(Header.m_Swaplen % sizeof(int) == 0 && SizeFix != 0 && HeaderSwaplen + SizeFix == FileSizeSwaplen)
582 {
583 log_warn("datafile", "fixing invalid header swaplen. swaplen=%d fix=+%" PRId64, Header.m_Swaplen, SizeFix);
584 Header.m_Swaplen += SizeFix;
585 }
586 else
587 {
588 io_close(io: File);
589 log_error("datafile", "invalid header swaplen or truncated file. swaplen=%" PRId64 " actual=%" PRId64, HeaderSwaplen, FileSizeSwaplen);
590 return false;
591 }
592 }
593
594 constexpr int64_t MaxAllocSize = (int64_t)2 * 1024 * 1024 * 1024;
595 int64_t AllocSize = Size;
596 AllocSize += sizeof(CDatafile); // add space for info structure
597 AllocSize += (int64_t)Header.m_NumRawData * sizeof(void *); // add space for data pointers
598 AllocSize += (int64_t)Header.m_NumRawData * sizeof(int); // add space for data sizes
599 if(AllocSize > MaxAllocSize)
600 {
601 io_close(io: File);
602 log_error("datafile", "file too large. alloc_size=%" PRId64 " max=%" PRId64, AllocSize, MaxAllocSize);
603 return false;
604 }
605
606 CDatafile *pTmpDataFile = static_cast<CDatafile *>(malloc(size: AllocSize));
607 if(pTmpDataFile == nullptr)
608 {
609 io_close(io: File);
610 log_error("datafile", "out of memory. could not allocate memory for datafile. alloc_size=%" PRId64, AllocSize);
611 return false;
612 }
613 pTmpDataFile->m_Header = Header;
614 pTmpDataFile->m_DataStartOffset = sizeof(CDatafileHeader) + Size;
615 pTmpDataFile->m_ppDataPtrs = (void **)(pTmpDataFile + 1);
616 pTmpDataFile->m_pDataSizes = (int *)(pTmpDataFile->m_ppDataPtrs + Header.m_NumRawData);
617 pTmpDataFile->m_pData = (char *)(pTmpDataFile->m_pDataSizes + Header.m_NumRawData);
618 pTmpDataFile->m_File = File;
619 str_copy(dst&: pTmpDataFile->m_aFullName, src: pFullName);
620 pTmpDataFile->m_pBaseName = fs_filename(path: pTmpDataFile->m_aFullName);
621 str_copy(dst&: pTmpDataFile->m_aPath, src: pPath);
622 pTmpDataFile->m_FileSize = FileSize;
623 pTmpDataFile->m_Sha256 = Sha256;
624 pTmpDataFile->m_Crc = Crc;
625
626 // clear the data pointers and sizes
627 mem_zero(block: pTmpDataFile->m_ppDataPtrs, size: Header.m_NumRawData * sizeof(void *));
628 mem_zero(block: pTmpDataFile->m_pDataSizes, size: Header.m_NumRawData * sizeof(int));
629
630 // read types, offsets, sizes and item data
631 const unsigned ReadSize = io_read(io: pTmpDataFile->m_File, buffer: pTmpDataFile->m_pData, size: Size);
632 if((int64_t)ReadSize != Size)
633 {
634 io_close(io: pTmpDataFile->m_File);
635 free(ptr: pTmpDataFile);
636 log_error("datafile", "truncation error. could not read all item data. wanted=%" PRId64 " got=%d", Size, ReadSize);
637 return false;
638 }
639
640 // The swap len also includes the size of the header (without the size offset), but the header was already swapped above.
641 const int64_t DataSwapLen = pTmpDataFile->m_Header.m_Swaplen - (int)(sizeof(Header) - Header.SizeOffset());
642 dbg_assert(DataSwapLen == Size, "Swap len and file size mismatch");
643 SwapEndianInPlace(pObj: pTmpDataFile->m_pData, Size: DataSwapLen);
644
645 pTmpDataFile->m_Info.m_pItemTypes = (CDatafileItemType *)pTmpDataFile->m_pData;
646 pTmpDataFile->m_Info.m_pItemOffsets = (int *)&pTmpDataFile->m_Info.m_pItemTypes[pTmpDataFile->m_Header.m_NumItemTypes];
647 pTmpDataFile->m_Info.m_pDataOffsets = &pTmpDataFile->m_Info.m_pItemOffsets[pTmpDataFile->m_Header.m_NumItems];
648 if(pTmpDataFile->m_Header.m_Version == 4) // v4 has uncompressed data sizes as well
649 {
650 pTmpDataFile->m_Info.m_pDataSizes = &pTmpDataFile->m_Info.m_pDataOffsets[pTmpDataFile->m_Header.m_NumRawData];
651 pTmpDataFile->m_Info.m_pItemStart = (char *)&pTmpDataFile->m_Info.m_pDataSizes[pTmpDataFile->m_Header.m_NumRawData];
652 }
653 else
654 {
655 pTmpDataFile->m_Info.m_pDataSizes = nullptr;
656 pTmpDataFile->m_Info.m_pItemStart = (char *)&pTmpDataFile->m_Info.m_pDataOffsets[pTmpDataFile->m_Header.m_NumRawData];
657 }
658 pTmpDataFile->m_Info.m_pDataStart = pTmpDataFile->m_Info.m_pItemStart + pTmpDataFile->m_Header.m_ItemSize;
659
660 if(!pTmpDataFile->Validate())
661 {
662 io_close(io: pTmpDataFile->m_File);
663 free(ptr: pTmpDataFile);
664 return false;
665 }
666
667 m_pDataFile = pTmpDataFile;
668 log_trace("datafile", "loading done. name='%s' path='%s'", pFullName, pPath);
669
670 return true;
671}
672
673bool CDataFileReader::Open(IStorage *pStorage, const char *pPath, int StorageType)
674{
675 char aFilename[IO_MAX_PATH_LENGTH];
676 fs_split_file_extension(filename: fs_filename(path: pPath), name: aFilename, name_size: sizeof(aFilename));
677 return Open(pFullName: aFilename, pStorage, pPath, StorageType);
678}
679
680void CDataFileReader::Close()
681{
682 if(!m_pDataFile)
683 {
684 return;
685 }
686
687 for(int i = 0; i < m_pDataFile->m_Header.m_NumRawData; i++)
688 {
689 free(ptr: m_pDataFile->m_ppDataPtrs[i]);
690 }
691
692 io_close(io: m_pDataFile->m_File);
693 free(ptr: m_pDataFile);
694 m_pDataFile = nullptr;
695}
696
697bool CDataFileReader::IsOpen() const
698{
699 return m_pDataFile != nullptr;
700}
701
702IOHANDLE CDataFileReader::File() const
703{
704 dbg_assert(m_pDataFile != nullptr, "File not open");
705
706 return m_pDataFile->m_File;
707}
708
709int CDataFileReader::GetDataSize(int Index) const
710{
711 dbg_assert(m_pDataFile != nullptr, "File not open");
712
713 return m_pDataFile->GetDataSize(Index);
714}
715
716void *CDataFileReader::GetData(int Index)
717{
718 dbg_assert(m_pDataFile != nullptr, "File not open");
719
720 return m_pDataFile->GetData(Index, Swap: false);
721}
722
723void *CDataFileReader::GetDataSwapped(int Index)
724{
725 dbg_assert(m_pDataFile != nullptr, "File not open");
726
727 return m_pDataFile->GetData(Index, Swap: true);
728}
729
730const char *CDataFileReader::GetDataString(int Index)
731{
732 dbg_assert(m_pDataFile != nullptr, "File not open");
733
734 if(Index == -1)
735 {
736 return "";
737 }
738
739 const int DataSize = GetDataSize(Index);
740 if(!DataSize)
741 {
742 return nullptr;
743 }
744
745 const char *pData = static_cast<const char *>(GetData(Index));
746 if(pData == nullptr || mem_has_null(block: pData, size: DataSize - 1) || pData[DataSize - 1] != '\0' || !str_utf8_check(str: pData))
747 {
748 return nullptr;
749 }
750 return pData;
751}
752
753void CDataFileReader::ReplaceData(int Index, char *pData, size_t Size)
754{
755 dbg_assert(m_pDataFile != nullptr, "File not open");
756 dbg_assert(Index >= 0 && Index < m_pDataFile->m_Header.m_NumRawData, "Index invalid: %d", Index);
757
758 free(ptr: m_pDataFile->m_ppDataPtrs[Index]);
759 m_pDataFile->m_ppDataPtrs[Index] = pData;
760 m_pDataFile->m_pDataSizes[Index] = Size;
761}
762
763void CDataFileReader::UnloadData(int Index)
764{
765 dbg_assert(m_pDataFile != nullptr, "File not open");
766
767 if(Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData)
768 return;
769
770 free(ptr: m_pDataFile->m_ppDataPtrs[Index]);
771 m_pDataFile->m_ppDataPtrs[Index] = nullptr;
772 m_pDataFile->m_pDataSizes[Index] = 0;
773}
774
775int CDataFileReader::NumData() const
776{
777 dbg_assert(m_pDataFile != nullptr, "File not open");
778
779 return m_pDataFile->m_Header.m_NumRawData;
780}
781
782int CDataFileReader::GetItemSize(int Index) const
783{
784 dbg_assert(m_pDataFile != nullptr, "File not open");
785
786 return m_pDataFile->GetItemSize(Index);
787}
788
789int CDataFileReader::GetExternalItemType(int InternalType, CUuid *pUuid)
790{
791 if(InternalType <= OFFSET_UUID_TYPE || InternalType == ITEMTYPE_EX)
792 {
793 if(pUuid)
794 {
795 *pUuid = UUID_ZEROED;
796 }
797 return InternalType;
798 }
799
800 const int TypeIndex = FindItemIndex(Type: ITEMTYPE_EX, Id: InternalType);
801 if(TypeIndex < 0 || GetItemSize(Index: TypeIndex) < (int)sizeof(CItemEx))
802 {
803 if(pUuid)
804 {
805 *pUuid = UUID_ZEROED;
806 }
807 return InternalType;
808 }
809
810 const CItemEx *pItemEx = static_cast<const CItemEx *>(GetItem(Index: TypeIndex));
811 const CUuid Uuid = pItemEx->ToUuid();
812 if(pUuid)
813 {
814 *pUuid = Uuid;
815 }
816 // Propagate UUID_UNKNOWN, it doesn't hurt.
817 return g_UuidManager.LookupUuid(Uuid);
818}
819
820int CDataFileReader::GetInternalItemType(int ExternalType)
821{
822 if(ExternalType < OFFSET_UUID)
823 {
824 return ExternalType;
825 }
826
827 const CUuid Uuid = g_UuidManager.GetUuid(Id: ExternalType);
828 int Start, Num;
829 GetType(Type: ITEMTYPE_EX, pStart: &Start, pNum: &Num);
830 for(int Index = Start; Index < Start + Num; Index++)
831 {
832 if(GetItemSize(Index) < (int)sizeof(CItemEx))
833 {
834 continue;
835 }
836 int Id;
837 if(Uuid == static_cast<const CItemEx *>(GetItem(Index, pType: nullptr, pId: &Id))->ToUuid())
838 {
839 return Id;
840 }
841 }
842 return -1;
843}
844
845void *CDataFileReader::GetItem(int Index, int *pType, int *pId, CUuid *pUuid)
846{
847 dbg_assert(m_pDataFile != nullptr, "File not open");
848
849 CDatafileItem *pItem = m_pDataFile->GetItem(Index);
850 const int ExternalType = GetExternalItemType(InternalType: pItem->Type(), pUuid);
851 if(pType)
852 {
853 *pType = ExternalType;
854 }
855 if(pId)
856 {
857 *pId = pItem->Id();
858 }
859 return static_cast<void *>(pItem + 1);
860}
861
862void CDataFileReader::GetType(int Type, int *pStart, int *pNum)
863{
864 dbg_assert(m_pDataFile != nullptr, "File not open");
865
866 *pStart = 0;
867 *pNum = 0;
868
869 const int InternalType = GetInternalItemType(ExternalType: Type);
870 for(int Index = 0; Index < m_pDataFile->m_Header.m_NumItemTypes; Index++)
871 {
872 const CDatafileItemType &ItemType = m_pDataFile->m_Info.m_pItemTypes[Index];
873 if(ItemType.m_Type == InternalType)
874 {
875 *pStart = ItemType.m_Start;
876 *pNum = ItemType.m_Num;
877 return;
878 }
879 }
880}
881
882int CDataFileReader::FindItemIndex(int Type, int Id)
883{
884 dbg_assert(m_pDataFile != nullptr, "File not open");
885
886 int Start, Num;
887 GetType(Type, pStart: &Start, pNum: &Num);
888 for(int Index = Start; Index < Start + Num; Index++)
889 {
890 const CDatafileItem *pItem = m_pDataFile->GetItem(Index);
891 if(pItem->Id() == Id)
892 {
893 return Index;
894 }
895 }
896 return -1;
897}
898
899void *CDataFileReader::FindItem(int Type, int Id)
900{
901 const int Index = FindItemIndex(Type, Id);
902 if(Index < 0)
903 {
904 return nullptr;
905 }
906 return GetItem(Index);
907}
908
909int CDataFileReader::NumItems() const
910{
911 dbg_assert(m_pDataFile != nullptr, "File not open");
912
913 return m_pDataFile->m_Header.m_NumItems;
914}
915
916const char *CDataFileReader::FullName() const
917{
918 dbg_assert(m_pDataFile != nullptr, "File not open");
919
920 return m_pDataFile->m_aFullName;
921}
922
923const char *CDataFileReader::BaseName() const
924{
925 dbg_assert(m_pDataFile != nullptr, "File not open");
926
927 return m_pDataFile->m_pBaseName;
928}
929
930const char *CDataFileReader::Path() const
931{
932 dbg_assert(m_pDataFile != nullptr, "File not open");
933
934 return m_pDataFile->m_aPath;
935}
936
937SHA256_DIGEST CDataFileReader::Sha256() const
938{
939 dbg_assert(m_pDataFile != nullptr, "File not open");
940
941 return m_pDataFile->m_Sha256;
942}
943
944unsigned CDataFileReader::Crc() const
945{
946 dbg_assert(m_pDataFile != nullptr, "File not open");
947
948 return m_pDataFile->m_Crc;
949}
950
951int CDataFileReader::Size() const
952{
953 dbg_assert(m_pDataFile != nullptr, "File not open");
954
955 return m_pDataFile->m_FileSize;
956}
957
958CDataFileWriter::CDataFileWriter()
959{
960 m_File = nullptr;
961}
962
963CDataFileWriter::~CDataFileWriter()
964{
965 if(m_File)
966 {
967 io_close(io: m_File);
968 m_File = nullptr;
969 }
970
971 for(CItemInfo &ItemInfo : m_vItems)
972 {
973 free(ptr: ItemInfo.m_pData);
974 }
975
976 for(CDataInfo &DataInfo : m_vDatas)
977 {
978 free(ptr: DataInfo.m_pUncompressedData);
979 free(ptr: DataInfo.m_pCompressedData);
980 }
981}
982
983bool CDataFileWriter::Open(class IStorage *pStorage, const char *pFilename, int StorageType)
984{
985 dbg_assert(!m_File, "File already open");
986 m_File = pStorage->OpenFile(pFilename, Flags: IOFLAG_WRITE, Type: StorageType);
987 return m_File != nullptr;
988}
989
990int CDataFileWriter::GetTypeFromIndex(int Index) const
991{
992 return ITEMTYPE_EX - Index - 1;
993}
994
995int CDataFileWriter::GetExtendedItemTypeIndex(int Type, const CUuid *pUuid)
996{
997 int Index = 0;
998 if(Type == -1)
999 {
1000 // Unknown type, search for UUID
1001 for(const auto &ExtendedItemType : m_vExtendedItemTypes)
1002 {
1003 if(ExtendedItemType.m_Uuid == *pUuid)
1004 {
1005 return Index;
1006 }
1007 ++Index;
1008 }
1009 }
1010 else
1011 {
1012 for(const auto &ExtendedItemType : m_vExtendedItemTypes)
1013 {
1014 if(ExtendedItemType.m_Type == Type)
1015 {
1016 return Index;
1017 }
1018 ++Index;
1019 }
1020 }
1021
1022 // Type not found, add it.
1023 const CUuid Uuid = Type == -1 ? *pUuid : g_UuidManager.GetUuid(Id: Type);
1024 CExtendedItemType ExtendedType;
1025 ExtendedType.m_Type = Type;
1026 ExtendedType.m_Uuid = Uuid;
1027 m_vExtendedItemTypes.emplace_back(args&: ExtendedType);
1028
1029 const CItemEx ItemEx = CItemEx::FromUuid(Uuid);
1030 AddItem(Type: ITEMTYPE_EX, Id: GetTypeFromIndex(Index), Size: sizeof(ItemEx), pData: &ItemEx);
1031 return Index;
1032}
1033
1034int CDataFileWriter::AddItem(int Type, int Id, size_t Size, const void *pData, const CUuid *pUuid)
1035{
1036 dbg_assert((Type >= 0 && Type <= MAX_ITEM_TYPE) || Type >= OFFSET_UUID || (Type == -1 && pUuid != nullptr), "Invalid Type: %d", Type);
1037 dbg_assert(Id >= 0 && Id <= MAX_ITEM_ID, "Invalid Id: %d", Id);
1038 dbg_assert(Size == 0 || pData != nullptr, "Data missing"); // Items without data are allowed
1039 dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
1040 dbg_assert(Size % sizeof(int) == 0, "Invalid data boundary");
1041 dbg_assert(m_vItems.size() < (size_t)std::numeric_limits<int>::max(), "Too many items");
1042
1043 if(Type == -1 || Type >= OFFSET_UUID)
1044 {
1045 Type = GetTypeFromIndex(Index: GetExtendedItemTypeIndex(Type, pUuid));
1046 }
1047
1048 const int NumItems = m_vItems.size();
1049 m_vItems.emplace_back();
1050 CItemInfo &Info = m_vItems.back();
1051 Info.m_Type = Type;
1052 Info.m_Id = Id;
1053 Info.m_Size = Size;
1054
1055 // copy data
1056 if(Size > 0)
1057 {
1058 Info.m_pData = malloc(size: Size);
1059 mem_copy(dest: Info.m_pData, source: pData, size: Size);
1060 }
1061 else
1062 {
1063 Info.m_pData = nullptr;
1064 }
1065
1066 // link
1067 CItemTypeInfo &ItemType = m_ItemTypes[Type];
1068 Info.m_Prev = ItemType.m_Last;
1069 Info.m_Next = -1;
1070
1071 if(ItemType.m_Last != -1)
1072 {
1073 m_vItems[ItemType.m_Last].m_Next = NumItems;
1074 }
1075 ItemType.m_Last = NumItems;
1076
1077 if(ItemType.m_First == -1)
1078 {
1079 ItemType.m_First = NumItems;
1080 }
1081
1082 ItemType.m_Num++;
1083 return NumItems;
1084}
1085
1086int CDataFileWriter::AddData(size_t Size, const void *pData, ECompressionLevel CompressionLevel)
1087{
1088 dbg_assert(Size > 0 && pData != nullptr, "Data missing");
1089 dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
1090 dbg_assert(m_vDatas.size() < (size_t)std::numeric_limits<int>::max(), "Too many data");
1091
1092 CDataInfo Info;
1093 Info.m_pUncompressedData = malloc(size: Size);
1094 mem_copy(dest: Info.m_pUncompressedData, source: pData, size: Size);
1095 Info.m_UncompressedSize = Size;
1096 Info.m_pCompressedData = nullptr;
1097 Info.m_CompressedSize = 0;
1098 Info.m_CompressionLevel = CompressionLevel;
1099 m_vDatas.emplace_back(args&: Info);
1100
1101 return m_vDatas.size() - 1;
1102}
1103
1104int CDataFileWriter::AddDataSwapped(size_t Size, const void *pData)
1105{
1106 dbg_assert(Size > 0 && pData != nullptr, "Data missing");
1107 dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
1108 dbg_assert(m_vDatas.size() < (size_t)std::numeric_limits<int>::max(), "Too many data");
1109 dbg_assert(Size % sizeof(int) == 0, "Invalid data boundary");
1110
1111#if defined(CONF_ARCH_ENDIAN_BIG)
1112 void *pSwapped = malloc(Size); // temporary buffer that we use during compression
1113 mem_copy(pSwapped, pData, Size);
1114 swap_endian(pSwapped, sizeof(int), Size / sizeof(int));
1115 int Index = AddData(Size, pSwapped);
1116 free(pSwapped);
1117 return Index;
1118#else
1119 return AddData(Size, pData);
1120#endif
1121}
1122
1123int CDataFileWriter::AddDataString(const char *pStr)
1124{
1125 dbg_assert(pStr != nullptr, "Data missing");
1126
1127 if(pStr[0] == '\0')
1128 {
1129 return -1;
1130 }
1131 return AddData(Size: str_length(str: pStr) + 1, pData: pStr);
1132}
1133
1134static int CompressionLevelToZlib(CDataFileWriter::ECompressionLevel CompressionLevel)
1135{
1136 switch(CompressionLevel)
1137 {
1138 case CDataFileWriter::COMPRESSION_DEFAULT:
1139 return Z_DEFAULT_COMPRESSION;
1140 case CDataFileWriter::COMPRESSION_BEST:
1141 return Z_BEST_COMPRESSION;
1142 default:
1143 dbg_assert_failed("Invalid CompressionLevel: %d", static_cast<int>(CompressionLevel));
1144 }
1145}
1146
1147void CDataFileWriter::Finish()
1148{
1149 dbg_assert((bool)m_File, "File not open");
1150
1151 // Compress data. This takes the majority of the time when saving a datafile,
1152 // so it's delayed until the end so it can be off-loaded to another thread.
1153 for(CDataInfo &DataInfo : m_vDatas)
1154 {
1155 unsigned long CompressedSize = compressBound(sourceLen: DataInfo.m_UncompressedSize);
1156 DataInfo.m_pCompressedData = malloc(size: CompressedSize);
1157 const int Result = compress2(dest: static_cast<Bytef *>(DataInfo.m_pCompressedData), destLen: &CompressedSize, source: static_cast<Bytef *>(DataInfo.m_pUncompressedData), sourceLen: DataInfo.m_UncompressedSize, level: CompressionLevelToZlib(CompressionLevel: DataInfo.m_CompressionLevel));
1158 DataInfo.m_CompressedSize = CompressedSize;
1159 free(ptr: DataInfo.m_pUncompressedData);
1160 DataInfo.m_pUncompressedData = nullptr;
1161 dbg_assert(Result == Z_OK, "datafile zlib compression failed with error %d", Result);
1162 }
1163
1164 // Calculate total size of items
1165 int64_t ItemSize = 0;
1166 for(const CItemInfo &ItemInfo : m_vItems)
1167 {
1168 ItemSize += ItemInfo.m_Size;
1169 ItemSize += sizeof(CDatafileItem);
1170 }
1171
1172 // Calculate total size of data
1173 int64_t DataSize = 0;
1174 for(const CDataInfo &DataInfo : m_vDatas)
1175 {
1176 DataSize += DataInfo.m_CompressedSize;
1177 }
1178
1179 // Calculate complete file size
1180 const int64_t TypesSize = m_ItemTypes.size() * sizeof(CDatafileItemType);
1181 const int64_t HeaderSize = sizeof(CDatafileHeader);
1182 const int64_t OffsetSize = (m_vItems.size() + m_vDatas.size() * 2) * sizeof(int); // ItemOffsets, DataOffsets, DataUncompressedSizes
1183 const int64_t SwapSize = HeaderSize + TypesSize + OffsetSize + ItemSize;
1184 const int64_t FileSize = SwapSize + DataSize;
1185
1186 // This also ensures that SwapSize, ItemSize and DataSize are valid.
1187 dbg_assert(FileSize <= (int64_t)std::numeric_limits<int>::max(), "File size too large");
1188
1189 // Construct and write header
1190 {
1191 CDatafileHeader Header;
1192 Header.m_aId[0] = 'D';
1193 Header.m_aId[1] = 'A';
1194 Header.m_aId[2] = 'T';
1195 Header.m_aId[3] = 'A';
1196 Header.m_Version = 4;
1197 Header.m_Size = FileSize - Header.SizeOffset();
1198 Header.m_Swaplen = SwapSize - Header.SizeOffset();
1199 Header.m_NumItemTypes = m_ItemTypes.size();
1200 Header.m_NumItems = m_vItems.size();
1201 Header.m_NumRawData = m_vDatas.size();
1202 Header.m_ItemSize = ItemSize;
1203 Header.m_DataSize = DataSize;
1204
1205 SwapEndianInPlace(pObj: &Header);
1206 io_write(io: m_File, buffer: &Header, size: sizeof(Header));
1207 }
1208
1209 // Write item types
1210 int ItemCount = 0;
1211 for(const auto &[Type, ItemType] : m_ItemTypes)
1212 {
1213 dbg_assert(ItemType.m_Num > 0, "Invalid ItemType.m_Num: %d", ItemType.m_Num);
1214
1215 CDatafileItemType Info;
1216 Info.m_Type = Type;
1217 Info.m_Start = ItemCount;
1218 Info.m_Num = ItemType.m_Num;
1219
1220 SwapEndianInPlace(pObj: &Info);
1221 io_write(io: m_File, buffer: &Info, size: sizeof(Info));
1222 ItemCount += ItemType.m_Num;
1223 }
1224
1225 // Write item offsets sorted by type
1226 int ItemOffset = 0;
1227 for(const auto &[Type, ItemType] : m_ItemTypes)
1228 {
1229 // Write all items offsets of this type
1230 for(int ItemIndex = ItemType.m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next)
1231 {
1232 const int ItemOffsetWrite = SwapEndianInt(Number: ItemOffset);
1233 io_write(io: m_File, buffer: &ItemOffsetWrite, size: sizeof(ItemOffsetWrite));
1234 ItemOffset += m_vItems[ItemIndex].m_Size + sizeof(CDatafileItem);
1235 }
1236 }
1237
1238 // Write data offsets
1239 int DataOffset = 0;
1240 for(const CDataInfo &DataInfo : m_vDatas)
1241 {
1242 const int DataOffsetWrite = SwapEndianInt(Number: DataOffset);
1243 io_write(io: m_File, buffer: &DataOffsetWrite, size: sizeof(DataOffsetWrite));
1244 DataOffset += DataInfo.m_CompressedSize;
1245 }
1246
1247 // Write data uncompressed sizes
1248 for(const CDataInfo &DataInfo : m_vDatas)
1249 {
1250 const int UncompressedSizeWrite = SwapEndianInt(Number: DataInfo.m_UncompressedSize);
1251 io_write(io: m_File, buffer: &UncompressedSizeWrite, size: sizeof(UncompressedSizeWrite));
1252 }
1253
1254 // Write items sorted by type
1255 for(const auto &[Type, ItemType] : m_ItemTypes)
1256 {
1257 // Write all items of this type
1258 for(int ItemIndex = ItemType.m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next)
1259 {
1260 CDatafileItem Item;
1261 Item.m_TypeAndId = ((unsigned)Type << 16u) | (unsigned)m_vItems[ItemIndex].m_Id;
1262 Item.m_Size = m_vItems[ItemIndex].m_Size;
1263
1264 SwapEndianInPlace(pObj: &Item);
1265 io_write(io: m_File, buffer: &Item, size: sizeof(Item));
1266
1267 if(m_vItems[ItemIndex].m_pData != nullptr)
1268 {
1269 SwapEndianInPlace(pObj: m_vItems[ItemIndex].m_pData, Size: m_vItems[ItemIndex].m_Size);
1270 io_write(io: m_File, buffer: m_vItems[ItemIndex].m_pData, size: m_vItems[ItemIndex].m_Size);
1271 free(ptr: m_vItems[ItemIndex].m_pData);
1272 m_vItems[ItemIndex].m_pData = nullptr;
1273 }
1274 }
1275 }
1276
1277 // Write data
1278 for(CDataInfo &DataInfo : m_vDatas)
1279 {
1280 io_write(io: m_File, buffer: DataInfo.m_pCompressedData, size: DataInfo.m_CompressedSize);
1281 free(ptr: DataInfo.m_pCompressedData);
1282 DataInfo.m_pCompressedData = nullptr;
1283 }
1284
1285 io_close(io: m_File);
1286 m_File = nullptr;
1287}
1288