1/* (c) Magnus Auvinen. See licence.txt in the root of the distribution for more information. */
2/* If you are missing that file, acquire a complete release at teeworlds.com. */
3
4#include "datafile.h"
5
6#include "uuid_manager.h"
7
8#include <base/bytes.h>
9#include <base/dbg.h>
10#include <base/fs.h>
11#include <base/hash_ctxt.h>
12#include <base/io.h>
13#include <base/log.h>
14#include <base/math.h>
15#include <base/mem.h>
16#include <base/str.h>
17
18#include <engine/storage.h>
19
20#include <zlib.h>
21
22#include <cstdlib>
23#include <limits>
24#include <unordered_set>
25
26static constexpr int MAX_ITEM_TYPE = 0xFFFF;
27static constexpr int MAX_ITEM_ID = 0xFFFF;
28static constexpr int OFFSET_UUID_TYPE = 0x8000;
29
30static inline void SwapEndianInPlace(void *pObj, size_t Size)
31{
32#if defined(CONF_ARCH_ENDIAN_BIG)
33 swap_endian(pObj, sizeof(int), Size / sizeof(int));
34#endif
35}
36
37template<typename T>
38static inline void SwapEndianInPlace(T *pObj)
39{
40 static_assert(sizeof(T) % sizeof(int) == 0);
41 SwapEndianInPlace(pObj, sizeof(T));
42}
43
44static inline int SwapEndianInt(int Number)
45{
46 SwapEndianInPlace(pObj: &Number);
47 return Number;
48}
49
50class CItemEx
51{
52public:
53 int m_aUuid[sizeof(CUuid) / sizeof(int32_t)];
54
55 static CItemEx FromUuid(CUuid Uuid)
56 {
57 CItemEx Result;
58 for(size_t i = 0; i < std::size(Result.m_aUuid); i++)
59 {
60 Result.m_aUuid[i] = bytes_be_to_uint(bytes: &Uuid.m_aData[i * sizeof(int32_t)]);
61 }
62 return Result;
63 }
64
65 CUuid ToUuid() const
66 {
67 CUuid Result;
68 for(size_t i = 0; i < std::size(m_aUuid); i++)
69 {
70 uint_to_bytes_be(bytes: &Result.m_aData[i * sizeof(int32_t)], value: m_aUuid[i]);
71 }
72 return Result;
73 }
74};
75
76class CDatafileItemType
77{
78public:
79 int m_Type;
80 int m_Start;
81 int m_Num;
82};
83
84class CDatafileItem
85{
86public:
87 unsigned m_TypeAndId;
88 int m_Size;
89
90 int Type() const
91 {
92 return (m_TypeAndId >> 16u) & MAX_ITEM_TYPE;
93 }
94
95 int Id() const
96 {
97 return m_TypeAndId & MAX_ITEM_ID;
98 }
99};
100
101class CDatafileHeader
102{
103public:
104 char m_aId[4];
105 int m_Version;
106 int m_Size;
107 int m_Swaplen;
108 int m_NumItemTypes;
109 int m_NumItems;
110 int m_NumRawData;
111 int m_ItemSize;
112 int m_DataSize;
113
114 constexpr size_t SizeOffset()
115 {
116 // The size of these members is not included in m_Size and m_Swaplen
117 return sizeof(m_aId) + sizeof(m_Version) + sizeof(m_Size) + sizeof(m_Swaplen);
118 }
119};
120
121class CDatafileInfo
122{
123public:
124 CDatafileItemType *m_pItemTypes;
125 int *m_pItemOffsets;
126 int *m_pDataOffsets;
127 int *m_pDataSizes;
128
129 char *m_pItemStart;
130 char *m_pDataStart;
131};
132
133class CDataProcessorWrapper
134{
135public:
136 FDataProcessor m_DataProcessor;
137};
138
139class CDatafile
140{
141public:
142 IOHANDLE m_File;
143 char m_aFullName[IO_MAX_PATH_LENGTH];
144 const char *m_pBaseName;
145 char m_aPath[IO_MAX_PATH_LENGTH];
146 unsigned m_FileSize;
147 SHA256_DIGEST m_Sha256;
148 unsigned m_Crc;
149
150 CDatafileInfo m_Info;
151 CDatafileHeader m_Header;
152 int m_DataStartOffset;
153 void **m_ppDataPtrs;
154 CDataProcessorWrapper **m_ppDataProcessors;
155 CDatafileItem **m_ppOverriddenItems;
156 int *m_pDataSizes;
157 char *m_pData;
158
159 int GetFileDataSize(int Index) const
160 {
161 dbg_assert(Index >= 0 && Index < m_Header.m_NumRawData, "Invalid Index: %d", Index);
162
163 if(Index == m_Header.m_NumRawData - 1)
164 {
165 return m_Header.m_DataSize - m_Info.m_pDataOffsets[Index];
166 }
167
168 return m_Info.m_pDataOffsets[Index + 1] - m_Info.m_pDataOffsets[Index];
169 }
170
171 int GetDataSize(int Index) const
172 {
173 // Invalid data indices may appear in map items
174 if(Index < 0 || Index >= m_Header.m_NumRawData)
175 {
176 return 0;
177 }
178
179 if(m_ppDataPtrs[Index] == nullptr)
180 {
181 if(m_Info.m_pDataSizes != nullptr)
182 {
183 return m_Info.m_pDataSizes[Index];
184 }
185 else
186 {
187 return GetFileDataSize(Index);
188 }
189 }
190
191 const int Size = m_pDataSizes[Index];
192 if(Size < 0)
193 {
194 return 0; // summarize all errors as zero size
195 }
196 return Size;
197 }
198
199 void *GetData(int Index, bool Swap) const
200 {
201 // Invalid data indices may appear in map items
202 if(Index < 0 || Index >= m_Header.m_NumRawData)
203 {
204 return nullptr;
205 }
206
207 // Data already loaded
208 if(m_ppDataPtrs[Index] != nullptr)
209 {
210 return m_ppDataPtrs[Index];
211 }
212
213 // Don't try to load the data again if it previously failed
214 if(m_pDataSizes[Index] < 0)
215 {
216 return nullptr;
217 }
218
219 const unsigned DataSize = GetFileDataSize(Index);
220 if(m_Info.m_pDataSizes != nullptr)
221 {
222 // v4 has compressed data
223 const unsigned OriginalUncompressedSize = m_Info.m_pDataSizes[Index];
224 log_trace("datafile", "loading data. index=%d size=%d uncompressed=%d", Index, DataSize, OriginalUncompressedSize);
225 if(OriginalUncompressedSize == 0)
226 {
227 log_error("datafile", "data size invalid. data will be ignored. index=%d size=%d uncompressed=%d", Index, DataSize, OriginalUncompressedSize);
228 m_ppDataPtrs[Index] = nullptr;
229 m_pDataSizes[Index] = -1;
230 return nullptr;
231 }
232
233 // read the compressed data
234 void *pCompressedData = malloc(size: DataSize);
235 if(pCompressedData == nullptr)
236 {
237 log_error("datafile", "out of memory. could not allocate memory for compressed data. index=%d size=%d", Index, DataSize);
238 m_ppDataPtrs[Index] = nullptr;
239 m_pDataSizes[Index] = -1;
240 return nullptr;
241 }
242 unsigned ActualDataSize = 0;
243 if(io_seek(io: m_File, offset: m_DataStartOffset + m_Info.m_pDataOffsets[Index], origin: EIoSeekOrigin::START) == 0)
244 {
245 ActualDataSize = io_read(io: m_File, buffer: pCompressedData, size: DataSize);
246 }
247 if(DataSize != ActualDataSize)
248 {
249 log_error("datafile", "truncation error. could not read all compressed data. index=%d wanted=%d got=%d", Index, DataSize, ActualDataSize);
250 free(ptr: pCompressedData);
251 m_ppDataPtrs[Index] = nullptr;
252 m_pDataSizes[Index] = -1;
253 return nullptr;
254 }
255
256 // decompress the data
257 m_ppDataPtrs[Index] = static_cast<char *>(malloc(size: OriginalUncompressedSize));
258 if(m_ppDataPtrs[Index] == nullptr)
259 {
260 free(ptr: pCompressedData);
261 log_error("datafile", "out of memory. could not allocate memory for uncompressed data. index=%d size=%d", Index, OriginalUncompressedSize);
262 m_pDataSizes[Index] = -1;
263 return nullptr;
264 }
265 unsigned long UncompressedSize = OriginalUncompressedSize;
266 const int Result = uncompress(dest: static_cast<Bytef *>(m_ppDataPtrs[Index]), destLen: &UncompressedSize, source: static_cast<Bytef *>(pCompressedData), sourceLen: DataSize);
267 free(ptr: pCompressedData);
268 if(Result != Z_OK || UncompressedSize != OriginalUncompressedSize)
269 {
270 log_error("datafile", "failed to uncompress data. index=%d result=%d wanted=%d got=%ld", Index, Result, OriginalUncompressedSize, UncompressedSize);
271 free(ptr: m_ppDataPtrs[Index]);
272 m_ppDataPtrs[Index] = nullptr;
273 m_pDataSizes[Index] = -1;
274 return nullptr;
275 }
276 m_pDataSizes[Index] = OriginalUncompressedSize;
277 }
278 else
279 {
280 log_trace("datafile", "loading data. index=%d size=%d", Index, DataSize);
281 m_ppDataPtrs[Index] = malloc(size: DataSize);
282 if(m_ppDataPtrs[Index] == nullptr)
283 {
284 log_error("datafile", "out of memory. could not allocate memory for uncompressed data. index=%d size=%d", Index, DataSize);
285 m_pDataSizes[Index] = -1;
286 return nullptr;
287 }
288 unsigned ActualDataSize = 0;
289 if(io_seek(io: m_File, offset: m_DataStartOffset + m_Info.m_pDataOffsets[Index], origin: EIoSeekOrigin::START) == 0)
290 {
291 ActualDataSize = io_read(io: m_File, buffer: m_ppDataPtrs[Index], size: DataSize);
292 }
293 if(DataSize != ActualDataSize)
294 {
295 log_error("datafile", "truncation error. could not read all uncompressed data. index=%d wanted=%d got=%d", Index, DataSize, ActualDataSize);
296 free(ptr: m_ppDataPtrs[Index]);
297 m_ppDataPtrs[Index] = nullptr;
298 m_pDataSizes[Index] = -1;
299 return nullptr;
300 }
301 m_pDataSizes[Index] = DataSize;
302 }
303
304 if(Swap)
305 {
306 SwapEndianInPlace(pObj: m_ppDataPtrs[Index], Size: m_pDataSizes[Index]);
307 }
308
309 if(m_ppDataProcessors[Index] != nullptr)
310 {
311 const auto [pNewData, NewSize] = m_ppDataProcessors[Index]->m_DataProcessor(m_ppDataPtrs[Index], m_pDataSizes[Index]);
312 if(pNewData == nullptr)
313 {
314 m_ppDataPtrs[Index] = nullptr;
315 m_pDataSizes[Index] = -1;
316 return nullptr;
317 }
318 m_ppDataPtrs[Index] = pNewData;
319 m_pDataSizes[Index] = NewSize;
320 }
321
322 return m_ppDataPtrs[Index];
323 }
324
325 void AddDataProcessor(int Index, FDataProcessor DataProcessor) // NOLINT(readability-make-member-function-const)
326 {
327 dbg_assert(Index >= 0 && Index < m_Header.m_NumRawData, "Index invalid: %d", Index);
328 dbg_assert(m_ppDataProcessors[Index] == nullptr, "Data already intercepted: %d", Index);
329
330 m_ppDataProcessors[Index] = new CDataProcessorWrapper;
331 m_ppDataProcessors[Index]->m_DataProcessor = std::move(DataProcessor);
332 }
333
334 int GetFileItemSize(int Index) const
335 {
336 dbg_assert(Index >= 0 && Index < m_Header.m_NumItems, "Invalid Index: %d", Index);
337
338 if(m_ppOverriddenItems[Index] != nullptr)
339 {
340 return m_ppOverriddenItems[Index]->m_Size + sizeof(CDatafileItem);
341 }
342
343 if(Index == m_Header.m_NumItems - 1)
344 {
345 return m_Header.m_ItemSize - m_Info.m_pItemOffsets[Index];
346 }
347
348 return m_Info.m_pItemOffsets[Index + 1] - m_Info.m_pItemOffsets[Index];
349 }
350
351 int GetItemSize(int Index) const
352 {
353 return GetFileItemSize(Index) - sizeof(CDatafileItem);
354 }
355
356 CDatafileItem *GetItem(int Index) const
357 {
358 dbg_assert(Index >= 0 && Index < m_Header.m_NumItems, "Invalid Index: %d", Index);
359
360 if(m_ppOverriddenItems[Index] != nullptr)
361 {
362 return m_ppOverriddenItems[Index];
363 }
364 return static_cast<CDatafileItem *>(static_cast<void *>(m_Info.m_pItemStart + m_Info.m_pItemOffsets[Index]));
365 }
366
367 bool OverrideItemData(int Index, const void *pData, size_t Size) // NOLINT(readability-make-member-function-const)
368 {
369 const CDatafileItem *pCurrentItem = GetItem(Index);
370 dbg_assert(m_ppOverriddenItems[Index] == nullptr, "Item already overridden: %d", Index);
371 dbg_assert(Size == 0 || pData != nullptr, "Data missing"); // Items without data are allowed
372 dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
373 dbg_assert(Size % sizeof(int) == 0, "Invalid data boundary");
374 m_ppOverriddenItems[Index] = static_cast<CDatafileItem *>(malloc(size: sizeof(CDatafileItem) + Size));
375 if(m_ppOverriddenItems[Index] == nullptr)
376 {
377 log_error("datafile", "out of memory. could not allocate memory for overridden item data. index=%d size=%" PRIzu, Index, Size);
378 return false;
379 }
380 m_ppOverriddenItems[Index]->m_TypeAndId = pCurrentItem->m_TypeAndId;
381 m_ppOverriddenItems[Index]->m_Size = Size;
382 if(pData)
383 {
384 mem_copy(dest: static_cast<void *>(m_ppOverriddenItems[Index] + 1), source: pData, size: Size);
385 }
386 return true;
387 }
388
389 bool Validate() const
390 {
391#define Check(Test, ErrorMessage, ...) \
392 do \
393 { \
394 if(!(Test)) \
395 { \
396 log_error("datafile", "invalid file information: " ErrorMessage, ##__VA_ARGS__); \
397 return false; \
398 } \
399 } while(false)
400
401 // validate item types
402 int64_t CountedItems = 0;
403 std::unordered_set<int> UsedItemTypes;
404 for(int Index = 0; Index < m_Header.m_NumItemTypes; Index++)
405 {
406 const CDatafileItemType &ItemType = m_Info.m_pItemTypes[Index];
407 Check(ItemType.m_Type >= 0 && ItemType.m_Type <= MAX_ITEM_TYPE, "item type has invalid type. index=%d type=%d", Index, ItemType.m_Type);
408 const auto [_, Inserted] = UsedItemTypes.insert(obj: ItemType.m_Type);
409 Check(Inserted, "item type has duplicate type. index=%d type=%d", Index, ItemType.m_Type);
410 Check(ItemType.m_Num > 0, "item type has invalid number of items. index=%d type=%d num=%d", Index, ItemType.m_Type, ItemType.m_Num);
411 Check(ItemType.m_Start == CountedItems, "item type has invalid start. index=%d type=%d start=%d", Index, ItemType.m_Type, ItemType.m_Start);
412 CountedItems += ItemType.m_Num;
413 if(CountedItems > m_Header.m_NumItems)
414 {
415 break;
416 }
417 }
418 Check(CountedItems == m_Header.m_NumItems, "mismatched number of items in item types. counted=%" PRId64 " header=%d", CountedItems, m_Header.m_NumItems);
419
420 // validate item offsets
421 int PrevItemOffset = -1;
422 for(int Index = 0; Index < m_Header.m_NumItems; Index++)
423 {
424 const int Offset = m_Info.m_pItemOffsets[Index];
425 if(Index == 0)
426 {
427 Check(Offset == 0, "first item offset is not zero. offset=%d", Offset);
428 }
429 else
430 {
431 Check(Offset > PrevItemOffset, "item offset not greater than previous. index=%d offset=%d previous=%d", Index, Offset, PrevItemOffset);
432 }
433 Check(Offset < m_Header.m_ItemSize, "item offset larger than total item size. index=%d offset=%d total=%d", Index, Offset, m_Header.m_ItemSize);
434 PrevItemOffset = Offset;
435 }
436
437 // validate item sizes, types and IDs
438 int64_t TotalItemSize = 0;
439 for(int TypeIndex = 0; TypeIndex < m_Header.m_NumItemTypes; TypeIndex++)
440 {
441 std::unordered_set<int> UsedItemIds;
442 const CDatafileItemType &ItemType = m_Info.m_pItemTypes[TypeIndex];
443 for(int ItemIndex = ItemType.m_Start; ItemIndex < ItemType.m_Start + ItemType.m_Num; ItemIndex++)
444 {
445 const int FileItemSize = GetFileItemSize(Index: ItemIndex);
446 Check(FileItemSize >= (int)sizeof(CDatafileItem), "map item too small for header. type_index=%d item_index=%d size=%d header=%d", TypeIndex, ItemIndex, FileItemSize, (int)sizeof(CDatafileItem));
447 const CDatafileItem *pItem = GetItem(Index: ItemIndex);
448 Check(pItem->Type() == ItemType.m_Type, "mismatched item type. type_index=%d item_index=%d type=%d expected=%d", TypeIndex, ItemIndex, pItem->Type(), ItemType.m_Type);
449 // Many old maps contain duplicate map items of type ITEMTYPE_EX due to a bug in DDNet tools.
450 if(pItem->Type() != ITEMTYPE_EX)
451 {
452 const auto [_, Inserted] = UsedItemIds.insert(obj: pItem->Id());
453 Check(Inserted, "map item has duplicate ID. type_index=%d item_index=%d type=%d ID=%d", TypeIndex, ItemIndex, pItem->Type(), pItem->Id());
454 }
455 Check(pItem->m_Size >= 0, "map item size invalid. type_index=%d item_index=%d size=%d", TypeIndex, ItemIndex, pItem->m_Size);
456 Check(pItem->m_Size % sizeof(int) == 0, "map item size not integer aligned. type_index=%d item_index=%d size=%d", TypeIndex, ItemIndex, pItem->m_Size);
457 Check(pItem->m_Size == GetItemSize(ItemIndex), "map item size does not match file. type_index=%d item_index=%d size=%d file_size=%d", TypeIndex, ItemIndex, pItem->m_Size, GetItemSize(ItemIndex));
458 TotalItemSize += FileItemSize;
459 if(TotalItemSize > m_Header.m_ItemSize)
460 {
461 break;
462 }
463 }
464 }
465 Check(TotalItemSize == m_Header.m_ItemSize, "mismatched total item size. expected=%" PRId64 " header=%d", TotalItemSize, m_Header.m_ItemSize);
466
467 // validate data offsets
468 int PrevDataOffset = -1;
469 for(int Index = 0; Index < m_Header.m_NumRawData; Index++)
470 {
471 const int Offset = m_Info.m_pDataOffsets[Index];
472 if(Index == 0)
473 {
474 Check(Offset == 0, "first data offset must be zero. offset=%d", Offset);
475 }
476 else
477 {
478 Check(Offset > PrevDataOffset, "data offset not greater than previous. index=%d offset=%d previous=%d", Index, Offset, PrevDataOffset);
479 }
480 Check(Offset < m_Header.m_DataSize, "data offset larger than total data size. index=%d offset=%d total=%d", Index, Offset, m_Header.m_DataSize);
481 PrevDataOffset = Offset;
482 }
483
484 // validate data sizes
485 if(m_Info.m_pDataSizes != nullptr)
486 {
487 for(int Index = 0; Index < m_Header.m_NumRawData; Index++)
488 {
489 const int Size = m_Info.m_pDataSizes[Index];
490 Check(Size >= 0, "data size invalid. index=%d size=%d", Index, Size);
491 if(Size == 0)
492 {
493 // Data of size zero is not allowed, but due to existing maps with this quirk we instead allow
494 // the file to be loaded and fail loading the data in the GetData function if the size is zero.
495 log_warn("datafile", "invalid file information: data size invalid. index=%d size=%d", Index, Size);
496 }
497 }
498 }
499
500 return true;
501#undef Check
502 }
503};
504
505CDataFileReader::~CDataFileReader()
506{
507 Close();
508}
509
510CDataFileReader &CDataFileReader::operator=(CDataFileReader &&Other)
511{
512 m_pDataFile = Other.m_pDataFile;
513 Other.m_pDataFile = nullptr;
514 return *this;
515}
516
517bool CDataFileReader::Open(const char *pFullName, IStorage *pStorage, const char *pPath, int StorageType)
518{
519 dbg_assert(m_pDataFile == nullptr, "File already open");
520
521 log_trace("datafile", "loading '%s' from '%s'", pFullName, pPath);
522
523 IOHANDLE File = pStorage->OpenFile(pFilename: pPath, Flags: IOFLAG_READ, Type: StorageType);
524 if(!File)
525 {
526 log_error("datafile", "failed to open file '%s' for reading", pPath);
527 return false;
528 }
529
530 // determine size and hashes of the file and store them
531 int64_t FileSize = 0;
532 unsigned Crc = 0;
533 SHA256_DIGEST Sha256;
534 {
535 SHA256_CTX Sha256Ctxt;
536 sha256_init(ctxt: &Sha256Ctxt);
537 unsigned char aBuffer[64 * 1024];
538 while(true)
539 {
540 const unsigned Bytes = io_read(io: File, buffer: aBuffer, size: sizeof(aBuffer));
541 if(Bytes == 0)
542 break;
543 FileSize += Bytes;
544 Crc = crc32(crc: Crc, buf: aBuffer, len: Bytes);
545 sha256_update(ctxt: &Sha256Ctxt, data: aBuffer, data_len: Bytes);
546 }
547 Sha256 = sha256_finish(ctxt: &Sha256Ctxt);
548 if(io_seek(io: File, offset: 0, origin: EIoSeekOrigin::START) != 0)
549 {
550 io_close(io: File);
551 log_error("datafile", "could not seek to start after calculating hashes");
552 return false;
553 }
554 }
555
556 // read header
557 CDatafileHeader Header;
558 if(io_read(io: File, buffer: &Header, size: sizeof(Header)) != sizeof(Header))
559 {
560 io_close(io: File);
561 log_error("datafile", "could not read file header. file truncated or not a datafile.");
562 return false;
563 }
564
565 // check header magic
566 if((Header.m_aId[0] != 'A' || Header.m_aId[1] != 'T' || Header.m_aId[2] != 'A' || Header.m_aId[3] != 'D') &&
567 (Header.m_aId[0] != 'D' || Header.m_aId[1] != 'A' || Header.m_aId[2] != 'T' || Header.m_aId[3] != 'A'))
568 {
569 io_close(io: File);
570 log_error("datafile", "wrong header magic. magic=%x%x%x%x", Header.m_aId[0], Header.m_aId[1], Header.m_aId[2], Header.m_aId[3]);
571 return false;
572 }
573
574 SwapEndianInPlace(pObj: &Header);
575
576 // check header version
577 if(Header.m_Version != 3 && Header.m_Version != 4)
578 {
579 io_close(io: File);
580 log_error("datafile", "unsupported header version. version=%d", Header.m_Version);
581 return false;
582 }
583
584 // validate header information
585 if(Header.m_NumItemTypes < 0 ||
586 Header.m_NumItemTypes > MAX_ITEM_TYPE + 1 ||
587 Header.m_NumItems < 0 ||
588 Header.m_NumRawData < 0 ||
589 Header.m_ItemSize < 0 ||
590 Header.m_ItemSize % sizeof(int) != 0 ||
591 Header.m_DataSize < 0)
592 {
593 io_close(io: File);
594 log_error("datafile", "invalid header information. num_types=%d num_items=%d num_data=%d item_size=%d data_size=%d",
595 Header.m_NumItemTypes, Header.m_NumItems, Header.m_NumRawData, Header.m_ItemSize, Header.m_DataSize);
596 return false;
597 }
598
599 // calculate and validate sizes
600 int64_t Size = 0;
601 Size += (int64_t)Header.m_NumItemTypes * sizeof(CDatafileItemType);
602 Size += (int64_t)Header.m_NumItems * sizeof(int);
603 Size += (int64_t)Header.m_NumRawData * sizeof(int);
604 int64_t SizeFix = 0;
605 if(Header.m_Version == 4) // v4 has uncompressed data sizes as well
606 {
607 // The size of the uncompressed data sizes was not included in
608 // Header.m_Size and Header.m_Swaplen of version 4 maps prior
609 // to commit 3dd1ea0d8f6cb442ac41bd223279f41d1ed1b2bb. We also
610 // support loading maps created prior to this commit by fixing
611 // the sizes transparently when loading.
612 SizeFix = (int64_t)Header.m_NumRawData * sizeof(int);
613 Size += SizeFix;
614 }
615 Size += Header.m_ItemSize;
616
617 if((int64_t)sizeof(Header) + Size + (int64_t)Header.m_DataSize != FileSize)
618 {
619 io_close(io: File);
620 log_error("datafile", "invalid header data size or truncated file. data_size=%d file_size=%" PRId64, Header.m_DataSize, FileSize);
621 return false;
622 }
623
624 const int64_t HeaderFileSize = (int64_t)Header.m_Size + Header.SizeOffset();
625 if(HeaderFileSize != FileSize)
626 {
627 if(SizeFix != 0 && HeaderFileSize + SizeFix == FileSize)
628 {
629 log_warn("datafile", "fixing invalid header size. size=%d fix=+%" PRId64, Header.m_Size, SizeFix);
630 Header.m_Size += SizeFix;
631 }
632 else
633 {
634 io_close(io: File);
635 log_error("datafile", "invalid header size or truncated file. size=%" PRId64 " actual=%" PRId64, HeaderFileSize, FileSize);
636 return false;
637 }
638 }
639
640 const int64_t HeaderSwaplen = (int64_t)Header.m_Swaplen + Header.SizeOffset();
641 const int64_t FileSizeSwaplen = FileSize - Header.m_DataSize;
642 if(HeaderSwaplen != FileSizeSwaplen)
643 {
644 if(Header.m_Swaplen % sizeof(int) == 0 && SizeFix != 0 && HeaderSwaplen + SizeFix == FileSizeSwaplen)
645 {
646 log_warn("datafile", "fixing invalid header swaplen. swaplen=%d fix=+%" PRId64, Header.m_Swaplen, SizeFix);
647 Header.m_Swaplen += SizeFix;
648 }
649 else
650 {
651 io_close(io: File);
652 log_error("datafile", "invalid header swaplen or truncated file. swaplen=%" PRId64 " actual=%" PRId64, HeaderSwaplen, FileSizeSwaplen);
653 return false;
654 }
655 }
656
657 constexpr int64_t MaxAllocSize = (int64_t)2 * 1024 * 1024 * 1024;
658 int64_t AllocSize = Size;
659 AllocSize += sizeof(CDatafile); // add space for info structure
660 AllocSize += (int64_t)Header.m_NumRawData * sizeof(void *); // add space for data pointers
661 AllocSize += (int64_t)Header.m_NumRawData * sizeof(CDataProcessorWrapper *); // add space for data interceptors
662 AllocSize += (int64_t)Header.m_NumItems * sizeof(CDatafileItem *); // add space for item overrides
663 AllocSize += (int64_t)Header.m_NumRawData * sizeof(int); // add space for data sizes
664 if(AllocSize > MaxAllocSize)
665 {
666 io_close(io: File);
667 log_error("datafile", "file too large. alloc_size=%" PRId64 " max=%" PRId64, AllocSize, MaxAllocSize);
668 return false;
669 }
670
671 CDatafile *pTmpDataFile = static_cast<CDatafile *>(malloc(size: AllocSize));
672 if(pTmpDataFile == nullptr)
673 {
674 io_close(io: File);
675 log_error("datafile", "out of memory. could not allocate memory for datafile. alloc_size=%" PRId64, AllocSize);
676 return false;
677 }
678 pTmpDataFile->m_Header = Header;
679 pTmpDataFile->m_DataStartOffset = sizeof(CDatafileHeader) + Size;
680 pTmpDataFile->m_ppDataPtrs = (void **)(pTmpDataFile + 1);
681 pTmpDataFile->m_ppDataProcessors = (CDataProcessorWrapper **)(pTmpDataFile->m_ppDataPtrs + Header.m_NumRawData);
682 pTmpDataFile->m_ppOverriddenItems = (CDatafileItem **)(pTmpDataFile->m_ppDataProcessors + Header.m_NumRawData);
683 pTmpDataFile->m_pDataSizes = (int *)(pTmpDataFile->m_ppOverriddenItems + Header.m_NumItems);
684 pTmpDataFile->m_pData = (char *)(pTmpDataFile->m_pDataSizes + Header.m_NumRawData);
685 pTmpDataFile->m_File = File;
686 str_copy(dst&: pTmpDataFile->m_aFullName, src: pFullName);
687 pTmpDataFile->m_pBaseName = fs_filename(path: pTmpDataFile->m_aFullName);
688 str_copy(dst&: pTmpDataFile->m_aPath, src: pPath);
689 pTmpDataFile->m_FileSize = FileSize;
690 pTmpDataFile->m_Sha256 = Sha256;
691 pTmpDataFile->m_Crc = Crc;
692
693 // clear the data pointers and sizes
694 mem_zero(block: pTmpDataFile->m_ppDataPtrs, size: Header.m_NumRawData * sizeof(void *));
695 mem_zero(block: pTmpDataFile->m_ppDataProcessors, size: Header.m_NumRawData * sizeof(CDataProcessorWrapper *));
696 mem_zero(block: pTmpDataFile->m_ppOverriddenItems, size: Header.m_NumItems * sizeof(CDatafileItem *));
697 mem_zero(block: pTmpDataFile->m_pDataSizes, size: Header.m_NumRawData * sizeof(int));
698
699 // read types, offsets, sizes and item data
700 const unsigned ReadSize = io_read(io: pTmpDataFile->m_File, buffer: pTmpDataFile->m_pData, size: Size);
701 if((int64_t)ReadSize != Size)
702 {
703 io_close(io: pTmpDataFile->m_File);
704 free(ptr: pTmpDataFile);
705 log_error("datafile", "truncation error. could not read all item data. wanted=%" PRId64 " got=%d", Size, ReadSize);
706 return false;
707 }
708
709 // The swap len also includes the size of the header (without the size offset), but the header was already swapped above.
710 const int64_t DataSwapLen = pTmpDataFile->m_Header.m_Swaplen - (int)(sizeof(Header) - Header.SizeOffset());
711 dbg_assert(DataSwapLen == Size, "Swap len and file size mismatch");
712 SwapEndianInPlace(pObj: pTmpDataFile->m_pData, Size: DataSwapLen);
713
714 pTmpDataFile->m_Info.m_pItemTypes = (CDatafileItemType *)pTmpDataFile->m_pData;
715 pTmpDataFile->m_Info.m_pItemOffsets = (int *)&pTmpDataFile->m_Info.m_pItemTypes[pTmpDataFile->m_Header.m_NumItemTypes];
716 pTmpDataFile->m_Info.m_pDataOffsets = &pTmpDataFile->m_Info.m_pItemOffsets[pTmpDataFile->m_Header.m_NumItems];
717 if(pTmpDataFile->m_Header.m_Version == 4) // v4 has uncompressed data sizes as well
718 {
719 pTmpDataFile->m_Info.m_pDataSizes = &pTmpDataFile->m_Info.m_pDataOffsets[pTmpDataFile->m_Header.m_NumRawData];
720 pTmpDataFile->m_Info.m_pItemStart = (char *)&pTmpDataFile->m_Info.m_pDataSizes[pTmpDataFile->m_Header.m_NumRawData];
721 }
722 else
723 {
724 pTmpDataFile->m_Info.m_pDataSizes = nullptr;
725 pTmpDataFile->m_Info.m_pItemStart = (char *)&pTmpDataFile->m_Info.m_pDataOffsets[pTmpDataFile->m_Header.m_NumRawData];
726 }
727 pTmpDataFile->m_Info.m_pDataStart = pTmpDataFile->m_Info.m_pItemStart + pTmpDataFile->m_Header.m_ItemSize;
728
729 if(!pTmpDataFile->Validate())
730 {
731 io_close(io: pTmpDataFile->m_File);
732 free(ptr: pTmpDataFile);
733 return false;
734 }
735
736 m_pDataFile = pTmpDataFile;
737 log_trace("datafile", "loading done. name='%s' path='%s'", pFullName, pPath);
738
739 return true;
740}
741
742bool CDataFileReader::Open(IStorage *pStorage, const char *pPath, int StorageType)
743{
744 char aFilename[IO_MAX_PATH_LENGTH];
745 fs_split_file_extension(filename: fs_filename(path: pPath), name: aFilename, name_size: sizeof(aFilename));
746 return Open(pFullName: aFilename, pStorage, pPath, StorageType);
747}
748
749void CDataFileReader::Close()
750{
751 if(!m_pDataFile)
752 {
753 return;
754 }
755
756 for(int i = 0; i < m_pDataFile->m_Header.m_NumRawData; i++)
757 {
758 free(ptr: m_pDataFile->m_ppDataPtrs[i]);
759 delete m_pDataFile->m_ppDataProcessors[i];
760 }
761
762 for(int i = 0; i < m_pDataFile->m_Header.m_NumItems; i++)
763 {
764 free(ptr: m_pDataFile->m_ppOverriddenItems[i]);
765 }
766
767 io_close(io: m_pDataFile->m_File);
768 free(ptr: m_pDataFile);
769 m_pDataFile = nullptr;
770}
771
772bool CDataFileReader::IsOpen() const
773{
774 return m_pDataFile != nullptr;
775}
776
777IOHANDLE CDataFileReader::File() const
778{
779 dbg_assert(m_pDataFile != nullptr, "File not open");
780
781 return m_pDataFile->m_File;
782}
783
784int CDataFileReader::GetDataSize(int Index) const
785{
786 dbg_assert(m_pDataFile != nullptr, "File not open");
787
788 return m_pDataFile->GetDataSize(Index);
789}
790
791void *CDataFileReader::GetData(int Index)
792{
793 dbg_assert(m_pDataFile != nullptr, "File not open");
794
795 return m_pDataFile->GetData(Index, Swap: false);
796}
797
798void *CDataFileReader::GetDataSwapped(int Index)
799{
800 dbg_assert(m_pDataFile != nullptr, "File not open");
801
802 return m_pDataFile->GetData(Index, Swap: true);
803}
804
805const char *CDataFileReader::GetDataString(int Index)
806{
807 dbg_assert(m_pDataFile != nullptr, "File not open");
808
809 if(Index == -1)
810 {
811 return "";
812 }
813
814 const int DataSize = GetDataSize(Index);
815 if(!DataSize)
816 {
817 return nullptr;
818 }
819
820 const char *pData = static_cast<const char *>(GetData(Index));
821 if(pData == nullptr || mem_has_null(block: pData, size: DataSize - 1) || pData[DataSize - 1] != '\0' || !str_utf8_check(str: pData))
822 {
823 return nullptr;
824 }
825 return pData;
826}
827
828void CDataFileReader::AddDataProcessor(int Index, FDataProcessor DataProcessor)
829{
830 dbg_assert(m_pDataFile != nullptr, "File not open");
831
832 m_pDataFile->AddDataProcessor(Index, DataProcessor: std::move(DataProcessor));
833}
834
835void CDataFileReader::UnloadData(int Index)
836{
837 dbg_assert(m_pDataFile != nullptr, "File not open");
838
839 if(Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData)
840 return;
841
842 // Don't try to load the data again if it previously failed
843 if(m_pDataFile->m_pDataSizes[Index] < 0)
844 return;
845
846 free(ptr: m_pDataFile->m_ppDataPtrs[Index]);
847 m_pDataFile->m_ppDataPtrs[Index] = nullptr;
848 m_pDataFile->m_pDataSizes[Index] = 0;
849}
850
851int CDataFileReader::NumData() const
852{
853 dbg_assert(m_pDataFile != nullptr, "File not open");
854
855 return m_pDataFile->m_Header.m_NumRawData;
856}
857
858int CDataFileReader::GetItemSize(int Index) const
859{
860 dbg_assert(m_pDataFile != nullptr, "File not open");
861
862 return m_pDataFile->GetItemSize(Index);
863}
864
865int CDataFileReader::GetExternalItemType(int InternalType, CUuid *pUuid)
866{
867 if(InternalType <= OFFSET_UUID_TYPE || InternalType == ITEMTYPE_EX)
868 {
869 if(pUuid)
870 {
871 *pUuid = UUID_ZEROED;
872 }
873 return InternalType;
874 }
875
876 const int TypeIndex = FindItemIndex(Type: ITEMTYPE_EX, Id: InternalType);
877 if(TypeIndex < 0 || GetItemSize(Index: TypeIndex) < (int)sizeof(CItemEx))
878 {
879 if(pUuid)
880 {
881 *pUuid = UUID_ZEROED;
882 }
883 return InternalType;
884 }
885
886 const CItemEx *pItemEx = static_cast<const CItemEx *>(GetItem(Index: TypeIndex));
887 const CUuid Uuid = pItemEx->ToUuid();
888 if(pUuid)
889 {
890 *pUuid = Uuid;
891 }
892 // Propagate UUID_UNKNOWN, it doesn't hurt.
893 return g_UuidManager.LookupUuid(Uuid);
894}
895
896int CDataFileReader::GetInternalItemType(int ExternalType)
897{
898 if(ExternalType < OFFSET_UUID)
899 {
900 return ExternalType;
901 }
902
903 const CUuid Uuid = g_UuidManager.GetUuid(Id: ExternalType);
904 int Start, Num;
905 GetType(Type: ITEMTYPE_EX, pStart: &Start, pNum: &Num);
906 for(int Index = Start; Index < Start + Num; Index++)
907 {
908 if(GetItemSize(Index) < (int)sizeof(CItemEx))
909 {
910 continue;
911 }
912 int Id;
913 if(Uuid == static_cast<const CItemEx *>(GetItem(Index, pType: nullptr, pId: &Id))->ToUuid())
914 {
915 return Id;
916 }
917 }
918 return -1;
919}
920
921void *CDataFileReader::GetItem(int Index, int *pType, int *pId, CUuid *pUuid)
922{
923 dbg_assert(m_pDataFile != nullptr, "File not open");
924
925 CDatafileItem *pItem = m_pDataFile->GetItem(Index);
926 const int ExternalType = GetExternalItemType(InternalType: pItem->Type(), pUuid);
927 if(pType)
928 {
929 *pType = ExternalType;
930 }
931 if(pId)
932 {
933 *pId = pItem->Id();
934 }
935 return static_cast<void *>(pItem + 1);
936}
937
938void CDataFileReader::GetType(int Type, int *pStart, int *pNum)
939{
940 dbg_assert(m_pDataFile != nullptr, "File not open");
941
942 *pStart = 0;
943 *pNum = 0;
944
945 const int InternalType = GetInternalItemType(ExternalType: Type);
946 for(int Index = 0; Index < m_pDataFile->m_Header.m_NumItemTypes; Index++)
947 {
948 const CDatafileItemType &ItemType = m_pDataFile->m_Info.m_pItemTypes[Index];
949 if(ItemType.m_Type == InternalType)
950 {
951 *pStart = ItemType.m_Start;
952 *pNum = ItemType.m_Num;
953 return;
954 }
955 }
956}
957
958int CDataFileReader::FindItemIndex(int Type, int Id)
959{
960 dbg_assert(m_pDataFile != nullptr, "File not open");
961
962 int Start, Num;
963 GetType(Type, pStart: &Start, pNum: &Num);
964 for(int Index = Start; Index < Start + Num; Index++)
965 {
966 const CDatafileItem *pItem = m_pDataFile->GetItem(Index);
967 if(pItem->Id() == Id)
968 {
969 return Index;
970 }
971 }
972 return -1;
973}
974
975void *CDataFileReader::FindItem(int Type, int Id)
976{
977 const int Index = FindItemIndex(Type, Id);
978 if(Index < 0)
979 {
980 return nullptr;
981 }
982 return GetItem(Index);
983}
984
985bool CDataFileReader::OverrideItemData(int Index, const void *pData, size_t Size)
986{
987 dbg_assert(m_pDataFile != nullptr, "File not open");
988
989 return m_pDataFile->OverrideItemData(Index, pData, Size);
990}
991
992int CDataFileReader::NumItems() const
993{
994 dbg_assert(m_pDataFile != nullptr, "File not open");
995
996 return m_pDataFile->m_Header.m_NumItems;
997}
998
999const char *CDataFileReader::FullName() const
1000{
1001 dbg_assert(m_pDataFile != nullptr, "File not open");
1002
1003 return m_pDataFile->m_aFullName;
1004}
1005
1006const char *CDataFileReader::BaseName() const
1007{
1008 dbg_assert(m_pDataFile != nullptr, "File not open");
1009
1010 return m_pDataFile->m_pBaseName;
1011}
1012
1013const char *CDataFileReader::Path() const
1014{
1015 dbg_assert(m_pDataFile != nullptr, "File not open");
1016
1017 return m_pDataFile->m_aPath;
1018}
1019
1020SHA256_DIGEST CDataFileReader::Sha256() const
1021{
1022 dbg_assert(m_pDataFile != nullptr, "File not open");
1023
1024 return m_pDataFile->m_Sha256;
1025}
1026
1027unsigned CDataFileReader::Crc() const
1028{
1029 dbg_assert(m_pDataFile != nullptr, "File not open");
1030
1031 return m_pDataFile->m_Crc;
1032}
1033
1034int CDataFileReader::Size() const
1035{
1036 dbg_assert(m_pDataFile != nullptr, "File not open");
1037
1038 return m_pDataFile->m_FileSize;
1039}
1040
1041CDataFileWriter::CDataFileWriter()
1042{
1043 m_File = nullptr;
1044}
1045
1046CDataFileWriter::~CDataFileWriter()
1047{
1048 if(m_File)
1049 {
1050 io_close(io: m_File);
1051 m_File = nullptr;
1052 }
1053
1054 for(CItemInfo &ItemInfo : m_vItems)
1055 {
1056 free(ptr: ItemInfo.m_pData);
1057 }
1058
1059 for(CDataInfo &DataInfo : m_vDatas)
1060 {
1061 free(ptr: DataInfo.m_pUncompressedData);
1062 free(ptr: DataInfo.m_pCompressedData);
1063 }
1064}
1065
1066bool CDataFileWriter::Open(class IStorage *pStorage, const char *pFilename, int StorageType)
1067{
1068 dbg_assert(!m_File, "File already open");
1069 m_File = pStorage->OpenFile(pFilename, Flags: IOFLAG_WRITE, Type: StorageType);
1070 return m_File != nullptr;
1071}
1072
1073int CDataFileWriter::GetTypeFromIndex(int Index) const
1074{
1075 return ITEMTYPE_EX - Index - 1;
1076}
1077
1078int CDataFileWriter::GetExtendedItemTypeIndex(int Type, const CUuid *pUuid)
1079{
1080 int Index = 0;
1081 if(Type == -1)
1082 {
1083 // Unknown type, search for UUID
1084 for(const auto &ExtendedItemType : m_vExtendedItemTypes)
1085 {
1086 if(ExtendedItemType.m_Uuid == *pUuid)
1087 {
1088 return Index;
1089 }
1090 ++Index;
1091 }
1092 }
1093 else
1094 {
1095 for(const auto &ExtendedItemType : m_vExtendedItemTypes)
1096 {
1097 if(ExtendedItemType.m_Type == Type)
1098 {
1099 return Index;
1100 }
1101 ++Index;
1102 }
1103 }
1104
1105 // Type not found, add it.
1106 const CUuid Uuid = Type == -1 ? *pUuid : g_UuidManager.GetUuid(Id: Type);
1107 CExtendedItemType ExtendedType;
1108 ExtendedType.m_Type = Type;
1109 ExtendedType.m_Uuid = Uuid;
1110 m_vExtendedItemTypes.emplace_back(args&: ExtendedType);
1111
1112 const CItemEx ItemEx = CItemEx::FromUuid(Uuid);
1113 AddItem(Type: ITEMTYPE_EX, Id: GetTypeFromIndex(Index), Size: sizeof(ItemEx), pData: &ItemEx);
1114 return Index;
1115}
1116
1117int CDataFileWriter::AddItem(int Type, int Id, size_t Size, const void *pData, const CUuid *pUuid)
1118{
1119 dbg_assert((Type >= 0 && Type <= MAX_ITEM_TYPE) || Type >= OFFSET_UUID || (Type == -1 && pUuid != nullptr), "Invalid Type: %d", Type);
1120 dbg_assert(Id >= 0 && Id <= MAX_ITEM_ID, "Invalid Id: %d", Id);
1121 dbg_assert(Size == 0 || pData != nullptr, "Data missing"); // Items without data are allowed
1122 dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
1123 dbg_assert(Size % sizeof(int) == 0, "Invalid data boundary");
1124 dbg_assert(m_vItems.size() < (size_t)std::numeric_limits<int>::max(), "Too many items");
1125
1126 if(Type == -1 || Type >= OFFSET_UUID)
1127 {
1128 Type = GetTypeFromIndex(Index: GetExtendedItemTypeIndex(Type, pUuid));
1129 }
1130
1131 const int NumItems = m_vItems.size();
1132 m_vItems.emplace_back();
1133 CItemInfo &Info = m_vItems.back();
1134 Info.m_Type = Type;
1135 Info.m_Id = Id;
1136 Info.m_Size = Size;
1137
1138 // copy data
1139 if(Size > 0)
1140 {
1141 Info.m_pData = malloc(size: Size);
1142 mem_copy(dest: Info.m_pData, source: pData, size: Size);
1143 }
1144 else
1145 {
1146 Info.m_pData = nullptr;
1147 }
1148
1149 // link
1150 CItemTypeInfo &ItemType = m_ItemTypes[Type];
1151 Info.m_Prev = ItemType.m_Last;
1152 Info.m_Next = -1;
1153
1154 if(ItemType.m_Last != -1)
1155 {
1156 m_vItems[ItemType.m_Last].m_Next = NumItems;
1157 }
1158 ItemType.m_Last = NumItems;
1159
1160 if(ItemType.m_First == -1)
1161 {
1162 ItemType.m_First = NumItems;
1163 }
1164
1165 ItemType.m_Num++;
1166 return NumItems;
1167}
1168
1169int CDataFileWriter::AddData(size_t Size, const void *pData, ECompressionLevel CompressionLevel)
1170{
1171 dbg_assert(Size > 0 && pData != nullptr, "Data missing");
1172 dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
1173 dbg_assert(m_vDatas.size() < (size_t)std::numeric_limits<int>::max(), "Too many data");
1174
1175 CDataInfo Info;
1176 Info.m_pUncompressedData = malloc(size: Size);
1177 mem_copy(dest: Info.m_pUncompressedData, source: pData, size: Size);
1178 Info.m_UncompressedSize = Size;
1179 Info.m_pCompressedData = nullptr;
1180 Info.m_CompressedSize = 0;
1181 Info.m_CompressionLevel = CompressionLevel;
1182 m_vDatas.emplace_back(args&: Info);
1183
1184 return m_vDatas.size() - 1;
1185}
1186
1187int CDataFileWriter::AddDataSwapped(size_t Size, const void *pData)
1188{
1189 dbg_assert(Size > 0 && pData != nullptr, "Data missing");
1190 dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
1191 dbg_assert(m_vDatas.size() < (size_t)std::numeric_limits<int>::max(), "Too many data");
1192 dbg_assert(Size % sizeof(int) == 0, "Invalid data boundary");
1193
1194#if defined(CONF_ARCH_ENDIAN_BIG)
1195 void *pSwapped = malloc(Size); // temporary buffer that we use during compression
1196 mem_copy(pSwapped, pData, Size);
1197 swap_endian(pSwapped, sizeof(int), Size / sizeof(int));
1198 int Index = AddData(Size, pSwapped);
1199 free(pSwapped);
1200 return Index;
1201#else
1202 return AddData(Size, pData);
1203#endif
1204}
1205
1206int CDataFileWriter::AddDataString(const char *pStr)
1207{
1208 dbg_assert(pStr != nullptr, "Data missing");
1209
1210 if(pStr[0] == '\0')
1211 {
1212 return -1;
1213 }
1214 return AddData(Size: str_length(str: pStr) + 1, pData: pStr);
1215}
1216
1217static int CompressionLevelToZlib(CDataFileWriter::ECompressionLevel CompressionLevel)
1218{
1219 switch(CompressionLevel)
1220 {
1221 case CDataFileWriter::COMPRESSION_DEFAULT:
1222 return Z_DEFAULT_COMPRESSION;
1223 case CDataFileWriter::COMPRESSION_BEST:
1224 return Z_BEST_COMPRESSION;
1225 default:
1226 dbg_assert_failed("Invalid CompressionLevel: %d", static_cast<int>(CompressionLevel));
1227 }
1228}
1229
1230void CDataFileWriter::Finish()
1231{
1232 dbg_assert((bool)m_File, "File not open");
1233
1234 // Compress data. This takes the majority of the time when saving a datafile,
1235 // so it's delayed until the end so it can be off-loaded to another thread.
1236 for(CDataInfo &DataInfo : m_vDatas)
1237 {
1238 unsigned long CompressedSize = compressBound(sourceLen: DataInfo.m_UncompressedSize);
1239 DataInfo.m_pCompressedData = malloc(size: CompressedSize);
1240 const int Result = compress2(dest: static_cast<Bytef *>(DataInfo.m_pCompressedData), destLen: &CompressedSize, source: static_cast<Bytef *>(DataInfo.m_pUncompressedData), sourceLen: DataInfo.m_UncompressedSize, level: CompressionLevelToZlib(CompressionLevel: DataInfo.m_CompressionLevel));
1241 DataInfo.m_CompressedSize = CompressedSize;
1242 free(ptr: DataInfo.m_pUncompressedData);
1243 DataInfo.m_pUncompressedData = nullptr;
1244 dbg_assert(Result == Z_OK, "datafile zlib compression failed with error %d", Result);
1245 }
1246
1247 // Calculate total size of items
1248 int64_t ItemSize = 0;
1249 for(const CItemInfo &ItemInfo : m_vItems)
1250 {
1251 ItemSize += ItemInfo.m_Size;
1252 ItemSize += sizeof(CDatafileItem);
1253 }
1254
1255 // Calculate total size of data
1256 int64_t DataSize = 0;
1257 for(const CDataInfo &DataInfo : m_vDatas)
1258 {
1259 DataSize += DataInfo.m_CompressedSize;
1260 }
1261
1262 // Calculate complete file size
1263 const int64_t TypesSize = m_ItemTypes.size() * sizeof(CDatafileItemType);
1264 const int64_t HeaderSize = sizeof(CDatafileHeader);
1265 const int64_t OffsetSize = (m_vItems.size() + m_vDatas.size() * 2) * sizeof(int); // ItemOffsets, DataOffsets, DataUncompressedSizes
1266 const int64_t SwapSize = HeaderSize + TypesSize + OffsetSize + ItemSize;
1267 const int64_t FileSize = SwapSize + DataSize;
1268
1269 // This also ensures that SwapSize, ItemSize and DataSize are valid.
1270 dbg_assert(FileSize <= (int64_t)std::numeric_limits<int>::max(), "File size too large");
1271
1272 // Construct and write header
1273 {
1274 CDatafileHeader Header;
1275 Header.m_aId[0] = 'D';
1276 Header.m_aId[1] = 'A';
1277 Header.m_aId[2] = 'T';
1278 Header.m_aId[3] = 'A';
1279 Header.m_Version = 4;
1280 Header.m_Size = FileSize - Header.SizeOffset();
1281 Header.m_Swaplen = SwapSize - Header.SizeOffset();
1282 Header.m_NumItemTypes = m_ItemTypes.size();
1283 Header.m_NumItems = m_vItems.size();
1284 Header.m_NumRawData = m_vDatas.size();
1285 Header.m_ItemSize = ItemSize;
1286 Header.m_DataSize = DataSize;
1287
1288 SwapEndianInPlace(pObj: &Header);
1289 io_write(io: m_File, buffer: &Header, size: sizeof(Header));
1290 }
1291
1292 // Write item types
1293 int ItemCount = 0;
1294 for(const auto &[Type, ItemType] : m_ItemTypes)
1295 {
1296 dbg_assert(ItemType.m_Num > 0, "Invalid ItemType.m_Num: %d", ItemType.m_Num);
1297
1298 CDatafileItemType Info;
1299 Info.m_Type = Type;
1300 Info.m_Start = ItemCount;
1301 Info.m_Num = ItemType.m_Num;
1302
1303 SwapEndianInPlace(pObj: &Info);
1304 io_write(io: m_File, buffer: &Info, size: sizeof(Info));
1305 ItemCount += ItemType.m_Num;
1306 }
1307
1308 // Write item offsets sorted by type
1309 int ItemOffset = 0;
1310 for(const auto &[Type, ItemType] : m_ItemTypes)
1311 {
1312 // Write all items offsets of this type
1313 for(int ItemIndex = ItemType.m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next)
1314 {
1315 const int ItemOffsetWrite = SwapEndianInt(Number: ItemOffset);
1316 io_write(io: m_File, buffer: &ItemOffsetWrite, size: sizeof(ItemOffsetWrite));
1317 ItemOffset += m_vItems[ItemIndex].m_Size + sizeof(CDatafileItem);
1318 }
1319 }
1320
1321 // Write data offsets
1322 int DataOffset = 0;
1323 for(const CDataInfo &DataInfo : m_vDatas)
1324 {
1325 const int DataOffsetWrite = SwapEndianInt(Number: DataOffset);
1326 io_write(io: m_File, buffer: &DataOffsetWrite, size: sizeof(DataOffsetWrite));
1327 DataOffset += DataInfo.m_CompressedSize;
1328 }
1329
1330 // Write data uncompressed sizes
1331 for(const CDataInfo &DataInfo : m_vDatas)
1332 {
1333 const int UncompressedSizeWrite = SwapEndianInt(Number: DataInfo.m_UncompressedSize);
1334 io_write(io: m_File, buffer: &UncompressedSizeWrite, size: sizeof(UncompressedSizeWrite));
1335 }
1336
1337 // Write items sorted by type
1338 for(const auto &[Type, ItemType] : m_ItemTypes)
1339 {
1340 // Write all items of this type
1341 for(int ItemIndex = ItemType.m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next)
1342 {
1343 CDatafileItem Item;
1344 Item.m_TypeAndId = ((unsigned)Type << 16u) | (unsigned)m_vItems[ItemIndex].m_Id;
1345 Item.m_Size = m_vItems[ItemIndex].m_Size;
1346
1347 SwapEndianInPlace(pObj: &Item);
1348 io_write(io: m_File, buffer: &Item, size: sizeof(Item));
1349
1350 if(m_vItems[ItemIndex].m_pData != nullptr)
1351 {
1352 SwapEndianInPlace(pObj: m_vItems[ItemIndex].m_pData, Size: m_vItems[ItemIndex].m_Size);
1353 io_write(io: m_File, buffer: m_vItems[ItemIndex].m_pData, size: m_vItems[ItemIndex].m_Size);
1354 free(ptr: m_vItems[ItemIndex].m_pData);
1355 m_vItems[ItemIndex].m_pData = nullptr;
1356 }
1357 }
1358 }
1359
1360 // Write data
1361 for(CDataInfo &DataInfo : m_vDatas)
1362 {
1363 io_write(io: m_File, buffer: DataInfo.m_pCompressedData, size: DataInfo.m_CompressedSize);
1364 free(ptr: DataInfo.m_pCompressedData);
1365 DataInfo.m_pCompressedData = nullptr;
1366 }
1367
1368 io_close(io: m_File);
1369 m_File = nullptr;
1370}
1371