1/* (c) Magnus Auvinen. See licence.txt in the root of the distribution for more information. */
2/* If you are missing that file, acquire a complete release at teeworlds.com. */
3
4#include "datafile.h"
5
6#include <base/hash_ctxt.h>
7#include <base/log.h>
8#include <base/math.h>
9#include <base/system.h>
10#include <engine/storage.h>
11
12#include "uuid_manager.h"
13
14#include <cstdlib>
15#include <limits>
16
17#include <zlib.h>
18
19static const int DEBUG = 0;
20
21enum
22{
23 OFFSET_UUID_TYPE = 0x8000,
24};
25
26struct CItemEx
27{
28 int m_aUuid[sizeof(CUuid) / sizeof(int32_t)];
29
30 static CItemEx FromUuid(CUuid Uuid)
31 {
32 CItemEx Result;
33 for(size_t i = 0; i < sizeof(CUuid) / sizeof(int32_t); i++)
34 Result.m_aUuid[i] = bytes_be_to_uint(bytes: &Uuid.m_aData[i * sizeof(int32_t)]);
35 return Result;
36 }
37
38 CUuid ToUuid() const
39 {
40 CUuid Result;
41 for(size_t i = 0; i < sizeof(CUuid) / sizeof(int32_t); i++)
42 uint_to_bytes_be(bytes: &Result.m_aData[i * sizeof(int32_t)], value: m_aUuid[i]);
43 return Result;
44 }
45};
46
47struct CDatafileItemType
48{
49 int m_Type;
50 int m_Start;
51 int m_Num;
52};
53
54struct CDatafileItem
55{
56 int m_TypeAndId;
57 int m_Size;
58};
59
60struct CDatafileHeader
61{
62 char m_aId[4];
63 int m_Version;
64 int m_Size;
65 int m_Swaplen;
66 int m_NumItemTypes;
67 int m_NumItems;
68 int m_NumRawData;
69 int m_ItemSize;
70 int m_DataSize;
71
72 constexpr size_t SizeOffset()
73 {
74 // The size of these members is not included in m_Size and m_Swaplen
75 return sizeof(m_aId) + sizeof(m_Version) + sizeof(m_Size) + sizeof(m_Swaplen);
76 }
77};
78
79struct CDatafileInfo
80{
81 CDatafileItemType *m_pItemTypes;
82 int *m_pItemOffsets;
83 int *m_pDataOffsets;
84 int *m_pDataSizes;
85
86 char *m_pItemStart;
87 char *m_pDataStart;
88};
89
90struct CDatafile
91{
92 IOHANDLE m_File;
93 SHA256_DIGEST m_Sha256;
94 unsigned m_Crc;
95 CDatafileInfo m_Info;
96 CDatafileHeader m_Header;
97 int m_DataStartOffset;
98 char **m_ppDataPtrs;
99 int *m_pDataSizes;
100 char *m_pData;
101};
102
103bool CDataFileReader::Open(class IStorage *pStorage, const char *pFilename, int StorageType)
104{
105 log_trace("datafile", "loading. filename='%s'", pFilename);
106
107 IOHANDLE File = pStorage->OpenFile(pFilename, Flags: IOFLAG_READ, Type: StorageType);
108 if(!File)
109 {
110 dbg_msg(sys: "datafile", fmt: "could not open '%s'", pFilename);
111 return false;
112 }
113
114 // take the CRC of the file and store it
115 unsigned Crc = 0;
116 SHA256_DIGEST Sha256;
117 {
118 enum
119 {
120 BUFFER_SIZE = 64 * 1024
121 };
122
123 SHA256_CTX Sha256Ctxt;
124 sha256_init(ctxt: &Sha256Ctxt);
125 unsigned char aBuffer[BUFFER_SIZE];
126
127 while(true)
128 {
129 unsigned Bytes = io_read(io: File, buffer: aBuffer, size: BUFFER_SIZE);
130 if(Bytes == 0)
131 break;
132 Crc = crc32(crc: Crc, buf: aBuffer, len: Bytes);
133 sha256_update(ctxt: &Sha256Ctxt, data: aBuffer, data_len: Bytes);
134 }
135 Sha256 = sha256_finish(ctxt: &Sha256Ctxt);
136
137 io_seek(io: File, offset: 0, origin: IOSEEK_START);
138 }
139
140 // TODO: change this header
141 CDatafileHeader Header;
142 if(sizeof(Header) != io_read(io: File, buffer: &Header, size: sizeof(Header)))
143 {
144 dbg_msg(sys: "datafile", fmt: "couldn't load header");
145 return false;
146 }
147 if(Header.m_aId[0] != 'A' || Header.m_aId[1] != 'T' || Header.m_aId[2] != 'A' || Header.m_aId[3] != 'D')
148 {
149 if(Header.m_aId[0] != 'D' || Header.m_aId[1] != 'A' || Header.m_aId[2] != 'T' || Header.m_aId[3] != 'A')
150 {
151 dbg_msg(sys: "datafile", fmt: "wrong signature. %x %x %x %x", Header.m_aId[0], Header.m_aId[1], Header.m_aId[2], Header.m_aId[3]);
152 return false;
153 }
154 }
155
156#if defined(CONF_ARCH_ENDIAN_BIG)
157 swap_endian(&Header, sizeof(int), sizeof(Header) / sizeof(int));
158#endif
159 if(Header.m_Version != 3 && Header.m_Version != 4)
160 {
161 dbg_msg(sys: "datafile", fmt: "wrong version. version=%x", Header.m_Version);
162 return false;
163 }
164
165 // read in the rest except the data
166 unsigned Size = 0;
167 Size += Header.m_NumItemTypes * sizeof(CDatafileItemType);
168 Size += (Header.m_NumItems + Header.m_NumRawData) * sizeof(int);
169 if(Header.m_Version == 4)
170 Size += Header.m_NumRawData * sizeof(int); // v4 has uncompressed data sizes as well
171 Size += Header.m_ItemSize;
172
173 unsigned AllocSize = Size;
174 AllocSize += sizeof(CDatafile); // add space for info structure
175 AllocSize += Header.m_NumRawData * sizeof(void *); // add space for data pointers
176 AllocSize += Header.m_NumRawData * sizeof(int); // add space for data sizes
177 if(Size > (((int64_t)1) << 31) || Header.m_NumItemTypes < 0 || Header.m_NumItems < 0 || Header.m_NumRawData < 0 || Header.m_ItemSize < 0)
178 {
179 io_close(io: File);
180 dbg_msg(sys: "datafile", fmt: "unable to load file, invalid file information");
181 return false;
182 }
183
184 CDatafile *pTmpDataFile = (CDatafile *)malloc(size: AllocSize);
185 pTmpDataFile->m_Header = Header;
186 pTmpDataFile->m_DataStartOffset = sizeof(CDatafileHeader) + Size;
187 pTmpDataFile->m_ppDataPtrs = (char **)(pTmpDataFile + 1);
188 pTmpDataFile->m_pDataSizes = (int *)(pTmpDataFile->m_ppDataPtrs + Header.m_NumRawData);
189 pTmpDataFile->m_pData = (char *)(pTmpDataFile->m_pDataSizes + Header.m_NumRawData);
190 pTmpDataFile->m_File = File;
191 pTmpDataFile->m_Sha256 = Sha256;
192 pTmpDataFile->m_Crc = Crc;
193
194 // clear the data pointers and sizes
195 mem_zero(block: pTmpDataFile->m_ppDataPtrs, size: Header.m_NumRawData * sizeof(void *));
196 mem_zero(block: pTmpDataFile->m_pDataSizes, size: Header.m_NumRawData * sizeof(int));
197
198 // read types, offsets, sizes and item data
199 unsigned ReadSize = io_read(io: File, buffer: pTmpDataFile->m_pData, size: Size);
200 if(ReadSize != Size)
201 {
202 io_close(io: pTmpDataFile->m_File);
203 free(ptr: pTmpDataFile);
204 dbg_msg(sys: "datafile", fmt: "couldn't load the whole thing, wanted=%d got=%d", Size, ReadSize);
205 return false;
206 }
207
208 Close();
209 m_pDataFile = pTmpDataFile;
210
211#if defined(CONF_ARCH_ENDIAN_BIG)
212 swap_endian(m_pDataFile->m_pData, sizeof(int), minimum(static_cast<unsigned>(Header.m_Swaplen), Size) / sizeof(int));
213#endif
214
215 if(DEBUG)
216 {
217 dbg_msg(sys: "datafile", fmt: "allocsize=%d", AllocSize);
218 dbg_msg(sys: "datafile", fmt: "readsize=%d", ReadSize);
219 dbg_msg(sys: "datafile", fmt: "swaplen=%d", Header.m_Swaplen);
220 dbg_msg(sys: "datafile", fmt: "item_size=%d", m_pDataFile->m_Header.m_ItemSize);
221 }
222
223 m_pDataFile->m_Info.m_pItemTypes = (CDatafileItemType *)m_pDataFile->m_pData;
224 m_pDataFile->m_Info.m_pItemOffsets = (int *)&m_pDataFile->m_Info.m_pItemTypes[m_pDataFile->m_Header.m_NumItemTypes];
225 m_pDataFile->m_Info.m_pDataOffsets = &m_pDataFile->m_Info.m_pItemOffsets[m_pDataFile->m_Header.m_NumItems];
226 m_pDataFile->m_Info.m_pDataSizes = &m_pDataFile->m_Info.m_pDataOffsets[m_pDataFile->m_Header.m_NumRawData];
227
228 if(Header.m_Version == 4)
229 m_pDataFile->m_Info.m_pItemStart = (char *)&m_pDataFile->m_Info.m_pDataSizes[m_pDataFile->m_Header.m_NumRawData];
230 else
231 m_pDataFile->m_Info.m_pItemStart = (char *)&m_pDataFile->m_Info.m_pDataOffsets[m_pDataFile->m_Header.m_NumRawData];
232 m_pDataFile->m_Info.m_pDataStart = m_pDataFile->m_Info.m_pItemStart + m_pDataFile->m_Header.m_ItemSize;
233
234 log_trace("datafile", "loading done. datafile='%s'", pFilename);
235
236 return true;
237}
238
239bool CDataFileReader::Close()
240{
241 if(!m_pDataFile)
242 return true;
243
244 // free the data that is loaded
245 for(int i = 0; i < m_pDataFile->m_Header.m_NumRawData; i++)
246 {
247 free(ptr: m_pDataFile->m_ppDataPtrs[i]);
248 m_pDataFile->m_ppDataPtrs[i] = nullptr;
249 m_pDataFile->m_pDataSizes[i] = 0;
250 }
251
252 io_close(io: m_pDataFile->m_File);
253 free(ptr: m_pDataFile);
254 m_pDataFile = nullptr;
255 return true;
256}
257
258IOHANDLE CDataFileReader::File() const
259{
260 if(!m_pDataFile)
261 return 0;
262 return m_pDataFile->m_File;
263}
264
265int CDataFileReader::NumData() const
266{
267 if(!m_pDataFile)
268 {
269 return 0;
270 }
271 return m_pDataFile->m_Header.m_NumRawData;
272}
273
274// returns the size in the file
275int CDataFileReader::GetFileDataSize(int Index) const
276{
277 if(!m_pDataFile)
278 {
279 return 0;
280 }
281
282 if(Index == m_pDataFile->m_Header.m_NumRawData - 1)
283 return m_pDataFile->m_Header.m_DataSize - m_pDataFile->m_Info.m_pDataOffsets[Index];
284
285 return m_pDataFile->m_Info.m_pDataOffsets[Index + 1] - m_pDataFile->m_Info.m_pDataOffsets[Index];
286}
287
288// returns the size of the resulting data
289int CDataFileReader::GetDataSize(int Index) const
290{
291 if(!m_pDataFile || Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData)
292 {
293 return 0;
294 }
295
296 if(!m_pDataFile->m_ppDataPtrs[Index])
297 {
298 if(m_pDataFile->m_Header.m_Version >= 4)
299 {
300 return m_pDataFile->m_Info.m_pDataSizes[Index];
301 }
302 else
303 {
304 return GetFileDataSize(Index);
305 }
306 }
307 const int Size = m_pDataFile->m_pDataSizes[Index];
308 if(Size < 0)
309 return 0; // summarize all errors as zero size
310 return Size;
311}
312
313void *CDataFileReader::GetDataImpl(int Index, bool Swap)
314{
315 if(!m_pDataFile)
316 {
317 return nullptr;
318 }
319
320 if(Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData)
321 return nullptr;
322
323 // load it if needed
324 if(!m_pDataFile->m_ppDataPtrs[Index])
325 {
326 // don't try to load again if it previously failed
327 if(m_pDataFile->m_pDataSizes[Index] < 0)
328 return nullptr;
329
330 // fetch the data size
331 unsigned DataSize = GetFileDataSize(Index);
332#if defined(CONF_ARCH_ENDIAN_BIG)
333 unsigned SwapSize = DataSize;
334#endif
335
336 if(m_pDataFile->m_Header.m_Version == 4)
337 {
338 // v4 has compressed data
339 const unsigned OriginalUncompressedSize = m_pDataFile->m_Info.m_pDataSizes[Index];
340 unsigned long UncompressedSize = OriginalUncompressedSize;
341
342 log_trace("datafile", "loading data. index=%d size=%u uncompressed=%u", Index, DataSize, OriginalUncompressedSize);
343
344 // read the compressed data
345 void *pCompressedData = malloc(size: DataSize);
346 unsigned ActualDataSize = 0;
347 if(io_seek(io: m_pDataFile->m_File, offset: m_pDataFile->m_DataStartOffset + m_pDataFile->m_Info.m_pDataOffsets[Index], origin: IOSEEK_START) == 0)
348 ActualDataSize = io_read(io: m_pDataFile->m_File, buffer: pCompressedData, size: DataSize);
349 if(DataSize != ActualDataSize)
350 {
351 log_error("datafile", "truncation error, could not read all data. index=%d wanted=%u got=%u", Index, DataSize, ActualDataSize);
352 free(ptr: pCompressedData);
353 m_pDataFile->m_ppDataPtrs[Index] = nullptr;
354 m_pDataFile->m_pDataSizes[Index] = -1;
355 return nullptr;
356 }
357
358 // decompress the data
359 m_pDataFile->m_ppDataPtrs[Index] = (char *)malloc(size: UncompressedSize);
360 m_pDataFile->m_pDataSizes[Index] = UncompressedSize;
361 const int Result = uncompress(dest: (Bytef *)m_pDataFile->m_ppDataPtrs[Index], destLen: &UncompressedSize, source: (Bytef *)pCompressedData, sourceLen: DataSize);
362 free(ptr: pCompressedData);
363 if(Result != Z_OK || UncompressedSize != OriginalUncompressedSize)
364 {
365 log_error("datafile", "uncompress error. result=%d wanted=%u got=%lu", Result, OriginalUncompressedSize, UncompressedSize);
366 free(ptr: m_pDataFile->m_ppDataPtrs[Index]);
367 m_pDataFile->m_ppDataPtrs[Index] = nullptr;
368 m_pDataFile->m_pDataSizes[Index] = -1;
369 return nullptr;
370 }
371
372#if defined(CONF_ARCH_ENDIAN_BIG)
373 SwapSize = UncompressedSize;
374#endif
375 }
376 else
377 {
378 // load the data
379 log_trace("datafile", "loading data. index=%d size=%d", Index, DataSize);
380 m_pDataFile->m_ppDataPtrs[Index] = static_cast<char *>(malloc(size: DataSize));
381 m_pDataFile->m_pDataSizes[Index] = DataSize;
382 unsigned ActualDataSize = 0;
383 if(io_seek(io: m_pDataFile->m_File, offset: m_pDataFile->m_DataStartOffset + m_pDataFile->m_Info.m_pDataOffsets[Index], origin: IOSEEK_START) == 0)
384 ActualDataSize = io_read(io: m_pDataFile->m_File, buffer: m_pDataFile->m_ppDataPtrs[Index], size: DataSize);
385 if(DataSize != ActualDataSize)
386 {
387 log_error("datafile", "truncation error, could not read all data. index=%d wanted=%u got=%u", Index, DataSize, ActualDataSize);
388 free(ptr: m_pDataFile->m_ppDataPtrs[Index]);
389 m_pDataFile->m_ppDataPtrs[Index] = nullptr;
390 m_pDataFile->m_pDataSizes[Index] = -1;
391 return nullptr;
392 }
393 }
394
395#if defined(CONF_ARCH_ENDIAN_BIG)
396 if(Swap && SwapSize)
397 swap_endian(m_pDataFile->m_ppDataPtrs[Index], sizeof(int), SwapSize / sizeof(int));
398#endif
399 }
400
401 return m_pDataFile->m_ppDataPtrs[Index];
402}
403
404void *CDataFileReader::GetData(int Index)
405{
406 return GetDataImpl(Index, Swap: false);
407}
408
409void *CDataFileReader::GetDataSwapped(int Index)
410{
411 return GetDataImpl(Index, Swap: true);
412}
413
414const char *CDataFileReader::GetDataString(int Index)
415{
416 if(Index == -1)
417 return "";
418 const int DataSize = GetDataSize(Index);
419 if(!DataSize)
420 return nullptr;
421 const char *pData = static_cast<char *>(GetData(Index));
422 if(pData == nullptr || mem_has_null(block: pData, size: DataSize - 1) || pData[DataSize - 1] != '\0' || !str_utf8_check(str: pData))
423 return nullptr;
424 return pData;
425}
426
427void CDataFileReader::ReplaceData(int Index, char *pData, size_t Size)
428{
429 dbg_assert(Index >= 0 && Index < m_pDataFile->m_Header.m_NumRawData, "Index invalid");
430
431 free(ptr: m_pDataFile->m_ppDataPtrs[Index]);
432 m_pDataFile->m_ppDataPtrs[Index] = pData;
433 m_pDataFile->m_pDataSizes[Index] = Size;
434}
435
436void CDataFileReader::UnloadData(int Index)
437{
438 if(Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData)
439 return;
440
441 free(ptr: m_pDataFile->m_ppDataPtrs[Index]);
442 m_pDataFile->m_ppDataPtrs[Index] = nullptr;
443 m_pDataFile->m_pDataSizes[Index] = 0;
444}
445
446int CDataFileReader::GetItemSize(int Index) const
447{
448 if(!m_pDataFile)
449 return 0;
450 if(Index == m_pDataFile->m_Header.m_NumItems - 1)
451 return m_pDataFile->m_Header.m_ItemSize - m_pDataFile->m_Info.m_pItemOffsets[Index] - sizeof(CDatafileItem);
452 return m_pDataFile->m_Info.m_pItemOffsets[Index + 1] - m_pDataFile->m_Info.m_pItemOffsets[Index] - sizeof(CDatafileItem);
453}
454
455int CDataFileReader::GetExternalItemType(int InternalType, CUuid *pUuid)
456{
457 if(InternalType <= OFFSET_UUID_TYPE || InternalType == ITEMTYPE_EX)
458 {
459 if(pUuid)
460 *pUuid = UUID_ZEROED;
461 return InternalType;
462 }
463 int TypeIndex = FindItemIndex(Type: ITEMTYPE_EX, Id: InternalType);
464 if(TypeIndex < 0 || GetItemSize(Index: TypeIndex) < (int)sizeof(CItemEx))
465 {
466 if(pUuid)
467 *pUuid = UUID_ZEROED;
468 return InternalType;
469 }
470 const CItemEx *pItemEx = (const CItemEx *)GetItem(Index: TypeIndex);
471 CUuid Uuid = pItemEx->ToUuid();
472 if(pUuid)
473 *pUuid = Uuid;
474 // Propagate UUID_UNKNOWN, it doesn't hurt.
475 return g_UuidManager.LookupUuid(Uuid);
476}
477
478int CDataFileReader::GetInternalItemType(int ExternalType)
479{
480 if(ExternalType < OFFSET_UUID)
481 {
482 return ExternalType;
483 }
484 CUuid Uuid = g_UuidManager.GetUuid(Id: ExternalType);
485 int Start, Num;
486 GetType(Type: ITEMTYPE_EX, pStart: &Start, pNum: &Num);
487 for(int i = Start; i < Start + Num; i++)
488 {
489 if(GetItemSize(Index: i) < (int)sizeof(CItemEx))
490 {
491 continue;
492 }
493 int Id;
494 if(Uuid == ((const CItemEx *)GetItem(Index: i, pType: nullptr, pId: &Id))->ToUuid())
495 {
496 return Id;
497 }
498 }
499 return -1;
500}
501
502void *CDataFileReader::GetItem(int Index, int *pType, int *pId, CUuid *pUuid)
503{
504 if(!m_pDataFile)
505 {
506 if(pType)
507 *pType = 0;
508 if(pId)
509 *pId = 0;
510 if(pUuid)
511 *pUuid = UUID_ZEROED;
512 return nullptr;
513 }
514
515 CDatafileItem *pItem = (CDatafileItem *)(m_pDataFile->m_Info.m_pItemStart + m_pDataFile->m_Info.m_pItemOffsets[Index]);
516
517 // remove sign extension
518 const int Type = GetExternalItemType(InternalType: (pItem->m_TypeAndId >> 16) & 0xffff, pUuid);
519 if(pType)
520 {
521 *pType = Type;
522 }
523 if(pId)
524 {
525 *pId = pItem->m_TypeAndId & 0xffff;
526 }
527 return (void *)(pItem + 1);
528}
529
530void CDataFileReader::GetType(int Type, int *pStart, int *pNum)
531{
532 *pStart = 0;
533 *pNum = 0;
534
535 if(!m_pDataFile)
536 return;
537
538 Type = GetInternalItemType(ExternalType: Type);
539 for(int i = 0; i < m_pDataFile->m_Header.m_NumItemTypes; i++)
540 {
541 if(m_pDataFile->m_Info.m_pItemTypes[i].m_Type == Type)
542 {
543 *pStart = m_pDataFile->m_Info.m_pItemTypes[i].m_Start;
544 *pNum = m_pDataFile->m_Info.m_pItemTypes[i].m_Num;
545 return;
546 }
547 }
548}
549
550int CDataFileReader::FindItemIndex(int Type, int Id)
551{
552 if(!m_pDataFile)
553 {
554 return -1;
555 }
556
557 int Start, Num;
558 GetType(Type, pStart: &Start, pNum: &Num);
559 for(int i = 0; i < Num; i++)
560 {
561 int ItemId;
562 GetItem(Index: Start + i, pType: nullptr, pId: &ItemId);
563 if(Id == ItemId)
564 {
565 return Start + i;
566 }
567 }
568 return -1;
569}
570
571void *CDataFileReader::FindItem(int Type, int Id)
572{
573 int Index = FindItemIndex(Type, Id);
574 if(Index < 0)
575 {
576 return nullptr;
577 }
578 return GetItem(Index);
579}
580
581int CDataFileReader::NumItems() const
582{
583 if(!m_pDataFile)
584 return 0;
585 return m_pDataFile->m_Header.m_NumItems;
586}
587
588SHA256_DIGEST CDataFileReader::Sha256() const
589{
590 if(!m_pDataFile)
591 {
592 SHA256_DIGEST Result;
593 for(unsigned char &d : Result.data)
594 {
595 d = 0xff;
596 }
597 return Result;
598 }
599 return m_pDataFile->m_Sha256;
600}
601
602unsigned CDataFileReader::Crc() const
603{
604 if(!m_pDataFile)
605 return 0xFFFFFFFF;
606 return m_pDataFile->m_Crc;
607}
608
609int CDataFileReader::MapSize() const
610{
611 if(!m_pDataFile)
612 return 0;
613 return m_pDataFile->m_Header.m_Size + m_pDataFile->m_Header.SizeOffset();
614}
615
616CDataFileWriter::CDataFileWriter()
617{
618 m_File = 0;
619 for(CItemTypeInfo &ItemTypeInfo : m_aItemTypes)
620 {
621 ItemTypeInfo.m_Num = 0;
622 ItemTypeInfo.m_First = -1;
623 ItemTypeInfo.m_Last = -1;
624 }
625}
626
627CDataFileWriter::~CDataFileWriter()
628{
629 if(m_File)
630 {
631 io_close(io: m_File);
632 m_File = 0;
633 }
634
635 for(CItemInfo &ItemInfo : m_vItems)
636 {
637 free(ptr: ItemInfo.m_pData);
638 }
639
640 for(CDataInfo &DataInfo : m_vDatas)
641 {
642 free(ptr: DataInfo.m_pUncompressedData);
643 free(ptr: DataInfo.m_pCompressedData);
644 }
645}
646
647bool CDataFileWriter::Open(class IStorage *pStorage, const char *pFilename, int StorageType)
648{
649 dbg_assert(!m_File, "File already open");
650 m_File = pStorage->OpenFile(pFilename, Flags: IOFLAG_WRITE, Type: StorageType);
651 return m_File != 0;
652}
653
654int CDataFileWriter::GetTypeFromIndex(int Index) const
655{
656 return ITEMTYPE_EX - Index - 1;
657}
658
659int CDataFileWriter::GetExtendedItemTypeIndex(int Type, const CUuid *pUuid)
660{
661 int Index = 0;
662 if(Type == -1)
663 {
664 // Unknown type, search for UUID
665 for(const auto &ExtendedItemType : m_vExtendedItemTypes)
666 {
667 if(ExtendedItemType.m_Uuid == *pUuid)
668 return Index;
669 ++Index;
670 }
671 }
672 else
673 {
674 for(const auto &ExtendedItemType : m_vExtendedItemTypes)
675 {
676 if(ExtendedItemType.m_Type == Type)
677 return Index;
678 ++Index;
679 }
680 }
681
682 // Type not found, add it.
683 CExtendedItemType ExtendedType;
684 ExtendedType.m_Type = Type;
685 ExtendedType.m_Uuid = Type == -1 ? *pUuid : g_UuidManager.GetUuid(Id: Type);
686 m_vExtendedItemTypes.push_back(x: ExtendedType);
687
688 CItemEx ItemEx = CItemEx::FromUuid(Uuid: ExtendedType.m_Uuid);
689 AddItem(Type: ITEMTYPE_EX, Id: GetTypeFromIndex(Index), Size: sizeof(ItemEx), pData: &ItemEx);
690 return Index;
691}
692
693int CDataFileWriter::AddItem(int Type, int Id, size_t Size, const void *pData, const CUuid *pUuid)
694{
695 dbg_assert((Type >= 0 && Type < MAX_ITEM_TYPES) || Type >= OFFSET_UUID || (Type == -1 && pUuid != nullptr), "Invalid type");
696 dbg_assert(Id >= 0 && Id <= ITEMTYPE_EX, "Invalid ID");
697 dbg_assert(Size == 0 || pData != nullptr, "Data missing"); // Items without data are allowed
698 dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
699 dbg_assert(Size % sizeof(int) == 0, "Invalid data boundary");
700
701 if(Type == -1 || Type >= OFFSET_UUID)
702 {
703 Type = GetTypeFromIndex(Index: GetExtendedItemTypeIndex(Type, pUuid));
704 }
705
706 const int NumItems = m_vItems.size();
707 m_vItems.emplace_back();
708 CItemInfo &Info = m_vItems.back();
709 Info.m_Type = Type;
710 Info.m_Id = Id;
711 Info.m_Size = Size;
712
713 // copy data
714 if(Size > 0)
715 {
716 Info.m_pData = malloc(size: Size);
717 mem_copy(dest: Info.m_pData, source: pData, size: Size);
718 }
719 else
720 Info.m_pData = nullptr;
721
722 // link
723 Info.m_Prev = m_aItemTypes[Type].m_Last;
724 Info.m_Next = -1;
725
726 if(m_aItemTypes[Type].m_Last != -1)
727 m_vItems[m_aItemTypes[Type].m_Last].m_Next = NumItems;
728 m_aItemTypes[Type].m_Last = NumItems;
729
730 if(m_aItemTypes[Type].m_First == -1)
731 m_aItemTypes[Type].m_First = NumItems;
732
733 m_aItemTypes[Type].m_Num++;
734 return NumItems;
735}
736
737int CDataFileWriter::AddData(size_t Size, const void *pData, ECompressionLevel CompressionLevel)
738{
739 dbg_assert(Size > 0 && pData != nullptr, "Data missing");
740 dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
741
742 m_vDatas.emplace_back();
743 CDataInfo &Info = m_vDatas.back();
744 Info.m_pUncompressedData = malloc(size: Size);
745 mem_copy(dest: Info.m_pUncompressedData, source: pData, size: Size);
746 Info.m_UncompressedSize = Size;
747 Info.m_pCompressedData = nullptr;
748 Info.m_CompressedSize = 0;
749 Info.m_CompressionLevel = CompressionLevel;
750
751 return m_vDatas.size() - 1;
752}
753
754int CDataFileWriter::AddDataSwapped(size_t Size, const void *pData)
755{
756 dbg_assert(Size > 0 && pData != nullptr, "Data missing");
757 dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
758 dbg_assert(Size % sizeof(int) == 0, "Invalid data boundary");
759
760#if defined(CONF_ARCH_ENDIAN_BIG)
761 void *pSwapped = malloc(Size); // temporary buffer that we use during compression
762 mem_copy(pSwapped, pData, Size);
763 swap_endian(pSwapped, sizeof(int), Size / sizeof(int));
764 int Index = AddData(Size, pSwapped);
765 free(pSwapped);
766 return Index;
767#else
768 return AddData(Size, pData);
769#endif
770}
771
772int CDataFileWriter::AddDataString(const char *pStr)
773{
774 dbg_assert(pStr != nullptr, "Data missing");
775
776 if(pStr[0] == '\0')
777 return -1;
778 return AddData(Size: str_length(str: pStr) + 1, pData: pStr);
779}
780
781static int CompressionLevelToZlib(CDataFileWriter::ECompressionLevel CompressionLevel)
782{
783 switch(CompressionLevel)
784 {
785 case CDataFileWriter::COMPRESSION_DEFAULT:
786 return Z_DEFAULT_COMPRESSION;
787 case CDataFileWriter::COMPRESSION_BEST:
788 return Z_BEST_COMPRESSION;
789 default:
790 dbg_assert(false, "CompressionLevel invalid");
791 dbg_break();
792 }
793}
794
795void CDataFileWriter::Finish()
796{
797 dbg_assert((bool)m_File, "File not open");
798
799 // Compress data. This takes the majority of the time when saving a datafile,
800 // so it's delayed until the end so it can be off-loaded to another thread.
801 for(CDataInfo &DataInfo : m_vDatas)
802 {
803 unsigned long CompressedSize = compressBound(sourceLen: DataInfo.m_UncompressedSize);
804 DataInfo.m_pCompressedData = malloc(size: CompressedSize);
805 const int Result = compress2(dest: (Bytef *)DataInfo.m_pCompressedData, destLen: &CompressedSize, source: (Bytef *)DataInfo.m_pUncompressedData, sourceLen: DataInfo.m_UncompressedSize, level: CompressionLevelToZlib(CompressionLevel: DataInfo.m_CompressionLevel));
806 DataInfo.m_CompressedSize = CompressedSize;
807 free(ptr: DataInfo.m_pUncompressedData);
808 DataInfo.m_pUncompressedData = nullptr;
809 if(Result != Z_OK)
810 {
811 char aError[32];
812 str_format(buffer: aError, buffer_size: sizeof(aError), format: "zlib compression error %d", Result);
813 dbg_assert(false, aError);
814 }
815 }
816
817 // Calculate total size of items
818 size_t ItemSize = 0;
819 for(const CItemInfo &ItemInfo : m_vItems)
820 {
821 ItemSize += ItemInfo.m_Size;
822 ItemSize += sizeof(CDatafileItem);
823 }
824
825 // Calculate total size of data
826 size_t DataSize = 0;
827 for(const CDataInfo &DataInfo : m_vDatas)
828 DataSize += DataInfo.m_CompressedSize;
829
830 // Count number of item types
831 int NumItemTypes = 0;
832 for(const CItemTypeInfo &ItemType : m_aItemTypes)
833 {
834 if(ItemType.m_Num > 0)
835 ++NumItemTypes;
836 }
837
838 // Calculate complete file size
839 const size_t TypesSize = NumItemTypes * sizeof(CDatafileItemType);
840 const size_t HeaderSize = sizeof(CDatafileHeader);
841 const size_t OffsetSize = (m_vItems.size() + m_vDatas.size() * 2) * sizeof(int); // ItemOffsets, DataOffsets, DataUncompressedSizes
842 const size_t SwapSize = HeaderSize + TypesSize + OffsetSize + ItemSize;
843 const size_t FileSize = SwapSize + DataSize;
844
845 if(DEBUG)
846 dbg_msg(sys: "datafile", fmt: "NumItemTypes=%d TypesSize=%" PRIzu " ItemSize=%" PRIzu " DataSize=%" PRIzu, NumItemTypes, TypesSize, ItemSize, DataSize);
847
848 // This also ensures that SwapSize, ItemSize and DataSize are valid.
849 dbg_assert(FileSize <= (size_t)std::numeric_limits<int>::max(), "File size too large");
850
851 // Construct and write header
852 {
853 CDatafileHeader Header;
854 Header.m_aId[0] = 'D';
855 Header.m_aId[1] = 'A';
856 Header.m_aId[2] = 'T';
857 Header.m_aId[3] = 'A';
858 Header.m_Version = 4;
859 Header.m_Size = FileSize - Header.SizeOffset();
860 Header.m_Swaplen = SwapSize - Header.SizeOffset();
861 Header.m_NumItemTypes = NumItemTypes;
862 Header.m_NumItems = m_vItems.size();
863 Header.m_NumRawData = m_vDatas.size();
864 Header.m_ItemSize = ItemSize;
865 Header.m_DataSize = DataSize;
866
867#if defined(CONF_ARCH_ENDIAN_BIG)
868 swap_endian(&Header, sizeof(int), sizeof(Header) / sizeof(int));
869#endif
870 io_write(io: m_File, buffer: &Header, size: sizeof(Header));
871 }
872
873 // Write item types
874 for(int Type = 0, Count = 0; Type < (int)m_aItemTypes.size(); ++Type)
875 {
876 if(!m_aItemTypes[Type].m_Num)
877 continue;
878
879 CDatafileItemType Info;
880 Info.m_Type = Type;
881 Info.m_Start = Count;
882 Info.m_Num = m_aItemTypes[Type].m_Num;
883
884 if(DEBUG)
885 dbg_msg(sys: "datafile", fmt: "writing item type. Type=%x Start=%d Num=%d", Info.m_Type, Info.m_Start, Info.m_Num);
886
887#if defined(CONF_ARCH_ENDIAN_BIG)
888 swap_endian(&Info, sizeof(int), sizeof(CDatafileItemType) / sizeof(int));
889#endif
890 io_write(io: m_File, buffer: &Info, size: sizeof(Info));
891 Count += m_aItemTypes[Type].m_Num;
892 }
893
894 // Write item offsets sorted by type
895 for(int Type = 0, Offset = 0; Type < (int)m_aItemTypes.size(); Type++)
896 {
897 // Write all items offsets of this type
898 for(int ItemIndex = m_aItemTypes[Type].m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next)
899 {
900 if(DEBUG)
901 dbg_msg(sys: "datafile", fmt: "writing item offset. Type=%d ItemIndex=%d Offset=%d", Type, ItemIndex, Offset);
902
903 int Temp = Offset;
904#if defined(CONF_ARCH_ENDIAN_BIG)
905 swap_endian(&Temp, sizeof(int), sizeof(Temp) / sizeof(int));
906#endif
907 io_write(io: m_File, buffer: &Temp, size: sizeof(Temp));
908 Offset += m_vItems[ItemIndex].m_Size + sizeof(CDatafileItem);
909 }
910 }
911
912 // Write data offsets
913 int Offset = 0, DataIndex = 0;
914 for(const CDataInfo &DataInfo : m_vDatas)
915 {
916 if(DEBUG)
917 dbg_msg(sys: "datafile", fmt: "writing data offset. DataIndex=%d Offset=%d", DataIndex, Offset);
918
919 int Temp = Offset;
920#if defined(CONF_ARCH_ENDIAN_BIG)
921 swap_endian(&Temp, sizeof(int), sizeof(Temp) / sizeof(int));
922#endif
923 io_write(io: m_File, buffer: &Temp, size: sizeof(Temp));
924 Offset += DataInfo.m_CompressedSize;
925 ++DataIndex;
926 }
927
928 // Write data uncompressed sizes
929 DataIndex = 0;
930 for(const CDataInfo &DataInfo : m_vDatas)
931 {
932 if(DEBUG)
933 dbg_msg(sys: "datafile", fmt: "writing data uncompressed size. DataIndex=%d UncompressedSize=%d", DataIndex, DataInfo.m_UncompressedSize);
934
935 int UncompressedSize = DataInfo.m_UncompressedSize;
936#if defined(CONF_ARCH_ENDIAN_BIG)
937 swap_endian(&UncompressedSize, sizeof(int), sizeof(UncompressedSize) / sizeof(int));
938#endif
939 io_write(io: m_File, buffer: &UncompressedSize, size: sizeof(UncompressedSize));
940 ++DataIndex;
941 }
942
943 // Write items sorted by type
944 for(int Type = 0; Type < (int)m_aItemTypes.size(); ++Type)
945 {
946 // Write all items of this type
947 for(int ItemIndex = m_aItemTypes[Type].m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next)
948 {
949 CDatafileItem Item;
950 Item.m_TypeAndId = (Type << 16) | m_vItems[ItemIndex].m_Id;
951 Item.m_Size = m_vItems[ItemIndex].m_Size;
952
953 if(DEBUG)
954 dbg_msg(sys: "datafile", fmt: "writing item. Type=%x ItemIndex=%d Id=%d Size=%d", Type, ItemIndex, m_vItems[ItemIndex].m_Id, m_vItems[ItemIndex].m_Size);
955
956#if defined(CONF_ARCH_ENDIAN_BIG)
957 swap_endian(&Item, sizeof(int), sizeof(Item) / sizeof(int));
958 if(m_vItems[ItemIndex].m_pData != nullptr)
959 swap_endian(m_vItems[ItemIndex].m_pData, sizeof(int), m_vItems[ItemIndex].m_Size / sizeof(int));
960#endif
961 io_write(io: m_File, buffer: &Item, size: sizeof(Item));
962 if(m_vItems[ItemIndex].m_pData != nullptr)
963 {
964 io_write(io: m_File, buffer: m_vItems[ItemIndex].m_pData, size: m_vItems[ItemIndex].m_Size);
965 free(ptr: m_vItems[ItemIndex].m_pData);
966 m_vItems[ItemIndex].m_pData = nullptr;
967 }
968 }
969 }
970
971 // Write data
972 DataIndex = 0;
973 for(CDataInfo &DataInfo : m_vDatas)
974 {
975 if(DEBUG)
976 dbg_msg(sys: "datafile", fmt: "writing data. DataIndex=%d CompressedSize=%d", DataIndex, DataInfo.m_CompressedSize);
977
978 io_write(io: m_File, buffer: DataInfo.m_pCompressedData, size: DataInfo.m_CompressedSize);
979 free(ptr: DataInfo.m_pCompressedData);
980 DataInfo.m_pCompressedData = nullptr;
981 ++DataIndex;
982 }
983
984 io_close(io: m_File);
985 m_File = 0;
986}
987