1 | /* (c) Magnus Auvinen. See licence.txt in the root of the distribution for more information. */ |
2 | /* If you are missing that file, acquire a complete release at teeworlds.com. */ |
3 | |
4 | #include "datafile.h" |
5 | |
6 | #include <base/hash_ctxt.h> |
7 | #include <base/log.h> |
8 | #include <base/math.h> |
9 | #include <base/system.h> |
10 | #include <engine/storage.h> |
11 | |
12 | #include "uuid_manager.h" |
13 | |
14 | #include <cstdlib> |
15 | #include <limits> |
16 | |
17 | #include <zlib.h> |
18 | |
19 | static const int DEBUG = 0; |
20 | |
21 | enum |
22 | { |
23 | OFFSET_UUID_TYPE = 0x8000, |
24 | }; |
25 | |
26 | struct CItemEx |
27 | { |
28 | int m_aUuid[sizeof(CUuid) / sizeof(int32_t)]; |
29 | |
30 | static CItemEx FromUuid(CUuid Uuid) |
31 | { |
32 | CItemEx Result; |
33 | for(size_t i = 0; i < sizeof(CUuid) / sizeof(int32_t); i++) |
34 | Result.m_aUuid[i] = bytes_be_to_uint(bytes: &Uuid.m_aData[i * sizeof(int32_t)]); |
35 | return Result; |
36 | } |
37 | |
38 | CUuid ToUuid() const |
39 | { |
40 | CUuid Result; |
41 | for(size_t i = 0; i < sizeof(CUuid) / sizeof(int32_t); i++) |
42 | uint_to_bytes_be(bytes: &Result.m_aData[i * sizeof(int32_t)], value: m_aUuid[i]); |
43 | return Result; |
44 | } |
45 | }; |
46 | |
47 | struct CDatafileItemType |
48 | { |
49 | int m_Type; |
50 | int m_Start; |
51 | int m_Num; |
52 | }; |
53 | |
54 | struct CDatafileItem |
55 | { |
56 | int m_TypeAndId; |
57 | int m_Size; |
58 | }; |
59 | |
60 | struct |
61 | { |
62 | char [4]; |
63 | int ; |
64 | int ; |
65 | int ; |
66 | int ; |
67 | int ; |
68 | int ; |
69 | int ; |
70 | int ; |
71 | |
72 | constexpr size_t () |
73 | { |
74 | // The size of these members is not included in m_Size and m_Swaplen |
75 | return sizeof(m_aId) + sizeof(m_Version) + sizeof(m_Size) + sizeof(m_Swaplen); |
76 | } |
77 | }; |
78 | |
79 | struct CDatafileInfo |
80 | { |
81 | CDatafileItemType *m_pItemTypes; |
82 | int *m_pItemOffsets; |
83 | int *m_pDataOffsets; |
84 | int *m_pDataSizes; |
85 | |
86 | char *m_pItemStart; |
87 | char *m_pDataStart; |
88 | }; |
89 | |
90 | struct CDatafile |
91 | { |
92 | IOHANDLE m_File; |
93 | SHA256_DIGEST m_Sha256; |
94 | unsigned m_Crc; |
95 | CDatafileInfo m_Info; |
96 | CDatafileHeader ; |
97 | int m_DataStartOffset; |
98 | char **m_ppDataPtrs; |
99 | int *m_pDataSizes; |
100 | char *m_pData; |
101 | }; |
102 | |
103 | bool CDataFileReader::Open(class IStorage *pStorage, const char *pFilename, int StorageType) |
104 | { |
105 | log_trace("datafile" , "loading. filename='%s'" , pFilename); |
106 | |
107 | IOHANDLE File = pStorage->OpenFile(pFilename, Flags: IOFLAG_READ, Type: StorageType); |
108 | if(!File) |
109 | { |
110 | dbg_msg(sys: "datafile" , fmt: "could not open '%s'" , pFilename); |
111 | return false; |
112 | } |
113 | |
114 | // take the CRC of the file and store it |
115 | unsigned Crc = 0; |
116 | SHA256_DIGEST Sha256; |
117 | { |
118 | enum |
119 | { |
120 | BUFFER_SIZE = 64 * 1024 |
121 | }; |
122 | |
123 | SHA256_CTX Sha256Ctxt; |
124 | sha256_init(ctxt: &Sha256Ctxt); |
125 | unsigned char aBuffer[BUFFER_SIZE]; |
126 | |
127 | while(true) |
128 | { |
129 | unsigned Bytes = io_read(io: File, buffer: aBuffer, size: BUFFER_SIZE); |
130 | if(Bytes == 0) |
131 | break; |
132 | Crc = crc32(crc: Crc, buf: aBuffer, len: Bytes); |
133 | sha256_update(ctxt: &Sha256Ctxt, data: aBuffer, data_len: Bytes); |
134 | } |
135 | Sha256 = sha256_finish(ctxt: &Sha256Ctxt); |
136 | |
137 | io_seek(io: File, offset: 0, origin: IOSEEK_START); |
138 | } |
139 | |
140 | // TODO: change this header |
141 | CDatafileHeader ; |
142 | if(sizeof(Header) != io_read(io: File, buffer: &Header, size: sizeof(Header))) |
143 | { |
144 | dbg_msg(sys: "datafile" , fmt: "couldn't load header" ); |
145 | return false; |
146 | } |
147 | if(Header.m_aId[0] != 'A' || Header.m_aId[1] != 'T' || Header.m_aId[2] != 'A' || Header.m_aId[3] != 'D') |
148 | { |
149 | if(Header.m_aId[0] != 'D' || Header.m_aId[1] != 'A' || Header.m_aId[2] != 'T' || Header.m_aId[3] != 'A') |
150 | { |
151 | dbg_msg(sys: "datafile" , fmt: "wrong signature. %x %x %x %x" , Header.m_aId[0], Header.m_aId[1], Header.m_aId[2], Header.m_aId[3]); |
152 | return false; |
153 | } |
154 | } |
155 | |
156 | #if defined(CONF_ARCH_ENDIAN_BIG) |
157 | swap_endian(&Header, sizeof(int), sizeof(Header) / sizeof(int)); |
158 | #endif |
159 | if(Header.m_Version != 3 && Header.m_Version != 4) |
160 | { |
161 | dbg_msg(sys: "datafile" , fmt: "wrong version. version=%x" , Header.m_Version); |
162 | return false; |
163 | } |
164 | |
165 | // read in the rest except the data |
166 | unsigned Size = 0; |
167 | Size += Header.m_NumItemTypes * sizeof(CDatafileItemType); |
168 | Size += (Header.m_NumItems + Header.m_NumRawData) * sizeof(int); |
169 | if(Header.m_Version == 4) |
170 | Size += Header.m_NumRawData * sizeof(int); // v4 has uncompressed data sizes as well |
171 | Size += Header.m_ItemSize; |
172 | |
173 | unsigned AllocSize = Size; |
174 | AllocSize += sizeof(CDatafile); // add space for info structure |
175 | AllocSize += Header.m_NumRawData * sizeof(void *); // add space for data pointers |
176 | AllocSize += Header.m_NumRawData * sizeof(int); // add space for data sizes |
177 | if(Size > (((int64_t)1) << 31) || Header.m_NumItemTypes < 0 || Header.m_NumItems < 0 || Header.m_NumRawData < 0 || Header.m_ItemSize < 0) |
178 | { |
179 | io_close(io: File); |
180 | dbg_msg(sys: "datafile" , fmt: "unable to load file, invalid file information" ); |
181 | return false; |
182 | } |
183 | |
184 | CDatafile *pTmpDataFile = (CDatafile *)malloc(size: AllocSize); |
185 | pTmpDataFile->m_Header = Header; |
186 | pTmpDataFile->m_DataStartOffset = sizeof(CDatafileHeader) + Size; |
187 | pTmpDataFile->m_ppDataPtrs = (char **)(pTmpDataFile + 1); |
188 | pTmpDataFile->m_pDataSizes = (int *)(pTmpDataFile->m_ppDataPtrs + Header.m_NumRawData); |
189 | pTmpDataFile->m_pData = (char *)(pTmpDataFile->m_pDataSizes + Header.m_NumRawData); |
190 | pTmpDataFile->m_File = File; |
191 | pTmpDataFile->m_Sha256 = Sha256; |
192 | pTmpDataFile->m_Crc = Crc; |
193 | |
194 | // clear the data pointers and sizes |
195 | mem_zero(block: pTmpDataFile->m_ppDataPtrs, size: Header.m_NumRawData * sizeof(void *)); |
196 | mem_zero(block: pTmpDataFile->m_pDataSizes, size: Header.m_NumRawData * sizeof(int)); |
197 | |
198 | // read types, offsets, sizes and item data |
199 | unsigned ReadSize = io_read(io: File, buffer: pTmpDataFile->m_pData, size: Size); |
200 | if(ReadSize != Size) |
201 | { |
202 | io_close(io: pTmpDataFile->m_File); |
203 | free(ptr: pTmpDataFile); |
204 | dbg_msg(sys: "datafile" , fmt: "couldn't load the whole thing, wanted=%d got=%d" , Size, ReadSize); |
205 | return false; |
206 | } |
207 | |
208 | Close(); |
209 | m_pDataFile = pTmpDataFile; |
210 | |
211 | #if defined(CONF_ARCH_ENDIAN_BIG) |
212 | swap_endian(m_pDataFile->m_pData, sizeof(int), minimum(static_cast<unsigned>(Header.m_Swaplen), Size) / sizeof(int)); |
213 | #endif |
214 | |
215 | if(DEBUG) |
216 | { |
217 | dbg_msg(sys: "datafile" , fmt: "allocsize=%d" , AllocSize); |
218 | dbg_msg(sys: "datafile" , fmt: "readsize=%d" , ReadSize); |
219 | dbg_msg(sys: "datafile" , fmt: "swaplen=%d" , Header.m_Swaplen); |
220 | dbg_msg(sys: "datafile" , fmt: "item_size=%d" , m_pDataFile->m_Header.m_ItemSize); |
221 | } |
222 | |
223 | m_pDataFile->m_Info.m_pItemTypes = (CDatafileItemType *)m_pDataFile->m_pData; |
224 | m_pDataFile->m_Info.m_pItemOffsets = (int *)&m_pDataFile->m_Info.m_pItemTypes[m_pDataFile->m_Header.m_NumItemTypes]; |
225 | m_pDataFile->m_Info.m_pDataOffsets = &m_pDataFile->m_Info.m_pItemOffsets[m_pDataFile->m_Header.m_NumItems]; |
226 | m_pDataFile->m_Info.m_pDataSizes = &m_pDataFile->m_Info.m_pDataOffsets[m_pDataFile->m_Header.m_NumRawData]; |
227 | |
228 | if(Header.m_Version == 4) |
229 | m_pDataFile->m_Info.m_pItemStart = (char *)&m_pDataFile->m_Info.m_pDataSizes[m_pDataFile->m_Header.m_NumRawData]; |
230 | else |
231 | m_pDataFile->m_Info.m_pItemStart = (char *)&m_pDataFile->m_Info.m_pDataOffsets[m_pDataFile->m_Header.m_NumRawData]; |
232 | m_pDataFile->m_Info.m_pDataStart = m_pDataFile->m_Info.m_pItemStart + m_pDataFile->m_Header.m_ItemSize; |
233 | |
234 | log_trace("datafile" , "loading done. datafile='%s'" , pFilename); |
235 | |
236 | return true; |
237 | } |
238 | |
239 | bool CDataFileReader::Close() |
240 | { |
241 | if(!m_pDataFile) |
242 | return true; |
243 | |
244 | // free the data that is loaded |
245 | for(int i = 0; i < m_pDataFile->m_Header.m_NumRawData; i++) |
246 | { |
247 | free(ptr: m_pDataFile->m_ppDataPtrs[i]); |
248 | m_pDataFile->m_ppDataPtrs[i] = nullptr; |
249 | m_pDataFile->m_pDataSizes[i] = 0; |
250 | } |
251 | |
252 | io_close(io: m_pDataFile->m_File); |
253 | free(ptr: m_pDataFile); |
254 | m_pDataFile = nullptr; |
255 | return true; |
256 | } |
257 | |
258 | IOHANDLE CDataFileReader::File() const |
259 | { |
260 | if(!m_pDataFile) |
261 | return 0; |
262 | return m_pDataFile->m_File; |
263 | } |
264 | |
265 | int CDataFileReader::NumData() const |
266 | { |
267 | if(!m_pDataFile) |
268 | { |
269 | return 0; |
270 | } |
271 | return m_pDataFile->m_Header.m_NumRawData; |
272 | } |
273 | |
274 | // returns the size in the file |
275 | int CDataFileReader::GetFileDataSize(int Index) const |
276 | { |
277 | if(!m_pDataFile) |
278 | { |
279 | return 0; |
280 | } |
281 | |
282 | if(Index == m_pDataFile->m_Header.m_NumRawData - 1) |
283 | return m_pDataFile->m_Header.m_DataSize - m_pDataFile->m_Info.m_pDataOffsets[Index]; |
284 | |
285 | return m_pDataFile->m_Info.m_pDataOffsets[Index + 1] - m_pDataFile->m_Info.m_pDataOffsets[Index]; |
286 | } |
287 | |
288 | // returns the size of the resulting data |
289 | int CDataFileReader::GetDataSize(int Index) const |
290 | { |
291 | if(!m_pDataFile || Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData) |
292 | { |
293 | return 0; |
294 | } |
295 | |
296 | if(!m_pDataFile->m_ppDataPtrs[Index]) |
297 | { |
298 | if(m_pDataFile->m_Header.m_Version >= 4) |
299 | { |
300 | return m_pDataFile->m_Info.m_pDataSizes[Index]; |
301 | } |
302 | else |
303 | { |
304 | return GetFileDataSize(Index); |
305 | } |
306 | } |
307 | const int Size = m_pDataFile->m_pDataSizes[Index]; |
308 | if(Size < 0) |
309 | return 0; // summarize all errors as zero size |
310 | return Size; |
311 | } |
312 | |
313 | void *CDataFileReader::GetDataImpl(int Index, bool Swap) |
314 | { |
315 | if(!m_pDataFile) |
316 | { |
317 | return nullptr; |
318 | } |
319 | |
320 | if(Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData) |
321 | return nullptr; |
322 | |
323 | // load it if needed |
324 | if(!m_pDataFile->m_ppDataPtrs[Index]) |
325 | { |
326 | // don't try to load again if it previously failed |
327 | if(m_pDataFile->m_pDataSizes[Index] < 0) |
328 | return nullptr; |
329 | |
330 | // fetch the data size |
331 | unsigned DataSize = GetFileDataSize(Index); |
332 | #if defined(CONF_ARCH_ENDIAN_BIG) |
333 | unsigned SwapSize = DataSize; |
334 | #endif |
335 | |
336 | if(m_pDataFile->m_Header.m_Version == 4) |
337 | { |
338 | // v4 has compressed data |
339 | const unsigned OriginalUncompressedSize = m_pDataFile->m_Info.m_pDataSizes[Index]; |
340 | unsigned long UncompressedSize = OriginalUncompressedSize; |
341 | |
342 | log_trace("datafile" , "loading data. index=%d size=%u uncompressed=%u" , Index, DataSize, OriginalUncompressedSize); |
343 | |
344 | // read the compressed data |
345 | void *pCompressedData = malloc(size: DataSize); |
346 | unsigned ActualDataSize = 0; |
347 | if(io_seek(io: m_pDataFile->m_File, offset: m_pDataFile->m_DataStartOffset + m_pDataFile->m_Info.m_pDataOffsets[Index], origin: IOSEEK_START) == 0) |
348 | ActualDataSize = io_read(io: m_pDataFile->m_File, buffer: pCompressedData, size: DataSize); |
349 | if(DataSize != ActualDataSize) |
350 | { |
351 | log_error("datafile" , "truncation error, could not read all data. index=%d wanted=%u got=%u" , Index, DataSize, ActualDataSize); |
352 | free(ptr: pCompressedData); |
353 | m_pDataFile->m_ppDataPtrs[Index] = nullptr; |
354 | m_pDataFile->m_pDataSizes[Index] = -1; |
355 | return nullptr; |
356 | } |
357 | |
358 | // decompress the data |
359 | m_pDataFile->m_ppDataPtrs[Index] = (char *)malloc(size: UncompressedSize); |
360 | m_pDataFile->m_pDataSizes[Index] = UncompressedSize; |
361 | const int Result = uncompress(dest: (Bytef *)m_pDataFile->m_ppDataPtrs[Index], destLen: &UncompressedSize, source: (Bytef *)pCompressedData, sourceLen: DataSize); |
362 | free(ptr: pCompressedData); |
363 | if(Result != Z_OK || UncompressedSize != OriginalUncompressedSize) |
364 | { |
365 | log_error("datafile" , "uncompress error. result=%d wanted=%u got=%lu" , Result, OriginalUncompressedSize, UncompressedSize); |
366 | free(ptr: m_pDataFile->m_ppDataPtrs[Index]); |
367 | m_pDataFile->m_ppDataPtrs[Index] = nullptr; |
368 | m_pDataFile->m_pDataSizes[Index] = -1; |
369 | return nullptr; |
370 | } |
371 | |
372 | #if defined(CONF_ARCH_ENDIAN_BIG) |
373 | SwapSize = UncompressedSize; |
374 | #endif |
375 | } |
376 | else |
377 | { |
378 | // load the data |
379 | log_trace("datafile" , "loading data. index=%d size=%d" , Index, DataSize); |
380 | m_pDataFile->m_ppDataPtrs[Index] = static_cast<char *>(malloc(size: DataSize)); |
381 | m_pDataFile->m_pDataSizes[Index] = DataSize; |
382 | unsigned ActualDataSize = 0; |
383 | if(io_seek(io: m_pDataFile->m_File, offset: m_pDataFile->m_DataStartOffset + m_pDataFile->m_Info.m_pDataOffsets[Index], origin: IOSEEK_START) == 0) |
384 | ActualDataSize = io_read(io: m_pDataFile->m_File, buffer: m_pDataFile->m_ppDataPtrs[Index], size: DataSize); |
385 | if(DataSize != ActualDataSize) |
386 | { |
387 | log_error("datafile" , "truncation error, could not read all data. index=%d wanted=%u got=%u" , Index, DataSize, ActualDataSize); |
388 | free(ptr: m_pDataFile->m_ppDataPtrs[Index]); |
389 | m_pDataFile->m_ppDataPtrs[Index] = nullptr; |
390 | m_pDataFile->m_pDataSizes[Index] = -1; |
391 | return nullptr; |
392 | } |
393 | } |
394 | |
395 | #if defined(CONF_ARCH_ENDIAN_BIG) |
396 | if(Swap && SwapSize) |
397 | swap_endian(m_pDataFile->m_ppDataPtrs[Index], sizeof(int), SwapSize / sizeof(int)); |
398 | #endif |
399 | } |
400 | |
401 | return m_pDataFile->m_ppDataPtrs[Index]; |
402 | } |
403 | |
404 | void *CDataFileReader::GetData(int Index) |
405 | { |
406 | return GetDataImpl(Index, Swap: false); |
407 | } |
408 | |
409 | void *CDataFileReader::GetDataSwapped(int Index) |
410 | { |
411 | return GetDataImpl(Index, Swap: true); |
412 | } |
413 | |
414 | const char *CDataFileReader::GetDataString(int Index) |
415 | { |
416 | if(Index == -1) |
417 | return "" ; |
418 | const int DataSize = GetDataSize(Index); |
419 | if(!DataSize) |
420 | return nullptr; |
421 | const char *pData = static_cast<char *>(GetData(Index)); |
422 | if(pData == nullptr || mem_has_null(block: pData, size: DataSize - 1) || pData[DataSize - 1] != '\0' || !str_utf8_check(str: pData)) |
423 | return nullptr; |
424 | return pData; |
425 | } |
426 | |
427 | void CDataFileReader::ReplaceData(int Index, char *pData, size_t Size) |
428 | { |
429 | dbg_assert(Index >= 0 && Index < m_pDataFile->m_Header.m_NumRawData, "Index invalid" ); |
430 | |
431 | free(ptr: m_pDataFile->m_ppDataPtrs[Index]); |
432 | m_pDataFile->m_ppDataPtrs[Index] = pData; |
433 | m_pDataFile->m_pDataSizes[Index] = Size; |
434 | } |
435 | |
436 | void CDataFileReader::UnloadData(int Index) |
437 | { |
438 | if(Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData) |
439 | return; |
440 | |
441 | free(ptr: m_pDataFile->m_ppDataPtrs[Index]); |
442 | m_pDataFile->m_ppDataPtrs[Index] = nullptr; |
443 | m_pDataFile->m_pDataSizes[Index] = 0; |
444 | } |
445 | |
446 | int CDataFileReader::GetItemSize(int Index) const |
447 | { |
448 | if(!m_pDataFile) |
449 | return 0; |
450 | if(Index == m_pDataFile->m_Header.m_NumItems - 1) |
451 | return m_pDataFile->m_Header.m_ItemSize - m_pDataFile->m_Info.m_pItemOffsets[Index] - sizeof(CDatafileItem); |
452 | return m_pDataFile->m_Info.m_pItemOffsets[Index + 1] - m_pDataFile->m_Info.m_pItemOffsets[Index] - sizeof(CDatafileItem); |
453 | } |
454 | |
455 | int CDataFileReader::GetExternalItemType(int InternalType, CUuid *pUuid) |
456 | { |
457 | if(InternalType <= OFFSET_UUID_TYPE || InternalType == ITEMTYPE_EX) |
458 | { |
459 | if(pUuid) |
460 | *pUuid = UUID_ZEROED; |
461 | return InternalType; |
462 | } |
463 | int TypeIndex = FindItemIndex(Type: ITEMTYPE_EX, Id: InternalType); |
464 | if(TypeIndex < 0 || GetItemSize(Index: TypeIndex) < (int)sizeof(CItemEx)) |
465 | { |
466 | if(pUuid) |
467 | *pUuid = UUID_ZEROED; |
468 | return InternalType; |
469 | } |
470 | const CItemEx *pItemEx = (const CItemEx *)GetItem(Index: TypeIndex); |
471 | CUuid Uuid = pItemEx->ToUuid(); |
472 | if(pUuid) |
473 | *pUuid = Uuid; |
474 | // Propagate UUID_UNKNOWN, it doesn't hurt. |
475 | return g_UuidManager.LookupUuid(Uuid); |
476 | } |
477 | |
478 | int CDataFileReader::GetInternalItemType(int ExternalType) |
479 | { |
480 | if(ExternalType < OFFSET_UUID) |
481 | { |
482 | return ExternalType; |
483 | } |
484 | CUuid Uuid = g_UuidManager.GetUuid(Id: ExternalType); |
485 | int Start, Num; |
486 | GetType(Type: ITEMTYPE_EX, pStart: &Start, pNum: &Num); |
487 | for(int i = Start; i < Start + Num; i++) |
488 | { |
489 | if(GetItemSize(Index: i) < (int)sizeof(CItemEx)) |
490 | { |
491 | continue; |
492 | } |
493 | int Id; |
494 | if(Uuid == ((const CItemEx *)GetItem(Index: i, pType: nullptr, pId: &Id))->ToUuid()) |
495 | { |
496 | return Id; |
497 | } |
498 | } |
499 | return -1; |
500 | } |
501 | |
502 | void *CDataFileReader::GetItem(int Index, int *pType, int *pId, CUuid *pUuid) |
503 | { |
504 | if(!m_pDataFile) |
505 | { |
506 | if(pType) |
507 | *pType = 0; |
508 | if(pId) |
509 | *pId = 0; |
510 | if(pUuid) |
511 | *pUuid = UUID_ZEROED; |
512 | return nullptr; |
513 | } |
514 | |
515 | CDatafileItem *pItem = (CDatafileItem *)(m_pDataFile->m_Info.m_pItemStart + m_pDataFile->m_Info.m_pItemOffsets[Index]); |
516 | |
517 | // remove sign extension |
518 | const int Type = GetExternalItemType(InternalType: (pItem->m_TypeAndId >> 16) & 0xffff, pUuid); |
519 | if(pType) |
520 | { |
521 | *pType = Type; |
522 | } |
523 | if(pId) |
524 | { |
525 | *pId = pItem->m_TypeAndId & 0xffff; |
526 | } |
527 | return (void *)(pItem + 1); |
528 | } |
529 | |
530 | void CDataFileReader::GetType(int Type, int *pStart, int *pNum) |
531 | { |
532 | *pStart = 0; |
533 | *pNum = 0; |
534 | |
535 | if(!m_pDataFile) |
536 | return; |
537 | |
538 | Type = GetInternalItemType(ExternalType: Type); |
539 | for(int i = 0; i < m_pDataFile->m_Header.m_NumItemTypes; i++) |
540 | { |
541 | if(m_pDataFile->m_Info.m_pItemTypes[i].m_Type == Type) |
542 | { |
543 | *pStart = m_pDataFile->m_Info.m_pItemTypes[i].m_Start; |
544 | *pNum = m_pDataFile->m_Info.m_pItemTypes[i].m_Num; |
545 | return; |
546 | } |
547 | } |
548 | } |
549 | |
550 | int CDataFileReader::FindItemIndex(int Type, int Id) |
551 | { |
552 | if(!m_pDataFile) |
553 | { |
554 | return -1; |
555 | } |
556 | |
557 | int Start, Num; |
558 | GetType(Type, pStart: &Start, pNum: &Num); |
559 | for(int i = 0; i < Num; i++) |
560 | { |
561 | int ItemId; |
562 | GetItem(Index: Start + i, pType: nullptr, pId: &ItemId); |
563 | if(Id == ItemId) |
564 | { |
565 | return Start + i; |
566 | } |
567 | } |
568 | return -1; |
569 | } |
570 | |
571 | void *CDataFileReader::FindItem(int Type, int Id) |
572 | { |
573 | int Index = FindItemIndex(Type, Id); |
574 | if(Index < 0) |
575 | { |
576 | return nullptr; |
577 | } |
578 | return GetItem(Index); |
579 | } |
580 | |
581 | int CDataFileReader::NumItems() const |
582 | { |
583 | if(!m_pDataFile) |
584 | return 0; |
585 | return m_pDataFile->m_Header.m_NumItems; |
586 | } |
587 | |
588 | SHA256_DIGEST CDataFileReader::Sha256() const |
589 | { |
590 | if(!m_pDataFile) |
591 | { |
592 | SHA256_DIGEST Result; |
593 | for(unsigned char &d : Result.data) |
594 | { |
595 | d = 0xff; |
596 | } |
597 | return Result; |
598 | } |
599 | return m_pDataFile->m_Sha256; |
600 | } |
601 | |
602 | unsigned CDataFileReader::Crc() const |
603 | { |
604 | if(!m_pDataFile) |
605 | return 0xFFFFFFFF; |
606 | return m_pDataFile->m_Crc; |
607 | } |
608 | |
609 | int CDataFileReader::MapSize() const |
610 | { |
611 | if(!m_pDataFile) |
612 | return 0; |
613 | return m_pDataFile->m_Header.m_Size + m_pDataFile->m_Header.SizeOffset(); |
614 | } |
615 | |
616 | CDataFileWriter::CDataFileWriter() |
617 | { |
618 | m_File = 0; |
619 | for(CItemTypeInfo &ItemTypeInfo : m_aItemTypes) |
620 | { |
621 | ItemTypeInfo.m_Num = 0; |
622 | ItemTypeInfo.m_First = -1; |
623 | ItemTypeInfo.m_Last = -1; |
624 | } |
625 | } |
626 | |
627 | CDataFileWriter::~CDataFileWriter() |
628 | { |
629 | if(m_File) |
630 | { |
631 | io_close(io: m_File); |
632 | m_File = 0; |
633 | } |
634 | |
635 | for(CItemInfo &ItemInfo : m_vItems) |
636 | { |
637 | free(ptr: ItemInfo.m_pData); |
638 | } |
639 | |
640 | for(CDataInfo &DataInfo : m_vDatas) |
641 | { |
642 | free(ptr: DataInfo.m_pUncompressedData); |
643 | free(ptr: DataInfo.m_pCompressedData); |
644 | } |
645 | } |
646 | |
647 | bool CDataFileWriter::Open(class IStorage *pStorage, const char *pFilename, int StorageType) |
648 | { |
649 | dbg_assert(!m_File, "File already open" ); |
650 | m_File = pStorage->OpenFile(pFilename, Flags: IOFLAG_WRITE, Type: StorageType); |
651 | return m_File != 0; |
652 | } |
653 | |
654 | int CDataFileWriter::GetTypeFromIndex(int Index) const |
655 | { |
656 | return ITEMTYPE_EX - Index - 1; |
657 | } |
658 | |
659 | int CDataFileWriter::GetExtendedItemTypeIndex(int Type, const CUuid *pUuid) |
660 | { |
661 | int Index = 0; |
662 | if(Type == -1) |
663 | { |
664 | // Unknown type, search for UUID |
665 | for(const auto &ExtendedItemType : m_vExtendedItemTypes) |
666 | { |
667 | if(ExtendedItemType.m_Uuid == *pUuid) |
668 | return Index; |
669 | ++Index; |
670 | } |
671 | } |
672 | else |
673 | { |
674 | for(const auto &ExtendedItemType : m_vExtendedItemTypes) |
675 | { |
676 | if(ExtendedItemType.m_Type == Type) |
677 | return Index; |
678 | ++Index; |
679 | } |
680 | } |
681 | |
682 | // Type not found, add it. |
683 | CExtendedItemType ExtendedType; |
684 | ExtendedType.m_Type = Type; |
685 | ExtendedType.m_Uuid = Type == -1 ? *pUuid : g_UuidManager.GetUuid(Id: Type); |
686 | m_vExtendedItemTypes.push_back(x: ExtendedType); |
687 | |
688 | CItemEx ItemEx = CItemEx::FromUuid(Uuid: ExtendedType.m_Uuid); |
689 | AddItem(Type: ITEMTYPE_EX, Id: GetTypeFromIndex(Index), Size: sizeof(ItemEx), pData: &ItemEx); |
690 | return Index; |
691 | } |
692 | |
693 | int CDataFileWriter::AddItem(int Type, int Id, size_t Size, const void *pData, const CUuid *pUuid) |
694 | { |
695 | dbg_assert((Type >= 0 && Type < MAX_ITEM_TYPES) || Type >= OFFSET_UUID || (Type == -1 && pUuid != nullptr), "Invalid type" ); |
696 | dbg_assert(Id >= 0 && Id <= ITEMTYPE_EX, "Invalid ID" ); |
697 | dbg_assert(Size == 0 || pData != nullptr, "Data missing" ); // Items without data are allowed |
698 | dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large" ); |
699 | dbg_assert(Size % sizeof(int) == 0, "Invalid data boundary" ); |
700 | |
701 | if(Type == -1 || Type >= OFFSET_UUID) |
702 | { |
703 | Type = GetTypeFromIndex(Index: GetExtendedItemTypeIndex(Type, pUuid)); |
704 | } |
705 | |
706 | const int NumItems = m_vItems.size(); |
707 | m_vItems.emplace_back(); |
708 | CItemInfo &Info = m_vItems.back(); |
709 | Info.m_Type = Type; |
710 | Info.m_Id = Id; |
711 | Info.m_Size = Size; |
712 | |
713 | // copy data |
714 | if(Size > 0) |
715 | { |
716 | Info.m_pData = malloc(size: Size); |
717 | mem_copy(dest: Info.m_pData, source: pData, size: Size); |
718 | } |
719 | else |
720 | Info.m_pData = nullptr; |
721 | |
722 | // link |
723 | Info.m_Prev = m_aItemTypes[Type].m_Last; |
724 | Info.m_Next = -1; |
725 | |
726 | if(m_aItemTypes[Type].m_Last != -1) |
727 | m_vItems[m_aItemTypes[Type].m_Last].m_Next = NumItems; |
728 | m_aItemTypes[Type].m_Last = NumItems; |
729 | |
730 | if(m_aItemTypes[Type].m_First == -1) |
731 | m_aItemTypes[Type].m_First = NumItems; |
732 | |
733 | m_aItemTypes[Type].m_Num++; |
734 | return NumItems; |
735 | } |
736 | |
737 | int CDataFileWriter::AddData(size_t Size, const void *pData, ECompressionLevel CompressionLevel) |
738 | { |
739 | dbg_assert(Size > 0 && pData != nullptr, "Data missing" ); |
740 | dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large" ); |
741 | |
742 | m_vDatas.emplace_back(); |
743 | CDataInfo &Info = m_vDatas.back(); |
744 | Info.m_pUncompressedData = malloc(size: Size); |
745 | mem_copy(dest: Info.m_pUncompressedData, source: pData, size: Size); |
746 | Info.m_UncompressedSize = Size; |
747 | Info.m_pCompressedData = nullptr; |
748 | Info.m_CompressedSize = 0; |
749 | Info.m_CompressionLevel = CompressionLevel; |
750 | |
751 | return m_vDatas.size() - 1; |
752 | } |
753 | |
754 | int CDataFileWriter::AddDataSwapped(size_t Size, const void *pData) |
755 | { |
756 | dbg_assert(Size > 0 && pData != nullptr, "Data missing" ); |
757 | dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large" ); |
758 | dbg_assert(Size % sizeof(int) == 0, "Invalid data boundary" ); |
759 | |
760 | #if defined(CONF_ARCH_ENDIAN_BIG) |
761 | void *pSwapped = malloc(Size); // temporary buffer that we use during compression |
762 | mem_copy(pSwapped, pData, Size); |
763 | swap_endian(pSwapped, sizeof(int), Size / sizeof(int)); |
764 | int Index = AddData(Size, pSwapped); |
765 | free(pSwapped); |
766 | return Index; |
767 | #else |
768 | return AddData(Size, pData); |
769 | #endif |
770 | } |
771 | |
772 | int CDataFileWriter::AddDataString(const char *pStr) |
773 | { |
774 | dbg_assert(pStr != nullptr, "Data missing" ); |
775 | |
776 | if(pStr[0] == '\0') |
777 | return -1; |
778 | return AddData(Size: str_length(str: pStr) + 1, pData: pStr); |
779 | } |
780 | |
781 | static int CompressionLevelToZlib(CDataFileWriter::ECompressionLevel CompressionLevel) |
782 | { |
783 | switch(CompressionLevel) |
784 | { |
785 | case CDataFileWriter::COMPRESSION_DEFAULT: |
786 | return Z_DEFAULT_COMPRESSION; |
787 | case CDataFileWriter::COMPRESSION_BEST: |
788 | return Z_BEST_COMPRESSION; |
789 | default: |
790 | dbg_assert(false, "CompressionLevel invalid" ); |
791 | dbg_break(); |
792 | } |
793 | } |
794 | |
795 | void CDataFileWriter::Finish() |
796 | { |
797 | dbg_assert((bool)m_File, "File not open" ); |
798 | |
799 | // Compress data. This takes the majority of the time when saving a datafile, |
800 | // so it's delayed until the end so it can be off-loaded to another thread. |
801 | for(CDataInfo &DataInfo : m_vDatas) |
802 | { |
803 | unsigned long CompressedSize = compressBound(sourceLen: DataInfo.m_UncompressedSize); |
804 | DataInfo.m_pCompressedData = malloc(size: CompressedSize); |
805 | const int Result = compress2(dest: (Bytef *)DataInfo.m_pCompressedData, destLen: &CompressedSize, source: (Bytef *)DataInfo.m_pUncompressedData, sourceLen: DataInfo.m_UncompressedSize, level: CompressionLevelToZlib(CompressionLevel: DataInfo.m_CompressionLevel)); |
806 | DataInfo.m_CompressedSize = CompressedSize; |
807 | free(ptr: DataInfo.m_pUncompressedData); |
808 | DataInfo.m_pUncompressedData = nullptr; |
809 | if(Result != Z_OK) |
810 | { |
811 | char aError[32]; |
812 | str_format(buffer: aError, buffer_size: sizeof(aError), format: "zlib compression error %d" , Result); |
813 | dbg_assert(false, aError); |
814 | } |
815 | } |
816 | |
817 | // Calculate total size of items |
818 | size_t ItemSize = 0; |
819 | for(const CItemInfo &ItemInfo : m_vItems) |
820 | { |
821 | ItemSize += ItemInfo.m_Size; |
822 | ItemSize += sizeof(CDatafileItem); |
823 | } |
824 | |
825 | // Calculate total size of data |
826 | size_t DataSize = 0; |
827 | for(const CDataInfo &DataInfo : m_vDatas) |
828 | DataSize += DataInfo.m_CompressedSize; |
829 | |
830 | // Count number of item types |
831 | int NumItemTypes = 0; |
832 | for(const CItemTypeInfo &ItemType : m_aItemTypes) |
833 | { |
834 | if(ItemType.m_Num > 0) |
835 | ++NumItemTypes; |
836 | } |
837 | |
838 | // Calculate complete file size |
839 | const size_t TypesSize = NumItemTypes * sizeof(CDatafileItemType); |
840 | const size_t = sizeof(CDatafileHeader); |
841 | const size_t OffsetSize = (m_vItems.size() + m_vDatas.size() * 2) * sizeof(int); // ItemOffsets, DataOffsets, DataUncompressedSizes |
842 | const size_t SwapSize = HeaderSize + TypesSize + OffsetSize + ItemSize; |
843 | const size_t FileSize = SwapSize + DataSize; |
844 | |
845 | if(DEBUG) |
846 | dbg_msg(sys: "datafile" , fmt: "NumItemTypes=%d TypesSize=%" PRIzu " ItemSize=%" PRIzu " DataSize=%" PRIzu, NumItemTypes, TypesSize, ItemSize, DataSize); |
847 | |
848 | // This also ensures that SwapSize, ItemSize and DataSize are valid. |
849 | dbg_assert(FileSize <= (size_t)std::numeric_limits<int>::max(), "File size too large" ); |
850 | |
851 | // Construct and write header |
852 | { |
853 | CDatafileHeader ; |
854 | Header.m_aId[0] = 'D'; |
855 | Header.m_aId[1] = 'A'; |
856 | Header.m_aId[2] = 'T'; |
857 | Header.m_aId[3] = 'A'; |
858 | Header.m_Version = 4; |
859 | Header.m_Size = FileSize - Header.SizeOffset(); |
860 | Header.m_Swaplen = SwapSize - Header.SizeOffset(); |
861 | Header.m_NumItemTypes = NumItemTypes; |
862 | Header.m_NumItems = m_vItems.size(); |
863 | Header.m_NumRawData = m_vDatas.size(); |
864 | Header.m_ItemSize = ItemSize; |
865 | Header.m_DataSize = DataSize; |
866 | |
867 | #if defined(CONF_ARCH_ENDIAN_BIG) |
868 | swap_endian(&Header, sizeof(int), sizeof(Header) / sizeof(int)); |
869 | #endif |
870 | io_write(io: m_File, buffer: &Header, size: sizeof(Header)); |
871 | } |
872 | |
873 | // Write item types |
874 | for(int Type = 0, Count = 0; Type < (int)m_aItemTypes.size(); ++Type) |
875 | { |
876 | if(!m_aItemTypes[Type].m_Num) |
877 | continue; |
878 | |
879 | CDatafileItemType Info; |
880 | Info.m_Type = Type; |
881 | Info.m_Start = Count; |
882 | Info.m_Num = m_aItemTypes[Type].m_Num; |
883 | |
884 | if(DEBUG) |
885 | dbg_msg(sys: "datafile" , fmt: "writing item type. Type=%x Start=%d Num=%d" , Info.m_Type, Info.m_Start, Info.m_Num); |
886 | |
887 | #if defined(CONF_ARCH_ENDIAN_BIG) |
888 | swap_endian(&Info, sizeof(int), sizeof(CDatafileItemType) / sizeof(int)); |
889 | #endif |
890 | io_write(io: m_File, buffer: &Info, size: sizeof(Info)); |
891 | Count += m_aItemTypes[Type].m_Num; |
892 | } |
893 | |
894 | // Write item offsets sorted by type |
895 | for(int Type = 0, Offset = 0; Type < (int)m_aItemTypes.size(); Type++) |
896 | { |
897 | // Write all items offsets of this type |
898 | for(int ItemIndex = m_aItemTypes[Type].m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next) |
899 | { |
900 | if(DEBUG) |
901 | dbg_msg(sys: "datafile" , fmt: "writing item offset. Type=%d ItemIndex=%d Offset=%d" , Type, ItemIndex, Offset); |
902 | |
903 | int Temp = Offset; |
904 | #if defined(CONF_ARCH_ENDIAN_BIG) |
905 | swap_endian(&Temp, sizeof(int), sizeof(Temp) / sizeof(int)); |
906 | #endif |
907 | io_write(io: m_File, buffer: &Temp, size: sizeof(Temp)); |
908 | Offset += m_vItems[ItemIndex].m_Size + sizeof(CDatafileItem); |
909 | } |
910 | } |
911 | |
912 | // Write data offsets |
913 | int Offset = 0, DataIndex = 0; |
914 | for(const CDataInfo &DataInfo : m_vDatas) |
915 | { |
916 | if(DEBUG) |
917 | dbg_msg(sys: "datafile" , fmt: "writing data offset. DataIndex=%d Offset=%d" , DataIndex, Offset); |
918 | |
919 | int Temp = Offset; |
920 | #if defined(CONF_ARCH_ENDIAN_BIG) |
921 | swap_endian(&Temp, sizeof(int), sizeof(Temp) / sizeof(int)); |
922 | #endif |
923 | io_write(io: m_File, buffer: &Temp, size: sizeof(Temp)); |
924 | Offset += DataInfo.m_CompressedSize; |
925 | ++DataIndex; |
926 | } |
927 | |
928 | // Write data uncompressed sizes |
929 | DataIndex = 0; |
930 | for(const CDataInfo &DataInfo : m_vDatas) |
931 | { |
932 | if(DEBUG) |
933 | dbg_msg(sys: "datafile" , fmt: "writing data uncompressed size. DataIndex=%d UncompressedSize=%d" , DataIndex, DataInfo.m_UncompressedSize); |
934 | |
935 | int UncompressedSize = DataInfo.m_UncompressedSize; |
936 | #if defined(CONF_ARCH_ENDIAN_BIG) |
937 | swap_endian(&UncompressedSize, sizeof(int), sizeof(UncompressedSize) / sizeof(int)); |
938 | #endif |
939 | io_write(io: m_File, buffer: &UncompressedSize, size: sizeof(UncompressedSize)); |
940 | ++DataIndex; |
941 | } |
942 | |
943 | // Write items sorted by type |
944 | for(int Type = 0; Type < (int)m_aItemTypes.size(); ++Type) |
945 | { |
946 | // Write all items of this type |
947 | for(int ItemIndex = m_aItemTypes[Type].m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next) |
948 | { |
949 | CDatafileItem Item; |
950 | Item.m_TypeAndId = (Type << 16) | m_vItems[ItemIndex].m_Id; |
951 | Item.m_Size = m_vItems[ItemIndex].m_Size; |
952 | |
953 | if(DEBUG) |
954 | dbg_msg(sys: "datafile" , fmt: "writing item. Type=%x ItemIndex=%d Id=%d Size=%d" , Type, ItemIndex, m_vItems[ItemIndex].m_Id, m_vItems[ItemIndex].m_Size); |
955 | |
956 | #if defined(CONF_ARCH_ENDIAN_BIG) |
957 | swap_endian(&Item, sizeof(int), sizeof(Item) / sizeof(int)); |
958 | if(m_vItems[ItemIndex].m_pData != nullptr) |
959 | swap_endian(m_vItems[ItemIndex].m_pData, sizeof(int), m_vItems[ItemIndex].m_Size / sizeof(int)); |
960 | #endif |
961 | io_write(io: m_File, buffer: &Item, size: sizeof(Item)); |
962 | if(m_vItems[ItemIndex].m_pData != nullptr) |
963 | { |
964 | io_write(io: m_File, buffer: m_vItems[ItemIndex].m_pData, size: m_vItems[ItemIndex].m_Size); |
965 | free(ptr: m_vItems[ItemIndex].m_pData); |
966 | m_vItems[ItemIndex].m_pData = nullptr; |
967 | } |
968 | } |
969 | } |
970 | |
971 | // Write data |
972 | DataIndex = 0; |
973 | for(CDataInfo &DataInfo : m_vDatas) |
974 | { |
975 | if(DEBUG) |
976 | dbg_msg(sys: "datafile" , fmt: "writing data. DataIndex=%d CompressedSize=%d" , DataIndex, DataInfo.m_CompressedSize); |
977 | |
978 | io_write(io: m_File, buffer: DataInfo.m_pCompressedData, size: DataInfo.m_CompressedSize); |
979 | free(ptr: DataInfo.m_pCompressedData); |
980 | DataInfo.m_pCompressedData = nullptr; |
981 | ++DataIndex; |
982 | } |
983 | |
984 | io_close(io: m_File); |
985 | m_File = 0; |
986 | } |
987 | |