TrinityCore
DB2FileLoader.cpp
Go to the documentation of this file.
1/*
2 * This file is part of the TrinityCore Project. See AUTHORS file for Copyright information
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the
6 * Free Software Foundation; either version 2 of the License, or (at your
7 * option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along
15 * with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include "DB2FileLoader.h"
19#include "ByteConverter.h"
20#include "DB2Meta.h"
21#include "Errors.h"
22#include "Log.h"
23#include <limits>
24#include <sstream>
25#include <system_error>
26#include <utility>
27#include <cstring>
28
29enum class DB2ColumnCompression : uint32
30{
31 None,
32 Immediate,
33 CommonData,
34 Pallet,
35 PalletArray,
36 SignedImmediate
37};
38
39#pragma pack(push, 1)
40
41struct DB2FieldEntry
42{
43 int16 UnusedBits;
44 uint16 Offset;
45};
46
47struct DB2CatalogEntry
48{
49 uint32 FileOffset;
50 uint16 RecordSize;
51};
52
53struct DB2ColumnMeta
54{
55 uint16 BitOffset;
56 uint16 BitSize;
57 uint32 AdditionalDataSize;
58 DB2ColumnCompression CompressionType;
59 union
60 {
61 struct
62 {
63 uint32 BitOffset;
64 uint32 BitWidth;
65 bool Signed;
66 } immediate;
67
68 struct
69 {
70 uint32 Value;
71 } commonData;
72
73 struct
74 {
75 uint32 BitOffset;
76 uint32 BitWidth;
77 uint32 ArraySize;
78 } pallet;
79 } CompressionData;
80};
81
82struct DB2CommonValue
83{
84 uint32 RecordId;
85 uint32 Value;
86};
87
88struct DB2PalletValue
89{
90 uint32 Value;
91};
92
93struct DB2IndexDataInfo
94{
95 uint32 NumEntries;
96 uint32 MinId;
97 uint32 MaxId;
98};
99
100struct DB2IndexEntry
101{
102 uint32 ParentId;
103 uint32 RecordIndex;
104};
105
106#pragma pack(pop)
107
108struct DB2IndexData
109{
110 std::vector<DB2IndexEntry> Entries;
111};
112
113static bool IsKnownTactId(uint64 tactId)
114{
115 return tactId == 0 || tactId == DUMMY_KNOWN_TACT_ID;
116}
117
118uint32 DB2FileLoadInfo::GetStringFieldCount(bool localizedOnly) const
119{
120 uint32 stringFields = 0;
121 for (std::size_t i = 0; i < FieldCount; ++i)
122 if (Fields[i].Type == FT_STRING || (Fields[i].Type == FT_STRING_NOT_LOCALIZED && !localizedOnly))
123 ++stringFields;
124
125 return stringFields;
126}
127
128std::pair<int32, int32> DB2FileLoadInfo::GetFieldIndexByName(char const* fieldName) const
129{
130 std::size_t ourIndex = Meta->HasIndexFieldInData() ? 0 : 1;
131 for (uint32 i = 0; i < Meta->FieldCount; ++i)
132 {
133 for (uint8 arr = 0; arr < Meta->Fields[i].ArraySize; ++arr)
134 {
135 if (!strcmp(Fields[ourIndex].Name, fieldName))
136 return std::make_pair(int32(i), int32(arr));
137
138 ++ourIndex;
139 }
140 }
141
142 return std::make_pair(-1, -1);
143}
144
145int32 DB2FileLoadInfo::GetFieldIndexByMetaIndex(uint32 metaIndex) const
146{
147 ASSERT(metaIndex < Meta->FieldCount);
148 int32 ourIndex = Meta->HasIndexFieldInData() ? 0 : 1;
149 for (uint32 i = 0; i < metaIndex; ++i)
150 ourIndex += Meta->Fields[i].ArraySize;
151
152 return ourIndex;
153}
154
155DB2FileSource::DB2FileSource() = default;
156DB2FileSource::~DB2FileSource() = default;
157
158class DB2FileLoaderImpl
159{
160public:
161 DB2FileLoaderImpl() = default;
162 DB2FileLoaderImpl(DB2FileLoaderImpl const& other) = delete;
163 DB2FileLoaderImpl(DB2FileLoaderImpl&& other) noexcept = delete;
164 DB2FileLoaderImpl& operator=(DB2FileLoaderImpl const& other) = delete;
165 DB2FileLoaderImpl& operator=(DB2FileLoaderImpl&& other) noexcept = delete;
166 virtual ~DB2FileLoaderImpl() = default;
167 virtual void LoadColumnData(std::unique_ptr<DB2SectionHeader[]> sections, std::unique_ptr<DB2FieldEntry[]> fields, std::unique_ptr<DB2ColumnMeta[]> columnMeta,
168 std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> palletValues, std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> palletArrayValues,
169 std::unique_ptr<std::unordered_map<uint32, uint32>[]> commonValues) = 0;
170 virtual void SkipEncryptedSection(uint32 section) = 0;
171 virtual bool LoadTableData(DB2FileSource* source, uint32 section) = 0;
172 virtual bool LoadCatalogData(DB2FileSource* source, uint32 section) = 0;
173 virtual void SetAdditionalData(std::vector<uint32> idTable, std::vector<DB2RecordCopy> copyTable, std::vector<std::vector<DB2IndexData>> parentIndexes) = 0;
174 virtual char* AutoProduceData(uint32& indexTableSize, char**& indexTable) = 0;
175 virtual char* AutoProduceStrings(char** indexTable, uint32 indexTableSize, uint32 locale) = 0;
176 virtual void AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable) = 0;
177 virtual DB2Record GetRecord(uint32 recordNumber) const = 0;
178 virtual DB2RecordCopy GetRecordCopy(uint32 copyNumber) const = 0;
179 virtual uint32 GetRecordCount() const = 0;
180 virtual uint32 GetRecordCopyCount() const = 0;
181 virtual uint32 GetMinId() const = 0;
182 virtual uint32 GetMaxId() const = 0;
183 virtual DB2FileLoadInfo const* GetLoadInfo() const = 0;
184 virtual DB2SectionHeader& GetSection(uint32 section) const = 0;
185 virtual bool IsSignedField(uint32 field) const = 0;
186 virtual char const* GetExpectedSignMismatchReason(uint32 field) const = 0;
187
188private:
189 friend class DB2Record;
190 virtual unsigned char const* GetRawRecordData(uint32 recordNumber, uint32 const* section) const = 0;
191 virtual uint32 RecordGetId(uint8 const* record, uint32 recordIndex) const = 0;
192 virtual uint8 RecordGetUInt8(uint8 const* record, uint32 field, uint32 arrayIndex) const = 0;
193 virtual uint16 RecordGetUInt16(uint8 const* record, uint32 field, uint32 arrayIndex) const = 0;
194 virtual uint32 RecordGetUInt32(uint8 const* record, uint32 field, uint32 arrayIndex) const = 0;
195 virtual int32 RecordGetInt32(uint8 const* record, uint32 field, uint32 arrayIndex) const = 0;
196 virtual uint64 RecordGetUInt64(uint8 const* record, uint32 field, uint32 arrayIndex) const = 0;
197 virtual float RecordGetFloat(uint8 const* record, uint32 field, uint32 arrayIndex) const = 0;
198 virtual char const* RecordGetString(uint8 const* record, uint32 field, uint32 arrayIndex) const = 0;
199 virtual std::size_t* RecordCreateDetachedFieldOffsets(std::size_t* oldOffsets) const = 0;
200 virtual std::size_t* RecordCopyDetachedFieldOffsets(std::size_t* oldOffsets) const = 0;
201 virtual void RecordDestroyFieldOffsets(std::size_t*& fieldOffsets) const = 0;
202};
203
204class DB2FileLoaderRegularImpl final : public DB2FileLoaderImpl
205{
206public:
207 DB2FileLoaderRegularImpl(char const* fileName, DB2FileLoadInfo const* loadInfo, DB2Header const* header);
208 DB2FileLoaderRegularImpl(DB2FileLoaderRegularImpl const& other) = delete;
209 DB2FileLoaderRegularImpl(DB2FileLoaderRegularImpl&& other) noexcept = delete;
210 DB2FileLoaderRegularImpl& operator=(DB2FileLoaderRegularImpl const& other) = delete;
211 DB2FileLoaderRegularImpl& operator=(DB2FileLoaderRegularImpl&& other) noexcept = delete;
212 ~DB2FileLoaderRegularImpl();
213
214 void LoadColumnData(std::unique_ptr<DB2SectionHeader[]> sections, std::unique_ptr<DB2FieldEntry[]> fields, std::unique_ptr<DB2ColumnMeta[]> columnMeta,
215 std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> palletValues, std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> palletArrayValues,
216 std::unique_ptr<std::unordered_map<uint32, uint32>[]> commonValues) override;
217 void SkipEncryptedSection(uint32 /*section*/) override { }
218 bool LoadTableData(DB2FileSource* source, uint32 section) override;
219 bool LoadCatalogData(DB2FileSource* /*source*/, uint32 /*section*/) override { return true; }
220 void SetAdditionalData(std::vector<uint32> idTable, std::vector<DB2RecordCopy> copyTable, std::vector<std::vector<DB2IndexData>> parentIndexes) override;
221 char* AutoProduceData(uint32& indexTableSize, char**& indexTable) override;
222 char* AutoProduceStrings(char** indexTable, uint32 indexTableSize, uint32 locale) override;
223 void AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable) override;
224 DB2Record GetRecord(uint32 recordNumber) const override;
225 DB2RecordCopy GetRecordCopy(uint32 copyNumber) const override;
226 uint32 GetRecordCount() const override;
227 uint32 GetRecordCopyCount() const override;
228 uint32 GetMinId() const override;
229 uint32 GetMaxId() const override;
230 DB2FileLoadInfo const* GetLoadInfo() const override;
231 DB2SectionHeader& GetSection(uint32 section) const override;
232 bool IsSignedField(uint32 field) const override;
233 char const* GetExpectedSignMismatchReason(uint32 field) const override;
234
235private:
236 void FillParentLookup(char* dataTable);
237 uint32 GetRecordSection(uint32 recordNumber) const;
238 unsigned char const* GetRawRecordData(uint32 recordNumber, uint32 const* section) const override;
239 uint32 RecordGetId(uint8 const* record, uint32 recordIndex) const override;
240 uint8 RecordGetUInt8(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
241 uint16 RecordGetUInt16(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
242 uint32 RecordGetUInt32(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
243 int32 RecordGetInt32(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
244 uint64 RecordGetUInt64(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
245 float RecordGetFloat(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
246 char const* RecordGetString(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
247 template<typename T>
248 T RecordGetVarInt(uint8 const* record, uint32 field, uint32 arrayIndex) const;
249 uint64 RecordGetPackedValue(uint8 const* packedRecordData, uint32 bitWidth, uint32 bitOffset) const;
250 uint16 GetFieldOffset(uint32 field) const;
251 std::size_t* RecordCreateDetachedFieldOffsets(std::size_t* oldOffsets) const override;
252 std::size_t* RecordCopyDetachedFieldOffsets(std::size_t* oldOffsets) const override;
253 void RecordDestroyFieldOffsets(std::size_t*& fieldOffsets) const override;
254
255 char const* _fileName;
256 DB2FileLoadInfo const* _loadInfo;
257 DB2Header const* _header;
258 std::unique_ptr<uint8[]> _data;
259 uint8* _stringTable;
260 std::unique_ptr<DB2SectionHeader[]> _sections;
261 std::unique_ptr<DB2ColumnMeta[]> _columnMeta;
262 std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> _palletValues;
263 std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> _palletArrayValues;
264 std::unique_ptr<std::unordered_map<uint32, uint32>[]> _commonValues;
265 std::vector<uint32> _idTable;
266 std::vector<DB2RecordCopy> _copyTable;
267 std::vector<std::vector<DB2IndexData>> _parentIndexes;
268};
269
270class DB2FileLoaderSparseImpl final : public DB2FileLoaderImpl
271{
272public:
273 DB2FileLoaderSparseImpl(char const* fileName, DB2FileLoadInfo const* loadInfo, DB2Header const* header, DB2FileSource* source);
274 DB2FileLoaderSparseImpl(DB2FileLoaderSparseImpl const& other) = delete;
275 DB2FileLoaderSparseImpl(DB2FileLoaderSparseImpl&& other) noexcept = delete;
276 DB2FileLoaderSparseImpl& operator=(DB2FileLoaderSparseImpl const& other) = delete;
277 DB2FileLoaderSparseImpl& operator=(DB2FileLoaderSparseImpl&& other) noexcept = delete;
278 ~DB2FileLoaderSparseImpl();
279
280 void LoadColumnData(std::unique_ptr<DB2SectionHeader[]> sections, std::unique_ptr<DB2FieldEntry[]> fields, std::unique_ptr<DB2ColumnMeta[]> columnMeta,
281 std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> palletValues, std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> palletArrayValues,
282 std::unique_ptr<std::unordered_map<uint32, uint32>[]> commonValues) override;
283 void SkipEncryptedSection(uint32 section) override;
284 bool LoadTableData(DB2FileSource* /*source*/, uint32 /*section*/) override { return true; }
285 bool LoadCatalogData(DB2FileSource* source, uint32 section) override;
286 void SetAdditionalData(std::vector<uint32> idTable, std::vector<DB2RecordCopy> copyTable, std::vector<std::vector<DB2IndexData>> parentIndexes) override;
287 char* AutoProduceData(uint32& indexTableSize, char**& indexTable) override;
288 char* AutoProduceStrings(char** indexTable, uint32 indexTableSize, uint32 locale) override;
289 void AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable) override;
290 DB2Record GetRecord(uint32 recordNumber) const override;
291 DB2RecordCopy GetRecordCopy(uint32 copyNumber) const override;
292 uint32 GetRecordCount() const override;
293 uint32 GetRecordCopyCount() const override;
294 uint32 GetMinId() const override;
295 uint32 GetMaxId() const override;
296 DB2FileLoadInfo const* GetLoadInfo() const override;
297 DB2SectionHeader& GetSection(uint32 section) const override;
298 bool IsSignedField(uint32 field) const override;
299 char const* GetExpectedSignMismatchReason(uint32 field) const override;
300
301private:
302 void FillParentLookup(char* dataTable);
303 uint32 GetRecordSection(uint32 recordNumber) const;
304 unsigned char const* GetRawRecordData(uint32 recordNumber, uint32 const* section) const override;
305 uint32 RecordGetId(uint8 const* record, uint32 recordIndex) const override;
306 uint8 RecordGetUInt8(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
307 uint16 RecordGetUInt16(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
308 uint32 RecordGetUInt32(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
309 int32 RecordGetInt32(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
310 uint64 RecordGetUInt64(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
311 float RecordGetFloat(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
312 char const* RecordGetString(uint8 const* record, uint32 field, uint32 arrayIndex) const override;
313 uint32 RecordGetVarInt(uint8 const* record, uint32 field, uint32 arrayIndex, bool isSigned) const;
314 uint16 GetFieldOffset(uint32 field, uint32 arrayIndex) const;
315 uint16 GetFieldSize(uint32 field) const;
316 std::size_t* RecordCreateDetachedFieldOffsets(std::size_t* oldOffsets) const override;
317 std::size_t* RecordCopyDetachedFieldOffsets(std::size_t* oldOffsets) const override;
318 void RecordDestroyFieldOffsets(std::size_t*& fieldOffsets) const override;
319 void CalculateAndStoreFieldOffsets(uint8 const* rawRecord) const;
320
321 char const* _fileName;
322 DB2FileLoadInfo const* _loadInfo;
323 DB2Header const* _header;
324 DB2FileSource* const _source;
325 std::size_t _totalRecordSize;
326 uint16 _maxRecordSize;
327 std::unique_ptr<uint8[]> _recordBuffer;
328 std::unique_ptr<DB2SectionHeader[]> _sections;
329 std::unique_ptr<DB2FieldEntry[]> _fields;
330 std::unique_ptr<std::size_t[]> _fieldAndArrayOffsets;
331 std::vector<uint32> _catalogIds;
332 std::vector<DB2CatalogEntry> _catalog;
333 std::vector<DB2RecordCopy> _copyTable;
334 std::vector<std::vector<DB2IndexData>> _parentIndexes;
335};
336
337DB2FileLoaderRegularImpl::DB2FileLoaderRegularImpl(char const* fileName, DB2FileLoadInfo const* loadInfo, DB2Header const* header) :
338 _fileName(fileName),
339 _loadInfo(loadInfo),
340 _header(header),
341 _stringTable(nullptr)
342{
343}
344
345void DB2FileLoaderRegularImpl::LoadColumnData(std::unique_ptr<DB2SectionHeader[]> sections, std::unique_ptr<DB2FieldEntry[]> /*fields*/, std::unique_ptr<DB2ColumnMeta[]> columnMeta,
346 std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> palletValues, std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> palletArrayValues,
347 std::unique_ptr<std::unordered_map<uint32, uint32>[]> commonValues)
348{
349 _sections = std::move(sections);
350 _columnMeta = std::move(columnMeta);
351 _palletValues = std::move(palletValues);
352 _palletArrayValues = std::move(palletArrayValues);
353 _commonValues = std::move(commonValues);
354}
355
356bool DB2FileLoaderRegularImpl::LoadTableData(DB2FileSource* source, uint32 section)
357{
358 if (!_data)
359 {
360 _data = std::make_unique<uint8[]>(_header->RecordSize * _header->RecordCount + _header->StringTableSize + 8);
361 _stringTable = &_data[_header->RecordSize * _header->RecordCount];
362 }
363
364 uint32 sectionDataStart = 0;
365 uint32 sectionStringTableStart = 0;
366 for (uint32 i = 0; i < section; ++i)
367 {
368 sectionDataStart += _header->RecordSize * _sections[i].RecordCount;
369 sectionStringTableStart += _sections[i].StringTableSize;
370 }
371
372 if (_sections[section].RecordCount && !source->Read(&_data[sectionDataStart], _header->RecordSize * _sections[section].RecordCount))
373 return false;
374
375 if (_sections[section].StringTableSize && !source->Read(&_stringTable[sectionStringTableStart], _sections[section].StringTableSize))
376 return false;
377
378 return true;
379}
380
381void DB2FileLoaderRegularImpl::SetAdditionalData(std::vector<uint32> idTable, std::vector<DB2RecordCopy> copyTable, std::vector<std::vector<DB2IndexData>> parentIndexes)
382{
383 _idTable = std::move(idTable);
384 _copyTable = std::move(copyTable);
385 _parentIndexes = std::move(parentIndexes);
386}
387
388DB2FileLoaderRegularImpl::~DB2FileLoaderRegularImpl() = default;
389
390static char const* const EmptyDb2String = "";
391
392char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& indexTableSize, char**& indexTable)
393{
394 //get struct size and index pos
395 uint32 recordsize = _loadInfo->Meta->GetRecordSize();
396
397 uint32 maxi = GetMaxId() + 1;
398
399 using index_entry_t = char*;
400
401 indexTableSize = maxi;
402 indexTable = new index_entry_t[maxi];
403 memset(indexTable, 0, maxi * sizeof(index_entry_t));
404
405 char* dataTable = new char[(_header->RecordCount + _copyTable.size()) * recordsize];
406
407 uint32 offset = 0;
408 uint32 recordIndex = 0;
409
410 for (uint32 section = 0; section < _header->SectionCount; ++section)
411 {
412 DB2SectionHeader const& sectionHeader = GetSection(section);
413 if (!IsKnownTactId(sectionHeader.TactId))
414 {
415 offset += recordsize * sectionHeader.RecordCount;
416 recordIndex += sectionHeader.RecordCount;
417 continue;
418 }
419
420 for (uint32 sr = 0; sr < sectionHeader.RecordCount; ++sr, ++recordIndex)
421 {
422 unsigned char const* rawRecord = GetRawRecordData(recordIndex, &section);
423 if (!rawRecord)
424 continue;
425
426 uint32 indexVal = RecordGetId(rawRecord, recordIndex);
427
428 indexTable[indexVal] = &dataTable[offset];
429
430 uint32 fieldIndex = 0;
431 if (!_loadInfo->Meta->HasIndexFieldInData())
432 {
433 *reinterpret_cast<uint32*>(&dataTable[offset]) = indexVal;
434 offset += 4;
435 ++fieldIndex;
436 }
437
438 for (uint32 x = 0; x < _header->FieldCount; ++x)
439 {
440 for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z)
441 {
442 switch (_loadInfo->Fields[fieldIndex].Type)
443 {
444 case FT_FLOAT:
445 *reinterpret_cast<float*>(&dataTable[offset]) = RecordGetFloat(rawRecord, x, z);
446 offset += 4;
447 break;
448 case FT_INT:
449 *reinterpret_cast<uint32*>(&dataTable[offset]) = RecordGetVarInt<uint32>(rawRecord, x, z);
450 offset += 4;
451 break;
452 case FT_BYTE:
453 *reinterpret_cast<uint8*>(&dataTable[offset]) = RecordGetUInt8(rawRecord, x, z);
454 offset += 1;
455 break;
456 case FT_SHORT:
457 *reinterpret_cast<uint16*>(&dataTable[offset]) = RecordGetUInt16(rawRecord, x, z);
458 offset += 2;
459 break;
460 case FT_LONG:
461 *reinterpret_cast<uint64*>(&dataTable[offset]) = RecordGetUInt64(rawRecord, x, z);
462 offset += 8;
463 break;
464 case FT_STRING:
465 for (char const*& localeStr : reinterpret_cast<LocalizedString*>(&dataTable[offset])->Str)
466 localeStr = EmptyDb2String;
467
468 offset += sizeof(LocalizedString);
469 break;
470 case FT_STRING_NOT_LOCALIZED:
471 *reinterpret_cast<char const**>(&dataTable[offset]) = EmptyDb2String;
472 offset += sizeof(char*);
473 break;
474 default:
475 ABORT_MSG("Unknown format character '%c' found in %s meta for field %s",
476 _loadInfo->Fields[fieldIndex].Type, _fileName, _loadInfo->Fields[fieldIndex].Name);
477 break;
478 }
479 ++fieldIndex;
480 }
481 }
482
483 for (uint32 x = _header->FieldCount; x < _loadInfo->Meta->FieldCount; ++x)
484 {
485 for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z)
486 {
487 switch (_loadInfo->Fields[fieldIndex].Type)
488 {
489 case FT_INT:
490 *reinterpret_cast<uint32*>(&dataTable[offset]) = 0;
491 offset += 4;
492 break;
493 case FT_BYTE:
494 *reinterpret_cast<uint8*>(&dataTable[offset]) = 0;
495 offset += 1;
496 break;
497 case FT_SHORT:
498 *reinterpret_cast<uint16*>(&dataTable[offset]) = 0;
499 offset += 2;
500 break;
501 default:
502 ABORT_MSG("Unknown format character '%c' found in %s meta for parent field %s",
503 _loadInfo->Fields[fieldIndex].Type, _fileName, _loadInfo->Fields[fieldIndex].Name);
504 break;
505 }
506 ++fieldIndex;
507 }
508 }
509 }
510 }
511
512 if (!_parentIndexes.empty())
513 FillParentLookup(dataTable);
514
515 return dataTable;
516}
517
518char* DB2FileLoaderRegularImpl::AutoProduceStrings(char** indexTable, uint32 indexTableSize, uint32 locale)
519{
520 if (!(_header->Locale & (1 << locale)))
521 {
522 char const* sep = "";
523 std::ostringstream str;
524 for (uint32 i = 0; i < TOTAL_LOCALES; ++i)
525 {
526 if (_header->Locale & (1 << i))
527 {
528 str << sep << localeNames[i];
529 sep = ", ";
530 }
531 }
532
533 TC_LOG_ERROR("", "Attempted to load {} which has locales {} as {}. Check if you placed your localized db2 files in correct directory.", _fileName, str.str(), localeNames[locale]);
534 return nullptr;
535 }
536
537 if (!_loadInfo->GetStringFieldCount(false))
538 return nullptr;
539
540 char* stringPool = new char[_header->StringTableSize];
541 memcpy(stringPool, _stringTable, _header->StringTableSize);
542
543 uint32 y = 0;
544
545 for (uint32 section = 0; section < _header->SectionCount; ++section)
546 {
547 DB2SectionHeader const& sectionHeader = GetSection(section);
548 if (!IsKnownTactId(sectionHeader.TactId))
549 {
550 y += sectionHeader.RecordCount;
551 continue;
552 }
553
554 for (uint32 sr = 0; sr < sectionHeader.RecordCount; ++sr, ++y)
555 {
556 unsigned char const* rawRecord = GetRawRecordData(y, &section);
557 if (!rawRecord)
558 continue;
559
560 uint32 indexVal = RecordGetId(rawRecord, y);
561 if (indexVal >= indexTableSize)
562 continue;
563
564 char* recordData = indexTable[indexVal];
565 if (!recordData)
566 continue;
567
568 uint32 offset = 0;
569 uint32 fieldIndex = 0;
570 if (!_loadInfo->Meta->HasIndexFieldInData())
571 {
572 offset += 4;
573 ++fieldIndex;
574 }
575
576 for (uint32 x = 0; x < _loadInfo->Meta->FieldCount; ++x)
577 {
578 for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z)
579 {
580 switch (_loadInfo->Fields[fieldIndex].Type)
581 {
582 case FT_FLOAT:
583 case FT_INT:
584 offset += 4;
585 break;
586 case FT_BYTE:
587 offset += 1;
588 break;
589 case FT_SHORT:
590 offset += 2;
591 break;
592 case FT_LONG:
593 offset += 8;
594 break;
595 case FT_STRING:
596 if (char const* string = RecordGetString(rawRecord, x, z))
597 reinterpret_cast<LocalizedString*>(&recordData[offset])->Str[locale] = stringPool + (string - reinterpret_cast<char const*>(_stringTable));
598
599 offset += sizeof(LocalizedString);
600 break;
601 case FT_STRING_NOT_LOCALIZED:
602 if (char const* string = RecordGetString(rawRecord, x, z))
603 *reinterpret_cast<char**>(&recordData[offset]) = stringPool + (string - reinterpret_cast<char const*>(_stringTable));
604
605 offset += sizeof(char*);
606 break;
607 default:
608 ABORT_MSG("Unknown format character '%c' found in %s meta for field %s",
609 _loadInfo->Fields[fieldIndex].Type, _fileName, _loadInfo->Fields[fieldIndex].Name);
610 break;
611 }
612 ++fieldIndex;
613 }
614 }
615 }
616 }
617
618 return stringPool;
619}
620
621void DB2FileLoaderRegularImpl::AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable)
622{
623 uint32 recordCopies = GetRecordCopyCount();
624 if (!recordCopies)
625 return;
626
627 uint32 recordsize = _loadInfo->Meta->GetRecordSize();
628 uint32 offset = _header->RecordCount * recordsize;
629 uint32 idFieldOffset = _loadInfo->Meta->HasIndexFieldInData() ? GetFieldOffset(_loadInfo->Meta->GetIndexField()) : 0;
630 for (uint32 c = 0; c < recordCopies; ++c)
631 {
632 DB2RecordCopy copy = GetRecordCopy(c);
633 if (copy.SourceRowId && copy.SourceRowId < records && copy.NewRowId < records && indexTable[copy.SourceRowId])
634 {
635 indexTable[copy.NewRowId] = &dataTable[offset];
636 memcpy(indexTable[copy.NewRowId], indexTable[copy.SourceRowId], recordsize);
637 *reinterpret_cast<uint32*>(&dataTable[offset + idFieldOffset]) = copy.NewRowId;
638 offset += recordsize;
639 }
640 }
641}
642
643void DB2FileLoaderRegularImpl::FillParentLookup(char* dataTable)
644{
645 int32 parentIdOffset = _loadInfo->Meta->GetParentIndexFieldOffset();
646 uint32 recordSize = _loadInfo->Meta->GetRecordSize();
647 uint32 recordIndexOffset = 0;
648 for (uint32 i = 0; i < _header->SectionCount; ++i)
649 {
650 DB2SectionHeader const& section = GetSection(i);
651 if (IsKnownTactId(section.TactId))
652 {
653 for (std::size_t j = 0; j < _parentIndexes[i][0].Entries.size(); ++j)
654 {
655 uint32 parentId = _parentIndexes[i][0].Entries[j].ParentId;
656 uint32 recordIndex = _parentIndexes[i][0].Entries[j].RecordIndex + recordIndexOffset;
657 char* recordData = &dataTable[recordIndex * recordSize];
658
659 switch (_loadInfo->Meta->Fields[_loadInfo->Meta->ParentIndexField].Type)
660 {
661 case FT_SHORT:
662 {
663 if (_loadInfo->Meta->ParentIndexField >= int32(_loadInfo->Meta->FileFieldCount))
664 {
665 // extra field at the end
666 *reinterpret_cast<uint32*>(&recordData[parentIdOffset]) = parentId;
667 }
668 else
669 {
670 // in data block, must fit
671 ASSERT(parentId <= std::numeric_limits<uint16>::max(), "ParentId value %u does not fit into uint16 field (%s in %s)",
672 parentId, _loadInfo->Fields[_loadInfo->GetFieldIndexByMetaIndex(_loadInfo->Meta->ParentIndexField)].Name, _fileName);
673 *reinterpret_cast<uint16*>(&recordData[parentIdOffset]) = parentId;
674 }
675 break;
676 }
677 case FT_BYTE:
678 {
679 if (_loadInfo->Meta->ParentIndexField >= int32(_loadInfo->Meta->FileFieldCount))
680 {
681 // extra field at the end
682 *reinterpret_cast<uint32*>(&recordData[parentIdOffset]) = parentId;
683 }
684 else
685 {
686 // in data block, must fit
687 ASSERT(parentId <= std::numeric_limits<uint8>::max(), "ParentId value %u does not fit into uint8 field (%s in %s)",
688 parentId, _loadInfo->Fields[_loadInfo->GetFieldIndexByMetaIndex(_loadInfo->Meta->ParentIndexField)].Name, _fileName);
689 *reinterpret_cast<uint8*>(&recordData[parentIdOffset]) = parentId;
690 }
691 break;
692 }
693 case FT_INT:
694 *reinterpret_cast<uint32*>(&recordData[parentIdOffset]) = parentId;
695 break;
696 default:
697 ABORT_MSG("Unhandled parent id type '%c' found in %s", _loadInfo->Meta->Fields[_loadInfo->Meta->ParentIndexField].Type, _fileName);
698 break;
699 }
700 }
701 }
702 recordIndexOffset += section.RecordCount;
703 }
704}
705
706DB2Record DB2FileLoaderRegularImpl::GetRecord(uint32 recordNumber) const
707{
708 return DB2Record(*this, recordNumber, nullptr);
709}
710
711DB2RecordCopy DB2FileLoaderRegularImpl::GetRecordCopy(uint32 copyNumber) const
712{
713 if (copyNumber >= GetRecordCopyCount())
714 return DB2RecordCopy{};
715
716 return _copyTable[copyNumber];
717}
718
719uint32 DB2FileLoaderRegularImpl::GetRecordCount() const
720{
721 return _header->RecordCount;
722}
723
724uint32 DB2FileLoaderRegularImpl::GetRecordCopyCount() const
725{
726 return _copyTable.size();
727}
728
729uint32 DB2FileLoaderRegularImpl::GetRecordSection(uint32 recordNumber) const
730{
731 uint32 section = 0;
732 for (; section < _header->SectionCount; ++section)
733 {
734 DB2SectionHeader const& sectionHeader = GetSection(section);
735 if (recordNumber < sectionHeader.RecordCount)
736 break;
737
738 recordNumber -= sectionHeader.RecordCount;
739 }
740
741 return section;
742}
743
744unsigned char const* DB2FileLoaderRegularImpl::GetRawRecordData(uint32 recordNumber, uint32 const* section) const
745{
746 if (recordNumber >= _header->RecordCount)
747 return nullptr;
748
749 if (!IsKnownTactId(GetSection(section ? *section : GetRecordSection(recordNumber)).TactId))
750 return nullptr;
751
752 return &_data[recordNumber * _header->RecordSize];
753}
754
755uint32 DB2FileLoaderRegularImpl::RecordGetId(uint8 const* record, uint32 recordIndex) const
756{
757 if (_loadInfo->Meta->HasIndexFieldInData())
758 return RecordGetVarInt<uint32>(record, _loadInfo->Meta->GetIndexField(), 0);
759
760 return _idTable[recordIndex];
761}
762
763uint8 DB2FileLoaderRegularImpl::RecordGetUInt8(uint8 const* record, uint32 field, uint32 arrayIndex) const
764{
765 return RecordGetVarInt<uint8>(record, field, arrayIndex);
766}
767
768uint16 DB2FileLoaderRegularImpl::RecordGetUInt16(uint8 const* record, uint32 field, uint32 arrayIndex) const
769{
770 return RecordGetVarInt<uint16>(record, field, arrayIndex);
771}
772
773uint32 DB2FileLoaderRegularImpl::RecordGetUInt32(uint8 const* record, uint32 field, uint32 arrayIndex) const
774{
775 return RecordGetVarInt<uint32>(record, field, arrayIndex);
776}
777
778int32 DB2FileLoaderRegularImpl::RecordGetInt32(uint8 const* record, uint32 field, uint32 arrayIndex) const
779{
780 return RecordGetVarInt<int32>(record, field, arrayIndex);
781}
782
783uint64 DB2FileLoaderRegularImpl::RecordGetUInt64(uint8 const* record, uint32 field, uint32 arrayIndex) const
784{
785 return RecordGetVarInt<uint64>(record, field, arrayIndex);
786}
787
788float DB2FileLoaderRegularImpl::RecordGetFloat(uint8 const* record, uint32 field, uint32 arrayIndex) const
789{
790 return RecordGetVarInt<float>(record, field, arrayIndex);
791}
792
793char const* DB2FileLoaderRegularImpl::RecordGetString(uint8 const* record, uint32 field, uint32 arrayIndex) const
794{
795 uint32 fieldOffset = GetFieldOffset(field) + sizeof(uint32) * arrayIndex;
796 uint32 stringOffset = RecordGetVarInt<uint32>(record, field, arrayIndex);
797 ASSERT(stringOffset < _header->RecordSize * _header->RecordCount + _header->StringTableSize);
798 return stringOffset ? reinterpret_cast<char const*>(record + fieldOffset + stringOffset) : nullptr;
799}
800
801template<typename T>
802T DB2FileLoaderRegularImpl::RecordGetVarInt(uint8 const* record, uint32 field, uint32 arrayIndex) const
803{
804 ASSERT(field < _header->FieldCount);
805 DB2ColumnCompression compressionType = _columnMeta ? _columnMeta[field].CompressionType : DB2ColumnCompression::None;
806 switch (compressionType)
807 {
808 case DB2ColumnCompression::None:
809 {
810 T val = *reinterpret_cast<T const*>(record + GetFieldOffset(field) + sizeof(T) * arrayIndex);
811 EndianConvert(val);
812 return val;
813 }
814 case DB2ColumnCompression::Immediate:
815 {
816 ASSERT(arrayIndex == 0);
817 uint64 immediateValue = RecordGetPackedValue(record + GetFieldOffset(field),
818 _columnMeta[field].CompressionData.immediate.BitWidth, _columnMeta[field].CompressionData.immediate.BitOffset);
819 EndianConvert(immediateValue);
820 T value;
821 memcpy(&value, &immediateValue, std::min(sizeof(T), sizeof(immediateValue)));
822 return value;
823 }
824 case DB2ColumnCompression::CommonData:
825 {
826 uint32 id = RecordGetId(record, (record - _data.get()) / _header->RecordSize);
827 T value;
828 auto valueItr = _commonValues[field].find(id);
829 if (valueItr != _commonValues[field].end())
830 memcpy(&value, &valueItr->second, std::min(sizeof(T), sizeof(uint32)));
831 else
832 memcpy(&value, &_columnMeta[field].CompressionData.commonData.Value, std::min(sizeof(T), sizeof(uint32)));
833 return value;
834 }
835 case DB2ColumnCompression::Pallet:
836 {
837 ASSERT(arrayIndex == 0);
838 uint64 palletIndex = RecordGetPackedValue(record + GetFieldOffset(field),
839 _columnMeta[field].CompressionData.pallet.BitWidth, _columnMeta[field].CompressionData.pallet.BitOffset);
840 EndianConvert(palletIndex);
841 uint32 palletValue = _palletValues[field][palletIndex].Value;
842 EndianConvert(palletValue);
843 T value;
844 memcpy(&value, &palletValue, std::min(sizeof(T), sizeof(palletValue)));
845 return value;
846 }
847 case DB2ColumnCompression::PalletArray:
848 {
849 uint64 palletIndex = RecordGetPackedValue(record + GetFieldOffset(field),
850 _columnMeta[field].CompressionData.pallet.BitWidth, _columnMeta[field].CompressionData.pallet.BitOffset);
851 EndianConvert(palletIndex);
852 uint32 palletValue = _palletArrayValues[field][palletIndex * _columnMeta[field].CompressionData.pallet.ArraySize + arrayIndex].Value;
853 EndianConvert(palletValue);
854 T value;
855 memcpy(&value, &palletValue, std::min(sizeof(T), sizeof(palletValue)));
856 return value;
857 }
858 case DB2ColumnCompression::SignedImmediate:
859 {
860 ASSERT(arrayIndex == 0);
861 uint64 immediateValue = RecordGetPackedValue(record + GetFieldOffset(field),
862 _columnMeta[field].CompressionData.immediate.BitWidth, _columnMeta[field].CompressionData.immediate.BitOffset);
863 EndianConvert(immediateValue);
864 uint64 mask = UI64LIT(1) << (_columnMeta[field].CompressionData.immediate.BitWidth - 1);
865 immediateValue = (immediateValue ^ mask) - mask;
866 T value;
867 memcpy(&value, &immediateValue, std::min(sizeof(T), sizeof(immediateValue)));
868 return value;
869 }
870 default:
871 ABORT_MSG("Unhandled compression type %u in %s", uint32(_columnMeta[field].CompressionType), _fileName);
872 break;
873 }
874
875 return 0;
876}
877
878uint64 DB2FileLoaderRegularImpl::RecordGetPackedValue(uint8 const* packedRecordData, uint32 bitWidth, uint32 bitOffset) const
879{
880 uint32 bitsToRead = bitOffset & 7;
881 return *reinterpret_cast<uint64 const*>(packedRecordData) << (64 - bitsToRead - bitWidth) >> (64 - bitWidth);
882}
883
884uint16 DB2FileLoaderRegularImpl::GetFieldOffset(uint32 field) const
885{
886 ASSERT(field < _header->FieldCount);
887 DB2ColumnCompression compressionType = _columnMeta ? _columnMeta[field].CompressionType : DB2ColumnCompression::None;
888 switch (compressionType)
889 {
890 case DB2ColumnCompression::None:
891 return _columnMeta[field].BitOffset / 8;
892 case DB2ColumnCompression::Immediate:
893 case DB2ColumnCompression::SignedImmediate:
894 return _columnMeta[field].CompressionData.immediate.BitOffset / 8 + _header->PackedDataOffset;
895 case DB2ColumnCompression::CommonData:
896 return 0xFFFF;
897 case DB2ColumnCompression::Pallet:
898 case DB2ColumnCompression::PalletArray:
899 return _columnMeta[field].CompressionData.pallet.BitOffset / 8 + _header->PackedDataOffset;
900 default:
901 ABORT_MSG("Unhandled compression type %u in %s", uint32(_columnMeta[field].CompressionType), _fileName);
902 break;
903 }
904
905 return 0;
906}
907
908std::size_t* DB2FileLoaderRegularImpl::RecordCreateDetachedFieldOffsets(std::size_t* /*oldOffsets*/) const
909{
910 return nullptr;
911}
912
913std::size_t* DB2FileLoaderRegularImpl::RecordCopyDetachedFieldOffsets(std::size_t* /*oldOffsets*/) const
914{
915 return nullptr;
916}
917
918void DB2FileLoaderRegularImpl::RecordDestroyFieldOffsets(std::size_t*& /*fieldOffsets*/) const
919{
920}
921
922uint32 DB2FileLoaderRegularImpl::GetMinId() const
923{
924 uint32 minId = std::numeric_limits<uint32>::max();
925 for (uint32 row = 0; row < _header->RecordCount; ++row)
926 {
927 unsigned char const* rawRecord = GetRawRecordData(row, nullptr);
928 if (!rawRecord)
929 continue;
930
931 uint32 id = RecordGetId(rawRecord, row);
932 if (id < minId)
933 minId = id;
934 }
935
936 for (uint32 copy = 0; copy < GetRecordCopyCount(); ++copy)
937 {
938 uint32 id = GetRecordCopy(copy).NewRowId;
939 if (id < minId)
940 minId = id;
941 }
942
943 if (minId == std::numeric_limits<uint32>::max())
944 minId = 0;
945
946 ASSERT(minId >= _header->MinId);
947 return minId;
948}
949
950uint32 DB2FileLoaderRegularImpl::GetMaxId() const
951{
952 uint32 maxId = 0;
953 for (uint32 row = 0; row < _header->RecordCount; ++row)
954 {
955 unsigned char const* rawRecord = GetRawRecordData(row, nullptr);
956 if (!rawRecord)
957 continue;
958
959 uint32 id = RecordGetId(rawRecord, row);
960 if (id > maxId)
961 maxId = id;
962 }
963
964 for (uint32 copy = 0; copy < GetRecordCopyCount(); ++copy)
965 {
966 uint32 id = GetRecordCopy(copy).NewRowId;
967 if (id > maxId)
968 maxId = id;
969 }
970
971 ASSERT(maxId <= _header->MaxId);
972 return maxId;
973}
974
975DB2FileLoadInfo const* DB2FileLoaderRegularImpl::GetLoadInfo() const
976{
977 return _loadInfo;
978}
979
980DB2SectionHeader& DB2FileLoaderRegularImpl::GetSection(uint32 section) const
981{
982 return _sections[section];
983}
984
985bool DB2FileLoaderRegularImpl::IsSignedField(uint32 field) const
986{
987 if (field >= _header->TotalFieldCount)
988 {
989 ASSERT(field == _header->TotalFieldCount);
990 ASSERT(int32(field) == _loadInfo->Meta->ParentIndexField);
991 return _loadInfo->Meta->IsSignedField(field);
992 }
993
994 DB2ColumnCompression compressionType = _columnMeta ? _columnMeta[field].CompressionType : DB2ColumnCompression::None;
995 switch (compressionType)
996 {
997 case DB2ColumnCompression::None:
998 case DB2ColumnCompression::CommonData:
999 case DB2ColumnCompression::Pallet:
1000 case DB2ColumnCompression::PalletArray:
1001 return _loadInfo->Meta->IsSignedField(field);
1002 case DB2ColumnCompression::SignedImmediate:
1003 return field != uint32(_loadInfo->Meta->IndexField);
1004 case DB2ColumnCompression::Immediate:
1005 return false;
1006 default:
1007 ABORT_MSG("Unhandled compression type %u in %s", uint32(_columnMeta[field].CompressionType), _fileName);
1008 break;
1009 }
1010
1011 return false;
1012}
1013
1014char const* DB2FileLoaderRegularImpl::GetExpectedSignMismatchReason(uint32 field) const
1015{
1016 if (field >= _header->TotalFieldCount)
1017 {
1018 ASSERT(field == _header->TotalFieldCount);
1019 ASSERT(int32(field) == _loadInfo->Meta->ParentIndexField);
1020 return " (ParentIndexField must always be unsigned)";
1021 }
1022
1023 DB2ColumnCompression compressionType = _columnMeta ? _columnMeta[field].CompressionType : DB2ColumnCompression::None;
1024 switch (compressionType)
1025 {
1026 case DB2ColumnCompression::None:
1027 case DB2ColumnCompression::CommonData:
1028 case DB2ColumnCompression::Pallet:
1029 case DB2ColumnCompression::PalletArray:
1030 if (int32(field) == _loadInfo->Meta->IndexField)
1031 return " (IndexField must always be unsigned)";
1032 if (int32(field) == _loadInfo->Meta->ParentIndexField)
1033 return " (ParentIndexField must always be unsigned)";
1034 return "";
1035 case DB2ColumnCompression::SignedImmediate:
1036 return " (CompressionType is SignedImmediate)";
1037 case DB2ColumnCompression::Immediate:
1038 return " (CompressionType is Immediate)";
1039 default:
1040 ABORT_MSG("Unhandled compression type %u in %s", uint32(_columnMeta[field].CompressionType), _fileName);
1041 break;
1042 }
1043
1044 return "";
1045}
1046
1047DB2FileLoaderSparseImpl::DB2FileLoaderSparseImpl(char const* fileName, DB2FileLoadInfo const* loadInfo, DB2Header const* header, DB2FileSource* source) :
1048 _fileName(fileName),
1049 _loadInfo(loadInfo),
1050 _header(header),
1051 _source(source),
1052 _totalRecordSize(0),
1053 _maxRecordSize(0),
1054 _fieldAndArrayOffsets(loadInfo ? (std::make_unique<std::size_t[]>(loadInfo->Meta->FieldCount + loadInfo->FieldCount - (!loadInfo->Meta->HasIndexFieldInData() ? 1 : 0))) : nullptr)
1055{
1056}
1057
1058DB2FileLoaderSparseImpl::~DB2FileLoaderSparseImpl() = default;
1059
1060void DB2FileLoaderSparseImpl::LoadColumnData(std::unique_ptr<DB2SectionHeader[]> sections, std::unique_ptr<DB2FieldEntry[]> fields, std::unique_ptr<DB2ColumnMeta[]> /*columnMeta*/,
1061 std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> /*palletValues*/, std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> /*palletArrayValues*/,
1062 std::unique_ptr<std::unordered_map<uint32, uint32>[]> /*commonValues*/)
1063{
1064 _sections = std::move(sections);
1065 _fields = std::move(fields);
1066}
1067
1068void DB2FileLoaderSparseImpl::SkipEncryptedSection(uint32 section)
1069{
1070 _catalogIds.resize(_catalogIds.size() + _sections[section].CatalogDataCount);
1071 _catalog.resize(_catalog.size() + _sections[section].CatalogDataCount);
1072}
1073
1074bool DB2FileLoaderSparseImpl::LoadCatalogData(DB2FileSource* source, uint32 section)
1075{
1076 source->SetPosition(_sections[section].CatalogDataOffset);
1077
1078 std::size_t oldSize = _catalog.size();
1079 _catalogIds.resize(oldSize + _sections[section].CatalogDataCount);
1080 if (!source->Read(&_catalogIds[oldSize], sizeof(uint32) * _sections[section].CatalogDataCount))
1081 return false;
1082
1083 if (_sections[section].CopyTableCount)
1084 {
1085 std::size_t oldCopyTableSize = _copyTable.size();
1086 _copyTable.resize(oldCopyTableSize + _sections[section].CopyTableCount);
1087 if (!source->Read(&_copyTable[oldCopyTableSize], sizeof(DB2RecordCopy) * _sections[section].CopyTableCount))
1088 return false;
1089 }
1090
1091 _catalog.resize(oldSize + _sections[section].CatalogDataCount);
1092 if (!source->Read(&_catalog[oldSize], sizeof(DB2CatalogEntry) * _sections[section].CatalogDataCount))
1093 return false;
1094
1095 for (uint32 i = 0; i < _sections[section].CatalogDataCount; ++i)
1096 {
1097 _totalRecordSize += _catalog[oldSize + i].RecordSize;
1098 _maxRecordSize = std::max(_maxRecordSize, _catalog[oldSize + i].RecordSize);
1099 }
1100
1101 return true;
1102}
1103
1104void DB2FileLoaderSparseImpl::SetAdditionalData(std::vector<uint32> /*idTable*/, std::vector<DB2RecordCopy> /*copyTable*/, std::vector<std::vector<DB2IndexData>> parentIndexes)
1105{
1106 _parentIndexes = std::move(parentIndexes);
1107 _recordBuffer = std::make_unique<uint8[]>(_maxRecordSize);
1108}
1109
1110char* DB2FileLoaderSparseImpl::AutoProduceData(uint32& indexTableSize, char**& indexTable)
1111{
1112 if (_loadInfo->Meta->FieldCount != _header->FieldCount)
1113 throw DB2FileLoadException(Trinity::StringFormat("Found unsupported parent index in sparse db2 {}", _fileName));
1114
1115 //get struct size and index pos
1116 uint32 recordsize = _loadInfo->Meta->GetRecordSize();
1117 uint32 records = _catalog.size();
1118
1119 using index_entry_t = char*;
1120
1121 indexTableSize = _header->MaxId + 1;
1122 indexTable = new index_entry_t[indexTableSize];
1123 memset(indexTable, 0, indexTableSize * sizeof(index_entry_t));
1124
1125 char* dataTable = new char[(records + _copyTable.size()) * recordsize];
1126 memset(dataTable, 0, (records + _copyTable.size()) * recordsize);
1127
1128 uint32 offset = 0;
1129 uint32 y = 0;
1130
1131 for (uint32 section = 0; section < _header->SectionCount; ++section)
1132 {
1133 DB2SectionHeader const& sectionHeader = GetSection(section);
1134 if (!IsKnownTactId(sectionHeader.TactId))
1135 {
1136 offset += recordsize * sectionHeader.RecordCount;
1137 y += sectionHeader.RecordCount;
1138 continue;
1139 }
1140
1141 for (uint32 sr = 0; sr < sectionHeader.CatalogDataCount; ++sr, ++y)
1142 {
1143 unsigned char const* rawRecord = GetRawRecordData(y, &section);
1144 if (!rawRecord)
1145 continue;
1146
1147 uint32 indexVal = _catalogIds[y];
1148 indexTable[indexVal] = &dataTable[offset];
1149
1150 uint32 fieldIndex = 0;
1151 if (!_loadInfo->Meta->HasIndexFieldInData())
1152 {
1153 *reinterpret_cast<uint32*>(&dataTable[offset]) = indexVal;
1154 offset += 4;
1155 ++fieldIndex;
1156 }
1157
1158 for (uint32 x = 0; x < _header->FieldCount; ++x)
1159 {
1160 for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z)
1161 {
1162 switch (_loadInfo->Fields[fieldIndex].Type)
1163 {
1164 case FT_FLOAT:
1165 *reinterpret_cast<float*>(&dataTable[offset]) = RecordGetFloat(rawRecord, x, z);
1166 offset += 4;
1167 break;
1168 case FT_INT:
1169 *reinterpret_cast<uint32*>(&dataTable[offset]) = RecordGetVarInt(rawRecord, x, z, _loadInfo->Fields[fieldIndex].IsSigned);
1170 offset += 4;
1171 break;
1172 case FT_BYTE:
1173 *reinterpret_cast<uint8*>(&dataTable[offset]) = RecordGetUInt8(rawRecord, x, z);
1174 offset += 1;
1175 break;
1176 case FT_SHORT:
1177 *reinterpret_cast<uint16*>(&dataTable[offset]) = RecordGetUInt16(rawRecord, x, z);
1178 offset += 2;
1179 break;
1180 case FT_LONG:
1181 *reinterpret_cast<uint64*>(&dataTable[offset]) = RecordGetUInt64(rawRecord, x, z);
1182 offset += 8;
1183 break;
1184 case FT_STRING:
1185 for (char const*& localeStr : reinterpret_cast<LocalizedString*>(&dataTable[offset])->Str)
1186 localeStr = EmptyDb2String;
1187
1188 offset += sizeof(LocalizedString);
1189 break;
1190 case FT_STRING_NOT_LOCALIZED:
1191 *reinterpret_cast<char const**>(&dataTable[offset]) = EmptyDb2String;
1192 offset += sizeof(char*);
1193 break;
1194 default:
1195 ABORT_MSG("Unknown format character '%c' found in %s meta for field %s",
1196 _loadInfo->Fields[fieldIndex].Type, _fileName, _loadInfo->Fields[fieldIndex].Name);
1197 break;
1198 }
1199 ++fieldIndex;
1200 }
1201 }
1202
1203 for (uint32 x = _header->FieldCount; x < _loadInfo->Meta->FieldCount; ++x)
1204 {
1205 for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z)
1206 {
1207 switch (_loadInfo->Fields[fieldIndex].Type)
1208 {
1209 case FT_INT:
1210 *reinterpret_cast<uint32*>(&dataTable[offset]) = 0;
1211 offset += 4;
1212 break;
1213 case FT_BYTE:
1214 *reinterpret_cast<uint8*>(&dataTable[offset]) = 0;
1215 offset += 1;
1216 break;
1217 case FT_SHORT:
1218 *reinterpret_cast<uint16*>(&dataTable[offset]) = 0;
1219 offset += 2;
1220 break;
1221 default:
1222 ABORT_MSG("Unknown format character '%c' found in %s meta for parent field %s",
1223 _loadInfo->Fields[fieldIndex].Type, _fileName, _loadInfo->Fields[fieldIndex].Name);
1224 break;
1225 }
1226 ++fieldIndex;
1227 }
1228 }
1229 }
1230 }
1231
1232 return dataTable;
1233}
1234
1235char* DB2FileLoaderSparseImpl::AutoProduceStrings(char** indexTable, uint32 indexTableSize, uint32 locale)
1236{
1237 if (_loadInfo->Meta->FieldCount != _header->FieldCount)
1238 throw DB2FileLoadException(Trinity::StringFormat("Found unsupported parent index in sparse db2 {}", _fileName));
1239
1240 if (!(_header->Locale & (1 << locale)))
1241 {
1242 char const* sep = "";
1243 std::ostringstream str;
1244 for (uint32 i = 0; i < TOTAL_LOCALES; ++i)
1245 {
1246 if (_header->Locale & (1 << i))
1247 {
1248 str << sep << localeNames[i];
1249 sep = ", ";
1250 }
1251 }
1252
1253 TC_LOG_ERROR("", "Attempted to load {} which has locales {} as {}. Check if you placed your localized db2 files in correct directory.", _fileName, str.str(), localeNames[locale]);
1254 return nullptr;
1255 }
1256
1257 uint32 records = _catalog.size();
1258 uint32 recordsize = _loadInfo->Meta->GetRecordSize();
1259 std::size_t stringFields = _loadInfo->GetStringFieldCount(false);
1260 std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true);
1261
1262 if (!stringFields)
1263 return nullptr;
1264
1265 std::size_t stringsInRecordSize = (stringFields - localizedStringFields) * sizeof(char*);
1266 std::size_t localizedStringsInRecordSize = localizedStringFields * sizeof(LocalizedString);
1267
1268 // string table size is "total size of all records" - RecordCount * "size of record without strings"
1269 std::size_t stringTableSize = _totalRecordSize - (records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringsInRecordSize - localizedStringsInRecordSize));
1270
1271 char* stringTable = new char[stringTableSize];
1272 memset(stringTable, 0, stringTableSize);
1273 char* stringPtr = stringTable;
1274
1275 uint32 y = 0;
1276
1277 for (uint32 section = 0; section < _header->SectionCount; ++section)
1278 {
1279 DB2SectionHeader const& sectionHeader = GetSection(section);
1280 if (!IsKnownTactId(sectionHeader.TactId))
1281 {
1282 y += sectionHeader.RecordCount;
1283 continue;
1284 }
1285
1286 for (uint32 sr = 0; sr < sectionHeader.CatalogDataCount; ++sr, ++y)
1287 {
1288 unsigned char const* rawRecord = GetRawRecordData(y, &section);
1289 if (!rawRecord)
1290 continue;
1291
1292 uint32 indexVal = _catalogIds[y];
1293 if (indexVal >= indexTableSize)
1294 continue;
1295
1296 char* recordData = indexTable[indexVal];
1297
1298 uint32 offset = 0;
1299 uint32 fieldIndex = 0;
1300 if (!_loadInfo->Meta->HasIndexFieldInData())
1301 {
1302 offset += 4;
1303 ++fieldIndex;
1304 }
1305
1306 for (uint32 x = 0; x < _header->FieldCount; ++x)
1307 {
1308 for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z)
1309 {
1310 switch (_loadInfo->Fields[fieldIndex].Type)
1311 {
1312 case FT_FLOAT:
1313 offset += 4;
1314 break;
1315 case FT_INT:
1316 offset += 4;
1317 break;
1318 case FT_BYTE:
1319 offset += 1;
1320 break;
1321 case FT_SHORT:
1322 offset += 2;
1323 break;
1324 case FT_LONG:
1325 offset += 8;
1326 break;
1327 case FT_STRING:
1328 {
1329 LocalizedString* db2str = reinterpret_cast<LocalizedString*>(&recordData[offset]);
1330 db2str->Str[locale] = stringPtr;
1331 strcpy(stringPtr, RecordGetString(rawRecord, x, z));
1332 stringPtr += strlen(stringPtr) + 1;
1333 offset += sizeof(LocalizedString);
1334 break;
1335 }
1336 case FT_STRING_NOT_LOCALIZED:
1337 {
1338 char const** db2str = reinterpret_cast<char const**>(&recordData[offset]);
1339 *db2str = stringPtr;
1340 strcpy(stringPtr, RecordGetString(rawRecord, x, z));
1341 stringPtr += strlen(stringPtr) + 1;
1342 offset += sizeof(char*);
1343 break;
1344 }
1345 default:
1346 ABORT_MSG("Unknown format character '%c' found in %s meta for field %s",
1347 _loadInfo->Fields[fieldIndex].Type, _fileName, _loadInfo->Fields[fieldIndex].Name);
1348 break;
1349 }
1350 ++fieldIndex;
1351 }
1352 }
1353 }
1354 }
1355
1356 return stringTable;
1357}
1358
1359void DB2FileLoaderSparseImpl::AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable)
1360{
1361 uint32 recordCopies = GetRecordCopyCount();
1362 if (!recordCopies)
1363 return;
1364
1365 uint32 recordsize = _loadInfo->Meta->GetRecordSize();
1366 uint32 offset = _header->RecordCount * recordsize;
1367 uint32 idFieldOffset = _loadInfo->Meta->HasIndexFieldInData() ? _loadInfo->Meta->GetIndexFieldOffset() : 0;
1368 for (uint32 c = 0; c < recordCopies; ++c)
1369 {
1370 DB2RecordCopy copy = GetRecordCopy(c);
1371 if (copy.SourceRowId && copy.SourceRowId < records && copy.NewRowId < records && indexTable[copy.SourceRowId])
1372 {
1373 indexTable[copy.NewRowId] = &dataTable[offset];
1374 memcpy(indexTable[copy.NewRowId], indexTable[copy.SourceRowId], recordsize);
1375 *reinterpret_cast<uint32*>(&dataTable[offset + idFieldOffset]) = copy.NewRowId;
1376 offset += recordsize;
1377 }
1378 }
1379}
1380
1381DB2Record DB2FileLoaderSparseImpl::GetRecord(uint32 recordNumber) const
1382{
1383 return DB2Record(*this, recordNumber, _fieldAndArrayOffsets.get());
1384}
1385
1386DB2RecordCopy DB2FileLoaderSparseImpl::GetRecordCopy(uint32 copyNumber) const
1387{
1388 if (copyNumber >= GetRecordCopyCount())
1389 return DB2RecordCopy{};
1390
1391 return _copyTable[copyNumber];
1392}
1393
1394uint32 DB2FileLoaderSparseImpl::GetRecordCount() const
1395{
1396 return _catalog.size();
1397}
1398
1399uint32 DB2FileLoaderSparseImpl::GetRecordCopyCount() const
1400{
1401 return _copyTable.size();
1402}
1403
1404void DB2FileLoaderSparseImpl::FillParentLookup(char* dataTable)
1405{
1406 int32 parentIdOffset = _loadInfo->Meta->GetParentIndexFieldOffset();
1407 uint32 recordSize = _loadInfo->Meta->GetRecordSize();
1408 uint32 recordIndexOffset = 0;
1409 for (uint32 i = 0; i < _header->SectionCount; ++i)
1410 {
1411 DB2SectionHeader const& section = GetSection(i);
1412 if (IsKnownTactId(section.TactId))
1413 {
1414 for (std::size_t j = 0; j < _parentIndexes[i][0].Entries.size(); ++j)
1415 {
1416 uint32 parentId = _parentIndexes[i][0].Entries[j].ParentId;
1417 char* recordData = &dataTable[(_parentIndexes[i][0].Entries[j].RecordIndex + recordIndexOffset) * recordSize];
1418
1419 switch (_loadInfo->Meta->Fields[_loadInfo->Meta->ParentIndexField].Type)
1420 {
1421 case FT_SHORT:
1422 {
1423 if (_loadInfo->Meta->ParentIndexField >= int32(_loadInfo->Meta->FileFieldCount))
1424 {
1425 // extra field at the end
1426 *reinterpret_cast<uint32*>(&recordData[parentIdOffset]) = parentId;
1427 }
1428 else
1429 {
1430 // in data block, must fit
1431 ASSERT(parentId <= 0xFFFF, "ParentId value %u does not fit into uint16 field (%s in %s)",
1432 parentId, _loadInfo->Fields[_loadInfo->GetFieldIndexByMetaIndex(_loadInfo->Meta->ParentIndexField)].Name, _fileName);
1433 *reinterpret_cast<uint16*>(&recordData[parentIdOffset]) = parentId;
1434 }
1435 break;
1436 }
1437 case FT_BYTE:
1438 {
1439 if (_loadInfo->Meta->ParentIndexField >= int32(_loadInfo->Meta->FileFieldCount))
1440 {
1441 // extra field at the end
1442 *reinterpret_cast<uint32*>(&recordData[parentIdOffset]) = parentId;
1443 }
1444 else
1445 {
1446 // in data block, must fit
1447 ASSERT(parentId <= 0xFF, "ParentId value %u does not fit into uint8 field (%s in %s)",
1448 parentId, _loadInfo->Fields[_loadInfo->GetFieldIndexByMetaIndex(_loadInfo->Meta->ParentIndexField)].Name, _fileName);
1449 *reinterpret_cast<uint8*>(&recordData[parentIdOffset]) = parentId;
1450 }
1451 break;
1452 }
1453 case FT_INT:
1454 *reinterpret_cast<uint32*>(&recordData[parentIdOffset]) = parentId;
1455 break;
1456 default:
1457 ABORT_MSG("Unhandled parent id type '%c' found in %s", _loadInfo->Meta->Fields[_loadInfo->Meta->ParentIndexField].Type, _fileName);
1458 break;
1459 }
1460 }
1461 }
1462 recordIndexOffset += section.RecordCount;
1463 }
1464}
1465
1466uint32 DB2FileLoaderSparseImpl::GetRecordSection(uint32 recordNumber) const
1467{
1468 uint32 section = 0;
1469 for (; section < _header->SectionCount; ++section)
1470 {
1471 DB2SectionHeader const& sectionHeader = GetSection(section);
1472 if (recordNumber < sectionHeader.CatalogDataCount)
1473 break;
1474
1475 recordNumber -= sectionHeader.CatalogDataCount;
1476 }
1477
1478 return section;
1479}
1480
1481unsigned char const* DB2FileLoaderSparseImpl::GetRawRecordData(uint32 recordNumber, uint32 const* section) const
1482{
1483 if (recordNumber >= _catalog.size())
1484 return nullptr;
1485
1486 if (!IsKnownTactId(GetSection(section ? *section : GetRecordSection(recordNumber)).TactId))
1487 return nullptr;
1488
1489 _source->SetPosition(_catalog[recordNumber].FileOffset);
1490 uint8* rawRecord = _recordBuffer.get();
1491 if (!_source->Read(rawRecord, _catalog[recordNumber].RecordSize))
1492 return nullptr;
1493
1494 CalculateAndStoreFieldOffsets(rawRecord);
1495 return rawRecord;
1496}
1497
1498uint32 DB2FileLoaderSparseImpl::RecordGetId(uint8 const* record, uint32 recordIndex) const
1499{
1500 if (_loadInfo->Meta->HasIndexFieldInData())
1501 return RecordGetVarInt(record, _loadInfo->Meta->GetIndexField(), 0, false);
1502
1503 return _catalogIds[recordIndex];
1504}
1505
1506uint8 DB2FileLoaderSparseImpl::RecordGetUInt8(uint8 const* record, uint32 field, uint32 arrayIndex) const
1507{
1508 ASSERT(field < _header->FieldCount);
1509 return *reinterpret_cast<uint8 const*>(record + GetFieldOffset(field, arrayIndex));
1510}
1511
1512uint16 DB2FileLoaderSparseImpl::RecordGetUInt16(uint8 const* record, uint32 field, uint32 arrayIndex) const
1513{
1514 ASSERT(field < _header->FieldCount);
1515 uint16 val = *reinterpret_cast<uint16 const*>(record + GetFieldOffset(field, arrayIndex));
1516 EndianConvert(val);
1517 return val;
1518}
1519
1520uint32 DB2FileLoaderSparseImpl::RecordGetUInt32(uint8 const* record, uint32 field, uint32 arrayIndex) const
1521{
1522 return RecordGetVarInt(record, field, arrayIndex, false);
1523}
1524
1525int32 DB2FileLoaderSparseImpl::RecordGetInt32(uint8 const* record, uint32 field, uint32 arrayIndex) const
1526{
1527 return int32(RecordGetVarInt(record, field, arrayIndex, true));
1528}
1529
1530uint64 DB2FileLoaderSparseImpl::RecordGetUInt64(uint8 const* record, uint32 field, uint32 arrayIndex) const
1531{
1532 ASSERT(field < _header->FieldCount);
1533 uint64 val = *reinterpret_cast<uint64 const*>(record + GetFieldOffset(field, arrayIndex));
1534 EndianConvert(val);
1535 return val;
1536}
1537
1538float DB2FileLoaderSparseImpl::RecordGetFloat(uint8 const* record, uint32 field, uint32 arrayIndex) const
1539{
1540 ASSERT(field < _header->FieldCount);
1541 float val = *reinterpret_cast<float const*>(record + GetFieldOffset(field, arrayIndex));
1542 EndianConvert(val);
1543 return val;
1544}
1545
1546char const* DB2FileLoaderSparseImpl::RecordGetString(uint8 const* record, uint32 field, uint32 arrayIndex) const
1547{
1548 ASSERT(field < _header->FieldCount);
1549 return reinterpret_cast<char const*>(record + GetFieldOffset(field, arrayIndex));
1550}
1551
1552uint32 DB2FileLoaderSparseImpl::RecordGetVarInt(uint8 const* record, uint32 field, uint32 arrayIndex, bool isSigned) const
1553{
1554 ASSERT(field < _header->FieldCount);
1555 uint32 val = 0;
1556 memcpy(&val, record + GetFieldOffset(field, arrayIndex), GetFieldSize(field));
1557 EndianConvert(val);
1558 if (isSigned)
1559 return int32(val) << _fields[field].UnusedBits >> _fields[field].UnusedBits;
1560
1561 return val << _fields[field].UnusedBits >> _fields[field].UnusedBits;
1562}
1563
1564uint16 DB2FileLoaderSparseImpl::GetFieldOffset(uint32 field, uint32 arrayIndex) const
1565{
1566 return uint16(_fieldAndArrayOffsets[_fieldAndArrayOffsets[field] + arrayIndex]);
1567}
1568
1569uint16 DB2FileLoaderSparseImpl::GetFieldSize(uint32 field) const
1570{
1571 ASSERT(field < _header->FieldCount);
1572 return 4 - _fields[field].UnusedBits / 8;
1573}
1574
1575std::size_t* DB2FileLoaderSparseImpl::RecordCreateDetachedFieldOffsets(std::size_t* oldOffsets) const
1576{
1577 if (oldOffsets != _fieldAndArrayOffsets.get())
1578 return oldOffsets;
1579
1580 std::size_t size = _loadInfo->Meta->FieldCount + _loadInfo->FieldCount - (!_loadInfo->Meta->HasIndexFieldInData() ? 1 : 0);
1581 std::size_t* newOffsets = new std::size_t[size];
1582 memcpy(newOffsets, _fieldAndArrayOffsets.get(), size * sizeof(std::size_t));
1583 return newOffsets;
1584}
1585
1586std::size_t* DB2FileLoaderSparseImpl::RecordCopyDetachedFieldOffsets(std::size_t* oldOffsets) const
1587{
1588 if (oldOffsets == _fieldAndArrayOffsets.get())
1589 return oldOffsets;
1590
1591 std::size_t size = _loadInfo->Meta->FieldCount + _loadInfo->FieldCount - (!_loadInfo->Meta->HasIndexFieldInData() ? 1 : 0);
1592 std::size_t* newOffsets = new std::size_t[size];
1593 memcpy(newOffsets, _fieldAndArrayOffsets.get(), size * sizeof(std::size_t));
1594 return newOffsets;
1595}
1596
1597void DB2FileLoaderSparseImpl::RecordDestroyFieldOffsets(std::size_t*& fieldOffsets) const
1598{
1599 if (fieldOffsets == _fieldAndArrayOffsets.get())
1600 return;
1601
1602 delete[] fieldOffsets;
1603 fieldOffsets = nullptr;
1604}
1605
1606void DB2FileLoaderSparseImpl::CalculateAndStoreFieldOffsets(uint8 const* rawRecord) const
1607{
1608 std::size_t offset = 0;
1609 uint32 combinedField = _loadInfo->Meta->FieldCount;
1610 for (uint32 field = 0; field < _loadInfo->Meta->FieldCount; ++field)
1611 {
1612 _fieldAndArrayOffsets[field] = combinedField;
1613 for (uint32 arr = 0; arr < _loadInfo->Meta->Fields[field].ArraySize; ++arr)
1614 {
1615 _fieldAndArrayOffsets[combinedField] = offset;
1616 switch (_loadInfo->Meta->Fields[field].Type)
1617 {
1618 case FT_BYTE:
1619 case FT_SHORT:
1620 case FT_INT:
1621 case FT_LONG:
1622 offset += GetFieldSize(field);
1623 break;
1624 case FT_FLOAT:
1625 offset += sizeof(float);
1626 break;
1627 case FT_STRING:
1628 case FT_STRING_NOT_LOCALIZED:
1629 offset += strlen(reinterpret_cast<char const*>(rawRecord) + offset) + 1;
1630 break;
1631 default:
1632 ABORT_MSG("Unknown format character '%c' found in %s meta", _loadInfo->Meta->Fields[field].Type, _fileName);
1633 break;
1634 }
1635 ++combinedField;
1636 }
1637 }
1638}
1639
1640uint32 DB2FileLoaderSparseImpl::GetMinId() const
1641{
1642 return _header->MinId;
1643}
1644
1645uint32 DB2FileLoaderSparseImpl::GetMaxId() const
1646{
1647 return _header->MaxId;
1648}
1649
1650DB2FileLoadInfo const* DB2FileLoaderSparseImpl::GetLoadInfo() const
1651{
1652 return _loadInfo;
1653}
1654
1655DB2SectionHeader& DB2FileLoaderSparseImpl::GetSection(uint32 section) const
1656{
1657 return _sections[section];
1658}
1659
1660bool DB2FileLoaderSparseImpl::IsSignedField(uint32 field) const
1661{
1662 ASSERT(field < _header->FieldCount);
1663 return _loadInfo->Meta->IsSignedField(field);
1664}
1665
1666char const* DB2FileLoaderSparseImpl::GetExpectedSignMismatchReason(uint32 field) const
1667{
1668 ASSERT(field < _header->FieldCount);
1669 if (int32(field) == _loadInfo->Meta->IndexField)
1670 return " (IndexField must always be unsigned)";
1671 if (int32(field) == _loadInfo->Meta->ParentIndexField)
1672 return " (ParentIndexField must always be unsigned)";
1673 return "";
1674}
1675
1676DB2Record::DB2Record(DB2FileLoaderImpl const& db2, uint32 recordIndex, std::size_t* fieldOffsets)
1677 : _db2(db2), _recordIndex(recordIndex), _recordData(db2.GetRawRecordData(recordIndex, nullptr)), _fieldOffsets(fieldOffsets)
1678{
1679}
1680
1681DB2Record::DB2Record(DB2Record const& other)
1682 : _db2(other._db2), _recordIndex(other._recordIndex), _recordData(other._recordData), _fieldOffsets(_db2.RecordCopyDetachedFieldOffsets(other._fieldOffsets))
1683{
1684}
1685
1686DB2Record::DB2Record(DB2Record&& other) noexcept
1687 : _db2(other._db2), _recordIndex(other._recordIndex), _recordData(other._recordData), _fieldOffsets(std::exchange(other._fieldOffsets, nullptr))
1688{
1689}
1690
1691DB2Record::~DB2Record()
1692{
1693 _db2.RecordDestroyFieldOffsets(_fieldOffsets);
1694}
1695
1696DB2Record::operator bool() const
1697{
1698 return _recordData != nullptr;
1699}
1700
1701uint32 DB2Record::GetId() const
1702{
1703 return _db2.RecordGetId(_recordData, _recordIndex);
1704}
1705
1706uint8 DB2Record::GetUInt8(uint32 field, uint32 arrayIndex) const
1707{
1708 return _db2.RecordGetUInt8(_recordData, field, arrayIndex);
1709}
1710
1711uint8 DB2Record::GetUInt8(char const* fieldName) const
1712{
1713 std::pair<int32, int32> fieldIndex = _db2.GetLoadInfo()->GetFieldIndexByName(fieldName);
1714 ASSERT(fieldIndex.first != -1, "Field with name %s does not exist!", fieldName);
1715 return _db2.RecordGetUInt8(_recordData, uint32(fieldIndex.first), uint32(fieldIndex.second));
1716}
1717
1718uint16 DB2Record::GetUInt16(uint32 field, uint32 arrayIndex) const
1719{
1720 return _db2.RecordGetUInt16(_recordData, field, arrayIndex);
1721}
1722
1723uint16 DB2Record::GetUInt16(char const* fieldName) const
1724{
1725 std::pair<int32, int32> fieldIndex = _db2.GetLoadInfo()->GetFieldIndexByName(fieldName);
1726 ASSERT(fieldIndex.first != -1, "Field with name %s does not exist!", fieldName);
1727 return _db2.RecordGetUInt16(_recordData, uint32(fieldIndex.first), uint32(fieldIndex.second));
1728}
1729
1730uint32 DB2Record::GetUInt32(uint32 field, uint32 arrayIndex) const
1731{
1732 return _db2.RecordGetUInt32(_recordData, field, arrayIndex);
1733}
1734
1735uint32 DB2Record::GetUInt32(char const* fieldName) const
1736{
1737 std::pair<int32, int32> fieldIndex = _db2.GetLoadInfo()->GetFieldIndexByName(fieldName);
1738 ASSERT(fieldIndex.first != -1, "Field with name %s does not exist!", fieldName);
1739 return _db2.RecordGetUInt32(_recordData, uint32(fieldIndex.first), uint32(fieldIndex.second));
1740}
1741
1742int32 DB2Record::GetInt32(uint32 field, uint32 arrayIndex) const
1743{
1744 return _db2.RecordGetInt32(_recordData, field, arrayIndex);
1745}
1746
1747int32 DB2Record::GetInt32(char const* fieldName) const
1748{
1749 std::pair<int32, int32> fieldIndex = _db2.GetLoadInfo()->GetFieldIndexByName(fieldName);
1750 ASSERT(fieldIndex.first != -1, "Field with name %s does not exist!", fieldName);
1751 return _db2.RecordGetInt32(_recordData, uint32(fieldIndex.first), uint32(fieldIndex.second));
1752}
1753
1754uint64 DB2Record::GetUInt64(uint32 field, uint32 arrayIndex) const
1755{
1756 return _db2.RecordGetUInt64(_recordData, field, arrayIndex);
1757}
1758
1759uint64 DB2Record::GetUInt64(char const* fieldName) const
1760{
1761 std::pair<int32, int32> fieldIndex = _db2.GetLoadInfo()->GetFieldIndexByName(fieldName);
1762 ASSERT(fieldIndex.first != -1, "Field with name %s does not exist!", fieldName);
1763 return _db2.RecordGetUInt64(_recordData, uint32(fieldIndex.first), uint32(fieldIndex.second));
1764}
1765
1766float DB2Record::GetFloat(uint32 field, uint32 arrayIndex) const
1767{
1768 return _db2.RecordGetFloat(_recordData, field, arrayIndex);
1769}
1770
1771float DB2Record::GetFloat(char const* fieldName) const
1772{
1773 std::pair<int32, int32> fieldIndex = _db2.GetLoadInfo()->GetFieldIndexByName(fieldName);
1774 ASSERT(fieldIndex.first != -1, "Field with name %s does not exist!", fieldName);
1775 return _db2.RecordGetFloat(_recordData, uint32(fieldIndex.first), uint32(fieldIndex.second));
1776}
1777
1778char const* DB2Record::GetString(uint32 field, uint32 arrayIndex) const
1779{
1780 return _db2.RecordGetString(_recordData, field, arrayIndex);
1781}
1782
1783char const* DB2Record::GetString(char const* fieldName) const
1784{
1785 std::pair<int32, int32> fieldIndex = _db2.GetLoadInfo()->GetFieldIndexByName(fieldName);
1786 ASSERT(fieldIndex.first != -1, "Field with name %s does not exist!", fieldName);
1787 return _db2.RecordGetString(_recordData, uint32(fieldIndex.first), uint32(fieldIndex.second));
1788}
1789
1790void DB2Record::MakePersistent()
1791{
1792 _fieldOffsets = _db2.RecordCreateDetachedFieldOffsets(_fieldOffsets);
1793}
1794
1795DB2FileLoader::DB2FileLoader() : _impl(nullptr), _header()
1796{
1797}
1798
1799DB2FileLoader::~DB2FileLoader()
1800{
1801 delete _impl;
1802}
1803
1804void DB2FileLoader::LoadHeaders(DB2FileSource* source, DB2FileLoadInfo const* loadInfo)
1805{
1806 if (!source->IsOpen())
1807 throw std::system_error(std::make_error_code(std::errc::no_such_file_or_directory));
1808
1809 if (!source->Read(&_header, sizeof(DB2Header)))
1810 throw DB2FileLoadException("Failed to read header");
1811
1812 EndianConvert(_header.Signature);
1813 EndianConvert(_header.Version);
1814 EndianConvert(_header.RecordCount);
1815 EndianConvert(_header.FieldCount);
1816 EndianConvert(_header.RecordSize);
1817 EndianConvert(_header.StringTableSize);
1818 EndianConvert(_header.TableHash);
1819 EndianConvert(_header.LayoutHash);
1820 EndianConvert(_header.MinId);
1821 EndianConvert(_header.MaxId);
1822 EndianConvert(_header.Locale);
1823 EndianConvert(_header.Flags);
1824 EndianConvert(_header.IndexField);
1825 EndianConvert(_header.TotalFieldCount);
1826 EndianConvert(_header.PackedDataOffset);
1827 EndianConvert(_header.ParentLookupCount);
1828 EndianConvert(_header.ColumnMetaSize);
1829 EndianConvert(_header.CommonDataSize);
1830 EndianConvert(_header.PalletDataSize);
1831 EndianConvert(_header.SectionCount);
1832
1833 if (_header.Signature != 0x35434457) //'WDC5'
1834 throw DB2FileLoadException(Trinity::StringFormat("Incorrect file signature in {}, expected 'WDC5', got {}{}{}{}", source->GetFileName(),
1835 char(_header.Signature & 0xFF), char((_header.Signature >> 8) & 0xFF), char((_header.Signature >> 16) & 0xFF), char((_header.Signature >> 24) & 0xFF)));
1836
1837 if (_header.Version != 5)
1838 throw DB2FileLoadException(Trinity::StringFormat("Incorrect version in {}, expected 5, got {} (possibly wrong client version)",
1839 source->GetFileName(), _header.Version));
1840
1841 if (loadInfo && _header.LayoutHash != loadInfo->Meta->LayoutHash)
1842 throw DB2FileLoadException(Trinity::StringFormat("Incorrect layout hash in {}, expected 0x{:08X}, got 0x{:08X} (possibly wrong client version)",
1843 source->GetFileName(), loadInfo->Meta->LayoutHash, _header.LayoutHash));
1844
1845 if (_header.ParentLookupCount > 1)
1846 throw DB2FileLoadException(Trinity::StringFormat("Too many parent lookups in {}, only one is allowed, got {}",
1847 source->GetFileName(), _header.ParentLookupCount));
1848
1849 if (loadInfo && (_header.TotalFieldCount + (loadInfo->Meta->ParentIndexField >= int32(_header.TotalFieldCount) ? 1 : 0) != loadInfo->Meta->FieldCount))
1850 throw DB2FileLoadException(Trinity::StringFormat("Incorrect number of fields in {}, expected {}, got {}",
1851 source->GetFileName(), loadInfo->Meta->FieldCount, _header.TotalFieldCount + (loadInfo->Meta->ParentIndexField >= int32(_header.TotalFieldCount) ? 1 : 0)));
1852
1853 if (loadInfo && (_header.ParentLookupCount && loadInfo->Meta->ParentIndexField == -1))
1854 throw DB2FileLoadException(Trinity::StringFormat("Unexpected parent lookup found in {}", source->GetFileName()));
1855
1856 std::unique_ptr<DB2SectionHeader[]> sections = std::make_unique<DB2SectionHeader[]>(_header.SectionCount);
1857 if (_header.SectionCount && !source->Read(sections.get(), sizeof(DB2SectionHeader) * _header.SectionCount))
1858 throw DB2FileLoadException(Trinity::StringFormat("Unable to read section headers from {}", source->GetFileName()));
1859
1860 std::unique_ptr<DB2FieldEntry[]> fieldData = std::make_unique<DB2FieldEntry[]>(_header.FieldCount);
1861 if (!source->Read(fieldData.get(), sizeof(DB2FieldEntry) * _header.FieldCount))
1862 throw DB2FileLoadException(Trinity::StringFormat("Unable to read field information from {}", source->GetFileName()));
1863
1864 std::unique_ptr<DB2ColumnMeta[]> columnMeta;
1865 std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> palletValues;
1866 std::unique_ptr<std::unique_ptr<DB2PalletValue[]>[]> palletArrayValues;
1867 std::unique_ptr<std::unordered_map<uint32, uint32>[]> commonValues;
1868 if (_header.ColumnMetaSize)
1869 {
1870 columnMeta = std::make_unique<DB2ColumnMeta[]>(_header.TotalFieldCount);
1871 if (!source->Read(columnMeta.get(), _header.ColumnMetaSize))
1872 throw DB2FileLoadException(Trinity::StringFormat("Unable to read field metadata from {}", source->GetFileName()));
1873
1874 if (loadInfo && loadInfo->Meta->HasIndexFieldInData())
1875 {
1876 if (columnMeta[loadInfo->Meta->IndexField].CompressionType != DB2ColumnCompression::None
1877 && columnMeta[loadInfo->Meta->IndexField].CompressionType != DB2ColumnCompression::Immediate
1878 && columnMeta[loadInfo->Meta->IndexField].CompressionType != DB2ColumnCompression::SignedImmediate)
1879 throw DB2FileLoadException(Trinity::StringFormat("Invalid compression type for index field in {}, expected one of None (0), Immediate (1), SignedImmediate (5), got {}",
1880 source->GetFileName(), uint32(columnMeta[loadInfo->Meta->IndexField].CompressionType)));
1881 }
1882
1883 palletValues = std::make_unique<std::unique_ptr<DB2PalletValue[]>[]>(_header.TotalFieldCount);
1884 for (uint32 i = 0; i < _header.TotalFieldCount; ++i)
1885 {
1886 if (columnMeta[i].CompressionType != DB2ColumnCompression::Pallet)
1887 continue;
1888
1889 palletValues[i] = std::make_unique<DB2PalletValue[]>(columnMeta[i].AdditionalDataSize / sizeof(DB2PalletValue));
1890 if (!source->Read(palletValues[i].get(), columnMeta[i].AdditionalDataSize))
1891 throw DB2FileLoadException(Trinity::StringFormat("Unable to read field pallet values from {} for field {}", source->GetFileName(), i));
1892 }
1893
1894 palletArrayValues = std::make_unique<std::unique_ptr<DB2PalletValue[]>[]>(_header.TotalFieldCount);
1895 for (uint32 i = 0; i < _header.TotalFieldCount; ++i)
1896 {
1897 if (columnMeta[i].CompressionType != DB2ColumnCompression::PalletArray)
1898 continue;
1899
1900 palletArrayValues[i] = std::make_unique<DB2PalletValue[]>(columnMeta[i].AdditionalDataSize / sizeof(DB2PalletValue));
1901 if (!source->Read(palletArrayValues[i].get(), columnMeta[i].AdditionalDataSize))
1902 throw DB2FileLoadException(Trinity::StringFormat("Unable to read field pallet array values from {} for field {}", source->GetFileName(), i));
1903 }
1904
1905 std::unique_ptr<std::unique_ptr<DB2CommonValue[]>[]> commonData = std::make_unique<std::unique_ptr<DB2CommonValue[]>[]>(_header.TotalFieldCount);
1906 commonValues = std::make_unique<std::unordered_map<uint32, uint32>[]>(_header.TotalFieldCount);
1907 for (uint32 i = 0; i < _header.TotalFieldCount; ++i)
1908 {
1909 if (columnMeta[i].CompressionType != DB2ColumnCompression::CommonData)
1910 continue;
1911
1912 if (!columnMeta[i].AdditionalDataSize)
1913 continue;
1914
1915 commonData[i] = std::make_unique<DB2CommonValue[]>(columnMeta[i].AdditionalDataSize / sizeof(DB2CommonValue));
1916 if (!source->Read(commonData[i].get(), columnMeta[i].AdditionalDataSize))
1917 throw DB2FileLoadException(Trinity::StringFormat("Unable to read field common values from {} for field {}", source->GetFileName(), i));
1918
1919 uint32 numExtraValuesForField = columnMeta[i].AdditionalDataSize / sizeof(DB2CommonValue);
1920 for (uint32 record = 0; record < numExtraValuesForField; ++record)
1921 {
1922 uint32 recordId = commonData[i][record].RecordId;
1923 uint32 value = commonData[i][record].Value;
1924 EndianConvert(value);
1925 commonValues[i][recordId] = value;
1926 }
1927 }
1928 }
1929
1930 if (!(_header.Flags & 0x1))
1931 _impl = new DB2FileLoaderRegularImpl(source->GetFileName(), loadInfo, &_header);
1932 else
1933 _impl = new DB2FileLoaderSparseImpl(source->GetFileName(), loadInfo, &_header, source);
1934
1935 _impl->LoadColumnData(std::move(sections), std::move(fieldData), std::move(columnMeta), std::move(palletValues), std::move(palletArrayValues), std::move(commonValues));
1936}
1937
1938void DB2FileLoader::Load(DB2FileSource* source, DB2FileLoadInfo const* loadInfo)
1939{
1940 LoadHeaders(source, loadInfo);
1941
1942 std::vector<uint32> idTable;
1943 std::vector<DB2RecordCopy> copyTable;
1944 std::vector<std::vector<DB2IndexData>> parentIndexes;
1945 if (loadInfo && !loadInfo->Meta->HasIndexFieldInData() && _header.RecordCount)
1946 idTable.reserve(_header.RecordCount);
1947
1948 if (_header.ParentLookupCount)
1949 {
1950 parentIndexes.resize(_header.SectionCount);
1951 for (std::vector<DB2IndexData>& parentIndexesForSection : parentIndexes)
1952 parentIndexesForSection.resize(_header.ParentLookupCount);
1953 }
1954
1955 for (uint32 i = 0; i < _header.SectionCount; ++i)
1956 {
1957 DB2SectionHeader& section = _impl->GetSection(i);
1958
1959 if (section.TactId)
1960 {
1961 uint32 encryptedIdCount;
1962 if (!source->Read(&encryptedIdCount, sizeof(encryptedIdCount)))
1963 throw DB2FileLoadException(Trinity::StringFormat("Unable to read number of encrypted records in {} for section {}", source->GetFileName(), i));
1964
1965 // it doesnt matter what the encrypted ids are, we have no use for them
1966 source->SetPosition(source->GetPosition() + sizeof(uint32) * encryptedIdCount);
1967 }
1968 }
1969
1970 if (loadInfo && !(_header.Flags & 0x1))
1971 {
1972 uint32 totalCopyTableSize = 0;
1973 uint32 totalParentLookupDataSize = 0;
1974 for (uint32 i = 0; i < _header.SectionCount; ++i)
1975 {
1976 DB2SectionHeader& section = _impl->GetSection(i);
1977 totalCopyTableSize += section.CopyTableCount * sizeof(DB2RecordCopy);
1978 totalParentLookupDataSize += section.ParentLookupDataSize;
1979 }
1980
1981 int64 expectedFileSize =
1982 source->GetPosition() +
1983 int64(_header.RecordSize) * _header.RecordCount +
1984 _header.StringTableSize +
1985 (loadInfo->Meta->IndexField == -1 ? sizeof(uint32) * _header.RecordCount : 0) +
1986 totalCopyTableSize +
1987 totalParentLookupDataSize;
1988
1989 if (source->GetFileSize() != expectedFileSize)
1990 throw DB2FileLoadException(Trinity::StringFormat("{} failed size consistency check, expected {}, got {}", source->GetFileName(), expectedFileSize, source->GetFileSize()));
1991 }
1992
1993 for (uint32 i = 0; i < _header.SectionCount; ++i)
1994 {
1995 DB2SectionHeader& section = _impl->GetSection(i);
1996
1997 if (!IsKnownTactId(section.TactId))
1998 {
1999 switch (source->HandleEncryptedSection(section))
2000 {
2001 case DB2EncryptedSectionHandling::Skip:
2002 _impl->SkipEncryptedSection(i);
2003 idTable.resize(idTable.size() + section.IdTableSize / sizeof(uint32));
2004 continue;
2005 case DB2EncryptedSectionHandling::Process:
2006 section.TactId = DUMMY_KNOWN_TACT_ID;
2007 break;
2008 default:
2009 break;
2010 }
2011 }
2012
2013 if (!source->SetPosition(section.FileOffset))
2014 throw DB2FileLoadException(Trinity::StringFormat("Unable to change {} read position for section {}", source->GetFileName(), i));
2015
2016 if (!_impl->LoadTableData(source, i))
2017 throw DB2FileLoadException(Trinity::StringFormat("Unable to read section table data from {} for section {}", source->GetFileName(), i));
2018
2019 if (!_impl->LoadCatalogData(source, i))
2020 throw DB2FileLoadException(Trinity::StringFormat("Unable to read section catalog data from {} for section {}", source->GetFileName(), i));
2021
2022 if (loadInfo)
2023 {
2024 if (loadInfo->Meta->HasIndexFieldInData())
2025 {
2026 if (section.IdTableSize != 0)
2027 throw DB2FileLoadException(Trinity::StringFormat("Unexpected id table found in {} for section {}", source->GetFileName(), i));
2028 }
2029 else if (section.IdTableSize != 4 * section.RecordCount)
2030 throw DB2FileLoadException(Trinity::StringFormat("Unexpected id table size in {} for section {}, expected {}, got {}",
2031 source->GetFileName(), i, 4 * section.RecordCount, section.IdTableSize));
2032 }
2033
2034 if (section.IdTableSize)
2035 {
2036 std::size_t idTableSize = idTable.size();
2037 idTable.resize(idTableSize + section.IdTableSize / sizeof(uint32));
2038 if (!source->Read(&idTable[idTableSize], section.IdTableSize))
2039 throw DB2FileLoadException(Trinity::StringFormat("Unable to read non-inline record ids from {} for section {}", source->GetFileName(), i));
2040
2041 // This is a hack fix for broken db2 files that have invalid id tables
2042 for (std::size_t i = idTableSize; i < idTable.size(); ++i)
2043 if (idTable[i] <= _header.MinId)
2044 idTable[i] = _header.MinId + i;
2045 }
2046
2047 if (!(_header.Flags & 0x1) && section.CopyTableCount)
2048 {
2049 std::size_t copyTableSize = copyTable.size();
2050 copyTable.resize(copyTableSize + section.CopyTableCount);
2051 if (!source->Read(&copyTable[copyTableSize], section.CopyTableCount * sizeof(DB2RecordCopy)))
2052 throw DB2FileLoadException(Trinity::StringFormat("Unable to read record copies from {} for section {}", source->GetFileName(), i));
2053 }
2054
2055 if (_header.ParentLookupCount)
2056 {
2057 std::vector<DB2IndexData>& parentIndexesForSection = parentIndexes[i];
2058 for (uint32 j = 0; j < _header.ParentLookupCount; ++j)
2059 {
2060 DB2IndexDataInfo indexInfo;
2061 if (!source->Read(&indexInfo, sizeof(DB2IndexDataInfo)))
2062 throw DB2FileLoadException(Trinity::StringFormat("Unable to read parent lookup info from {} for section {}", source->GetFileName(), i));
2063
2064 if (!indexInfo.NumEntries)
2065 continue;
2066
2067 parentIndexesForSection[j].Entries.resize(indexInfo.NumEntries);
2068 if (!source->Read(parentIndexesForSection[j].Entries.data(), sizeof(DB2IndexEntry) * indexInfo.NumEntries))
2069 throw DB2FileLoadException(Trinity::StringFormat("Unable to read parent lookup content from {} for section {}", source->GetFileName(), i));
2070 }
2071 }
2072 }
2073
2074 _impl->SetAdditionalData(std::move(idTable), std::move(copyTable), std::move(parentIndexes));
2075
2076 if (loadInfo)
2077 {
2078 uint32 fieldIndex = 0;
2079 if (!loadInfo->Meta->HasIndexFieldInData())
2080 {
2081 ASSERT(!loadInfo->Fields[0].IsSigned, "ID must be unsigned in %s", source->GetFileName());
2082 ++fieldIndex;
2083 }
2084 for (uint32 f = 0; f < loadInfo->Meta->FieldCount; ++f)
2085 {
2086 ASSERT(loadInfo->Fields[fieldIndex].IsSigned == _impl->IsSignedField(f),
2087 "Field %s in %s must be %s%s", loadInfo->Fields[fieldIndex].Name, source->GetFileName(), _impl->IsSignedField(f) ? "signed" : "unsigned",
2088 _impl->GetExpectedSignMismatchReason(f));
2089
2090 fieldIndex += loadInfo->Meta->Fields[f].ArraySize;
2091 }
2092 }
2093}
2094
2095char* DB2FileLoader::AutoProduceData(uint32& indexTableSize, char**& indexTable)
2096{
2097 return _impl->AutoProduceData(indexTableSize, indexTable);
2098}
2099
2100char* DB2FileLoader::AutoProduceStrings(char** indexTable, uint32 indexTableSize, LocaleConstant locale)
2101{
2102 return _impl->AutoProduceStrings(indexTable, indexTableSize, locale);
2103}
2104
2105void DB2FileLoader::AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable)
2106{
2107 _impl->AutoProduceRecordCopies(records, indexTable, dataTable);
2108}
2109
2110uint32 DB2FileLoader::GetRecordCount() const
2111{
2112 return _impl->GetRecordCount();
2113}
2114
2115uint32 DB2FileLoader::GetRecordCopyCount() const
2116{
2117 return _impl->GetRecordCopyCount();
2118}
2119
2120uint32 DB2FileLoader::GetMinId() const
2121{
2122 return _impl->GetMinId();
2123}
2124
2125uint32 DB2FileLoader::GetMaxId() const
2126{
2127 return _impl->GetMaxId();
2128}
2129
2130DB2SectionHeader const& DB2FileLoader::GetSectionHeader(uint32 section) const
2131{
2132 return _impl->GetSection(section);
2133}
2134
2135DB2Record DB2FileLoader::GetRecord(uint32 recordNumber) const
2136{
2137 return _impl->GetRecord(recordNumber);
2138}
2139
2140DB2RecordCopy DB2FileLoader::GetRecordCopy(uint32 copyNumber) const
2141{
2142 return _impl->GetRecordCopy(copyNumber);
2143}
EndianConvert
void EndianConvert(T &val)
Definition: ByteConverter.h:48
localeNames
char const * localeNames[TOTAL_LOCALES]
Definition: Common.cpp:20
LocaleConstant
LocaleConstant
Definition: Common.h:48
TOTAL_LOCALES
@ TOTAL_LOCALES
Definition: Common.h:62
DB2ColumnCompression
DB2ColumnCompression
Definition: DB2FileLoader.cpp:30
IsKnownTactId
static bool IsKnownTactId(uint64 tactId)
Definition: DB2FileLoader.cpp:113
EmptyDb2String
static char const *const EmptyDb2String
Definition: DB2FileLoader.cpp:390
DUMMY_KNOWN_TACT_ID
constinit uint64 DUMMY_KNOWN_TACT_ID
Definition: DB2FileLoader.h:71
uint8
uint8_t uint8
Definition: Define.h:144
int64
int64_t int64
Definition: Define.h:137
int16
int16_t int16
Definition: Define.h:139
int32
int32_t int32
Definition: Define.h:138
uint64
uint64_t uint64
Definition: Define.h:141
UI64LIT
#define UI64LIT(N)
Definition: Define.h:127
uint16
uint16_t uint16
Definition: Define.h:143
FT_FLOAT
@ FT_FLOAT
Definition: Define.h:150
FT_SHORT
@ FT_SHORT
Definition: Define.h:153
FT_STRING
@ FT_STRING
Definition: Define.h:148
FT_INT
@ FT_INT
Definition: Define.h:151
FT_STRING_NOT_LOCALIZED
@ FT_STRING_NOT_LOCALIZED
Definition: Define.h:149
FT_BYTE
@ FT_BYTE
Definition: Define.h:152
FT_LONG
@ FT_LONG
Definition: Define.h:154
uint32
uint32_t uint32
Definition: Define.h:142
ABORT_MSG
#define ABORT_MSG
Definition: Errors.h:75
ASSERT
#define ASSERT
Definition: Errors.h:68
TC_LOG_ERROR
#define TC_LOG_ERROR(filterType__,...)
Definition: Log.h:165
DB2FileLoaderImpl::AutoProduceData
virtual char * AutoProduceData(uint32 &indexTableSize, char **&indexTable)=0
DB2FileLoaderImpl::RecordGetUInt64
virtual uint64 RecordGetUInt64(uint8 const *record, uint32 field, uint32 arrayIndex) const =0
DB2FileLoaderImpl::GetMaxId
virtual uint32 GetMaxId() const =0
DB2FileLoaderImpl::GetRecordCopyCount
virtual uint32 GetRecordCopyCount() const =0
DB2FileLoaderImpl::GetSection
virtual DB2SectionHeader & GetSection(uint32 section) const =0
DB2FileLoaderImpl::DB2FileLoaderImpl
DB2FileLoaderImpl()=default
DB2FileLoaderImpl::IsSignedField
virtual bool IsSignedField(uint32 field) const =0
DB2FileLoaderImpl::~DB2FileLoaderImpl
virtual ~DB2FileLoaderImpl()=default
DB2FileLoaderImpl::GetLoadInfo
virtual DB2FileLoadInfo const * GetLoadInfo() const =0
DB2FileLoaderImpl::RecordGetId
virtual uint32 RecordGetId(uint8 const *record, uint32 recordIndex) const =0
DB2FileLoaderImpl::RecordGetFloat
virtual float RecordGetFloat(uint8 const *record, uint32 field, uint32 arrayIndex) const =0
DB2FileLoaderImpl::SetAdditionalData
virtual void SetAdditionalData(std::vector< uint32 > idTable, std::vector< DB2RecordCopy > copyTable, std::vector< std::vector< DB2IndexData > > parentIndexes)=0
DB2FileLoaderImpl::LoadColumnData
virtual void LoadColumnData(std::unique_ptr< DB2SectionHeader[]> sections, std::unique_ptr< DB2FieldEntry[]> fields, std::unique_ptr< DB2ColumnMeta[]> columnMeta, std::unique_ptr< std::unique_ptr< DB2PalletValue[]>[]> palletValues, std::unique_ptr< std::unique_ptr< DB2PalletValue[]>[]> palletArrayValues, std::unique_ptr< std::unordered_map< uint32, uint32 >[]> commonValues)=0
DB2FileLoaderImpl::RecordCopyDetachedFieldOffsets
virtual std::size_t * RecordCopyDetachedFieldOffsets(std::size_t *oldOffsets) const =0
DB2FileLoaderImpl::RecordCreateDetachedFieldOffsets
virtual std::size_t * RecordCreateDetachedFieldOffsets(std::size_t *oldOffsets) const =0
DB2FileLoaderImpl::GetExpectedSignMismatchReason
virtual char const * GetExpectedSignMismatchReason(uint32 field) const =0
DB2FileLoaderImpl::GetMinId
virtual uint32 GetMinId() const =0
DB2FileLoaderImpl::GetRecord
virtual DB2Record GetRecord(uint32 recordNumber) const =0
DB2FileLoaderImpl::GetRecordCopy
virtual DB2RecordCopy GetRecordCopy(uint32 copyNumber) const =0
DB2FileLoaderImpl::RecordGetUInt32
virtual uint32 RecordGetUInt32(uint8 const *record, uint32 field, uint32 arrayIndex) const =0
DB2FileLoaderImpl::RecordGetString
virtual char const * RecordGetString(uint8 const *record, uint32 field, uint32 arrayIndex) const =0
DB2FileLoaderImpl::DB2FileLoaderImpl
DB2FileLoaderImpl(DB2FileLoaderImpl const &other)=delete
DB2FileLoaderImpl::LoadCatalogData
virtual bool LoadCatalogData(DB2FileSource *source, uint32 section)=0
DB2FileLoaderImpl::LoadTableData
virtual bool LoadTableData(DB2FileSource *source, uint32 section)=0
DB2FileLoaderImpl::DB2FileLoaderImpl
DB2FileLoaderImpl(DB2FileLoaderImpl &&other) noexcept=delete
DB2FileLoaderImpl::AutoProduceRecordCopies
virtual void AutoProduceRecordCopies(uint32 records, char **indexTable, char *dataTable)=0
DB2FileLoaderImpl::SkipEncryptedSection
virtual void SkipEncryptedSection(uint32 section)=0
DB2FileLoaderImpl::DB2Record
friend class DB2Record
Definition: DB2FileLoader.cpp:189
DB2FileLoaderImpl::RecordGetInt32
virtual int32 RecordGetInt32(uint8 const *record, uint32 field, uint32 arrayIndex) const =0
DB2FileLoaderImpl::GetRecordCount
virtual uint32 GetRecordCount() const =0
DB2FileLoaderImpl::RecordGetUInt8
virtual uint8 RecordGetUInt8(uint8 const *record, uint32 field, uint32 arrayIndex) const =0
DB2FileLoaderImpl::GetRawRecordData
virtual unsigned char const * GetRawRecordData(uint32 recordNumber, uint32 const *section) const =0
DB2FileLoaderImpl::RecordDestroyFieldOffsets
virtual void RecordDestroyFieldOffsets(std::size_t *&fieldOffsets) const =0
DB2FileLoaderImpl::AutoProduceStrings
virtual char * AutoProduceStrings(char **indexTable, uint32 indexTableSize, uint32 locale)=0
DB2FileLoaderImpl::RecordGetUInt16
virtual uint16 RecordGetUInt16(uint8 const *record, uint32 field, uint32 arrayIndex) const =0
DB2FileLoaderImpl::operator=
DB2FileLoaderImpl & operator=(DB2FileLoaderImpl const &other)=delete
DB2FileLoaderImpl::operator=
DB2FileLoaderImpl & operator=(DB2FileLoaderImpl &&other) noexcept=delete
DB2FileLoaderRegularImpl::_columnMeta
std::unique_ptr< DB2ColumnMeta[]> _columnMeta
Definition: DB2FileLoader.cpp:261
DB2FileLoaderRegularImpl::FillParentLookup
void FillParentLookup(char *dataTable)
Definition: DB2FileLoader.cpp:643
DB2FileLoaderRegularImpl::AutoProduceData
char * AutoProduceData(uint32 &indexTableSize, char **&indexTable) override
Definition: DB2FileLoader.cpp:392
DB2FileLoaderRegularImpl::_loadInfo
DB2FileLoadInfo const * _loadInfo
Definition: DB2FileLoader.cpp:256
DB2FileLoaderRegularImpl::RecordGetPackedValue
uint64 RecordGetPackedValue(uint8 const *packedRecordData, uint32 bitWidth, uint32 bitOffset) const
Definition: DB2FileLoader.cpp:878
DB2FileLoaderRegularImpl::DB2FileLoaderRegularImpl
DB2FileLoaderRegularImpl(char const *fileName, DB2FileLoadInfo const *loadInfo, DB2Header const *header)
Definition: DB2FileLoader.cpp:337
DB2FileLoaderRegularImpl::operator=
DB2FileLoaderRegularImpl & operator=(DB2FileLoaderRegularImpl &&other) noexcept=delete
DB2FileLoaderRegularImpl::_header
DB2Header const * _header
Definition: DB2FileLoader.cpp:257
DB2FileLoaderRegularImpl::GetRecordSection
uint32 GetRecordSection(uint32 recordNumber) const
Definition: DB2FileLoader.cpp:729
DB2FileLoaderRegularImpl::GetSection
DB2SectionHeader & GetSection(uint32 section) const override
Definition: DB2FileLoader.cpp:980
DB2FileLoaderRegularImpl::RecordCreateDetachedFieldOffsets
std::size_t * RecordCreateDetachedFieldOffsets(std::size_t *oldOffsets) const override
Definition: DB2FileLoader.cpp:908
DB2FileLoaderRegularImpl::LoadCatalogData
bool LoadCatalogData(DB2FileSource *, uint32) override
Definition: DB2FileLoader.cpp:219
DB2FileLoaderRegularImpl::_palletArrayValues
std::unique_ptr< std::unique_ptr< DB2PalletValue[]>[]> _palletArrayValues
Definition: DB2FileLoader.cpp:263
DB2FileLoaderRegularImpl::GetMaxId
uint32 GetMaxId() const override
Definition: DB2FileLoader.cpp:950
DB2FileLoaderRegularImpl::RecordGetUInt8
uint8 RecordGetUInt8(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:763
DB2FileLoaderRegularImpl::GetRawRecordData
unsigned char const * GetRawRecordData(uint32 recordNumber, uint32 const *section) const override
Definition: DB2FileLoader.cpp:744
DB2FileLoaderRegularImpl::GetRecordCopyCount
uint32 GetRecordCopyCount() const override
Definition: DB2FileLoader.cpp:724
DB2FileLoaderRegularImpl::_sections
std::unique_ptr< DB2SectionHeader[]> _sections
Definition: DB2FileLoader.cpp:260
DB2FileLoaderRegularImpl::RecordGetUInt32
uint32 RecordGetUInt32(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:773
DB2FileLoaderRegularImpl::RecordGetFloat
float RecordGetFloat(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:788
DB2FileLoaderRegularImpl::SkipEncryptedSection
void SkipEncryptedSection(uint32) override
Definition: DB2FileLoader.cpp:217
DB2FileLoaderRegularImpl::SetAdditionalData
void SetAdditionalData(std::vector< uint32 > idTable, std::vector< DB2RecordCopy > copyTable, std::vector< std::vector< DB2IndexData > > parentIndexes) override
Definition: DB2FileLoader.cpp:381
DB2FileLoaderRegularImpl::LoadColumnData
void LoadColumnData(std::unique_ptr< DB2SectionHeader[]> sections, std::unique_ptr< DB2FieldEntry[]> fields, std::unique_ptr< DB2ColumnMeta[]> columnMeta, std::unique_ptr< std::unique_ptr< DB2PalletValue[]>[]> palletValues, std::unique_ptr< std::unique_ptr< DB2PalletValue[]>[]> palletArrayValues, std::unique_ptr< std::unordered_map< uint32, uint32 >[]> commonValues) override
Definition: DB2FileLoader.cpp:345
DB2FileLoaderRegularImpl::GetExpectedSignMismatchReason
char const * GetExpectedSignMismatchReason(uint32 field) const override
Definition: DB2FileLoader.cpp:1014
DB2FileLoaderRegularImpl::RecordGetString
char const * RecordGetString(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:793
DB2FileLoaderRegularImpl::RecordDestroyFieldOffsets
void RecordDestroyFieldOffsets(std::size_t *&fieldOffsets) const override
Definition: DB2FileLoader.cpp:918
DB2FileLoaderRegularImpl::_copyTable
std::vector< DB2RecordCopy > _copyTable
Definition: DB2FileLoader.cpp:266
DB2FileLoaderRegularImpl::_palletValues
std::unique_ptr< std::unique_ptr< DB2PalletValue[]>[]> _palletValues
Definition: DB2FileLoader.cpp:262
DB2FileLoaderRegularImpl::GetRecord
DB2Record GetRecord(uint32 recordNumber) const override
Definition: DB2FileLoader.cpp:706
DB2FileLoaderRegularImpl::_data
std::unique_ptr< uint8[]> _data
Definition: DB2FileLoader.cpp:258
DB2FileLoaderRegularImpl::GetRecordCopy
DB2RecordCopy GetRecordCopy(uint32 copyNumber) const override
Definition: DB2FileLoader.cpp:711
DB2FileLoaderRegularImpl::operator=
DB2FileLoaderRegularImpl & operator=(DB2FileLoaderRegularImpl const &other)=delete
DB2FileLoaderRegularImpl::GetLoadInfo
DB2FileLoadInfo const * GetLoadInfo() const override
Definition: DB2FileLoader.cpp:975
DB2FileLoaderRegularImpl::RecordGetId
uint32 RecordGetId(uint8 const *record, uint32 recordIndex) const override
Definition: DB2FileLoader.cpp:755
DB2FileLoaderRegularImpl::AutoProduceStrings
char * AutoProduceStrings(char **indexTable, uint32 indexTableSize, uint32 locale) override
Definition: DB2FileLoader.cpp:518
DB2FileLoaderRegularImpl::IsSignedField
bool IsSignedField(uint32 field) const override
Definition: DB2FileLoader.cpp:985
DB2FileLoaderRegularImpl::GetFieldOffset
uint16 GetFieldOffset(uint32 field) const
Definition: DB2FileLoader.cpp:884
DB2FileLoaderRegularImpl::RecordGetUInt64
uint64 RecordGetUInt64(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:783
DB2FileLoaderRegularImpl::DB2FileLoaderRegularImpl
DB2FileLoaderRegularImpl(DB2FileLoaderRegularImpl const &other)=delete
DB2FileLoaderRegularImpl::_parentIndexes
std::vector< std::vector< DB2IndexData > > _parentIndexes
Definition: DB2FileLoader.cpp:267
DB2FileLoaderRegularImpl::GetRecordCount
uint32 GetRecordCount() const override
Definition: DB2FileLoader.cpp:719
DB2FileLoaderRegularImpl::AutoProduceRecordCopies
void AutoProduceRecordCopies(uint32 records, char **indexTable, char *dataTable) override
Definition: DB2FileLoader.cpp:621
DB2FileLoaderRegularImpl::RecordCopyDetachedFieldOffsets
std::size_t * RecordCopyDetachedFieldOffsets(std::size_t *oldOffsets) const override
Definition: DB2FileLoader.cpp:913
DB2FileLoaderRegularImpl::DB2FileLoaderRegularImpl
DB2FileLoaderRegularImpl(DB2FileLoaderRegularImpl &&other) noexcept=delete
DB2FileLoaderRegularImpl::RecordGetInt32
int32 RecordGetInt32(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:778
DB2FileLoaderRegularImpl::_idTable
std::vector< uint32 > _idTable
Definition: DB2FileLoader.cpp:265
DB2FileLoaderRegularImpl::_commonValues
std::unique_ptr< std::unordered_map< uint32, uint32 >[]> _commonValues
Definition: DB2FileLoader.cpp:264
DB2FileLoaderRegularImpl::RecordGetUInt16
uint16 RecordGetUInt16(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:768
DB2FileLoaderRegularImpl::LoadTableData
bool LoadTableData(DB2FileSource *source, uint32 section) override
Definition: DB2FileLoader.cpp:356
DB2FileLoaderRegularImpl::GetMinId
uint32 GetMinId() const override
Definition: DB2FileLoader.cpp:922
DB2FileLoaderRegularImpl::RecordGetVarInt
T RecordGetVarInt(uint8 const *record, uint32 field, uint32 arrayIndex) const
Definition: DB2FileLoader.cpp:802
DB2FileLoaderSparseImpl::_recordBuffer
std::unique_ptr< uint8[]> _recordBuffer
Definition: DB2FileLoader.cpp:327
DB2FileLoaderSparseImpl::_source
DB2FileSource *const _source
Definition: DB2FileLoader.cpp:324
DB2FileLoaderSparseImpl::FillParentLookup
void FillParentLookup(char *dataTable)
Definition: DB2FileLoader.cpp:1404
DB2FileLoaderSparseImpl::CalculateAndStoreFieldOffsets
void CalculateAndStoreFieldOffsets(uint8 const *rawRecord) const
Definition: DB2FileLoader.cpp:1606
DB2FileLoaderSparseImpl::DB2FileLoaderSparseImpl
DB2FileLoaderSparseImpl(DB2FileLoaderSparseImpl const &other)=delete
DB2FileLoaderSparseImpl::GetExpectedSignMismatchReason
char const * GetExpectedSignMismatchReason(uint32 field) const override
Definition: DB2FileLoader.cpp:1666
DB2FileLoaderSparseImpl::LoadTableData
bool LoadTableData(DB2FileSource *, uint32) override
Definition: DB2FileLoader.cpp:284
DB2FileLoaderSparseImpl::RecordDestroyFieldOffsets
void RecordDestroyFieldOffsets(std::size_t *&fieldOffsets) const override
Definition: DB2FileLoader.cpp:1597
DB2FileLoaderSparseImpl::_loadInfo
DB2FileLoadInfo const * _loadInfo
Definition: DB2FileLoader.cpp:322
DB2FileLoaderSparseImpl::_copyTable
std::vector< DB2RecordCopy > _copyTable
Definition: DB2FileLoader.cpp:333
DB2FileLoaderSparseImpl::GetSection
DB2SectionHeader & GetSection(uint32 section) const override
Definition: DB2FileLoader.cpp:1655
DB2FileLoaderSparseImpl::GetRecordSection
uint32 GetRecordSection(uint32 recordNumber) const
Definition: DB2FileLoader.cpp:1466
DB2FileLoaderSparseImpl::GetMinId
uint32 GetMinId() const override
Definition: DB2FileLoader.cpp:1640
DB2FileLoaderSparseImpl::_parentIndexes
std::vector< std::vector< DB2IndexData > > _parentIndexes
Definition: DB2FileLoader.cpp:334
DB2FileLoaderSparseImpl::GetMaxId
uint32 GetMaxId() const override
Definition: DB2FileLoader.cpp:1645
DB2FileLoaderSparseImpl::_fields
std::unique_ptr< DB2FieldEntry[]> _fields
Definition: DB2FileLoader.cpp:329
DB2FileLoaderSparseImpl::GetRawRecordData
unsigned char const * GetRawRecordData(uint32 recordNumber, uint32 const *section) const override
Definition: DB2FileLoader.cpp:1481
DB2FileLoaderSparseImpl::GetRecord
DB2Record GetRecord(uint32 recordNumber) const override
Definition: DB2FileLoader.cpp:1381
DB2FileLoaderSparseImpl::GetRecordCopyCount
uint32 GetRecordCopyCount() const override
Definition: DB2FileLoader.cpp:1399
DB2FileLoaderSparseImpl::AutoProduceData
char * AutoProduceData(uint32 &indexTableSize, char **&indexTable) override
Definition: DB2FileLoader.cpp:1110
DB2FileLoaderSparseImpl::RecordGetString
char const * RecordGetString(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:1546
DB2FileLoaderSparseImpl::DB2FileLoaderSparseImpl
DB2FileLoaderSparseImpl(char const *fileName, DB2FileLoadInfo const *loadInfo, DB2Header const *header, DB2FileSource *source)
Definition: DB2FileLoader.cpp:1047
DB2FileLoaderSparseImpl::SkipEncryptedSection
void SkipEncryptedSection(uint32 section) override
Definition: DB2FileLoader.cpp:1068
DB2FileLoaderSparseImpl::RecordGetVarInt
uint32 RecordGetVarInt(uint8 const *record, uint32 field, uint32 arrayIndex, bool isSigned) const
Definition: DB2FileLoader.cpp:1552
DB2FileLoaderSparseImpl::GetRecordCopy
DB2RecordCopy GetRecordCopy(uint32 copyNumber) const override
Definition: DB2FileLoader.cpp:1386
DB2FileLoaderSparseImpl::RecordGetFloat
float RecordGetFloat(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:1538
DB2FileLoaderSparseImpl::RecordGetUInt64
uint64 RecordGetUInt64(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:1530
DB2FileLoaderSparseImpl::LoadCatalogData
bool LoadCatalogData(DB2FileSource *source, uint32 section) override
Definition: DB2FileLoader.cpp:1074
DB2FileLoaderSparseImpl::DB2FileLoaderSparseImpl
DB2FileLoaderSparseImpl(DB2FileLoaderSparseImpl &&other) noexcept=delete
DB2FileLoaderSparseImpl::RecordGetId
uint32 RecordGetId(uint8 const *record, uint32 recordIndex) const override
Definition: DB2FileLoader.cpp:1498
DB2FileLoaderSparseImpl::RecordGetUInt32
uint32 RecordGetUInt32(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:1520
DB2FileLoaderSparseImpl::_header
DB2Header const * _header
Definition: DB2FileLoader.cpp:323
DB2FileLoaderSparseImpl::GetLoadInfo
DB2FileLoadInfo const * GetLoadInfo() const override
Definition: DB2FileLoader.cpp:1650
DB2FileLoaderSparseImpl::IsSignedField
bool IsSignedField(uint32 field) const override
Definition: DB2FileLoader.cpp:1660
DB2FileLoaderSparseImpl::operator=
DB2FileLoaderSparseImpl & operator=(DB2FileLoaderSparseImpl const &other)=delete
DB2FileLoaderSparseImpl::_sections
std::unique_ptr< DB2SectionHeader[]> _sections
Definition: DB2FileLoader.cpp:328
DB2FileLoaderSparseImpl::RecordCopyDetachedFieldOffsets
std::size_t * RecordCopyDetachedFieldOffsets(std::size_t *oldOffsets) const override
Definition: DB2FileLoader.cpp:1586
DB2FileLoaderSparseImpl::operator=
DB2FileLoaderSparseImpl & operator=(DB2FileLoaderSparseImpl &&other) noexcept=delete
DB2FileLoaderSparseImpl::_fieldAndArrayOffsets
std::unique_ptr< std::size_t[]> _fieldAndArrayOffsets
Definition: DB2FileLoader.cpp:330
DB2FileLoaderSparseImpl::SetAdditionalData
void SetAdditionalData(std::vector< uint32 > idTable, std::vector< DB2RecordCopy > copyTable, std::vector< std::vector< DB2IndexData > > parentIndexes) override
Definition: DB2FileLoader.cpp:1104
DB2FileLoaderSparseImpl::_catalog
std::vector< DB2CatalogEntry > _catalog
Definition: DB2FileLoader.cpp:332
DB2FileLoaderSparseImpl::_catalogIds
std::vector< uint32 > _catalogIds
Definition: DB2FileLoader.cpp:331
DB2FileLoaderSparseImpl::GetFieldOffset
uint16 GetFieldOffset(uint32 field, uint32 arrayIndex) const
Definition: DB2FileLoader.cpp:1564
DB2FileLoaderSparseImpl::RecordGetUInt8
uint8 RecordGetUInt8(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:1506
DB2FileLoaderSparseImpl::RecordGetInt32
int32 RecordGetInt32(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:1525
DB2FileLoaderSparseImpl::RecordCreateDetachedFieldOffsets
std::size_t * RecordCreateDetachedFieldOffsets(std::size_t *oldOffsets) const override
Definition: DB2FileLoader.cpp:1575
DB2FileLoaderSparseImpl::LoadColumnData
void LoadColumnData(std::unique_ptr< DB2SectionHeader[]> sections, std::unique_ptr< DB2FieldEntry[]> fields, std::unique_ptr< DB2ColumnMeta[]> columnMeta, std::unique_ptr< std::unique_ptr< DB2PalletValue[]>[]> palletValues, std::unique_ptr< std::unique_ptr< DB2PalletValue[]>[]> palletArrayValues, std::unique_ptr< std::unordered_map< uint32, uint32 >[]> commonValues) override
Definition: DB2FileLoader.cpp:1060
DB2FileLoaderSparseImpl::GetFieldSize
uint16 GetFieldSize(uint32 field) const
Definition: DB2FileLoader.cpp:1569
DB2FileLoaderSparseImpl::GetRecordCount
uint32 GetRecordCount() const override
Definition: DB2FileLoader.cpp:1394
DB2FileLoaderSparseImpl::RecordGetUInt16
uint16 RecordGetUInt16(uint8 const *record, uint32 field, uint32 arrayIndex) const override
Definition: DB2FileLoader.cpp:1512
DB2FileLoaderSparseImpl::AutoProduceStrings
char * AutoProduceStrings(char **indexTable, uint32 indexTableSize, uint32 locale) override
Definition: DB2FileLoader.cpp:1235
DB2FileLoaderSparseImpl::AutoProduceRecordCopies
void AutoProduceRecordCopies(uint32 records, char **indexTable, char *dataTable) override
Definition: DB2FileLoader.cpp:1359
DB2FileLoader::_header
DB2Header _header
Definition: DB2FileLoader.h:221
DB2FileLoader::GetMaxId
uint32 GetMaxId() const
Definition: DB2FileLoader.cpp:2125
DB2FileLoader::GetMinId
uint32 GetMinId() const
Definition: DB2FileLoader.cpp:2120
DB2FileLoader::Load
void Load(DB2FileSource *source, DB2FileLoadInfo const *loadInfo)
Definition: DB2FileLoader.cpp:1938
DB2FileLoader::GetRecord
DB2Record GetRecord(uint32 recordNumber) const
Definition: DB2FileLoader.cpp:2135
DB2FileLoader::GetRecordCopy
DB2RecordCopy GetRecordCopy(uint32 copyNumber) const
Definition: DB2FileLoader.cpp:2140
DB2FileLoader::AutoProduceData
char * AutoProduceData(uint32 &indexTableSize, char **&indexTable)
Definition: DB2FileLoader.cpp:2095
DB2FileLoader::AutoProduceStrings
char * AutoProduceStrings(char **indexTable, uint32 indexTableSize, LocaleConstant locale)
Definition: DB2FileLoader.cpp:2100
DB2FileLoader::LoadHeaders
void LoadHeaders(DB2FileSource *source, DB2FileLoadInfo const *loadInfo)
Definition: DB2FileLoader.cpp:1804
DB2FileLoader::GetSectionHeader
DB2SectionHeader const & GetSectionHeader(uint32 section) const
Definition: DB2FileLoader.cpp:2130
DB2FileLoader::GetRecordCopyCount
uint32 GetRecordCopyCount() const
Definition: DB2FileLoader.cpp:2115
DB2FileLoader::_impl
DB2FileLoaderImpl * _impl
Definition: DB2FileLoader.h:220
DB2FileLoader::AutoProduceRecordCopies
void AutoProduceRecordCopies(uint32 records, char **indexTable, char *dataTable)
Definition: DB2FileLoader.cpp:2105
DB2FileLoader::GetRecordCount
uint32 GetRecordCount() const
Definition: DB2FileLoader.cpp:2110
DB2Record::GetUInt64
uint64 GetUInt64(uint32 field, uint32 arrayIndex) const
Definition: DB2FileLoader.cpp:1754
DB2Record::GetFloat
float GetFloat(uint32 field, uint32 arrayIndex) const
Definition: DB2FileLoader.cpp:1766
DB2Record::DB2Record
DB2Record(DB2FileLoaderImpl const &db2, uint32 recordIndex, std::size_t *fieldOffsets)
Definition: DB2FileLoader.cpp:1676
DB2Record::_db2
DB2FileLoaderImpl const & _db2
Definition: DB2FileLoader.h:164
DB2Record::GetInt32
int32 GetInt32(uint32 field, uint32 arrayIndex) const
Definition: DB2FileLoader.cpp:1742
DB2Record::_fieldOffsets
std::size_t * _fieldOffsets
Definition: DB2FileLoader.h:167
DB2Record::GetUInt32
uint32 GetUInt32(uint32 field, uint32 arrayIndex) const
Definition: DB2FileLoader.cpp:1730
DB2Record::GetUInt16
uint16 GetUInt16(uint32 field, uint32 arrayIndex) const
Definition: DB2FileLoader.cpp:1718
DB2Record::GetUInt8
uint8 GetUInt8(uint32 field, uint32 arrayIndex) const
Definition: DB2FileLoader.cpp:1706
DB2Record::GetId
uint32 GetId() const
Definition: DB2FileLoader.cpp:1701
DB2Record::GetString
char const * GetString(uint32 field, uint32 arrayIndex) const
Definition: DB2FileLoader.cpp:1778
DB2Record::_recordData
unsigned char const * _recordData
Definition: DB2FileLoader.h:166
DB2Record::_recordIndex
uint32 _recordIndex
Definition: DB2FileLoader.h:165
DB2Record::MakePersistent
void MakePersistent()
Definition: DB2FileLoader.cpp:1790
Trinity::StringFormat
std::string StringFormat(FormatString< Args... > fmt, Args &&... args)
Default TC string format function.
Definition: StringFormat.h:38
UF::size
constexpr std::size_t size()
Definition: UpdateField.h:796
std
STL namespace.
DB2ColumnMeta::immediate
struct DB2ColumnMeta::@1::@2 immediate
DB2ColumnMeta::CompressionData
union DB2ColumnMeta::@1 CompressionData
DB2ColumnMeta::CompressionType
DB2ColumnCompression CompressionType
Definition: DB2FileLoader.cpp:58
DB2ColumnMeta::commonData
struct DB2ColumnMeta::@1::@3 commonData
DB2ColumnMeta::AdditionalDataSize
uint32 AdditionalDataSize
Definition: DB2FileLoader.cpp:57
DB2ColumnMeta::pallet
struct DB2ColumnMeta::@1::@4 pallet
DB2FieldMeta::Name
char const * Name
Definition: DB2FileLoader.h:77
DB2FieldMeta::Type
DBCFormer Type
Definition: DB2FileLoader.h:76
DB2FileLoadInfo::FieldCount
std::size_t FieldCount
Definition: DB2FileLoader.h:90
DB2FileLoadInfo::GetFieldIndexByName
std::pair< int32, int32 > GetFieldIndexByName(char const *fieldName) const
Definition: DB2FileLoader.cpp:128
DB2FileLoadInfo::Meta
DB2Meta const * Meta
Definition: DB2FileLoader.h:91
DB2FileLoadInfo::GetStringFieldCount
uint32 GetStringFieldCount(bool localizedOnly) const
Definition: DB2FileLoader.cpp:118
DB2FileLoadInfo::GetFieldIndexByMetaIndex
int32 GetFieldIndexByMetaIndex(uint32 metaIndex) const
Definition: DB2FileLoader.cpp:145
DB2FileLoadInfo::Fields
DB2FieldMeta const * Fields
Definition: DB2FileLoader.h:89
DB2FileSource::GetFileSize
virtual int64 GetFileSize() const =0
DB2FileSource::HandleEncryptedSection
virtual DB2EncryptedSectionHandling HandleEncryptedSection(DB2SectionHeader const &sectionHeader) const =0
DB2FileSource::GetFileName
virtual char const * GetFileName() const =0
DB2FileSource::IsOpen
virtual bool IsOpen() const =0
DB2FileSource::SetPosition
virtual bool SetPosition(int64 position)=0
DB2FileSource::~DB2FileSource
virtual ~DB2FileSource()
DB2FileSource::Read
virtual bool Read(void *buffer, std::size_t numBytes)=0
DB2FileSource::GetPosition
virtual int64 GetPosition() const =0
DB2Header::TotalFieldCount
uint32 TotalFieldCount
Definition: DB2FileLoader.h:47
DB2Header::SectionCount
uint32 SectionCount
Definition: DB2FileLoader.h:53
DB2Header::RecordCount
uint32 RecordCount
Definition: DB2FileLoader.h:36
DB2Header::PackedDataOffset
uint32 PackedDataOffset
Definition: DB2FileLoader.h:48
DB2Header::CommonDataSize
uint32 CommonDataSize
Definition: DB2FileLoader.h:51
DB2Header::StringTableSize
uint32 StringTableSize
Definition: DB2FileLoader.h:39
DB2Header::MaxId
uint32 MaxId
Definition: DB2FileLoader.h:43
DB2Header::PalletDataSize
uint32 PalletDataSize
Definition: DB2FileLoader.h:52
DB2Header::ColumnMetaSize
uint32 ColumnMetaSize
Definition: DB2FileLoader.h:50
DB2Header::Signature
uint32 Signature
Definition: DB2FileLoader.h:33
DB2Header::Flags
uint16 Flags
Definition: DB2FileLoader.h:45
DB2Header::TableHash
uint32 TableHash
Definition: DB2FileLoader.h:40
DB2Header::FieldCount
uint32 FieldCount
Definition: DB2FileLoader.h:37
DB2Header::LayoutHash
uint32 LayoutHash
Definition: DB2FileLoader.h:41
DB2Header::RecordSize
uint32 RecordSize
Definition: DB2FileLoader.h:38
DB2Header::IndexField
int16 IndexField
Definition: DB2FileLoader.h:46
DB2Header::ParentLookupCount
uint32 ParentLookupCount
Definition: DB2FileLoader.h:49
DB2Header::MinId
uint32 MinId
Definition: DB2FileLoader.h:42
DB2Header::Version
uint32 Version
Definition: DB2FileLoader.h:34
DB2Header::Locale
uint32 Locale
Definition: DB2FileLoader.h:44
DB2IndexData::Entries
std::vector< DB2IndexEntry > Entries
Definition: DB2FileLoader.cpp:110
DB2MetaField::ArraySize
uint8 ArraySize
Definition: DB2Meta.h:26
DB2MetaField::Type
DBCFormer Type
Definition: DB2Meta.h:25
DB2Meta::FileFieldCount
uint32 FileFieldCount
Definition: DB2Meta.h:58
DB2Meta::IndexField
int32 IndexField
Definition: DB2Meta.h:55
DB2Meta::Fields
DB2MetaField const * Fields
Definition: DB2Meta.h:60
DB2Meta::GetIndexField
uint32 GetIndexField() const
Definition: DB2Meta.cpp:27
DB2Meta::GetRecordSize
uint32 GetRecordSize() const
Definition: DB2Meta.cpp:32
DB2Meta::HasIndexFieldInData
bool HasIndexFieldInData() const
Definition: DB2Meta.cpp:22
DB2Meta::GetIndexFieldOffset
uint32 GetIndexFieldOffset() const
Definition: DB2Meta.cpp:79
DB2Meta::FieldCount
uint32 FieldCount
Definition: DB2Meta.h:57
DB2Meta::LayoutHash
uint32 LayoutHash
Definition: DB2Meta.h:59
DB2Meta::IsSignedField
bool IsSignedField(uint32 field) const
Definition: DB2Meta.cpp:195
DB2Meta::ParentIndexField
int32 ParentIndexField
Definition: DB2Meta.h:56
DB2Meta::GetParentIndexFieldOffset
int32 GetParentIndexFieldOffset() const
Definition: DB2Meta.cpp:127
DB2RecordCopy::SourceRowId
uint32 SourceRowId
Definition: DB2FileLoader.h:174
DB2SectionHeader::CatalogDataCount
uint32 CatalogDataCount
Definition: DB2FileLoader.h:65
DB2SectionHeader::ParentLookupDataSize
uint32 ParentLookupDataSize
Definition: DB2FileLoader.h:64
LocalizedString::Str
std::array< char const *, TOTAL_LOCALES > Str
Definition: Common.h:109