From cde0dec060edc3f90caea8fde649c211a95b5954 Mon Sep 17 00:00:00 2001 From: jsmall-nvidia Date: Tue, 2 Oct 2018 17:22:15 -0400 Subject: [PATCH] Feature/ir serial debug (#657) * * Change the layout of IROp such that 'main' IROps are 0-x. * Removed MANUAL_RANGE instuction types, as no longer needed. * Work in prog on optimizing. * * Constant time lookup for IROpInfo * Refactor and document a little more the IROp layout * Mark ops that use 'other' bits * Fix typo in definition of kIROpFlag_UseOther * First pass at working out serialization structure. * Work in progress on ir-serialize * Storing strings in IRSerialInfo Split out IRSerialInfo from the IRSerializer - to make more explicit what is actually saved. * First pass at serializing out data. * First pass at serialize reading. * Fix riff fourcc mark order. * First pass at reconstructing IRInst / IRDecoration from serialized data. * Handling of TextureBaseType * Deserializing of constants. * Small changes around ir serialization. * Changed StringIndex indexing to not be an offset into the m_strings array, but an index into strings in order. Doing so makes cache lookup much faster, and makes the 'indicies' themselves smaller and therefore more compressible. * Removed the need for m_arena in IRSerialWriter. Previously it's purpose was to store the string contents that were being used to lookup UnownedStringSlice. Now we keep the StringRepresentation in scope and reference that, and so don't need the copy. * Don't need to construct the IRModuleInst as is created and set on createModule call. * Remove test code for testing serialization. * Fix problem with release build in ir-serialize causing warning. * Use SLANG_OFFSET_OF for offsets in non pod classes to avoid gcc/clang warning. Give storage to integral static variables to avoid linkage problems with gcc/clang. * Fix warnings under x86 win32 debug. * Small improvements around IR serialization. * * Support for serializing SourceLoc. * Small improvements around serialization. * RawSourceLoc allows for regular SourceLoc information to be held (and serialized) as is. This is only really useful for the 'passthru' mode as there needs to be a more compact mechanism to encode source locations. * Small fixes around comments for SourceLoc serializing. --- source/slang/ir-serialize.cpp | 157 ++++++++++++++++++++++++++++------ source/slang/ir-serialize.h | 106 +++++++++++++---------- 2 files changed, 190 insertions(+), 73 deletions(-) diff --git a/source/slang/ir-serialize.cpp b/source/slang/ir-serialize.cpp index 2645e445db..10caaeb5bf 100644 --- a/source/slang/ir-serialize.cpp +++ b/source/slang/ir-serialize.cpp @@ -87,6 +87,43 @@ static UnownedStringSlice asStringSlice(const PrefixString* prefixString) return UnownedStringSlice(reader.m_pos, len); } +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialData !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +template +static size_t _calcArraySize(const List& list) +{ + return list.Count() * sizeof(T); +} + +size_t IRSerialData::calcSizeInBytes() const +{ + return + _calcArraySize(m_insts) + + _calcArraySize(m_childRuns) + + _calcArraySize(m_decorationRuns) + + _calcArraySize(m_externalOperands) + + _calcArraySize(m_rawSourceLocs) + + _calcArraySize(m_strings); +} + +void IRSerialData::clear() +{ + // First Instruction is null + m_insts.SetSize(1); + memset(&m_insts[0], 0, sizeof(Inst)); + + m_childRuns.Clear(); + m_decorationRuns.Clear(); + m_externalOperands.Clear(); + m_rawSourceLocs.Clear(); + + m_strings.SetSize(2); + m_strings[int(kNullStringIndex)] = 0; + m_strings[int(kEmptyStringIndex)] = 0; + + m_decorationBaseIndex = 0; +} + // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IRSerialWriter !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! void IRSerialWriter::_addInstruction(IRInst* inst) @@ -137,8 +174,12 @@ UnownedStringSlice IRSerialWriter::getStringSlice(Ser::StringIndex index) const return asStringSlice((const PrefixString*)(m_serialData->m_strings.begin() + int(offset))); } -Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) +Result IRSerialWriter::write(IRModule* module, SourceManager* sourceManager, OptionFlags options, IRSerialData* serialData) { + typedef Ser::Inst::PayloadType PayloadType; + + SLANG_UNUSED(sourceManager); + m_serialData = serialData; serialData->clear(); @@ -256,7 +297,7 @@ Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) SLANG_ASSERT(!isPseudoOp(srcInst->op)); dstInst.m_op = uint8_t(srcInst->op & kIROpMeta_OpMask); - dstInst.m_payloadType = Ser::Inst::PayloadType::Empty; + dstInst.m_payloadType = PayloadType::Empty; dstInst.m_resultTypeIndex = getInstIndex(srcInst->getFullType()); @@ -269,25 +310,25 @@ Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) case kIROp_StringLit: { auto stringLit = static_cast(srcInst); - dstInst.m_payloadType = Ser::Inst::PayloadType::String_1; + dstInst.m_payloadType = PayloadType::String_1; dstInst.m_payload.m_stringIndices[0] = getStringIndex(stringLit->getStringSlice()); break; } case kIROp_IntLit: { - dstInst.m_payloadType = Ser::Inst::PayloadType::Int64; + dstInst.m_payloadType = PayloadType::Int64; dstInst.m_payload.m_int64 = irConst->value.intVal; break; } case kIROp_FloatLit: { - dstInst.m_payloadType = Ser::Inst::PayloadType::Float64; + dstInst.m_payloadType = PayloadType::Float64; dstInst.m_payload.m_float64 = irConst->value.floatVal; break; } case kIROp_boolConst: { - dstInst.m_payloadType = Ser::Inst::PayloadType::UInt32; + dstInst.m_payloadType = PayloadType::UInt32; dstInst.m_payload.m_uint32 = irConst->value.intVal ? 1 : 0; break; } @@ -302,7 +343,7 @@ Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) IRGlobalValue* globValue = as(srcInst); if (globValue) { - dstInst.m_payloadType = Ser::Inst::PayloadType::String_1; + dstInst.m_payloadType = PayloadType::String_1; dstInst.m_payload.m_stringIndices[0] = getStringIndex(globValue->mangledName); continue; } @@ -310,7 +351,7 @@ Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) IRTextureTypeBase* textureBase = as(srcInst); if (textureBase) { - dstInst.m_payloadType = Ser::Inst::PayloadType::OperandAndUInt32; + dstInst.m_payloadType = PayloadType::OperandAndUInt32; dstInst.m_payload.m_operandAndUInt32.m_uint32 = uint32_t(srcInst->op) >> kIROpMeta_OtherShift; dstInst.m_payload.m_operandAndUInt32.m_operand = getInstIndex(textureBase->getElementType()); continue; @@ -322,14 +363,17 @@ Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) const int numOperands = int(srcInst->operandCount); Ser::InstIndex* dstOperands = nullptr; - if (numOperands <= Ser::kNumOperands) + if (numOperands <= Ser::Inst::kMaxOperands) { + // Checks the compile below is valid + SLANG_COMPILE_TIME_ASSERT(PayloadType(0) == PayloadType::Empty && PayloadType(1) == PayloadType::Operand_1 && PayloadType(2) == PayloadType::Operand_2); + + dstInst.m_payloadType = PayloadType(numOperands); dstOperands = dstInst.m_payload.m_operands; - dstInst.m_payloadType = Ser::Inst::PayloadType(numOperands); } else { - dstInst.m_payloadType = Ser::Inst::PayloadType::OperandExternal; + dstInst.m_payloadType = PayloadType::OperandExternal; int operandArrayBaseIndex = int(m_serialData->m_externalOperands.Count()); m_serialData->m_externalOperands.SetSize(operandArrayBaseIndex + numOperands); @@ -384,7 +428,7 @@ Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) { auto loopDecor = static_cast(srcDecor); - dstInst.m_payloadType = Ser::Inst::PayloadType::UInt32; + dstInst.m_payloadType = PayloadType::UInt32; dstInst.m_payload.m_uint32 = uint32_t(loopDecor->mode); break; } @@ -392,14 +436,14 @@ Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) { auto targetDecor = static_cast(srcDecor); - dstInst.m_payloadType = Ser::Inst::PayloadType::String_1; + dstInst.m_payloadType = PayloadType::String_1; dstInst.m_payload.m_stringIndices[0] = getStringIndex(targetDecor->targetName); break; } case kIRDecorationOp_TargetIntrinsic: { auto targetDecor = static_cast(srcDecor); - dstInst.m_payloadType = Ser::Inst::PayloadType::String_2; + dstInst.m_payloadType = PayloadType::String_2; dstInst.m_payload.m_stringIndices[0] = getStringIndex(targetDecor->targetName); dstInst.m_payload.m_stringIndices[1] = getStringIndex(targetDecor->definition); @@ -408,7 +452,7 @@ Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) case kIRDecorationOp_GLSLOuterArray: { auto arrayDecor = static_cast(srcDecor); - dstInst.m_payloadType = Ser::Inst::PayloadType::String_1; + dstInst.m_payloadType = PayloadType::String_1; dstInst.m_payload.m_stringIndices[0] = getStringIndex(arrayDecor->outerArrayName); break; @@ -417,14 +461,14 @@ Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) { auto semanticDecor = static_cast(srcDecor); - dstInst.m_payloadType = Ser::Inst::PayloadType::String_1; + dstInst.m_payloadType = PayloadType::String_1; dstInst.m_payload.m_stringIndices[0] = getStringIndex(semanticDecor->semanticName); break; } case kIRDecorationOp_InterpolationMode: { auto semanticDecor = static_cast(srcDecor); - dstInst.m_payloadType = Ser::Inst::PayloadType::UInt32; + dstInst.m_payloadType = PayloadType::UInt32; dstInst.m_payload.m_uint32 = uint32_t(semanticDecor->mode); break; } @@ -432,7 +476,7 @@ Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) { auto nameDecor = static_cast(srcDecor); - dstInst.m_payloadType = Ser::Inst::PayloadType::String_1; + dstInst.m_payloadType = PayloadType::String_1; dstInst.m_payload.m_stringIndices[0] = getStringIndex(nameDecor->name); break; } @@ -445,6 +489,22 @@ Result IRSerialWriter::write(IRModule* module, IRSerialData* serialData) } } + // If the option to use RawSourceLocations is enabled, serialize out as is + if (options & OptionFlag::RawSourceLocation) + { + const int numInsts = int(m_insts.Count()); + serialData->m_rawSourceLocs.SetSize(numInsts); + + Ser::RawSourceLoc* dstLocs = serialData->m_rawSourceLocs.begin(); + // 0 is null, just mark as no location + dstLocs[0] = Ser::RawSourceLoc(0); + for (int i = 1; i < numInsts; ++i) + { + IRInst* srcInst = m_insts[i]; + dstLocs[i] = Ser::RawSourceLoc(srcInst->sourceLoc.getRaw()); + } + } + m_serialData = nullptr; return SLANG_OK; } @@ -578,7 +638,8 @@ Result _writeArrayChunk(uint32_t chunkId, const List& array, Stream* stream) _calcChunkSize(data.m_childRuns) + _calcChunkSize(data.m_decorationRuns) + _calcChunkSize(data.m_externalOperands) + - _calcChunkSize(data.m_strings); + _calcChunkSize(data.m_strings) + + _calcChunkSize(data.m_rawSourceLocs); { Bin::Chunk riffHeader; @@ -601,7 +662,12 @@ Result _writeArrayChunk(uint32_t chunkId, const List& array, Stream* stream) _writeArrayChunk(Bin::kDecoratorRunFourCc, data.m_decorationRuns, stream); _writeArrayChunk(Bin::kExternalOperandsFourCc, data.m_externalOperands, stream); _writeArrayChunk(Bin::kStringFourCc, data.m_strings, stream); - + + { + uint32_t fourCc = sizeof(IRSerialData::RawSourceLoc) == 4 ? Bin::kUInt32SourceLocFourCc : Bin::kUInt64SourceLocFourCc; + _writeArrayChunk(fourCc, data.m_rawSourceLocs, stream); + } + return SLANG_OK; } @@ -645,6 +711,20 @@ int64_t _calcChunkTotalSize(const IRSerialBinary::Chunk& chunk) return (size + 3) & ~int64_t(3); } +/* static */Result IRSerialReader::_skip(const IRSerialBinary::Chunk& chunk, Stream* stream, int64_t* remainingBytesInOut) +{ + typedef IRSerialBinary Bin; + int64_t chunkSize = _calcChunkTotalSize(chunk); + if (remainingBytesInOut) + { + *remainingBytesInOut -= chunkSize; + } + + // Skip the payload (we don't need to skip the Chunk because that was already read + stream->Seek(SeekOrigin::Current, chunkSize - sizeof(IRSerialBinary::Chunk)); + return SLANG_OK; +} + /* static */Result IRSerialReader::readStream(Stream* stream, IRSerialData* dataOut) { typedef IRSerialBinary Bin; @@ -717,13 +797,24 @@ int64_t _calcChunkTotalSize(const IRSerialBinary::Chunk& chunk) remainingBytes -= _calcChunkTotalSize(chunk); break; } + case Bin::kUInt32SourceLocFourCc: + case Bin::kUInt64SourceLocFourCc: + { + if ((sizeof(IRSerialData::RawSourceLoc) == 4 && chunk.m_type == Bin::kUInt32SourceLocFourCc) || + (sizeof(IRSerialData::RawSourceLoc) == 8 && chunk.m_type == Bin::kUInt64SourceLocFourCc)) + { + SLANG_RETURN_ON_FAIL(_readArrayChunk(chunk, stream, dataOut->m_rawSourceLocs)); + remainingBytes -= _calcChunkTotalSize(chunk); + } + else + { + SLANG_RETURN_ON_FAIL(_skip(chunk, stream, &remainingBytes)); + } + break; + } default: { - remainingBytes -= _calcChunkTotalSize(chunk); - - // Unhandled chunk... skip it - int skipSize = (chunk.m_size + 3) & ~3; - stream->Seek(SeekOrigin::Current, skipSize); + SLANG_RETURN_ON_FAIL(_skip(chunk, stream, &remainingBytes)); break; } } @@ -1135,17 +1226,29 @@ IRDecoration* IRSerialReader::_createDecoration(const Ser::Inst& srcInst) } } + // Re-add source locations, if they are defined + if (int(m_serialData->m_rawSourceLocs.Count()) == numInsts) + { + const Ser::RawSourceLoc* srcLocs = m_serialData->m_rawSourceLocs.begin(); + for (int i = 1; i < numInsts; ++i) + { + IRInst* dstInst = insts[i]; + + dstInst->sourceLoc.setRaw(Slang::SourceLoc::RawValue(srcLocs[i])); + } + } + return SLANG_OK; } // !!!!!!!!!!!!!!!!!!!!!!!!!!!! Free functions !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Result serializeModule(IRModule* module, Stream* stream) +Result serializeModule(IRModule* module, SourceManager* sourceManager, Stream* stream) { IRSerialWriter serializer; IRSerialData serialData; - SLANG_RETURN_ON_FAIL(serializer.write(module, &serialData)); + SLANG_RETURN_ON_FAIL(serializer.write(module, sourceManager, IRSerialWriter::OptionFlag::RawSourceLocation, &serialData)); if (stream) { diff --git a/source/slang/ir-serialize.h b/source/slang/ir-serialize.h index 31e8aa6ee2..356e140558 100644 --- a/source/slang/ir-serialize.h +++ b/source/slang/ir-serialize.h @@ -20,7 +20,8 @@ struct IRSerialData enum class InstIndex : uint32_t; enum class StringIndex : uint32_t; enum class ArrayIndex : uint32_t; - enum class SourceLoc : uint32_t; + + enum class RawSourceLoc : SourceLoc::RawValue; ///< This is just to copy over source loc data (ie not strictly serialize) enum class StringOffset : uint32_t; ///< Offset into the m_stringsBuffer typedef uint32_t SizeType; @@ -28,11 +29,6 @@ struct IRSerialData static const StringIndex kNullStringIndex = StringIndex(0); static const StringIndex kEmptyStringIndex = StringIndex(1); - enum - { - kNumOperands = 2, - }; - /// A run of instructions struct InstRun { @@ -53,12 +49,19 @@ struct IRSerialData // Decoration information is stored in m_decorationRuns struct Inst { - // NOTE! Can't change order or list without changing approprite s_payloadInfos + enum + { + kMaxOperands = 2, ///< Maximum number of operands that can be held in an instruction (otherwise held 'externally') + }; + + // NOTE! Can't change order or list without changing appropriate s_payloadInfos enum class PayloadType : uint8_t { + // First 3 must be in this order so a cast from 0-2 is directly represented as number of operands Empty, ///< Has no payload (or operands) Operand_1, ///< 1 Operand Operand_2, ///< 2 Operands + OperandAndUInt32, ///< 1 Operand and a single UInt32 OperandExternal, ///< Operands are held externally String_1, ///< 1 String @@ -71,10 +74,7 @@ struct IRSerialData }; /// Get the number of operands - SLANG_FORCE_INLINE int getNumOperands() const - { - return (m_payloadType == PayloadType::OperandExternal) ? m_payload.m_externalOperand.m_size : s_payloadInfos[int(m_payloadType)].m_numOperands; - } + SLANG_FORCE_INLINE int getNumOperands() const; uint8_t m_op; ///< For now one of IROp PayloadType m_payloadType; ///< The type of payload @@ -101,8 +101,8 @@ struct IRSerialData uint32_t m_uint32; ///< Unsigned integral value IRFloatingPointValue m_float; ///< Floating point value IRIntegerValue m_int; ///< Integral value - InstIndex m_operands[kNumOperands]; ///< For items that 2 or less operands it can use this. - StringIndex m_stringIndices[kNumOperands]; + InstIndex m_operands[kMaxOperands]; ///< For items that 2 or less operands it can use this. + StringIndex m_stringIndices[kMaxOperands]; ExternalOperandPayload m_externalOperand; ///< Operands are stored in an an index of an operand array OperandAndUInt32 m_operandAndUInt32; }; @@ -110,44 +110,19 @@ struct IRSerialData Payload m_payload; }; - /// Clear to initial state - void clear() - { - // First Instruction is null - m_insts.SetSize(1); - memset(&m_insts[0], 0, sizeof(Inst)); - - m_childRuns.Clear(); - m_decorationRuns.Clear(); - m_externalOperands.Clear(); - - m_strings.SetSize(2); - m_strings[int(kNullStringIndex)] = 0; - m_strings[int(kEmptyStringIndex)] = 0; - - m_decorationBaseIndex = 0; - } + /// Clear to initial state + void clear(); + /// Get the operands of an instruction + SLANG_FORCE_INLINE int getOperands(const Inst& inst, const InstIndex** operandsOut) const; - SLANG_FORCE_INLINE int getOperands(const Inst& inst, const InstIndex** operandsOut) const - { - if (inst.m_payloadType == Inst::PayloadType::OperandExternal) - { - *operandsOut = m_externalOperands.begin() + int(inst.m_payload.m_externalOperand.m_arrayIndex); - return int(inst.m_payload.m_externalOperand.m_size); - } - else - { - *operandsOut = inst.m_payload.m_operands; - return s_payloadInfos[int(inst.m_payloadType)].m_numOperands; - } - } + /// Calculate the amount of memory used by this IRSerialData + size_t calcSizeInBytes() const; /// Ctor IRSerialData() : m_decorationBaseIndex(0) {} - List m_insts; ///< The instructions List m_childRuns; ///< Holds the information about children that belong to an instruction @@ -157,11 +132,35 @@ struct IRSerialData List m_strings; ///< All strings. Indexed into by StringIndex + List m_rawSourceLocs; ///< A source location per instruction (saved without modification from IRInst)s + static const PayloadInfo s_payloadInfos[int(Inst::PayloadType::CountOf)]; int m_decorationBaseIndex; ///< All decorations insts are at indices >= to this value }; +// -------------------------------------------------------------------------- +SLANG_FORCE_INLINE int IRSerialData::Inst::getNumOperands() const +{ + return (m_payloadType == PayloadType::OperandExternal) ? m_payload.m_externalOperand.m_size : s_payloadInfos[int(m_payloadType)].m_numOperands; +} + +// -------------------------------------------------------------------------- +SLANG_FORCE_INLINE int IRSerialData::getOperands(const Inst& inst, const InstIndex** operandsOut) const +{ + if (inst.m_payloadType == Inst::PayloadType::OperandExternal) + { + *operandsOut = m_externalOperands.begin() + int(inst.m_payload.m_externalOperand.m_arrayIndex); + return int(inst.m_payload.m_externalOperand.m_size); + } + else + { + *operandsOut = inst.m_payload.m_operands; + return s_payloadInfos[int(inst.m_payloadType)].m_numOperands; + } +} + + #define SLANG_FOUR_CC(c0, c1, c2, c3) ((uint32_t(c0) << 0) | (uint32_t(c1) << 8) | (uint32_t(c2) << 16) | (uint32_t(c3) << 24)) struct IRSerialBinary @@ -182,6 +181,10 @@ struct IRSerialBinary static const uint32_t kChildRunFourCc = SLANG_FOUR_CC('S', 'L', 'c', 'r'); static const uint32_t kExternalOperandsFourCc = SLANG_FOUR_CC('S', 'L', 'e', 'o'); static const uint32_t kStringFourCc = SLANG_FOUR_CC('S', 'L', 's', 't'); + /// 4 bytes per entry + static const uint32_t kUInt32SourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '4'); + /// 8 bytes per entry + static const uint32_t kUInt64SourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '8'); struct SlangHeader { @@ -200,7 +203,17 @@ struct IRSerialWriter { typedef IRSerialData Ser; - Result write(IRModule* module, IRSerialData* serialData); + struct OptionFlag + { + typedef uint32_t Type; + enum Enum: Type + { + RawSourceLocation = 1, + }; + }; + typedef OptionFlag::Type OptionFlags; + + Result write(IRModule* module, SourceManager* sourceManager, OptionFlags options, IRSerialData* serialData); static Result writeStream(const IRSerialData& data, Stream* stream); @@ -268,6 +281,7 @@ struct IRSerialReader void _calcStringStarts(); IRDecoration* _createDecoration(const Ser::Inst& srcIns); + static Result _skip(const IRSerialBinary::Chunk& chunk, Stream* stream, int64_t* remainingBytesInOut); List m_stringStarts; List m_stringRepresentationCache; @@ -277,7 +291,7 @@ struct IRSerialReader }; -Result serializeModule(IRModule* module, Stream* stream); +Result serializeModule(IRModule* module, SourceManager* sourceManager, Stream* stream); Result readModule(Session* session, Stream* stream, RefPtr& moduleOut); } // namespace Slang