From da1731bf73166cd662e47434bc86dda70e31a2d6 Mon Sep 17 00:00:00 2001 From: Sven van Haastregt Date: Wed, 27 Nov 2024 12:07:51 +0100 Subject: [PATCH 01/11] [SPIR-V 1.2] SPIRVReader: Add AlignmentId support (#2869) If there is no `OpDecorate .. Alignment` in the input, see if there is an `OpDecorateId .. AlignmentId` and take the alignment from the referenced constant instead. Once `AlignmentId` has been translated to LLVM IR, it is indistinguishable from an (non-ID) `Alignment` decoration. --- lib/SPIRV/SPIRVReader.cpp | 39 ++++++++++++++++++++++++++++++--------- lib/SPIRV/SPIRVReader.h | 7 +++++++ test/AlignmentId.spvasm | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 test/AlignmentId.spvasm diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp index 959c246ae..681140148 100644 --- a/lib/SPIRV/SPIRVReader.cpp +++ b/lib/SPIRV/SPIRVReader.cpp @@ -288,6 +288,29 @@ Value *SPIRVToLLVM::mapFunction(SPIRVFunction *BF, Function *F) { return F; } +std::optional SPIRVToLLVM::transIdAsConstant(SPIRVId Id) { + auto *V = BM->get(Id); + const auto *ConstValue = + dyn_cast(transValue(V, nullptr, nullptr)); + if (!ConstValue) + return {}; + return ConstValue->getZExtValue(); +} + +std::optional SPIRVToLLVM::getAlignment(SPIRVValue *V) { + SPIRVWord AlignmentBytes = 0; + if (V->hasAlignment(&AlignmentBytes)) { + return AlignmentBytes; + } + + // If there was no Alignment decoration, look for AlignmentId instead. + SPIRVId AlignId; + if (V->hasDecorateId(DecorationAlignmentId, 0, &AlignId)) { + return transIdAsConstant(AlignId); + } + return {}; +} + Type *SPIRVToLLVM::transFPType(SPIRVType *T) { switch (T->getFloatBitWidth()) { case 16: @@ -3154,9 +3177,9 @@ void SPIRVToLLVM::transFunctionAttrs(SPIRVFunction *BF, Function *F) { SPIRVWord MaxOffset = 0; if (BA->hasDecorate(DecorationMaxByteOffset, 0, &MaxOffset)) Builder.addDereferenceableAttr(MaxOffset); - SPIRVWord AlignmentBytes = 0; - if (BA->hasAlignment(&AlignmentBytes)) - Builder.addAlignmentAttr(AlignmentBytes); + if (auto Alignment = getAlignment(BA)) { + Builder.addAlignmentAttr(*Alignment); + } I->addAttrs(Builder); } BF->foreachReturnValueAttr([&](SPIRVFuncParamAttrKind Kind) { @@ -4931,15 +4954,13 @@ bool SPIRVToLLVM::transFPGAFunctionMetadata(SPIRVFunction *BF, Function *F) { bool SPIRVToLLVM::transAlign(SPIRVValue *BV, Value *V) { if (auto *AL = dyn_cast(V)) { - SPIRVWord Align = 0; - if (BV->hasAlignment(&Align)) - AL->setAlignment(llvm::Align(Align)); + if (auto Align = getAlignment(BV)) + AL->setAlignment(llvm::Align(*Align)); return true; } if (auto *GV = dyn_cast(V)) { - SPIRVWord Align = 0; - if (BV->hasAlignment(&Align)) - GV->setAlignment(MaybeAlign(Align)); + if (auto Align = getAlignment(BV)) + GV->setAlignment(MaybeAlign(*Align)); return true; } return true; diff --git a/lib/SPIRV/SPIRVReader.h b/lib/SPIRV/SPIRVReader.h index c1b74874c..75dfd52c0 100644 --- a/lib/SPIRV/SPIRVReader.h +++ b/lib/SPIRV/SPIRVReader.h @@ -215,6 +215,13 @@ class SPIRVToLLVM : private BuiltinCallHelper { bool isDirectlyTranslatedToOCL(Op OpCode) const; MDString *transOCLKernelArgTypeName(SPIRVFunctionParameter *); + + // Attempt to translate Id as a (specialization) constant. + std::optional transIdAsConstant(SPIRVId Id); + + // Return the value of an Alignment or AlignmentId decoration for V. + std::optional getAlignment(SPIRVValue *V); + Value *mapFunction(SPIRVFunction *BF, Function *F); Value *getTranslatedValue(SPIRVValue *BV); IntrinsicInst *getLifetimeStartIntrinsic(Instruction *I); diff --git a/test/AlignmentId.spvasm b/test/AlignmentId.spvasm new file mode 100644 index 000000000..aa1b28eda --- /dev/null +++ b/test/AlignmentId.spvasm @@ -0,0 +1,34 @@ +; REQUIRES: spirv-as + +; RUN: spirv-as %s --target-env spv1.2 -o %t.spv +; RUN: spirv-val %t.spv +; RUN: llvm-spirv -r -o %t.rev.bc %t.spv +; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s + + OpCapability Addresses + OpCapability Linkage + OpCapability Kernel + OpMemoryModel Physical64 OpenCL + OpEntryPoint Kernel %fn "testAlignmentId" + OpName %p "p" + OpDecorate %x LinkageAttributes "x" Export + OpDecorateId %x AlignmentId %al + OpDecorateId %p AlignmentId %al_spec + %void = OpTypeVoid + %uint = OpTypeInt 32 0 + %ptr = OpTypePointer CrossWorkgroup %uint + %fnTy = OpTypeFunction %void %ptr + %al = OpConstant %uint 16 + %al_spec = OpSpecConstantOp %uint IAdd %al %al + %uint_42 = OpConstant %uint 42 +; Verify alignment of variable. +; CHECK: @x = addrspace(1) global i32 42, align 16 + %x = OpVariable %ptr CrossWorkgroup %uint_42 + + %fn = OpFunction %void None %fnTy +; Verify alignment of function parameter. +; CHECK: define spir_kernel void @testAlignmentId(ptr addrspace(1) align 32 %p) + %p = OpFunctionParameter %ptr + %entry = OpLabel + OpReturn + OpFunctionEnd From 926a92a57d73ccc0b5af88185767e1fb42be5819 Mon Sep 17 00:00:00 2001 From: Viktoria Maximova Date: Thu, 28 Nov 2024 09:32:27 +0100 Subject: [PATCH 02/11] Fix reverse translation of `OpPtrDiff` with untyped pointers (#2858) We should not lose operand type when SPV_KHR_untyped_pointers is used. --- lib/SPIRV/SPIRVReader.cpp | 13 ++++++++++--- test/transcoding/ptr_diff.ll | 17 ++++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp index 681140148..6e0ae3a6e 100644 --- a/lib/SPIRV/SPIRVReader.cpp +++ b/lib/SPIRV/SPIRVReader.cpp @@ -2305,10 +2305,17 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, case OpPtrDiff: { auto *BC = static_cast(BV); - auto Ops = transValue(BC->getOperands(), F, BB); + auto SPVOps = BC->getOperands(); + auto Ops = transValue(SPVOps, F, BB); IRBuilder<> Builder(BB); - Type *ElemTy = - transType(BC->getOperands()[0]->getType()->getPointerElementType()); + + Type *ElemTy = nullptr; + if (SPVOps[0]->isUntypedVariable()) + ElemTy = transType( + static_cast(SPVOps[0])->getDataType()); + else + ElemTy = transType(SPVOps[0]->getType()->getPointerElementType()); + Value *V = Builder.CreatePtrDiff(ElemTy, Ops[0], Ops[1]); return mapValue(BV, V); } diff --git a/test/transcoding/ptr_diff.ll b/test/transcoding/ptr_diff.ll index 1f05f6b1a..9daedc62d 100644 --- a/test/transcoding/ptr_diff.ll +++ b/test/transcoding/ptr_diff.ll @@ -5,7 +5,16 @@ ; RUN: llvm-spirv %t.bc -o %t.spv ; RUN: llvm-spirv %t.spv -to-text -o %t.spt -; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV +; RUN: FileCheck < %t.spt %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-TYPED-PTR +; RUN: spirv-val %t.spv + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis %t.rev.bc +; RUN: FileCheck < %t.rev.ll %s --check-prefix=CHECK-LLVM + +; RUN: llvm-spirv %t.bc -o %t.spv --spirv-ext=+SPV_KHR_untyped_pointers +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-UNTYPED-PTR ; RUN: spirv-val %t.spv ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc @@ -19,9 +28,11 @@ ; CHECK-SPIRV: 66560 ; CHECK-SPIRV: TypeInt [[#TypeInt:]] 32 0 ; CHECK-SPIRV: TypeFloat [[#TypeFloat:]] 32 -; CHECK-SPIRV: TypePointer [[#TypePointer:]] [[#]] [[#TypeFloat]] +; CHECK-SPIRV-TYPED-PTR: TypePointer [[#TypePointer:]] [[#]] [[#TypeFloat]] +; CHECK-SPIRV-UNTYPED-PTR: TypeUntypedPointerKHR [[#TypePointer:]] [[#]] -; CHECK-SPIRV: Variable [[#TypePointer]] [[#Var:]] +; CHECK-SPIRV-TYPED-PTR: Variable [[#TypePointer]] [[#Var:]] +; CHECK-SPIRV-UNTYPED-PTR: UntypedVariableKHR [[#TypePointer]] [[#Var:]] [[#]] [[#TypeFloat]] ; CHECK-SPIRV: PtrDiff [[#TypeInt]] [[#]] [[#Var]] [[#Var]] target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" From 44bd69ee70b79ed8fe1cb11d728644b8dc99f8bf Mon Sep 17 00:00:00 2001 From: Viktoria Maximova Date: Thu, 28 Nov 2024 09:39:01 +0100 Subject: [PATCH 03/11] Translate OpIAddCarry and OpISubBorrow back to llvm intrinsics (#2877) Semantically we can translate `OpIAddCarry` and `OpISubBorrow` back into `@llvm.uadd.with.overflow` and `@llvm.usub.with.overflow` respectively. It require small transformation as there is a difference between return types in SPIR-V and LLVM IR - e.g., SPIR-V instructions return {i32, i32} struct, but LLVM intrinsics return {i32, i1}. --- lib/SPIRV/SPIRVReader.cpp | 34 ++++++- test/builtin_returns_struct.spvasm | 4 +- test/iaddcarry_builtin.ll | 48 ++++++++-- test/isubborrow_builtin.ll | 48 ++++++++-- test/llvm-intrinsics/uadd.with.overflow.ll | 104 +++++++++++++-------- test/llvm-intrinsics/usub.with.overflow.ll | 104 +++++++++++++-------- 6 files changed, 243 insertions(+), 99 deletions(-) diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp index 6e0ae3a6e..c0cdcd38a 100644 --- a/lib/SPIRV/SPIRVReader.cpp +++ b/lib/SPIRV/SPIRVReader.cpp @@ -2714,13 +2714,37 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, case OpSignBitSet: return mapValue(BV, transRelational(static_cast(BV), BB)); - case OpIAddCarry: { - auto *BC = static_cast(BV); - return mapValue(BV, transBuiltinFromInst("__spirv_IAddCarry", BC, BB)); - } + case OpIAddCarry: case OpISubBorrow: { + IRBuilder Builder(BB); auto *BC = static_cast(BV); - return mapValue(BV, transBuiltinFromInst("__spirv_ISubBorrow", BC, BB)); + Intrinsic::ID ID = OC == OpIAddCarry ? Intrinsic::uadd_with_overflow + : Intrinsic::usub_with_overflow; + auto *Inst = + Builder.CreateBinaryIntrinsic(ID, transValue(BC->getOperand(0), F, BB), + transValue(BC->getOperand(1), F, BB)); + + // Extract components of the result. + auto *Result = Builder.CreateExtractValue(Inst, 0); // iN result + auto *Carry = Builder.CreateExtractValue(Inst, 1); // i1 overflow + + // Convert {iN, i1} into {iN, iN} for SPIR-V compatibility. + Value *CarryInt; + if (Carry->getType()->isVectorTy()) { + CarryInt = Builder.CreateZExt( + Carry, VectorType::get( + cast(Result->getType())->getElementType(), + cast(Carry->getType())->getElementCount())); + } else { + CarryInt = Builder.CreateZExt(Carry, Result->getType()); + } + auto *ResultStruct = + Builder.CreateInsertValue(UndefValue::get(StructType::get( + Result->getType(), CarryInt->getType())), + Result, 0); + ResultStruct = Builder.CreateInsertValue(ResultStruct, CarryInt, 1); + + return mapValue(BV, ResultStruct); } case OpSMulExtended: { auto *BC = static_cast(BV); diff --git a/test/builtin_returns_struct.spvasm b/test/builtin_returns_struct.spvasm index 48e4973b7..fe16db24b 100644 --- a/test/builtin_returns_struct.spvasm +++ b/test/builtin_returns_struct.spvasm @@ -34,7 +34,7 @@ OpBranch %20 %20 = OpLabel %e = OpPhi %uint %a %19 %math %21 - %16 = OpIAddCarry %_struct_7 %e %uint_1 + %16 = OpUMulExtended %_struct_7 %e %uint_1 %math = OpCompositeExtract %uint %16 0 %17 = OpCompositeExtract %uint %16 1 %ov = OpINotEqual %bool %17 %10 @@ -47,6 +47,6 @@ OpFunctionEnd ; CHECK: %[[#Var:]] = alloca %structtype, align 8 -; CHECK: call spir_func void @_Z17__spirv_IAddCarryii(ptr sret(%structtype) %[[#Var:]] +; CHECK: call spir_func void @_Z20__spirv_UMulExtendedii(ptr sret(%structtype) %[[#Var:]] ; CHECK: %[[#Load:]] = load %structtype, ptr %[[#Var]], align 4 ; CHECK-2: extractvalue %structtype %[[#Load:]] diff --git a/test/iaddcarry_builtin.ll b/test/iaddcarry_builtin.ll index d0fb1290b..a68bb1f42 100644 --- a/test/iaddcarry_builtin.ll +++ b/test/iaddcarry_builtin.ll @@ -57,7 +57,13 @@ define spir_func void @test_builtin_iaddcarrycc(i8 %a, i8 %b) { ; CHECK-SPIRV: OpFunctionEnd ; CHECK-LLVM: %0 = alloca [[i8struct]], align 8 -; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarrycc(ptr sret([[i8struct]]) %0, i8 %a, i8 %b) +; CHECK-LLVM: %1 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %a, i8 %b) +; CHECK-LLVM: %2 = extractvalue { i8, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i8, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i8 +; CHECK-LLVM: %5 = insertvalue { i8, i8 } undef, i8 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i8, i8 } %5, i8 %4, 1 +; CHECK-LLVM: store { i8, i8 } %6, ptr %0, align 1 ; CHECK-LLVM: ret void define spir_func void @test_builtin_iaddcarryss(i16 %a, i16 %b) { entry: @@ -75,7 +81,13 @@ define spir_func void @test_builtin_iaddcarryss(i16 %a, i16 %b) { ; CHECK-SPIRV: OpFunctionEnd ; CHECK-LLVM: %0 = alloca [[i16struct]], align 8 -; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryss(ptr sret([[i16struct]]) %0, i16 %a, i16 %b) +; CHECK-LLVM: %1 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 %a, i16 %b) +; CHECK-LLVM: %2 = extractvalue { i16, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i16, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i16 +; CHECK-LLVM: %5 = insertvalue { i16, i16 } undef, i16 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i16, i16 } %5, i16 %4, 1 +; CHECK-LLVM: store { i16, i16 } %6, ptr %0, align 2 ; CHECK-LLVM: ret void define spir_func void @test_builtin_iaddcarryii(i32 %a, i32 %b) { entry: @@ -93,7 +105,13 @@ define spir_func void @test_builtin_iaddcarryii(i32 %a, i32 %b) { ; CHECK-SPIRV: OpFunctionEnd ; CHECK-LLVM: %0 = alloca [[i32struct]], align 8 -; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryii(ptr sret([[i32struct]]) %0, i32 %a, i32 %b) +; CHECK-LLVM: %1 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b) +; CHECK-LLVM: %2 = extractvalue { i32, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i32, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i32 +; CHECK-LLVM: %5 = insertvalue { i32, i32 } undef, i32 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i32, i32 } %5, i32 %4, 1 +; CHECK-LLVM: store { i32, i32 } %6, ptr %0, align 4 ; CHECK-LLVM: ret void define spir_func void @test_builtin_iaddcarryll(i64 %a, i64 %b) { entry: @@ -111,7 +129,13 @@ define spir_func void @test_builtin_iaddcarryll(i64 %a, i64 %b) { ; CHECK-SPIRV: OpFunctionEnd ; CHECK-LLVM: %0 = alloca [[i64struct]] -; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryll(ptr sret([[i64struct]]) %0, i64 %a, i64 %b) +; CHECK-LLVM: %1 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %a, i64 %b) +; CHECK-LLVM: %2 = extractvalue { i64, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i64, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i64 +; CHECK-LLVM: %5 = insertvalue { i64, i64 } undef, i64 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i64, i64 } %5, i64 %4, 1 +; CHECK-LLVM: store { i64, i64 } %6, ptr %0, align 8 ; CHECK-LLVM: ret void define spir_func void @test_builtin_iaddcarryDv4_xS_(<4 x i32> %a, <4 x i32> %b) { entry: @@ -129,7 +153,13 @@ define spir_func void @test_builtin_iaddcarryDv4_xS_(<4 x i32> %a, <4 x i32> %b) ; CHECK-SPIRV: OpFunctionEnd ; CHECK-LLVM: %0 = alloca [[vecstruct]] -; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret([[vecstruct]]) %0, <4 x i32> %a, <4 x i32> %b) +; CHECK-LLVM: %1 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> %a, <4 x i32> %b) +; CHECK-LLVM: %2 = extractvalue { <4 x i32>, <4 x i1> } %1, 0 +; CHECK-LLVM: %3 = extractvalue { <4 x i32>, <4 x i1> } %1, 1 +; CHECK-LLVM: %4 = zext <4 x i1> %3 to <4 x i32> +; CHECK-LLVM: %5 = insertvalue { <4 x i32>, <4 x i32> } undef, <4 x i32> %2, 0 +; CHECK-LLVM: %6 = insertvalue { <4 x i32>, <4 x i32> } %5, <4 x i32> %4, 1 +; CHECK-LLVM: store { <4 x i32>, <4 x i32> } %6, ptr %0, align 16 ; CHECK-LLVM: ret void %struct.anon = type { i32, i32 } @@ -151,7 +181,13 @@ define spir_func void @test_builtin_iaddcarry_anon(i32 %a, i32 %b) { ; CHECK-LLVM: %0 = alloca [[struct_anon]], align 8 ; CHECK-LLVM: %1 = addrspacecast ptr %0 to ptr addrspace(4) -; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryii.1(ptr addrspace(4) sret([[struct_anon]]) %1, i32 %a, i32 %b) +; CHECK-LLVM: %2 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b) +; CHECK-LLVM: %3 = extractvalue { i32, i1 } %2, 0 +; CHECK-LLVM: %4 = extractvalue { i32, i1 } %2, 1 +; CHECK-LLVM: %5 = zext i1 %4 to i32 +; CHECK-LLVM: %6 = insertvalue { i32, i32 } undef, i32 %3, 0 +; CHECK-LLVM: %7 = insertvalue { i32, i32 } %6, i32 %5, 1 +; CHECK-LLVM: store { i32, i32 } %7, ptr addrspace(4) %1, align 4 ; CHECK-LLVM: ret void declare void @_Z17__spirv_IAddCarryIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4, i32, i32) diff --git a/test/isubborrow_builtin.ll b/test/isubborrow_builtin.ll index a9c143d06..dfcc37883 100644 --- a/test/isubborrow_builtin.ll +++ b/test/isubborrow_builtin.ll @@ -58,7 +58,13 @@ define spir_func void @test_builtin_isubborrowcc(i8 %a, i8 %b) { ; CHECK-SPIRV: OpFunctionEnd ; CHECK-LLVM: %0 = alloca [[i8struct]], align 8 -; CHECK-LLVM: call spir_func void @_Z18__spirv_ISubBorrowcc(ptr sret([[i8struct]]) %0, i8 %a, i8 %b) +; CHECK-LLVM: %1 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 %a, i8 %b) +; CHECK-LLVM: %2 = extractvalue { i8, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i8, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i8 +; CHECK-LLVM: %5 = insertvalue { i8, i8 } undef, i8 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i8, i8 } %5, i8 %4, 1 +; CHECK-LLVM: store { i8, i8 } %6, ptr %0, align 1 ; CHECK-LLVM: ret void define spir_func void @test_builtin_isubborrowss(i16 %a, i16 %b) { entry: @@ -76,7 +82,13 @@ define spir_func void @test_builtin_isubborrowss(i16 %a, i16 %b) { ; CHECK-SPIRV: OpFunctionEnd ; CHECK-LLVM: %0 = alloca [[i16struct]], align 8 -; CHECK-LLVM: call spir_func void @_Z18__spirv_ISubBorrowss(ptr sret([[i16struct]]) %0, i16 %a, i16 %b) +; CHECK-LLVM: %1 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 %a, i16 %b) +; CHECK-LLVM: %2 = extractvalue { i16, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i16, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i16 +; CHECK-LLVM: %5 = insertvalue { i16, i16 } undef, i16 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i16, i16 } %5, i16 %4, 1 +; CHECK-LLVM: store { i16, i16 } %6, ptr %0, align 2 ; CHECK-LLVM: ret void define spir_func void @test_builtin_isubborrowii(i32 %a, i32 %b) { entry: @@ -94,7 +106,13 @@ define spir_func void @test_builtin_isubborrowii(i32 %a, i32 %b) { ; CHECK-SPIRV: OpFunctionEnd ; CHECK-LLVM: %0 = alloca [[i32struct]], align 8 -; CHECK-LLVM: call spir_func void @_Z18__spirv_ISubBorrowii(ptr sret([[i32struct]]) %0, i32 %a, i32 %b) +; CHECK-LLVM: %1 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %a, i32 %b) +; CHECK-LLVM: %2 = extractvalue { i32, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i32, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i32 +; CHECK-LLVM: %5 = insertvalue { i32, i32 } undef, i32 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i32, i32 } %5, i32 %4, 1 +; CHECK-LLVM: store { i32, i32 } %6, ptr %0, align 4 ; CHECK-LLVM: ret void define spir_func void @test_builtin_isubborrowll(i64 %a, i64 %b) { entry: @@ -112,7 +130,13 @@ define spir_func void @test_builtin_isubborrowll(i64 %a, i64 %b) { ; CHECK-SPIRV: OpFunctionEnd ; CHECK-LLVM: %0 = alloca [[i64struct]] -; CHECK-LLVM: call spir_func void @_Z18__spirv_ISubBorrowll(ptr sret([[i64struct]]) %0, i64 %a, i64 %b) +; CHECK-LLVM: %1 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %a, i64 %b) +; CHECK-LLVM: %2 = extractvalue { i64, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i64, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i64 +; CHECK-LLVM: %5 = insertvalue { i64, i64 } undef, i64 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i64, i64 } %5, i64 %4, 1 +; CHECK-LLVM: store { i64, i64 } %6, ptr %0, align 8 ; CHECK-LLVM: ret void define spir_func void @test_builtin_isubborrowDv4_xS_(<4 x i32> %a, <4 x i32> %b) { entry: @@ -130,7 +154,13 @@ define spir_func void @test_builtin_isubborrowDv4_xS_(<4 x i32> %a, <4 x i32> %b ; CHECK-SPIRV: OpFunctionEnd ; CHECK-LLVM: %0 = alloca [[vecstruct]] -; CHECK-LLVM: call spir_func void @_Z18__spirv_ISubBorrowDv4_iS_(ptr sret([[vecstruct]]) %0, <4 x i32> %a, <4 x i32> %b) +; CHECK-LLVM: %1 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> %a, <4 x i32> %b) +; CHECK-LLVM: %2 = extractvalue { <4 x i32>, <4 x i1> } %1, 0 +; CHECK-LLVM: %3 = extractvalue { <4 x i32>, <4 x i1> } %1, 1 +; CHECK-LLVM: %4 = zext <4 x i1> %3 to <4 x i32> +; CHECK-LLVM: %5 = insertvalue { <4 x i32>, <4 x i32> } undef, <4 x i32> %2, 0 +; CHECK-LLVM: %6 = insertvalue { <4 x i32>, <4 x i32> } %5, <4 x i32> %4, 1 +; CHECK-LLVM: store { <4 x i32>, <4 x i32> } %6, ptr %0, align 16 ; CHECK-LLVM: ret void @@ -151,7 +181,13 @@ define spir_func void @test_builtin_isubborrow_anon(i32 %a, i32 %b) { ; CHECK-LLVM: %0 = alloca [[struct_anon]], align 8 ; CHECK-LLVM: %1 = addrspacecast ptr %0 to ptr addrspace(4) -; CHECK-LLVM: call spir_func void @_Z18__spirv_ISubBorrowii.1(ptr addrspace(4) sret([[struct_anon]]) %1, i32 %a, i32 %b) +; CHECK-LLVM: %2 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %a, i32 %b) +; CHECK-LLVM: %3 = extractvalue { i32, i1 } %2, 0 +; CHECK-LLVM: %4 = extractvalue { i32, i1 } %2, 1 +; CHECK-LLVM: %5 = zext i1 %4 to i32 +; CHECK-LLVM: %6 = insertvalue { i32, i32 } undef, i32 %3, 0 +; CHECK-LLVM: %7 = insertvalue { i32, i32 } %6, i32 %5, 1 +; CHECK-LLVM: store { i32, i32 } %7, ptr addrspace(4) %1, align 4 ; CHECK-LLVM: ret void declare void @_Z18__spirv_ISubBorrowIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4, i32, i32) diff --git a/test/llvm-intrinsics/uadd.with.overflow.ll b/test/llvm-intrinsics/uadd.with.overflow.ll index 33fec83f0..8ed038536 100644 --- a/test/llvm-intrinsics/uadd.with.overflow.ll +++ b/test/llvm-intrinsics/uadd.with.overflow.ll @@ -73,16 +73,22 @@ entry: ; CHECK-SPIRV: OpReturnValue [[var_23]] ; CHECK-LLVM: %0 = alloca [[structtype]], align 8 -; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryss(ptr sret([[structtype]]) %0, i16 %a, i16 %b) -; CHECK-LLVM: %1 = load [[structtype]], ptr %0, align 2 -; CHECK-LLVM: %2 = extractvalue [[structtype]] %1, 0 -; CHECK-LLVM: %3 = extractvalue [[structtype]] %1, 1 -; CHECK-LLVM: %4 = icmp ne i16 %3, 0 -; CHECK-LLVM: %5 = insertvalue [[structtype_0]] undef, i16 %2, 0 -; CHECK-LLVM: %6 = insertvalue [[structtype_0]] %5, i1 %4, 1 -; CHECK-LLVM: %7 = extractvalue [[structtype_0]] %6, 0 -; CHECK-LLVM: %8 = extractvalue [[structtype_0]] %6, 1 -; CHECK-LLVM: ret i1 %8 +; CHECK-LLVM: %1 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 %a, i16 %b) +; CHECK-LLVM: %2 = extractvalue { i16, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i16, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i16 +; CHECK-LLVM: %5 = insertvalue { i16, i16 } undef, i16 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i16, i16 } %5, i16 %4, 1 +; CHECK-LLVM: store { i16, i16 } %6, ptr %0, align 2 +; CHECK-LLVM: %7 = load %structtype, ptr %0, align 2 +; CHECK-LLVM: %8 = extractvalue [[structtype]] %7, 0 +; CHECK-LLVM: %9 = extractvalue [[structtype]] %7, 1 +; CHECK-LLVM: %10 = icmp ne i16 %9, 0 +; CHECK-LLVM: %11 = insertvalue [[structtype_0]] undef, i16 %8, 0 +; CHECK-LLVM: %12 = insertvalue [[structtype_0]] %11, i1 %10, 1 +; CHECK-LLVM: %13 = extractvalue [[structtype_0]] %12, 0 +; CHECK-LLVM: %14 = extractvalue [[structtype_0]] %12, 1 +; CHECK-LLVM: ret i1 %14 define spir_func i1 @test_uadd_with_overflow_i32(i32 %a, i32 %b) { entry: %res = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a, i32 %b) @@ -109,16 +115,22 @@ entry: ; CHECK-LLVM: %0 = alloca [[structtype_1]], align 8 -; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryii(ptr sret([[structtype_1]]) %0, i32 %a, i32 %b) -; CHECK-LLVM: %1 = load [[structtype_1]], ptr %0, align 4 -; CHECK-LLVM: %2 = extractvalue [[structtype_1]] %1, 0 -; CHECK-LLVM: %3 = extractvalue [[structtype_1]] %1, 1 -; CHECK-LLVM: %4 = icmp ne i32 %3, 0 -; CHECK-LLVM: %5 = insertvalue [[structtype_2]] undef, i32 %2, 0 -; CHECK-LLVM: %6 = insertvalue [[structtype_2]] %5, i1 %4, 1 -; CHECK-LLVM: %7 = extractvalue [[structtype_2]] %6, 0 -; CHECK-LLVM: %8 = extractvalue [[structtype_2]] %6, 1 -; CHECK-LLVM: ret i1 %8 +; CHECK-LLVM: %1 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b) +; CHECK-LLVM: %2 = extractvalue { i32, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i32, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i32 +; CHECK-LLVM: %5 = insertvalue { i32, i32 } undef, i32 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i32, i32 } %5, i32 %4, 1 +; CHECK-LLVM: store { i32, i32 } %6, ptr %0, align 4 +; CHECK-LLVM: %7 = load [[structtype_1]], ptr %0, align 4 +; CHECK-LLVM: %8 = extractvalue [[structtype_1]] %7, 0 +; CHECK-LLVM: %9 = extractvalue [[structtype_1]] %7, 1 +; CHECK-LLVM: %10 = icmp ne i32 %9, 0 +; CHECK-LLVM: %11 = insertvalue [[structtype_2]] undef, i32 %8, 0 +; CHECK-LLVM: %12 = insertvalue [[structtype_2]] %11, i1 %10, 1 +; CHECK-LLVM: %13 = extractvalue [[structtype_2]] %12, 0 +; CHECK-LLVM: %14 = extractvalue [[structtype_2]] %12, 1 +; CHECK-LLVM: ret i1 %14 define spir_func i1 @test_uadd_with_overflow_i64(i64 %a, i64 %b) { entry: %res = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %a, i64 %b) @@ -144,16 +156,22 @@ entry: ; CHECK-SPIRV: OpReturnValue [[var_65]] ; CHECK-LLVM: %0 = alloca [[structtype_3]], align 8 -; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryll(ptr sret([[structtype_3]]) %0, i64 %a, i64 %b) -; CHECK-LLVM: %1 = load [[structtype_3]], ptr %0, align 4 -; CHECK-LLVM: %2 = extractvalue [[structtype_3]] %1, 0 -; CHECK-LLVM: %3 = extractvalue [[structtype_3]] %1, 1 -; CHECK-LLVM: %4 = icmp ne i64 %3, 0 -; CHECK-LLVM: %5 = insertvalue [[structtype_4]] undef, i64 %2, 0 -; CHECK-LLVM: %6 = insertvalue [[structtype_4]] %5, i1 %4, 1 -; CHECK-LLVM: %7 = extractvalue [[structtype_4]] %6, 0 -; CHECK-LLVM: %8 = extractvalue [[structtype_4]] %6, 1 -; CHECK-LLVM: ret i1 %8 +; CHECK-LLVM: %1 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %a, i64 %b) +; CHECK-LLVM: %2 = extractvalue { i64, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i64, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i64 +; CHECK-LLVM: %5 = insertvalue { i64, i64 } undef, i64 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i64, i64 } %5, i64 %4, 1 +; CHECK-LLVM: store { i64, i64 } %6, ptr %0, align 8 +; CHECK-LLVM: %7 = load [[structtype_3]], ptr %0, align 4 +; CHECK-LLVM: %8 = extractvalue [[structtype_3]] %7, 0 +; CHECK-LLVM: %9 = extractvalue [[structtype_3]] %7, 1 +; CHECK-LLVM: %10 = icmp ne i64 %9, 0 +; CHECK-LLVM: %11 = insertvalue [[structtype_4]] undef, i64 %8, 0 +; CHECK-LLVM: %12 = insertvalue [[structtype_4]] %11, i1 %10, 1 +; CHECK-LLVM: %13 = extractvalue [[structtype_4]] %12, 0 +; CHECK-LLVM: %14 = extractvalue [[structtype_4]] %12, 1 +; CHECK-LLVM: ret i1 %14 define spir_func <4 x i1> @test_uadd_with_overflow_v4i32(<4 x i32> %a, <4 x i32> %b) { entry: %res = call {<4 x i32>, <4 x i1>} @llvm.uadd.with.overflow.v4i32(<4 x i32> %a, <4 x i32> %b) @@ -179,16 +197,22 @@ entry: ; CHECK-SPIRV: OpReturnValue [[var_87]] ; CHECK-LLVM: %0 = alloca [[structtype_5]], align 16 -; CHECK-LLVM: call spir_func void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret([[structtype_5]]) %0, <4 x i32> %a, <4 x i32> %b) -; CHECK-LLVM: %1 = load [[structtype_5]], ptr %0, align 16 -; CHECK-LLVM: %2 = extractvalue [[structtype_5]] %1, 0 -; CHECK-LLVM: %3 = extractvalue [[structtype_5]] %1, 1 -; CHECK-LLVM: %4 = icmp ne <4 x i32> %3, zeroinitializer -; CHECK-LLVM: %5 = insertvalue [[structtype_6]] undef, <4 x i32> %2, 0 -; CHECK-LLVM: %6 = insertvalue [[structtype_6]] %5, <4 x i1> %4, 1 -; CHECK-LLVM: %7 = extractvalue [[structtype_6]] %6, 0 -; CHECK-LLVM: %8 = extractvalue [[structtype_6]] %6, 1 -; CHECK-LLVM: ret <4 x i1> %8 +; CHECK-LLVM: %1 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> %a, <4 x i32> %b) +; CHECK-LLVM: %2 = extractvalue { <4 x i32>, <4 x i1> } %1, 0 +; CHECK-LLVM: %3 = extractvalue { <4 x i32>, <4 x i1> } %1, 1 +; CHECK-LLVM: %4 = zext <4 x i1> %3 to <4 x i32> +; CHECK-LLVM: %5 = insertvalue { <4 x i32>, <4 x i32> } undef, <4 x i32> %2, 0 +; CHECK-LLVM: %6 = insertvalue { <4 x i32>, <4 x i32> } %5, <4 x i32> %4, 1 +; CHECK-LLVM: store { <4 x i32>, <4 x i32> } %6, ptr %0, align 16 +; CHECK-LLVM: %7 = load [[structtype_5]], ptr %0, align 16 +; CHECK-LLVM: %8 = extractvalue [[structtype_5]] %7, 0 +; CHECK-LLVM: %9 = extractvalue [[structtype_5]] %7, 1 +; CHECK-LLVM: %10 = icmp ne <4 x i32> %9, zeroinitializer +; CHECK-LLVM: %11 = insertvalue [[structtype_6]] undef, <4 x i32> %8, 0 +; CHECK-LLVM: %12 = insertvalue [[structtype_6]] %11, <4 x i1> %10, 1 +; CHECK-LLVM: %13 = extractvalue [[structtype_6]] %12, 0 +; CHECK-LLVM: %14 = extractvalue [[structtype_6]] %12, 1 +; CHECK-LLVM: ret <4 x i1> %14 declare {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a, i16 %b) declare {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a, i32 %b) declare {i64, i1} @llvm.uadd.with.overflow.i64(i64 %a, i64 %b) diff --git a/test/llvm-intrinsics/usub.with.overflow.ll b/test/llvm-intrinsics/usub.with.overflow.ll index 5225345ad..8505c395f 100644 --- a/test/llvm-intrinsics/usub.with.overflow.ll +++ b/test/llvm-intrinsics/usub.with.overflow.ll @@ -73,16 +73,22 @@ entry: ; CHECK-SPIRV: OpReturnValue [[var_23]] ; CHECK-LLVM: %0 = alloca [[structtype]], align 8 -; CHECK-LLVM: call spir_func void @_Z18__spirv_ISubBorrowss(ptr sret([[structtype]]) %0, i16 %a, i16 %b) -; CHECK-LLVM: %1 = load [[structtype]], ptr %0, align 2 -; CHECK-LLVM: %2 = extractvalue [[structtype]] %1, 0 -; CHECK-LLVM: %3 = extractvalue [[structtype]] %1, 1 -; CHECK-LLVM: %4 = icmp ne i16 %3, 0 -; CHECK-LLVM: %5 = insertvalue [[structtype_0]] undef, i16 %2, 0 -; CHECK-LLVM: %6 = insertvalue [[structtype_0]] %5, i1 %4, 1 -; CHECK-LLVM: %7 = extractvalue [[structtype_0]] %6, 0 -; CHECK-LLVM: %8 = extractvalue [[structtype_0]] %6, 1 -; CHECK-LLVM: ret i1 %8 +; CHECK-LLVM: %1 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 %a, i16 %b) +; CHECK-LLVM: %2 = extractvalue { i16, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i16, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i16 +; CHECK-LLVM: %5 = insertvalue { i16, i16 } undef, i16 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i16, i16 } %5, i16 %4, 1 +; CHECK-LLVM: store { i16, i16 } %6, ptr %0, align 2 +; CHECK-LLVM: %7 = load %structtype, ptr %0, align 2 +; CHECK-LLVM: %8 = extractvalue [[structtype]] %7, 0 +; CHECK-LLVM: %9 = extractvalue [[structtype]] %7, 1 +; CHECK-LLVM: %10 = icmp ne i16 %9, 0 +; CHECK-LLVM: %11 = insertvalue [[structtype_0]] undef, i16 %8, 0 +; CHECK-LLVM: %12 = insertvalue [[structtype_0]] %11, i1 %10, 1 +; CHECK-LLVM: %13 = extractvalue [[structtype_0]] %12, 0 +; CHECK-LLVM: %14 = extractvalue [[structtype_0]] %12, 1 +; CHECK-LLVM: ret i1 %14 define spir_func i1 @test_usub_with_overflow_i32(i32 %a, i32 %b) { entry: %res = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a, i32 %b) @@ -109,16 +115,22 @@ entry: ; CHECK-LLVM: %0 = alloca [[structtype_1]], align 8 -; CHECK-LLVM: call spir_func void @_Z18__spirv_ISubBorrowii(ptr sret([[structtype_1]]) %0, i32 %a, i32 %b) -; CHECK-LLVM: %1 = load [[structtype_1]], ptr %0, align 4 -; CHECK-LLVM: %2 = extractvalue [[structtype_1]] %1, 0 -; CHECK-LLVM: %3 = extractvalue [[structtype_1]] %1, 1 -; CHECK-LLVM: %4 = icmp ne i32 %3, 0 -; CHECK-LLVM: %5 = insertvalue [[structtype_2]] undef, i32 %2, 0 -; CHECK-LLVM: %6 = insertvalue [[structtype_2]] %5, i1 %4, 1 -; CHECK-LLVM: %7 = extractvalue [[structtype_2]] %6, 0 -; CHECK-LLVM: %8 = extractvalue [[structtype_2]] %6, 1 -; CHECK-LLVM: ret i1 %8 +; CHECK-LLVM: %1 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %a, i32 %b) +; CHECK-LLVM: %2 = extractvalue { i32, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i32, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i32 +; CHECK-LLVM: %5 = insertvalue { i32, i32 } undef, i32 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i32, i32 } %5, i32 %4, 1 +; CHECK-LLVM: store { i32, i32 } %6, ptr %0, align 4 +; CHECK-LLVM: %7 = load [[structtype_1]], ptr %0, align 4 +; CHECK-LLVM: %8 = extractvalue [[structtype_1]] %7, 0 +; CHECK-LLVM: %9 = extractvalue [[structtype_1]] %7, 1 +; CHECK-LLVM: %10 = icmp ne i32 %9, 0 +; CHECK-LLVM: %11 = insertvalue [[structtype_2]] undef, i32 %8, 0 +; CHECK-LLVM: %12 = insertvalue [[structtype_2]] %11, i1 %10, 1 +; CHECK-LLVM: %13 = extractvalue [[structtype_2]] %12, 0 +; CHECK-LLVM: %14 = extractvalue [[structtype_2]] %12, 1 +; CHECK-LLVM: ret i1 %14 define spir_func i1 @test_usub_with_overflow_i64(i64 %a, i64 %b) { entry: %res = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %a, i64 %b) @@ -144,16 +156,22 @@ entry: ; CHECK-SPIRV: OpReturnValue [[var_65]] ; CHECK-LLVM: %0 = alloca [[structtype_3]], align 8 -; CHECK-LLVM: call spir_func void @_Z18__spirv_ISubBorrowll(ptr sret([[structtype_3]]) %0, i64 %a, i64 %b) -; CHECK-LLVM: %1 = load [[structtype_3]], ptr %0, align 4 -; CHECK-LLVM: %2 = extractvalue [[structtype_3]] %1, 0 -; CHECK-LLVM: %3 = extractvalue [[structtype_3]] %1, 1 -; CHECK-LLVM: %4 = icmp ne i64 %3, 0 -; CHECK-LLVM: %5 = insertvalue [[structtype_4]] undef, i64 %2, 0 -; CHECK-LLVM: %6 = insertvalue [[structtype_4]] %5, i1 %4, 1 -; CHECK-LLVM: %7 = extractvalue [[structtype_4]] %6, 0 -; CHECK-LLVM: %8 = extractvalue [[structtype_4]] %6, 1 -; CHECK-LLVM: ret i1 %8 +; CHECK-LLVM: %1 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %a, i64 %b) +; CHECK-LLVM: %2 = extractvalue { i64, i1 } %1, 0 +; CHECK-LLVM: %3 = extractvalue { i64, i1 } %1, 1 +; CHECK-LLVM: %4 = zext i1 %3 to i64 +; CHECK-LLVM: %5 = insertvalue { i64, i64 } undef, i64 %2, 0 +; CHECK-LLVM: %6 = insertvalue { i64, i64 } %5, i64 %4, 1 +; CHECK-LLVM: store { i64, i64 } %6, ptr %0, align 8 +; CHECK-LLVM: %7 = load [[structtype_3]], ptr %0, align 4 +; CHECK-LLVM: %8 = extractvalue [[structtype_3]] %7, 0 +; CHECK-LLVM: %9 = extractvalue [[structtype_3]] %7, 1 +; CHECK-LLVM: %10 = icmp ne i64 %9, 0 +; CHECK-LLVM: %11 = insertvalue [[structtype_4]] undef, i64 %8, 0 +; CHECK-LLVM: %12 = insertvalue [[structtype_4]] %11, i1 %10, 1 +; CHECK-LLVM: %13 = extractvalue [[structtype_4]] %12, 0 +; CHECK-LLVM: %14 = extractvalue [[structtype_4]] %12, 1 +; CHECK-LLVM: ret i1 %14 define spir_func <4 x i1> @test_usub_with_overflow_v4i32(<4 x i32> %a, <4 x i32> %b) { entry: %res = call {<4 x i32>, <4 x i1>} @llvm.usub.with.overflow.v4i32(<4 x i32> %a, <4 x i32> %b) @@ -179,16 +197,22 @@ entry: ; CHECK-SPIRV: OpReturnValue [[var_87]] ; CHECK-LLVM: %0 = alloca [[structtype_5]], align 16 -; CHECK-LLVM: call spir_func void @_Z18__spirv_ISubBorrowDv4_iS_(ptr sret([[structtype_5]]) %0, <4 x i32> %a, <4 x i32> %b) -; CHECK-LLVM: %1 = load [[structtype_5]], ptr %0, align 16 -; CHECK-LLVM: %2 = extractvalue [[structtype_5]] %1, 0 -; CHECK-LLVM: %3 = extractvalue [[structtype_5]] %1, 1 -; CHECK-LLVM: %4 = icmp ne <4 x i32> %3, zeroinitializer -; CHECK-LLVM: %5 = insertvalue [[structtype_6]] undef, <4 x i32> %2, 0 -; CHECK-LLVM: %6 = insertvalue [[structtype_6]] %5, <4 x i1> %4, 1 -; CHECK-LLVM: %7 = extractvalue [[structtype_6]] %6, 0 -; CHECK-LLVM: %8 = extractvalue [[structtype_6]] %6, 1 -; CHECK-LLVM: ret <4 x i1> %8 +; CHECK-LLVM: %1 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> %a, <4 x i32> %b) +; CHECK-LLVM: %2 = extractvalue { <4 x i32>, <4 x i1> } %1, 0 +; CHECK-LLVM: %3 = extractvalue { <4 x i32>, <4 x i1> } %1, 1 +; CHECK-LLVM: %4 = zext <4 x i1> %3 to <4 x i32> +; CHECK-LLVM: %5 = insertvalue { <4 x i32>, <4 x i32> } undef, <4 x i32> %2, 0 +; CHECK-LLVM: %6 = insertvalue { <4 x i32>, <4 x i32> } %5, <4 x i32> %4, 1 +; CHECK-LLVM: store { <4 x i32>, <4 x i32> } %6, ptr %0, align 16 +; CHECK-LLVM: %7 = load [[structtype_5]], ptr %0, align 16 +; CHECK-LLVM: %8 = extractvalue [[structtype_5]] %7, 0 +; CHECK-LLVM: %9 = extractvalue [[structtype_5]] %7, 1 +; CHECK-LLVM: %10 = icmp ne <4 x i32> %9, zeroinitializer +; CHECK-LLVM: %11 = insertvalue [[structtype_6]] undef, <4 x i32> %8, 0 +; CHECK-LLVM: %12 = insertvalue [[structtype_6]] %11, <4 x i1> %10, 1 +; CHECK-LLVM: %13 = extractvalue [[structtype_6]] %12, 0 +; CHECK-LLVM: %14 = extractvalue [[structtype_6]] %12, 1 +; CHECK-LLVM: ret <4 x i1> %14 declare {i16, i1} @llvm.usub.with.overflow.i16(i16 %a, i16 %b) declare {i32, i1} @llvm.usub.with.overflow.i32(i32 %a, i32 %b) declare {i64, i1} @llvm.usub.with.overflow.i64(i64 %a, i64 %b) From 0332a1e50c3b878b0b776f41ee60890f42c75a49 Mon Sep 17 00:00:00 2001 From: Sven van Haastregt Date: Fri, 29 Nov 2024 11:26:04 +0100 Subject: [PATCH 04/11] [SPIR-V 1.2] SPIRVReader: Add MaxByteOffsetId support (#2884) If there is no `OpDecorate .. MaxByteOffset` in the input, see if there is an `OpDecorateId .. MaxByteOffsetId` and take the value for the LLVM `dereferenceable` attribute from the referenced constant instead. Once `MaxByteOffsetId` has been translated to LLVM IR, it is indistinguishable from a (non-ID) `MaxByteOffset` decoration. --- lib/SPIRV/SPIRVReader.cpp | 8 ++++++++ test/MaxByteOffsetId.spvasm | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 test/MaxByteOffsetId.spvasm diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp index c0cdcd38a..87cb7f61e 100644 --- a/lib/SPIRV/SPIRVReader.cpp +++ b/lib/SPIRV/SPIRVReader.cpp @@ -3208,6 +3208,14 @@ void SPIRVToLLVM::transFunctionAttrs(SPIRVFunction *BF, Function *F) { SPIRVWord MaxOffset = 0; if (BA->hasDecorate(DecorationMaxByteOffset, 0, &MaxOffset)) Builder.addDereferenceableAttr(MaxOffset); + else { + SPIRVId MaxOffsetId; + if (BA->hasDecorateId(DecorationMaxByteOffsetId, 0, &MaxOffsetId)) { + if (auto MaxOffsetVal = transIdAsConstant(MaxOffsetId)) { + Builder.addDereferenceableAttr(*MaxOffsetVal); + } + } + } if (auto Alignment = getAlignment(BA)) { Builder.addAlignmentAttr(*Alignment); } diff --git a/test/MaxByteOffsetId.spvasm b/test/MaxByteOffsetId.spvasm new file mode 100644 index 000000000..d71fbe43e --- /dev/null +++ b/test/MaxByteOffsetId.spvasm @@ -0,0 +1,33 @@ +; REQUIRES: spirv-as + +; RUN: spirv-as %s --target-env spv1.2 -o %t.spv +; RUN: spirv-val %t.spv +; RUN: llvm-spirv -r -o %t.rev.bc %t.spv +; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s + +; CHECK: define spir_kernel void @testMaxByteOffsetId( +; CHECK-SAME: ptr addrspace(1) dereferenceable(24) %p, +; CHECK-SAME: ptr addrspace(1) dereferenceable(48) %q) + + OpCapability Addresses + OpCapability Kernel + OpMemoryModel Physical64 OpenCL + OpEntryPoint Kernel %fn "testMaxByteOffsetId" + OpName %p "p" + OpName %q "q" + OpDecorateId %p MaxByteOffsetId %mbo + OpDecorateId %q MaxByteOffsetId %spec + %void = OpTypeVoid + %i32 = OpTypeInt 32 0 + %ptr = OpTypePointer CrossWorkgroup %i32 + %fnTy = OpTypeFunction %void %ptr %ptr + %mbo = OpConstant %i32 24 + %spec = OpSpecConstantOp %i32 IAdd %mbo %mbo + + %fn = OpFunction %void None %fnTy + %p = OpFunctionParameter %ptr + %q = OpFunctionParameter %ptr + %entry = OpLabel + + OpReturn + OpFunctionEnd From 8af0eb5e9f537c03c517d3c1510e46fd8835db1b Mon Sep 17 00:00:00 2001 From: Marcos Maronas Date: Mon, 2 Dec 2024 05:23:47 -0800 Subject: [PATCH 05/11] Initialize uninitialized class members. (#2899) --- lib/SPIRV/libSPIRV/SPIRVInstruction.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/SPIRV/libSPIRV/SPIRVInstruction.h b/lib/SPIRV/libSPIRV/SPIRVInstruction.h index d29193aef..9e72281c7 100644 --- a/lib/SPIRV/libSPIRV/SPIRVInstruction.h +++ b/lib/SPIRV/libSPIRV/SPIRVInstruction.h @@ -4280,7 +4280,9 @@ class SPIRVUntypedPrefetchKHR : public SPIRVInstruction { validate(); } - SPIRVUntypedPrefetchKHR() : SPIRVInstruction(OC) { + SPIRVUntypedPrefetchKHR() + : SPIRVInstruction(OC), PtrTy(SPIRVID_INVALID), + NumBytes(SPIRVID_INVALID) { setHasNoId(); setHasNoType(); } From c1828a93d050afe8a44cc239a2908270c578c345 Mon Sep 17 00:00:00 2001 From: Wenju He Date: Mon, 2 Dec 2024 15:13:41 +0000 Subject: [PATCH 06/11] [SPIRVToOCL20] Fix __spirv_ocl_printf translation assert error (#2891) When SPIRVToOCL20Pass is used in OCL CPU backend to translate SPV-IR builtin to OCL builtin, __spirv_ocl_printf call has bigger number of arguments than its parameters. --- lib/SPIRV/SPIRVBuiltinHelper.cpp | 3 ++- test/printf-spv-ir-to-ocl.ll | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 test/printf-spv-ir-to-ocl.ll diff --git a/lib/SPIRV/SPIRVBuiltinHelper.cpp b/lib/SPIRV/SPIRVBuiltinHelper.cpp index 2c6986ad0..7dceb9a17 100644 --- a/lib/SPIRV/SPIRVBuiltinHelper.cpp +++ b/lib/SPIRV/SPIRVBuiltinHelper.cpp @@ -90,7 +90,8 @@ BuiltinCallMutator::BuiltinCallMutator(BuiltinCallMutator &&Other) Value *BuiltinCallMutator::doConversion() { assert(CI && "Need to have a call instruction to do the conversion"); auto Mangler = makeMangler(CI, Rules); - for (unsigned I = 0; I < Args.size(); I++) { + for (unsigned I = 0, E = std::min(Args.size(), PointerTypes.size()); I < E; + I++) { Mangler->getTypeMangleInfo(I).PointerTy = dyn_cast(PointerTypes[I]); } diff --git a/test/printf-spv-ir-to-ocl.ll b/test/printf-spv-ir-to-ocl.ll new file mode 100644 index 000000000..c42af7dc4 --- /dev/null +++ b/test/printf-spv-ir-to-ocl.ll @@ -0,0 +1,14 @@ +; REQUIRES: pass-plugin +; UNSUPPORTED: target={{.*windows.*}} + +; RUN: opt %load_spirv_lib -passes=spirv-to-ocl20 %s -S -o - | FileCheck %s + +target triple = "spir64-unknown-unknown" + +declare spir_func i32 @_Z18__spirv_ocl_printfPU3AS2Kcz(ptr addrspace(2), ...) + +define spir_func void @__asan_set_shadow_dynamic_local() { +; CHECK: call spir_func i32 (ptr addrspace(2), ...) @printf(ptr addrspace(2) null, i32 0, i32 0) + %call = call spir_func i32 (ptr addrspace(2), ...) @_Z18__spirv_ocl_printfPU3AS2Kcz(ptr addrspace(2) null, i32 0, i32 0) + ret void +} From d20ca5dd1c92493740e7bdbc7373bfa7c6ad4284 Mon Sep 17 00:00:00 2001 From: Sven van Haastregt Date: Wed, 4 Dec 2024 10:27:53 +0100 Subject: [PATCH 07/11] [SPIR-V 1.2] SPIRVReader: Support LocalSizeId (#2898) If there is no `OpExecutionMode .. LocalSize` in the input, see if there is an `OpExecutionModeId .. LocalSizeId` and take the value for the `reqd_work_group_size` metadata from the referenced constants instead. Once `LocalSizeId` has been translated to LLVM IR, it is indistinguishable from a (non-ID) `LocalSize` execution mode. --- lib/SPIRV/SPIRVReader.cpp | 8 ++++++++ lib/SPIRV/libSPIRV/SPIRVEntry.cpp | 1 + lib/SPIRV/libSPIRV/SPIRVEntry.h | 6 ++++++ test/LocalSizeId.spvasm | 27 +++++++++++++++++++++++++++ 4 files changed, 42 insertions(+) create mode 100644 test/LocalSizeId.spvasm diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp index 87cb7f61e..2b0b05946 100644 --- a/lib/SPIRV/SPIRVReader.cpp +++ b/lib/SPIRV/SPIRVReader.cpp @@ -4519,6 +4519,14 @@ bool SPIRVToLLVM::transMetadata() { if (auto *EM = BF->getExecutionMode(ExecutionModeLocalSize)) { F->setMetadata(kSPIR2MD::WGSize, getMDNodeStringIntVec(Context, EM->getLiterals())); + } else if (auto *EM = BF->getExecutionModeId(ExecutionModeLocalSizeId)) { + std::vector Values; + for (const auto Id : EM->getLiterals()) { + if (auto Val = transIdAsConstant(Id)) { + Values.emplace_back(static_cast(*Val)); + } + } + F->setMetadata(kSPIR2MD::WGSize, getMDNodeStringIntVec(Context, Values)); } // Generate metadata for work_group_size_hint if (auto *EM = BF->getExecutionMode(ExecutionModeLocalSizeHint)) { diff --git a/lib/SPIRV/libSPIRV/SPIRVEntry.cpp b/lib/SPIRV/libSPIRV/SPIRVEntry.cpp index 473535ba0..459109b4c 100644 --- a/lib/SPIRV/libSPIRV/SPIRVEntry.cpp +++ b/lib/SPIRV/libSPIRV/SPIRVEntry.cpp @@ -656,6 +656,7 @@ void SPIRVExecutionMode::decode(std::istream &I) { getDecoder(I) >> Target >> ExecMode; switch (static_cast(ExecMode)) { case ExecutionModeLocalSize: + case ExecutionModeLocalSizeId: case ExecutionModeLocalSizeHint: case ExecutionModeMaxWorkgroupSizeINTEL: WordLiterals.resize(3); diff --git a/lib/SPIRV/libSPIRV/SPIRVEntry.h b/lib/SPIRV/libSPIRV/SPIRVEntry.h index f1b1ea79c..847a434c7 100644 --- a/lib/SPIRV/libSPIRV/SPIRVEntry.h +++ b/lib/SPIRV/libSPIRV/SPIRVEntry.h @@ -807,6 +807,12 @@ class SPIRVComponentExecutionModes { return nullptr; return Loc->second; } + SPIRVExecutionModeId *getExecutionModeId(SPIRVExecutionModeKind EMK) const { + auto Loc = ExecModes.find(EMK); + if (Loc == ExecModes.end()) + return nullptr; + return static_cast(Loc->second); + } SPIRVExecutionModeRange getExecutionModeRange(SPIRVExecutionModeKind EMK) const { return ExecModes.equal_range(EMK); diff --git a/test/LocalSizeId.spvasm b/test/LocalSizeId.spvasm new file mode 100644 index 000000000..24f0ea3f6 --- /dev/null +++ b/test/LocalSizeId.spvasm @@ -0,0 +1,27 @@ +; REQUIRES: spirv-as + +; RUN: spirv-as %s --target-env spv1.2 -o %t.spv +; RUN: spirv-val %t.spv +; RUN: llvm-spirv -r -o %t.rev.bc %t.spv +; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s + + OpCapability Addresses + OpCapability Linkage + OpCapability Kernel + OpMemoryModel Physical64 OpenCL + OpEntryPoint Kernel %fn "testLocalSizeId" + OpExecutionModeId %fn LocalSizeId %uint_64 %uint_1 %uint_1sco + %void = OpTypeVoid + %uint = OpTypeInt 32 0 + %uint_1 = OpConstant %uint 1 + %uint_64 = OpConstant %uint 64 + %uint_1sco = OpSpecConstantOp %uint UDiv %uint_64 %uint_64 + %fnTy = OpTypeFunction %void + +; CHECK: define spir_kernel void @testLocalSizeId() {{.*}} !reqd_work_group_size ![[MD:[0-9]+]] +; CHECK: ![[MD]] = !{i32 64, i32 1, i32 1} + + %fn = OpFunction %void None %fnTy + %entry = OpLabel + OpReturn + OpFunctionEnd From a1f6742cd6a2ce2109e9756dd3a7c1cb4d5bb750 Mon Sep 17 00:00:00 2001 From: Viktoria Maximova Date: Wed, 4 Dec 2024 17:05:22 +0100 Subject: [PATCH 08/11] Improve matrices translation with SPV_KHR_untyped_pointers (#2857) Ensure that we do correct translation of matrices (SPV_KHR_cooperative_matrix and SPV_INTEL_joint_matrix extensions) when untyped pointers are enabled. This mainly fixes mangling issues in reverse translation for the untyped pointer. Also added handling for typed and untyped SPIR-V friendly access chain instructions in forward translation (a point to review and discuss). --- lib/SPIRV/SPIRVReader.cpp | 59 ++++++++++++++--- lib/SPIRV/SPIRVWriter.cpp | 20 ++++++ lib/SPIRV/libSPIRV/SPIRVInstruction.h | 10 +++ lib/SPIRV/libSPIRV/SPIRVOpCode.h | 7 ++ .../array_of_matrices.ll | 7 ++ .../bf16_conversion_instructions.ll | 13 ++++ .../cooperative_matrix_apply.ll | 7 ++ .../cooperative_matrix_checked.ll | 7 ++ .../cooperative_matrix_prefetch.ll | 7 ++ .../SPV_INTEL_joint_matrix/joint_matrix.ll | 7 ++ .../joint_matrix_bfloat16.ll | 7 ++ .../joint_matrix_checked.ll | 7 ++ .../joint_matrix_half.ll | 7 ++ .../access_store.ll | 22 +++++-- .../access_store_untyped.ll | 66 +++++++++++++++++++ .../arithmetic_instructions.ll | 9 +++ .../array_of_matrices.ll | 7 ++ .../conversion_instructions.ll | 9 +++ .../cooperative_matrix.ll | 16 +++++ 19 files changed, 280 insertions(+), 14 deletions(-) create mode 100644 test/extensions/KHR/SPV_KHR_cooperative_matrix/access_store_untyped.ll diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp index 2b0b05946..584dd894c 100644 --- a/lib/SPIRV/SPIRVReader.cpp +++ b/lib/SPIRV/SPIRVReader.cpp @@ -2239,8 +2239,10 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, auto *AC = static_cast(BV); auto *Base = transValue(AC->getBase(), F, BB); SPIRVType *BaseSPVTy = AC->getBaseType(); - if (BaseSPVTy->isTypePointer() && - BaseSPVTy->getPointerElementType()->isTypeCooperativeMatrixKHR()) { + if ((BaseSPVTy->isTypePointer() && + BaseSPVTy->getPointerElementType()->isTypeCooperativeMatrixKHR()) || + (isUntypedAccessChainOpCode(OC) && + BaseSPVTy->isTypeCooperativeMatrixKHR())) { return mapValue(BV, transSPIRVBuiltinFromInst(AC, BB)); } Type *BaseTy = @@ -3488,23 +3490,64 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName, BasicBlock *BB) { std::string MangledName; auto Ops = BI->getOperands(); + Op OC = BI->getOpCode(); + if (isUntypedAccessChainOpCode(OC)) { + auto *AC = static_cast(BI); + if (AC->getBaseType()->isTypeCooperativeMatrixKHR()) + Ops.erase(Ops.begin()); + } Type *RetTy = BI->hasType() ? transType(BI->getType()) : Type::getVoidTy(*Context); transOCLBuiltinFromInstPreproc(BI, RetTy, Ops); std::vector ArgTys = transTypeVector(SPIRVInstruction::getOperandTypes(Ops), true); - // Special handling for "truly" untyped pointers to preserve correct - // builtin mangling of atomic operations. auto Ptr = findFirstPtrType(ArgTys); if (Ptr < ArgTys.size() && BI->getValueType(Ops[Ptr]->getId())->isTypeUntypedPointerKHR()) { - if (isAtomicOpCodeUntypedPtrSupported(BI->getOpCode())) { + // Special handling for "truly" untyped pointers to preserve correct + // builtin mangling of atomic and matrix operations. + if (isAtomicOpCodeUntypedPtrSupported(OC)) { auto *AI = static_cast(BI); ArgTys[Ptr] = TypedPointerType::get( transType(AI->getSemanticType()), SPIRSPIRVAddrSpaceMap::rmap( BI->getValueType(Ops[Ptr]->getId())->getPointerStorageClass())); + } else if (OC == spv::OpCooperativeMatrixStoreKHR || + OC == spv::internal::OpJointMatrixStoreINTEL || + OC == spv::internal::OpCooperativeMatrixStoreCheckedINTEL || + OC == spv::internal::OpJointMatrixLoadINTEL || + OC == spv::OpCompositeConstruct || + OC == spv::internal::OpCooperativeMatrixApplyFunctionINTEL) { + auto *Val = transValue(Ops[Ptr], BB->getParent(), BB); + Val = Val->stripPointerCasts(); + if (auto *GEP = dyn_cast(Val)) + ArgTys[Ptr] = TypedPointerType::get( + GEP->getSourceElementType(), + SPIRSPIRVAddrSpaceMap::rmap( + BI->getValueType(Ops[Ptr]->getId())->getPointerStorageClass())); + else if (auto *AI = dyn_cast(Val)) + ArgTys[Ptr] = TypedPointerType::get( + AI->getAllocatedType(), + SPIRSPIRVAddrSpaceMap::rmap( + BI->getValueType(Ops[Ptr]->getId())->getPointerStorageClass())); + else if (isa(Val) && !RetTy->isVoidTy()) { + // Pointer could be a function parameter. Assume that the type of the + // pointer is the same as the return type. + Type *Ty = nullptr; + // it return type is array type, assign its element type to Ty + if (RetTy->isArrayTy()) + Ty = RetTy->getArrayElementType(); + else if (RetTy->isVectorTy()) + Ty = cast(RetTy)->getElementType(); + else + Ty = RetTy; + + ArgTys[Ptr] = TypedPointerType::get( + Ty, + SPIRSPIRVAddrSpaceMap::rmap( + BI->getValueType(Ops[Ptr]->getId())->getPointerStorageClass())); + } } } @@ -3517,8 +3560,7 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName, if (BM->getDesiredBIsRepresentation() != BIsRepresentation::SPIRVFriendlyIR) mangleOpenClBuiltin(FuncName, ArgTys, MangledName); else - MangledName = - getSPIRVFriendlyIRFunctionName(FuncName, BI->getOpCode(), ArgTys, Ops); + MangledName = getSPIRVFriendlyIRFunctionName(FuncName, OC, ArgTys, Ops); opaquifyTypedPointers(ArgTys); @@ -3541,14 +3583,13 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName, Func->setCallingConv(CallingConv::SPIR_FUNC); if (isFuncNoUnwind()) Func->addFnAttr(Attribute::NoUnwind); - auto OC = BI->getOpCode(); if (isGroupOpCode(OC) || isGroupNonUniformOpcode(OC) || isIntelSubgroupOpCode(OC) || isSplitBarrierINTELOpCode(OC) || OC == OpControlBarrier) Func->addFnAttr(Attribute::Convergent); } CallInst *Call; - if (BI->getOpCode() == OpCooperativeMatrixLengthKHR && + if (OC == OpCooperativeMatrixLengthKHR && Ops[0]->getOpCode() == OpTypeCooperativeMatrixKHR) { // OpCooperativeMatrixLengthKHR needs special handling as its operand is // a Type instead of a Value. diff --git a/lib/SPIRV/SPIRVWriter.cpp b/lib/SPIRV/SPIRVWriter.cpp index 55af2aad6..6ce0a3f31 100644 --- a/lib/SPIRV/SPIRVWriter.cpp +++ b/lib/SPIRV/SPIRVWriter.cpp @@ -6836,8 +6836,28 @@ LLVMToSPIRVBase::transBuiltinToInstWithoutDecoration(Op OC, CallInst *CI, SPRetTy = transType(F->getParamStructRetType(0)); Args.erase(Args.begin()); } + if (RetTy->isPointerTy() && + BM->isAllowedToUseExtension(ExtensionID::SPV_KHR_untyped_pointers)) { + if (OC == OpAccessChain) + OC = OpUntypedAccessChainKHR; + else if (OC == OpInBoundsAccessChain) + OC = OpUntypedInBoundsAccessChainKHR; + else if (OC == OpPtrAccessChain) + OC = OpUntypedPtrAccessChainKHR; + else if (OC == OpInBoundsPtrAccessChain) + OC = OpUntypedInBoundsPtrAccessChainKHR; + } auto *SPI = SPIRVInstTemplateBase::create(OC); std::vector SPArgs; + if (isUntypedAccessChainOpCode(OC)) { + // Untyped access chain instructions have an additional argument BaseTy. + Type *Ty = Scavenger->getScavengedType(Args[0]); + SPIRVType *PtrTy = nullptr; + if (auto *TPT = dyn_cast(Ty)) { + PtrTy = transType(TPT->getElementType()); + SPArgs.push_back(PtrTy->getId()); + } + } for (size_t I = 0, E = Args.size(); I != E; ++I) { if (Args[I]->getType()->isPointerTy()) { Value *Pointee = Args[I]->stripPointerCasts(); diff --git a/lib/SPIRV/libSPIRV/SPIRVInstruction.h b/lib/SPIRV/libSPIRV/SPIRVInstruction.h index 9e72281c7..f6d714294 100644 --- a/lib/SPIRV/libSPIRV/SPIRVInstruction.h +++ b/lib/SPIRV/libSPIRV/SPIRVInstruction.h @@ -1747,6 +1747,16 @@ class SPIRVAccessChainBase : public SPIRVInstTemplateBase { OpCode == OpUntypedPtrAccessChainKHR || OpCode == OpUntypedInBoundsPtrAccessChainKHR; } + SPIRVCapVec getRequiredCapability() const override { + if (isUntyped()) + return getVec(CapabilityUntypedPointersKHR); + return {}; + } + std::optional getRequiredExtension() const override { + if (isUntyped()) + return ExtensionID::SPV_KHR_untyped_pointers; + return {}; + } }; template diff --git a/lib/SPIRV/libSPIRV/SPIRVOpCode.h b/lib/SPIRV/libSPIRV/SPIRVOpCode.h index 3541a94bf..8e0ff6190 100644 --- a/lib/SPIRV/libSPIRV/SPIRVOpCode.h +++ b/lib/SPIRV/libSPIRV/SPIRVOpCode.h @@ -150,6 +150,13 @@ inline bool isAccessChainOpCode(Op OpCode) { return OpCode == OpAccessChain || OpCode == OpInBoundsAccessChain; } +inline bool isUntypedAccessChainOpCode(Op OpCode) { + return OpCode == OpUntypedAccessChainKHR || + OpCode == OpUntypedInBoundsAccessChainKHR || + OpCode == OpUntypedPtrAccessChainKHR || + OpCode == OpUntypedInBoundsPtrAccessChainKHR; +} + inline bool hasExecScope(Op OpCode) { unsigned OC = OpCode; return (OpGroupWaitEvents <= OC && OC <= OpGroupSMax) || diff --git a/test/extensions/INTEL/SPV_INTEL_joint_matrix/array_of_matrices.ll b/test/extensions/INTEL/SPV_INTEL_joint_matrix/array_of_matrices.ll index 0571af5dd..cb6a0e586 100644 --- a/test/extensions/INTEL/SPV_INTEL_joint_matrix/array_of_matrices.ll +++ b/test/extensions/INTEL/SPV_INTEL_joint_matrix/array_of_matrices.ll @@ -9,6 +9,13 @@ ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_joint_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM + ; CHECK-SPIRV-DAG: Capability JointMatrixINTEL ; CHECK-SPIRV-DAG: Extension "SPV_INTEL_joint_matrix" ; CHECK-SPIRV: TypeInt [[#Int16Ty:]] 16 0 diff --git a/test/extensions/INTEL/SPV_INTEL_joint_matrix/bf16_conversion_instructions.ll b/test/extensions/INTEL/SPV_INTEL_joint_matrix/bf16_conversion_instructions.ll index 237c05688..ae853abae 100644 --- a/test/extensions/INTEL/SPV_INTEL_joint_matrix/bf16_conversion_instructions.ll +++ b/test/extensions/INTEL/SPV_INTEL_joint_matrix/bf16_conversion_instructions.ll @@ -12,6 +12,19 @@ ; RUN: not llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_bfloat16_conversion 2>&1 \ ; RUN: | FileCheck %s --check-prefix=CHECK-ERROR +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix,+SPV_INTEL_bfloat16_conversion,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-OCL-IR + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc --spirv-target-env=SPV-IR +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-SPV-IR + +; RUN: not llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_bfloat16_conversion,+SPV_KHR_untyped_pointers 2>&1 \ +; RUN: | FileCheck %s --check-prefix=CHECK-ERROR + ; CHECK-ERROR: InvalidInstruction: Can't translate llvm instruction: ; CHECK-ERROR-NEXT: ConvertFToBF16INTEL ; CHECK-ERROR-NEXT: Can be used with cooperative matrices only when SPV_INTEL_joint_matrix is enabled diff --git a/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_apply.ll b/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_apply.ll index b0f97b74d..59451d0c7 100644 --- a/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_apply.ll +++ b/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_apply.ll @@ -8,6 +8,13 @@ ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM + ; CHECK-SPIRV-DAG: Capability CooperativeMatrixKHR ; CHECK-SPIRV-DAG: Capability CooperativeMatrixInvocationInstructionsINTEL ; CHECK-SPIRV-DAG: Extension "SPV_INTEL_joint_matrix" diff --git a/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll b/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll index f6e4ba727..1c43d2f30 100644 --- a/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll +++ b/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_checked.ll @@ -8,6 +8,13 @@ ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM + ; CHECK-SPIRV-DAG: Capability CooperativeMatrixKHR ; CHECK-SPIRV-DAG: Capability CooperativeMatrixCheckedInstructionsINTEL ; CHECK-SPIRV-DAG: Extension "SPV_KHR_cooperative_matrix" diff --git a/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll b/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll index 17f8c3f59..53f6a51a7 100644 --- a/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll +++ b/test/extensions/INTEL/SPV_INTEL_joint_matrix/cooperative_matrix_prefetch.ll @@ -8,6 +8,13 @@ ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_INTEL_joint_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM + ; CHECK-SPIRV-DAG: Capability CooperativeMatrixKHR ; CHECK-SPIRV-DAG: Capability CooperativeMatrixPrefetchINTEL ; CHECK-SPIRV-DAG: Extension "SPV_KHR_cooperative_matrix" diff --git a/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix.ll b/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix.ll index 7c8ec53eb..066ea5aeb 100644 --- a/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix.ll +++ b/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix.ll @@ -6,6 +6,13 @@ ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_joint_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM + ; CHECK-SPIRV-DAG: Capability JointMatrixINTEL ; CHECK-SPIRV-DAG: Extension "SPV_INTEL_joint_matrix" ; CHECK-SPIRV-DAG: TypeInt [[#Int8Ty:]] 8 0 diff --git a/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_bfloat16.ll b/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_bfloat16.ll index 80c014b68..b7f2404f2 100644 --- a/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_bfloat16.ll +++ b/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_bfloat16.ll @@ -9,6 +9,13 @@ ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_bfloat16_conversion,+SPV_INTEL_joint_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM + ; CHECK-SPIRV-DAG: TypeInt [[#SHORT:]] 16 ; CHECK-SPIRV-DAG: TypeInt [[#INT:]] 32 ; CHECK-SPIRV-DAG: TypeFloat [[#Float:]] 32 diff --git a/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_checked.ll b/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_checked.ll index d948844ef..509b6815f 100644 --- a/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_checked.ll +++ b/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_checked.ll @@ -8,6 +8,13 @@ ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_joint_matrix,+SPV_KHR_untyped_pointers,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM + ; CHECK-SPIRV-DAG: Capability JointMatrixINTEL ; CHECK-SPIRV-DAG: Capability CooperativeMatrixKHR ; CHECK-SPIRV-DAG: Capability CooperativeMatrixCheckedInstructionsINTEL diff --git a/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_half.ll b/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_half.ll index 6b5c380de..8eccca433 100644 --- a/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_half.ll +++ b/test/extensions/INTEL/SPV_INTEL_joint_matrix/joint_matrix_half.ll @@ -9,6 +9,13 @@ ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_bfloat16_conversion,+SPV_INTEL_joint_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM + ; CHECK-SPIRV-DAG: TypeInt [[#INT:]] 32 ; CHECK-SPIRV-DAG: TypeFloat [[#Half:]] 16 ; CHECK-SPIRV-DAG: TypeFloat [[#Float:]] 32 diff --git a/test/extensions/KHR/SPV_KHR_cooperative_matrix/access_store.ll b/test/extensions/KHR/SPV_KHR_cooperative_matrix/access_store.ll index 62f3f8407..da7d76e7c 100644 --- a/test/extensions/KHR/SPV_KHR_cooperative_matrix/access_store.ll +++ b/test/extensions/KHR/SPV_KHR_cooperative_matrix/access_store.ll @@ -1,10 +1,19 @@ ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix -o %t.spv +; RUN: spirv-val %t.spv ; RUN: llvm-spirv %t.spv -to-text -o %t.spt -; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV +; RUN: FileCheck < %t.spt %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-TYPED-PTR ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc -; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefixes=CHECK-LLVM,CHECK-LLVM-TYPED-PTR + +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: spirv-val %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefixes=CHECK-SPIRV,CHECK-SPIRV-UNTYPED-PTR + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefixes=CHECK-LLVM,CHECK-LLVM-UNTYPED-PTR ; CHECK-SPIRV: TypeInt [[#TypeInt:]] 32 0 ; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const0:]] 0 @@ -15,18 +24,21 @@ ; CHECK-SPIRV: TypeCooperativeMatrixKHR [[#TypeMatrix:]] [[#TypeInt]] [[#Const3]] [[#Const12]] [[#Const12]] [[#Const0]] ; CHECK-SPIRV: TypePointer [[#TypeMatrixPtr:]] 7 [[#TypeMatrix]] -; CHECK-SPIRV: TypePointer [[#TypeIntPtr:]] 7 [[#TypeInt]] +; CHECK-SPIRV-TYPED-PTR: TypePointer [[#TypeIntPtr:]] 7 [[#TypeInt]] +; CHECK-SPIRV-UNTYPED-PTR: TypeUntypedPointerKHR [[#TypePtr:]] 7 ; CHECK-SPIRV: Variable [[#TypeMatrixPtr]] [[#VarMatrixPtr:]] 7 ; CHECK-SPIRV: CompositeConstruct [[#TypeMatrix]] [[#Composite:]] [[#Const0]] ; CHECK-SPIRV: Store [[#VarMatrixPtr]] [[#Composite]] -; CHECK-SPIRV: AccessChain [[#TypeIntPtr]] [[#Res:]] [[#VarMatrixPtr]] [[#Const1]] +; CHECK-SPIRV-TYPED-PTR: AccessChain [[#TypeIntPtr]] [[#Res:]] [[#VarMatrixPtr]] [[#Const1]] +; CHECK-SPIRV-UNTYPED-PTR: UntypedAccessChainKHR [[#TypePtr]] [[#Res:]] [[#TypeMatrix]] [[#VarMatrixPtr]] [[#Const1]] ; CHECK-SPIRV: Store [[#Res]] [[#Const42]] ; CHECK-LLVM: %0 = alloca target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 0) ; CHECK-LLVM: %Obj = call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 0) @_Z26__spirv_CompositeConstructi(i32 0) ; CHECK-LLVM: store target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 0) %Obj, ptr %0 -; CHECK-LLVM: %call = call spir_func ptr @_Z19__spirv_AccessChainPPU3AS144__spirv_CooperativeMatrixKHR__uint_3_12_12_0i(ptr %0, i32 1) +; CHECK-LLVM-TYPED-PTR: %call = call spir_func ptr @_Z19__spirv_AccessChainPPU3AS144__spirv_CooperativeMatrixKHR__uint_3_12_12_0i(ptr %0, i32 1) +; CHECK-LLVM-UNTYPED-PTR: %call = call spir_func ptr @_Z29__spirv_UntypedAccessChainKHRPPU3AS144__spirv_CooperativeMatrixKHR__uint_3_12_12_0i(ptr %0, i32 1) ; CHECK-LLVM: store i32 42, ptr %call target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" diff --git a/test/extensions/KHR/SPV_KHR_cooperative_matrix/access_store_untyped.ll b/test/extensions/KHR/SPV_KHR_cooperative_matrix/access_store_untyped.ll new file mode 100644 index 000000000..f070ad80b --- /dev/null +++ b/test/extensions/KHR/SPV_KHR_cooperative_matrix/access_store_untyped.ll @@ -0,0 +1,66 @@ +; RUN: llvm-as < %s -o %t.bc +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: spirv-val %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM + +; CHECK-SPIRV: TypeInt [[#TypeInt:]] 32 0 +; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const0:]] 0 +; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const1:]] 1 {{$}} +; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const3:]] 3 +; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const12:]] 12 +; CHECK-SPIRV-DAG: Constant [[#TypeInt]] [[#Const42:]] 42 + +; CHECK-SPIRV: TypeCooperativeMatrixKHR [[#TypeMatrix:]] [[#TypeInt]] [[#Const3]] [[#Const12]] [[#Const12]] [[#Const0]] +; CHECK-SPIRV: TypePointer [[#TypeMatrixPtr:]] 7 [[#TypeMatrix]] +; CHECK-SPIRV: TypeUntypedPointerKHR [[#TypePtr:]] 7 + +; CHECK-SPIRV: Variable [[#TypeMatrixPtr]] [[#VarMatrixPtr:]] 7 +; CHECK-SPIRV: CompositeConstruct [[#TypeMatrix]] [[#Composite:]] [[#Const0]] +; CHECK-SPIRV: Store [[#VarMatrixPtr]] [[#Composite]] +; CHECK-SPIRV: UntypedAccessChainKHR [[#TypePtr]] [[#Res:]] [[#TypeMatrix]] [[#VarMatrixPtr]] [[#Const1]] +; CHECK-SPIRV: Store [[#Res]] [[#Const42]] + +; CHECK-LLVM: %0 = alloca target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 0) +; CHECK-LLVM: %Obj = call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 0) @_Z26__spirv_CompositeConstructi(i32 0) +; CHECK-LLVM: store target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 0) %Obj, ptr %0 +; CHECK-LLVM: %call = call spir_func ptr @_Z29__spirv_UntypedAccessChainKHRPPU3AS144__spirv_CooperativeMatrixKHR__uint_3_12_12_0i(ptr %0, i32 1) +; CHECK-LLVM: store i32 42, ptr %call + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "spir64-unknown-unknown" + +; Function Attrs: mustprogress uwtable +define dso_local void @_Z3fooi(i32 noundef %idx) local_unnamed_addr #0 { +entry: + %0 = alloca target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 0), align 8 + %Obj = tail call spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 0) @_Z26__spirv_CompositeConstruct(i32 noundef 0) #4 + store target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 0) %Obj, ptr %0, align 8 + %call = call noundef ptr @_Z29__spirv_UntypedAccessChainKHRP6Matrixii(ptr %0, i32 noundef 1) + call void @_Z13__spirv_StorePii(ptr noundef %call, i32 noundef 42) + ret void +} + +declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 0) @_Z26__spirv_CompositeConstruct(i32 noundef) local_unnamed_addr #2 + +declare noundef ptr @_Z29__spirv_UntypedAccessChainKHRP6Matrixii(ptr noundef, i32 noundef) local_unnamed_addr #2 + +declare void @_Z13__spirv_StorePii(ptr noundef, i32 noundef) local_unnamed_addr #2 + +attributes #0 = { mustprogress uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" } +attributes #1 = { mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" } +attributes #3 = { nounwind } + +!llvm.module.flags = !{!0, !1, !2, !3, !4} +!llvm.ident = !{!5} + +!0 = !{i32 7, !"Dwarf Version", i32 4} +!1 = !{i32 1, !"wchar_size", i32 4} +!2 = !{i32 8, !"PIC Level", i32 2} +!3 = !{i32 7, !"PIE Level", i32 2} +!4 = !{i32 7, !"uwtable", i32 2} +!5 = !{!"clang version 16.0.0 (https://github.com/llvm/llvm-project.git 08d094a0e457360ad8b94b017d2dc277e697ca76)"} diff --git a/test/extensions/KHR/SPV_KHR_cooperative_matrix/arithmetic_instructions.ll b/test/extensions/KHR/SPV_KHR_cooperative_matrix/arithmetic_instructions.ll index cac16270c..38d712bcf 100644 --- a/test/extensions/KHR/SPV_KHR_cooperative_matrix/arithmetic_instructions.ll +++ b/test/extensions/KHR/SPV_KHR_cooperative_matrix/arithmetic_instructions.ll @@ -1,8 +1,17 @@ ; RUN: llvm-as < %s -o %t.bc ; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix -o %t.spv +; RUN: spirv-val %t.spv ; RUN: llvm-spirv %t.spv -to-text -o %t.spt ; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r --spirv-target-env=SPV-IR %t.spv -o %t.rev.bc +; RUN: llvm-dis %t.rev.bc +; RUN: FileCheck < %t.rev.ll %s --check-prefix=CHECK-LLVM + +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_untyped_pointers -o %t.spv ; RUN: spirv-val %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV ; RUN: llvm-spirv -r --spirv-target-env=SPV-IR %t.spv -o %t.rev.bc ; RUN: llvm-dis %t.rev.bc diff --git a/test/extensions/KHR/SPV_KHR_cooperative_matrix/array_of_matrices.ll b/test/extensions/KHR/SPV_KHR_cooperative_matrix/array_of_matrices.ll index 28979b4eb..7df3d61f6 100644 --- a/test/extensions/KHR/SPV_KHR_cooperative_matrix/array_of_matrices.ll +++ b/test/extensions/KHR/SPV_KHR_cooperative_matrix/array_of_matrices.ll @@ -10,6 +10,13 @@ ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r --spirv-target-env=SPV-IR %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM + ; CHECK-SPIRV-DAG: Capability CooperativeMatrixKHR ; CHECK-SPIRV-DAG: Extension "SPV_KHR_cooperative_matrix" ; CHECK-SPIRV: TypeInt [[#Int16Ty:]] 16 0 diff --git a/test/extensions/KHR/SPV_KHR_cooperative_matrix/conversion_instructions.ll b/test/extensions/KHR/SPV_KHR_cooperative_matrix/conversion_instructions.ll index 39f103abd..9130ac91a 100644 --- a/test/extensions/KHR/SPV_KHR_cooperative_matrix/conversion_instructions.ll +++ b/test/extensions/KHR/SPV_KHR_cooperative_matrix/conversion_instructions.ll @@ -8,6 +8,15 @@ ; RUN: llvm-dis %t.rev.bc ; RUN: FileCheck < %t.rev.ll %s --check-prefix=CHECK-LLVM +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: spirv-val %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r --spirv-target-env=SPV-IR %t.spv -o %t.rev.bc +; RUN: llvm-dis %t.rev.bc +; RUN: FileCheck < %t.rev.ll %s --check-prefix=CHECK-LLVM + ; CHECK-SPIRV: TypeInt [[#TypeInt32:]] 32 0 ; CHECK-SPIRV: TypeInt [[#TypeInt16:]] 16 0 ; CHECK-SPIRV: TypeInt [[#TypeInt8:]] 8 0 diff --git a/test/extensions/KHR/SPV_KHR_cooperative_matrix/cooperative_matrix.ll b/test/extensions/KHR/SPV_KHR_cooperative_matrix/cooperative_matrix.ll index cb12dc52a..71d7139ee 100644 --- a/test/extensions/KHR/SPV_KHR_cooperative_matrix/cooperative_matrix.ll +++ b/test/extensions/KHR/SPV_KHR_cooperative_matrix/cooperative_matrix.ll @@ -8,6 +8,13 @@ ; RUN: llvm-spirv -r %t.spv -o %t.rev.bc ; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_KHR_cooperative_matrix,+SPV_KHR_untyped_pointers -o %t.spv +; RUN: llvm-spirv %t.spv -to-text -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv -r %t.spv -o %t.rev.bc +; RUN: llvm-dis < %t.rev.bc | FileCheck %s --check-prefix=CHECK-LLVM + ; CHECK-SPIRV-DAG: Capability CooperativeMatrixKHR ; CHECK-SPIRV-DAG: Extension "SPV_KHR_cooperative_matrix" ; CHECK-SPIRV-DAG: TypeInt [[#Int8Ty:]] 8 0 @@ -36,6 +43,7 @@ ; CHECK-LLVM: call spir_func target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) @_Z86__spirv_CooperativeMatrixLoadKHR_RPU3AS144__spirv_CooperativeMatrixKHR__char_2_48_12_1PU3AS4cil ; CHECK-LLVM: call spir_func target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z34__spirv_CooperativeMatrixMulAddKHRPU3AS144__spirv_CooperativeMatrixKHR__char_0_12_48_0PU3AS144__spirv_CooperativeMatrixKHR__char_2_48_12_1PU3AS144__spirv_CooperativeMatrixKHR__uint_3_12_12_2i(target("spirv.CooperativeMatrixKHR", i8, 0, 12, 48, 0) %{{.*}}, target("spirv.CooperativeMatrixKHR", i8, 2, 48, 12, 1) %{{.*}}, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) ; CHECK-LLVM: call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHRPU3AS4iPU3AS144__spirv_CooperativeMatrixKHR__uint_3_12_12_2ili(ptr addrspace(4) %call.ascast.i.i, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) +; CHECK-LLVM: call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHRPU3AS1cPU3AS144__spirv_CooperativeMatrixKHR__uint_3_12_12_2ili(ptr addrspace(1) %_arg_accC, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) ; ModuleID = 'test-matrix-opaque.bc' source_filename = "matrix-int8-test.cpp" @@ -132,6 +140,13 @@ _ZZZ15matrix_multiplyIiaLm24ELm96ELm24ELm96ELm24ELm24EEvR10big_matrixIT_XT5_EXT6 ret void } +define weak_odr dso_local spir_func void @matrix_store(ptr addrspace(1) noundef align 4 %_arg_accC, i64 noundef %_arg_N) local_unnamed_addr #0 { + %sub_c.sroa.0.i = alloca target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), align 8 + %sub_c.sroa.0.i.0 = load target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2), ptr %sub_c.sroa.0.i, align 8 + tail call spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR_2(ptr addrspace(1) noundef %_arg_accC, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef %sub_c.sroa.0.i.0, i32 noundef 0, i64 noundef %_arg_N, i32 noundef 1) #4 + ret void +} + ; Function Attrs: convergent declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) @_Z26__spirv_CompositeConstruct(i32 noundef) local_unnamed_addr #2 @@ -149,6 +164,7 @@ declare dso_local spir_func noundef target("spirv.CooperativeMatrixKHR", i32, 3, ; Function Attrs: convergent declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR(ptr addrspace(4) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #2 +declare dso_local spir_func void @_Z33__spirv_CooperativeMatrixStoreKHR_2(ptr addrspace(1) noundef, target("spirv.CooperativeMatrixKHR", i32, 3, 12, 12, 2) noundef, i32 noundef, i64 noundef, i32 noundef) local_unnamed_addr #2 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #3 From 8ef84d23d874791ee0f7344153d0ea28cf326241 Mon Sep 17 00:00:00 2001 From: bwlodarcz Date: Thu, 5 Dec 2024 11:34:14 +0100 Subject: [PATCH 09/11] Add support for llvm.lround.* (#2904) --- lib/SPIRV/SPIRVWriter.cpp | 8 ++++++++ test/llvm-intrinsics/lround.ll | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 test/llvm-intrinsics/lround.ll diff --git a/lib/SPIRV/SPIRVWriter.cpp b/lib/SPIRV/SPIRVWriter.cpp index 6ce0a3f31..06a23e240 100644 --- a/lib/SPIRV/SPIRVWriter.cpp +++ b/lib/SPIRV/SPIRVWriter.cpp @@ -4225,6 +4225,14 @@ SPIRVValue *LLVMToSPIRVBase::transIntrinsicInst(IntrinsicInst *II, return BM->addExtInst(STy, BM->getExtInstSetId(SPIRVEIS_OpenCL), ExtOp, Ops, BB); } + case Intrinsic::lround: { + SPIRVType *SourceTy = transType(II->getArgOperand(0)->getType()); + SPIRVType *DestTy = transType(II->getType()); + SPIRVValue *Rounded = BM->addExtInst( + SourceTy, BM->getExtInstSetId(SPIRVEIS_OpenCL), OpenCLLIB::Round, + {transValue(II->getArgOperand(0), BB)}, BB); + return BM->addUnaryInst(OpConvertFToS, DestTy, Rounded, BB); + } case Intrinsic::frexp: { if (!checkTypeForSPIRVExtendedInstLowering(II, BM)) break; diff --git a/test/llvm-intrinsics/lround.ll b/test/llvm-intrinsics/lround.ll new file mode 100644 index 000000000..c164f6236 --- /dev/null +++ b/test/llvm-intrinsics/lround.ll @@ -0,0 +1,32 @@ +; REQUIRES: spirv-dis +; RUN: llvm-as %s -o %t.bc +; RUN: llvm-spirv %t.bc -o %t.spv +; RUN: spirv-dis --raw-id %t.spv | FileCheck --check-prefix CHECK-SPIRV %s +; RUN: spirv-val %t.spv + +; CHECK-SPIRV: [[opencl:%[0-9]+]] = OpExtInstImport "OpenCL.std" +; CHECK-SPIRV-DAG: [[i32:%[0-9]+]] = OpTypeInt 32 0 +; CHECK-SPIRV-DAG: [[i64:%[0-9]+]] = OpTypeInt 64 0 +; CHECK-SPIRV-DAG: [[f32:%[0-9]+]] = OpTypeFloat 32 +; CHECK-SPIRV-DAG: [[f64:%[0-9]+]] = OpTypeFloat 64 +; CHECK-SPIRV: [[rounded_f32:%[0-9]+]] = OpExtInst [[f32]] [[opencl]] round +; CHECK-SPIRV: OpConvertFToS [[i32]] [[rounded_f32]] +; CHECK-SPIRV: [[rounded_f64:%[0-9]+]] = OpExtInst [[f64]] [[opencl]] round +; CHECK-SPIRV: OpConvertFToS [[i64]] [[rounded_f64]] + +target triple = "spir64-unknown-unknown" + +define spir_func i32 @test_0(float %arg0) { +entry: + %0 = call i32 @llvm.lround.i32.f32(float %arg0) + ret i32 %0 +} + +define spir_func i64 @test_1(double %arg0) { +entry: + %0 = call i64 @llvm.lround.i64.f64(double %arg0) + ret i64 %0 +} + +declare i32 @llvm.lround.i32.f32(float) +declare i64 @llvm.lround.i64.f64(double) From 420f5dd41a21d5a7e5429a88e8c70182d6792e90 Mon Sep 17 00:00:00 2001 From: Sven van Haastregt Date: Thu, 5 Dec 2024 12:20:00 +0100 Subject: [PATCH 10/11] [SPIR-V 1.2] SPIRVReader: Support LocalSizeHintId (#2907) If there is no `OpExecutionMode .. LocalSizeHint` in the input, see if there is an `OpExecutionModeId .. LocalSizeHintId` and take the value for the `work_group_size_hint` metadata from the referenced constants instead. Once `LocalSizeHintId` has been translated to LLVM IR, it is indistinguishable from a (non-ID) `LocalSizeHint` execution mode. --- lib/SPIRV/SPIRVReader.cpp | 10 ++++++++++ lib/SPIRV/libSPIRV/SPIRVEntry.cpp | 1 + test/LocalSizeHintId.spvasm | 27 +++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 test/LocalSizeHintId.spvasm diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp index 584dd894c..8e8877519 100644 --- a/lib/SPIRV/SPIRVReader.cpp +++ b/lib/SPIRV/SPIRVReader.cpp @@ -4573,6 +4573,16 @@ bool SPIRVToLLVM::transMetadata() { if (auto *EM = BF->getExecutionMode(ExecutionModeLocalSizeHint)) { F->setMetadata(kSPIR2MD::WGSizeHint, getMDNodeStringIntVec(Context, EM->getLiterals())); + } else if (auto *EM = + BF->getExecutionModeId(ExecutionModeLocalSizeHintId)) { + std::vector Values; + for (const auto Id : EM->getLiterals()) { + if (auto Val = transIdAsConstant(Id)) { + Values.emplace_back(static_cast(*Val)); + } + } + F->setMetadata(kSPIR2MD::WGSizeHint, + getMDNodeStringIntVec(Context, Values)); } // Generate metadata for vec_type_hint if (auto *EM = BF->getExecutionMode(ExecutionModeVecTypeHint)) { diff --git a/lib/SPIRV/libSPIRV/SPIRVEntry.cpp b/lib/SPIRV/libSPIRV/SPIRVEntry.cpp index 459109b4c..5a18cfd7e 100644 --- a/lib/SPIRV/libSPIRV/SPIRVEntry.cpp +++ b/lib/SPIRV/libSPIRV/SPIRVEntry.cpp @@ -658,6 +658,7 @@ void SPIRVExecutionMode::decode(std::istream &I) { case ExecutionModeLocalSize: case ExecutionModeLocalSizeId: case ExecutionModeLocalSizeHint: + case ExecutionModeLocalSizeHintId: case ExecutionModeMaxWorkgroupSizeINTEL: WordLiterals.resize(3); break; diff --git a/test/LocalSizeHintId.spvasm b/test/LocalSizeHintId.spvasm new file mode 100644 index 000000000..c6beebd74 --- /dev/null +++ b/test/LocalSizeHintId.spvasm @@ -0,0 +1,27 @@ +; REQUIRES: spirv-as + +; RUN: spirv-as %s --target-env spv1.2 -o %t.spv +; RUN: spirv-val %t.spv +; RUN: llvm-spirv -r -o %t.rev.bc %t.spv +; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s + + OpCapability Addresses + OpCapability Linkage + OpCapability Kernel + OpMemoryModel Physical64 OpenCL + OpEntryPoint Kernel %fn "testLocalSizeHintId" + OpExecutionModeId %fn LocalSizeHintId %uint_64 %uint_1 %uint_1sco + %void = OpTypeVoid + %uint = OpTypeInt 32 0 + %uint_1 = OpConstant %uint 1 + %uint_64 = OpConstant %uint 64 + %uint_1sco = OpSpecConstantOp %uint UDiv %uint_64 %uint_64 + %fnTy = OpTypeFunction %void + +; CHECK: define spir_kernel void @testLocalSizeHintId() {{.*}} !work_group_size_hint ![[MD:[0-9]+]] +; CHECK: ![[MD]] = !{i32 64, i32 1, i32 1} + + %fn = OpFunction %void None %fnTy + %entry = OpLabel + OpReturn + OpFunctionEnd From 50af2c70832745be1c0ce1562c4038162c319e5c Mon Sep 17 00:00:00 2001 From: bwlodarcz Date: Fri, 6 Dec 2024 01:02:08 +0100 Subject: [PATCH 11/11] [DebugInfo] Fix EnumClass support (#2876) This commit addresses issues with EnumClass handling in DebugEnumType. In reverse translation, a bug caused the EnumClass flag to be incorrectly applied to every DebugEnumType with an UnderlyingType. Additionally, in forward translation, the support for EnumClassFlag was missing entirely. --- lib/SPIRV/LLVMToSPIRVDbgTran.cpp | 2 ++ lib/SPIRV/SPIRVToLLVMDbgTran.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/SPIRV/LLVMToSPIRVDbgTran.cpp b/lib/SPIRV/LLVMToSPIRVDbgTran.cpp index a3a82d8b9..1ef244674 100644 --- a/lib/SPIRV/LLVMToSPIRVDbgTran.cpp +++ b/lib/SPIRV/LLVMToSPIRVDbgTran.cpp @@ -500,6 +500,8 @@ SPIRVWord LLVMToSPIRVDbgTran::mapDebugFlags(DINode::DIFlags DFlags) { if (BM->getDebugInfoEIS() == SPIRVEIS_NonSemantic_Shader_DebugInfo_200) if (DFlags & DINode::FlagBitField) Flags |= SPIRVDebug::FlagBitField; + if (DFlags & DINode::FlagEnumClass) + Flags |= SPIRVDebug::FlagIsEnumClass; return Flags; } diff --git a/lib/SPIRV/SPIRVToLLVMDbgTran.cpp b/lib/SPIRV/SPIRVToLLVMDbgTran.cpp index dfd842211..7b94b2aeb 100644 --- a/lib/SPIRV/SPIRVToLLVMDbgTran.cpp +++ b/lib/SPIRV/SPIRVToLLVMDbgTran.cpp @@ -811,7 +811,7 @@ DINode *SPIRVToLLVMDbgTran::transTypeEnum(const SPIRVExtInst *DebugInst) { UnderlyingType = transDebugInst(static_cast(E)); return getDIBuilder(DebugInst).createEnumerationType( Scope, Name, File, LineNo, SizeInBits, AlignInBits, Enumerators, - UnderlyingType, 0, "", UnderlyingType); + UnderlyingType, 0, "", Flags & SPIRVDebug::FlagIsEnumClass); } }