diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td index 909165f4c3fa..18c26a82b289 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIROps.td +++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td @@ -3927,44 +3927,6 @@ def AtomicCmpXchg : CIR_Op<"atomic.cmp_xchg", let hasVerifier = 0; } - -def AtomicOrderingNotAtomic : I32EnumAttrCase<"NotAtomic", 0, "not_atomic">; -def AtomicOrderingUnordered : I32EnumAttrCase<"Unordered", 1, "unordered">; -def AtomicOrderingMonotonic : I32EnumAttrCase<"Monotonic", 2, "monotonic">; -def AtomicOrderingAcquire : I32EnumAttrCase<"Acquire", 4, "acquire">; -def AtomicOrderingRelease : I32EnumAttrCase<"Release", 5, "release">; -def AtomicOrderingAcqRel : I32EnumAttrCase<"AcquireRelease", 6, "acq_rel">; -def AtomicOrderingSeqCst : I32EnumAttrCase<"SequentiallyConsistent", 7, "seq_cst">; - -def AtomicOrdering : I32EnumAttr< - "AtomicOrdering", - "Atomicity levels for atomic operations", - [AtomicOrderingNotAtomic, AtomicOrderingUnordered, AtomicOrderingMonotonic, - AtomicOrderingAcquire, AtomicOrderingRelease, AtomicOrderingAcqRel, - AtomicOrderingSeqCst ]> { - let cppNamespace = "::mlir::cir"; -} - -def AtomicRMWType : AnyTypeOf<[CIR_AnyFloat, CIR_PointerType, CIR_IntType]>; - -def AtomicRMWOp : CIR_Op<"atomic.rmw", - [AllTypesMatch<["val", "res"]>]> { - - let summary = ""; - let description = [{ }]; - - let arguments = (ins - AtomicFetchKind:$bin_op, - CIR_PointerType:$ptr, - AtomicRMWType:$val, - AtomicOrdering:$ordering, - OptionalAttr:$alignment, - UnitAttr:$isVolatile); - let results = (outs AtomicRMWType:$res); - - let hasVerifier = 0; - -} //===----------------------------------------------------------------------===// // Operations Lowered Directly to LLVM IR // diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/clang/lib/CIR/CodeGen/CIRGenBuilder.h index 247caba62e8e..e49f0f7b32db 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuilder.h +++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.h @@ -888,20 +888,6 @@ class CIRGenBuilderTy : public CIRBaseBuilderTy { return create(loc, resultTy, objectPtr, memberPtr); } - - mlir::cir::AtomicRMWOp createAtomicRMW(mlir::Location loc, - mlir::cir::AtomicFetchKind kind, - mlir::Value ptr, - mlir::Value val, - mlir::cir::AtomicOrdering ordering, - unsigned alignment, - bool isVolatile) { - auto alignAttr = - mlir::IntegerAttr::get(mlir::IntegerType::get(getContext(), 64), alignment); - return create( - loc, kind, ptr, val, ordering, alignAttr, isVolatile); - } - }; } // namespace cir diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp index e1b6506f38bd..a30ff4299e5c 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp @@ -183,31 +183,40 @@ static Address checkAtomicAlignment(CIRGenFunction &CGF, const CallExpr *E) { /// Utility to insert an atomic instruction based on Intrinsic::ID /// and the expression node. static mlir::Value makeBinaryAtomicValue( - CIRGenFunction &CGF, mlir::cir::AtomicFetchKind Kind, const CallExpr *E, - mlir::cir::AtomicOrdering Ordering = - mlir::cir::AtomicOrdering::SequentiallyConsistent) { + CIRGenFunction &cgf, mlir::cir::AtomicFetchKind kind, const CallExpr *expr, + mlir::cir::MemOrder ordering = + mlir::cir::MemOrder::SequentiallyConsistent) { - QualType T = E->getType(); + QualType typ = expr->getType(); - assert(E->getArg(0)->getType()->isPointerType()); - assert(CGF.getContext().hasSameUnqualifiedType(T, - E->getArg(0)->getType()->getPointeeType())); - assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType())); + assert(expr->getArg(0)->getType()->isPointerType()); + assert(cgf.getContext().hasSameUnqualifiedType(typ, + expr->getArg(0)->getType()->getPointeeType())); + assert(cgf.getContext().hasSameUnqualifiedType(typ, expr->getArg(1)->getType())); - Address destAddr = checkAtomicAlignment(CGF, E); - auto& builder = CGF.getBuilder(); + Address destAddr = checkAtomicAlignment(cgf, expr); + auto& builder = cgf.getBuilder(); auto* ctxt = builder.getContext(); - auto intType = builder.getSIntNTy(CGF.getContext().getTypeSize(T)); + auto intType = builder.getSIntNTy(cgf.getContext().getTypeSize(typ)); - mlir::Value val = CGF.buildScalarExpr(E->getArg(1)); + mlir::Value val = cgf.buildScalarExpr(expr->getArg(1)); mlir::Type valueType = val.getType(); - val = buildToInt(CGF, val, T, intType); + val = buildToInt(cgf, val, typ, intType); - auto loc = CGF.getLoc(E->getSourceRange()); - auto result = builder.createAtomicRMW(loc, Kind, destAddr.emitRawPointer(), - val, Ordering, destAddr.getAlignment().getAsAlign().value(), false); - - return buildFromInt(CGF, result, T, valueType); + auto loc = cgf.getLoc(expr->getSourceRange()); + auto op = mlir::cir::AtomicFetch::getOperationName(); + SmallVector atomicOperands = {destAddr.emitRawPointer(), val}; + SmallVector atomicResTys = {val.getType()}; + auto fetchAttr = mlir::cir::AtomicFetchKindAttr::get(builder.getContext(), kind); + auto rmwi = builder.create(loc, builder.getStringAttr(op), atomicOperands, + atomicResTys, {}); + auto orderAttr = mlir::cir::MemOrderAttr::get(builder.getContext(), ordering); + rmwi->setAttr("binop", fetchAttr); + rmwi->setAttr("mem_order", orderAttr); + rmwi->setAttr("fetch_first", mlir::UnitAttr::get(builder.getContext())); + auto result = rmwi->getResult(0); + + return buildFromInt(cgf, result, typ, valueType); } static RValue buildBinaryAtomic(CIRGenFunction &CGF, diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index ab7799e8a4d8..881f61a2ac8f 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -2449,41 +2449,6 @@ class CIRAtomicXchgLowering } }; - -static mlir::LLVM::AtomicBinOp getLLVMAtomicBinOp(mlir::cir::AtomicFetchKind k, - bool isInt, - bool isSignedInt) { - switch (k) { - case mlir::cir::AtomicFetchKind::Add: - return isInt ? mlir::LLVM::AtomicBinOp::add - : mlir::LLVM::AtomicBinOp::fadd; - case mlir::cir::AtomicFetchKind::Sub: - return isInt ? mlir::LLVM::AtomicBinOp::sub - : mlir::LLVM::AtomicBinOp::fsub; - case mlir::cir::AtomicFetchKind::And: - return mlir::LLVM::AtomicBinOp::_and; - case mlir::cir::AtomicFetchKind::Xor: - return mlir::LLVM::AtomicBinOp::_xor; - case mlir::cir::AtomicFetchKind::Or: - return mlir::LLVM::AtomicBinOp::_or; - case mlir::cir::AtomicFetchKind::Nand: - return mlir::LLVM::AtomicBinOp::nand; - case mlir::cir::AtomicFetchKind::Max: { - if (!isInt) - return mlir::LLVM::AtomicBinOp::fmax; - return isSignedInt ? mlir::LLVM::AtomicBinOp::max - : mlir::LLVM::AtomicBinOp::umax; - } - case mlir::cir::AtomicFetchKind::Min: { - if (!isInt) - return mlir::LLVM::AtomicBinOp::fmin; - return isSignedInt ? mlir::LLVM::AtomicBinOp::min - : mlir::LLVM::AtomicBinOp::umin; - } - } - llvm_unreachable("Unknown atomic fetch opcode"); -} - class CIRAtomicFetchLowering : public mlir::OpConversionPattern { public: @@ -2546,6 +2511,40 @@ class CIRAtomicFetchLowering llvm_unreachable("Unknown atomic fetch opcode"); } + mlir::LLVM::AtomicBinOp getLLVMAtomicBinOp(mlir::cir::AtomicFetchKind k, + bool isInt, + bool isSignedInt) const { + switch (k) { + case mlir::cir::AtomicFetchKind::Add: + return isInt ? mlir::LLVM::AtomicBinOp::add + : mlir::LLVM::AtomicBinOp::fadd; + case mlir::cir::AtomicFetchKind::Sub: + return isInt ? mlir::LLVM::AtomicBinOp::sub + : mlir::LLVM::AtomicBinOp::fsub; + case mlir::cir::AtomicFetchKind::And: + return mlir::LLVM::AtomicBinOp::_and; + case mlir::cir::AtomicFetchKind::Xor: + return mlir::LLVM::AtomicBinOp::_xor; + case mlir::cir::AtomicFetchKind::Or: + return mlir::LLVM::AtomicBinOp::_or; + case mlir::cir::AtomicFetchKind::Nand: + return mlir::LLVM::AtomicBinOp::nand; + case mlir::cir::AtomicFetchKind::Max: { + if (!isInt) + return mlir::LLVM::AtomicBinOp::fmax; + return isSignedInt ? mlir::LLVM::AtomicBinOp::max + : mlir::LLVM::AtomicBinOp::umax; + } + case mlir::cir::AtomicFetchKind::Min: { + if (!isInt) + return mlir::LLVM::AtomicBinOp::fmin; + return isSignedInt ? mlir::LLVM::AtomicBinOp::min + : mlir::LLVM::AtomicBinOp::umin; + } + } + llvm_unreachable("Unknown atomic fetch opcode"); + } + mlir::LogicalResult matchAndRewrite(mlir::cir::AtomicFetch op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const override { @@ -3162,66 +3161,6 @@ class CIRCmpThreeWayOpLowering } }; - -static mlir::LLVM::AtomicOrdering getLLVMAtomicOrder(mlir::cir::AtomicOrdering ord) { - switch (ord) { - case mlir::cir::AtomicOrdering::NotAtomic: - return mlir::LLVM::AtomicOrdering::not_atomic; - case mlir::cir::AtomicOrdering::Unordered: - return mlir::LLVM::AtomicOrdering::unordered; - case mlir::cir::AtomicOrdering::Monotonic: - return mlir::LLVM::AtomicOrdering::monotonic; - case mlir::cir::AtomicOrdering::Acquire: - return mlir::LLVM::AtomicOrdering::acquire; - case mlir::cir::AtomicOrdering::Release: - return mlir::LLVM::AtomicOrdering::release; - case mlir::cir::AtomicOrdering::AcquireRelease: - return mlir::LLVM::AtomicOrdering::acq_rel; - case mlir::cir::AtomicOrdering::SequentiallyConsistent: - return mlir::LLVM::AtomicOrdering::seq_cst; - } - llvm_unreachable("shouldn't get here"); -} - -class CIRAtomicRMWOpLowering - : public mlir::OpConversionPattern { - - using mlir::OpConversionPattern::OpConversionPattern; - - mlir::LogicalResult - matchAndRewrite(mlir::cir::AtomicRMWOp op, OpAdaptor adaptor, - mlir::ConversionPatternRewriter &rewriter) const override { - - auto val = op.getVal(); - bool isInt = false; - bool isSigned = false; - - if (auto typ = dyn_cast(val.getType())) { - isInt = true; - isSigned = typ.isSigned(); - } - - auto newOp = rewriter.replaceOpWithNewOp( - op, - getLLVMAtomicBinOp(op.getBinOp(), isInt, isSigned), - adaptor.getPtr(), - adaptor.getVal(), - getLLVMAtomicOrder(op.getOrdering()) - ); - - newOp.dump(); - - if (auto al = op.getAlignmentAttr()) - newOp.setAlignmentAttr(al); - - if (auto v = op.getIsVolatile()) - newOp.setVolatile_(true); - - return mlir::success(); - } -}; - - void populateCIRToLLVMConversionPatterns(mlir::RewritePatternSet &patterns, mlir::TypeConverter &converter) { patterns.add(patterns.getContext()); @@ -3246,7 +3185,7 @@ void populateCIRToLLVMConversionPatterns(mlir::RewritePatternSet &patterns, CIRStackRestoreLowering, CIRUnreachableLowering, CIRTrapLowering, CIRInlineAsmOpLowering, CIRSetBitfieldLowering, CIRGetBitfieldLowering, CIRPrefetchLowering, CIRObjSizeOpLowering, CIRIsConstantOpLowering, - CIRCmpThreeWayOpLowering, CIRAtomicRMWOpLowering>(converter, patterns.getContext()); + CIRCmpThreeWayOpLowering>(converter, patterns.getContext()); } namespace {