diff --git a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp index 4f6567d1fd15..e4e5c718c16b 100644 --- a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp @@ -369,6 +369,82 @@ static bool isCstWeak(mlir::Value weakVal, bool &val) { return false; } +// Functions that help with the creation of compiler-generated switch +// statements that are used to implement non-constant memory order parameters. + +// Create a new region. Create a block within the region. Add a "break" +// statement to the block. Set the builder's insertion point to before the +// "break" statement. Add the new region to the given container. +template +static void startRegion(mlir::OpBuilder &builder, RegionsCont &Regions, + mlir::Location loc) { + + Regions.push_back(std::make_unique()); + mlir::Region *Region = Regions.back().get(); + mlir::Block *Block = builder.createBlock(Region); + builder.setInsertionPointToEnd(Block); + auto Break = builder.create(loc); + builder.setInsertionPoint(Break); +} + +// Create a "default:" label and add it to the given collection of case labels. +// Create the region that will hold the body of the "default:" block. +template +static void buildDefaultCase(mlir::OpBuilder &builder, CaseAttrsCont &CaseAttrs, + RegionsCont &Regions, mlir::Location loc) { + + auto Context = builder.getContext(); + auto EmptyArrayAttr = builder.getArrayAttr({}); + auto DefaultKind = + mlir::cir::CaseOpKindAttr::get(Context, mlir::cir::CaseOpKind::Default); + auto DefaultAttr = + mlir::cir::CaseAttr::get(Context, EmptyArrayAttr, DefaultKind); + CaseAttrs.push_back(DefaultAttr); + startRegion(builder, Regions, loc); +} + +// Create a single "case" label with the given MemOrder as its value. Add the +// "case" label to the given collection of case labels. Create the region that +// will hold the body of the "case" block. +template +static void +buildSingleMemOrderCase(mlir::OpBuilder &builder, CaseAttrsCont &CaseAttrs, + RegionsCont &Regions, mlir::Location loc, + mlir::Type Type, mlir::cir::MemOrder Order) { + + auto Context = builder.getContext(); + SmallVector OneOrder{ + mlir::cir::IntAttr::get(Type, static_cast(Order))}; + auto OneAttribute = builder.getArrayAttr(OneOrder); + auto CaseKind = + mlir::cir::CaseOpKindAttr::get(Context, mlir::cir::CaseOpKind::Equal); + auto CaseAttr = mlir::cir::CaseAttr::get(Context, OneAttribute, CaseKind); + CaseAttrs.push_back(CaseAttr); + startRegion(builder, Regions, loc); +} + +// Create a pair of "case" labels with the given MemOrders as their values. +// Add the combined "case" attribute to the given collection of case labels. +// Create the region that will hold the body of the "case" block. +template +static void buildDoubleMemOrderCase(mlir::OpBuilder &builder, + CaseAttrsCont &CaseAttrs, + RegionsCont &Regions, mlir::Location loc, + mlir::Type Type, mlir::cir::MemOrder Order1, + mlir::cir::MemOrder Order2) { + + auto Context = builder.getContext(); + SmallVector TwoOrders{ + mlir::cir::IntAttr::get(Type, static_cast(Order1)), + mlir::cir::IntAttr::get(Type, static_cast(Order2))}; + auto TwoAttributes = builder.getArrayAttr(TwoOrders); + auto CaseKind = + mlir::cir::CaseOpKindAttr::get(Context, mlir::cir::CaseOpKind::Anyof); + auto CaseAttr = mlir::cir::CaseAttr::get(Context, TwoAttributes, CaseKind); + CaseAttrs.push_back(CaseAttr); + startRegion(builder, Regions, loc); +} + static void buildAtomicCmpXchg(CIRGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, uint64_t Size, @@ -446,7 +522,49 @@ static void buildAtomicCmpXchgFailureSet( return; } - llvm_unreachable("NYI"); + // The failure memory order is not a compile-time value. The CIR atomic ops + // can't handle a runtime value; all memory orders must be hard coded. + // Generate a "switch" statement that converts the runtime value into a + // compile-time value. + CGF.getBuilder().create( + FailureOrderVal.getLoc(), FailureOrderVal, + [&](mlir::OpBuilder &builder, mlir::Location loc, + mlir::OperationState &os) { + SmallVector CaseAttrs; + SmallVector, 3> Regions; + + // default: + // Unsupported memory orders get generated as memory_order_relaxed, + // because there is no practical way to report an error at runtime. + buildDefaultCase(builder, CaseAttrs, Regions, loc); + buildAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, + SuccessOrder, mlir::cir::MemOrder::Relaxed, Scope); + + // case consume: + // case acquire: + // memory_order_consume is not implemented and always falls back to + // memory_order_acquire + buildDoubleMemOrderCase( + builder, CaseAttrs, Regions, loc, FailureOrderVal.getType(), + mlir::cir::MemOrder::Consume, mlir::cir::MemOrder::Acquire); + buildAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, + SuccessOrder, mlir::cir::MemOrder::Acquire, Scope); + + // A failed compare-exchange is a read-only operation. So + // memory_order_release and memory_order_acq_rel are not supported for + // the failure memory order. They fall back to memory_order_relaxed. + + // case seq_cst: + buildSingleMemOrderCase(builder, CaseAttrs, Regions, loc, + FailureOrderVal.getType(), + mlir::cir::MemOrder::SequentiallyConsistent); + buildAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, + SuccessOrder, + mlir::cir::MemOrder::SequentiallyConsistent, Scope); + + os.addRegions(Regions); + os.addAttribute("cases", builder.getArrayAttr(CaseAttrs)); + }); } static void buildAtomicOp(CIRGenFunction &CGF, AtomicExpr *E, Address Dest, @@ -1149,8 +1267,74 @@ RValue CIRGenFunction::buildAtomicExpr(AtomicExpr *E) { RValTy, E->getExprLoc()); } - // Long case, when Order isn't obviously constant. - llvm_unreachable("NYI"); + // The memory order is not known at compile-time. The atomic operations + // can't handle runtime memory orders; the memory order must be hard coded. + // Generate a "switch" statement that converts a runtime value into a + // compile-time value. + builder.create( + Order.getLoc(), Order, + [&](mlir::OpBuilder &builder, mlir::Location loc, + mlir::OperationState &os) { + llvm::SmallVector CaseAttrs; + llvm::SmallVector, 6> Regions; + + // default: + // Use memory_order_relaxed for relaxed operations and for any memory + // order value that is not supported. There is no good way to report + // an unsupported memory order at runtime, hence the fallback to + // memory_order_relaxed. + buildDefaultCase(builder, CaseAttrs, Regions, loc); + buildAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, + mlir::cir::MemOrder::Relaxed, Scope); + + if (!IsStore) { + // case consume: + // case acquire: + // memory_order_consume is not implemented; it is always treated like + // memory_order_acquire. These memory orders are not valid for + // write-only operations. + buildDoubleMemOrderCase(builder, CaseAttrs, Regions, loc, + Order.getType(), mlir::cir::MemOrder::Consume, + mlir::cir::MemOrder::Acquire); + buildAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, mlir::cir::MemOrder::Acquire, Scope); + } + + if (!IsLoad) { + // case release: + // memory_order_release is not valid for read-only operations. + buildSingleMemOrderCase(builder, CaseAttrs, Regions, loc, + Order.getType(), + mlir::cir::MemOrder::Release); + buildAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, mlir::cir::MemOrder::Release, Scope); + } + + if (!IsLoad && !IsStore) { + // case acq_rel: + // memory_order_acq_rel is only valid for read-write operations. + buildSingleMemOrderCase(builder, CaseAttrs, Regions, loc, + Order.getType(), + mlir::cir::MemOrder::AcquireRelease); + buildAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, + Size, mlir::cir::MemOrder::AcquireRelease, Scope); + } + + // case seq_cst: + buildSingleMemOrderCase(builder, CaseAttrs, Regions, loc, + Order.getType(), + mlir::cir::MemOrder::SequentiallyConsistent); + buildAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, + mlir::cir::MemOrder::SequentiallyConsistent, Scope); + + os.addRegions(Regions); + os.addAttribute("cases", builder.getArrayAttr(CaseAttrs)); + }); + + if (RValTy->isVoidType()) + return RValue::get(nullptr); + return convertTempToRValue(Dest.withElementType(convertTypeForMem(RValTy)), + RValTy, E->getExprLoc()); } void CIRGenFunction::buildAtomicStore(RValue rvalue, LValue lvalue, diff --git a/clang/test/CIR/CodeGen/atomic-runtime.cpp b/clang/test/CIR/CodeGen/atomic-runtime.cpp new file mode 100644 index 000000000000..dfe74a9e77c9 --- /dev/null +++ b/clang/test/CIR/CodeGen/atomic-runtime.cpp @@ -0,0 +1,309 @@ +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir +// RUN: FileCheck --input-file=%t.cir %s + +// Test __atomic_* built-ins that have a memory order parameter with a runtime +// value. This requires generating a switch statement, so the amount of +// generated code is surprisingly large. +// +// Only a representative sample of atomic operations are tested: one read-only +// operation (atomic_load), one write-only operation (atomic_store), one +// read-write operation (atomic_exchange), and the most complex operation +// (atomic_compare_exchange). + +int runtime_load(int *ptr, int order) { + return __atomic_load_n(ptr, order); +} + +// CHECK: %[[ptr:.*]] = cir.load %[[ptr_var:.*]] : !cir.ptr>, !cir.ptr +// CHECK: %[[order:.*]] = cir.load %[[order_var:.*]] : !cir.ptr, !s32i +// CHECK: cir.switch (%[[order]] : !s32i) [ +// CHECK: case (default) { +// CHECK: %[[T8:.*]] = cir.load atomic(relaxed) %[[ptr]] : !cir.ptr, !s32i +// CHECK: cir.store %[[T8]], %[[temp_var:.*]] : !s32i, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (anyof, [1, 2] : !s32i) { +// CHECK: %[[T8:.*]] = cir.load atomic(acquire) %[[ptr]] : !cir.ptr, !s32i +// CHECK: cir.store %[[T8]], %[[temp_var]] : !s32i, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 5) { +// CHECK: %[[T8:.*]] = cir.load atomic(seq_cst) %[[ptr]] : !cir.ptr, !s32i +// CHECK: cir.store %[[T8]], %[[temp_var]] : !s32i, !cir.ptr +// CHECK: cir.break +// CHECK: } +// CHECK: ] + +void atomic_store_n(int* ptr, int val, int order) { + __atomic_store_n(ptr, val, order); +} + +// CHECK: %[[ptr:.*]] = cir.load %[[ptr_var:.*]] : !cir.ptr>, !cir.ptr +// CHECK: %[[order:.*]] = cir.load %[[order_var:.*]] : !cir.ptr, !s32i +// CHECK: %[[val:.*]] = cir.load %[[val_var:.*]] : !cir.ptr, !s32i +// CHECK: cir.store %[[val]], %[[temp_var:.*]] : !s32i, !cir.ptr +// CHECK: cir.switch (%[[order]] : !s32i) [ +// CHECK: case (default) { +// CHECK: %[[T7:.*]] = cir.load %[[temp_var:.*]] : !cir.ptr, !s32i +// CHECK: cir.store atomic(relaxed) %[[T7]], %[[ptr]] : !s32i, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 3) { +// CHECK: %[[T7:.*]] = cir.load %[[temp_var:.*]] : !cir.ptr, !s32i +// CHECK: cir.store atomic(release) %[[T7]], %[[ptr]] : !s32i, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 5) { +// CHECK: %[[T7:.*]] = cir.load %[[temp_var:.*]] : !cir.ptr, !s32i +// CHECK: cir.store atomic(seq_cst) %[[T7]], %[[ptr]] : !s32i, !cir.ptr +// CHECK: cir.break +// CHECK: } +// CHECK: ] + +int atomic_exchange_n(int* ptr, int val, int order) { + return __atomic_exchange_n(ptr, val, order); +} + +// CHECK: %[[ptr:.*]] = cir.load %[[ptr_var:.*]] : !cir.ptr>, !cir.ptr +// CHECK: %[[order:.*]] = cir.load %[[order_var:.*]] : !cir.ptr, !s32i +// CHECK: %[[val:.*]] = cir.load %[[val_var:.*]] : !cir.ptr, !s32i +// CHECK: cir.store %[[val]], %[[temp_var:.*]] : !s32i, !cir.ptr +// CHECK: cir.switch (%[[order]] : !s32i) [ +// CHECK: case (default) { +// CHECK: %[[T11:.*]] = cir.load %[[temp_var]] : !cir.ptr, !s32i +// CHECK: %[[T12:.*]] = cir.atomic.xchg(%[[ptr]] : !cir.ptr, %[[T11]] : !s32i, relaxed) : !s32i +// CHECK: cir.store %[[T12]], %[[result:.*]] : !s32i, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (anyof, [1, 2] : !s32i) { +// CHECK: %[[T11:.*]] = cir.load %[[temp_var]] : !cir.ptr, !s32i +// CHECK: %[[T12:.*]] = cir.atomic.xchg(%[[ptr]] : !cir.ptr, %[[T11]] : !s32i, acquire) : !s32i +// CHECK: cir.store %[[T12]], %[[result]] : !s32i, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 3) { +// CHECK: %[[T11:.*]] = cir.load %[[temp_var]] : !cir.ptr, !s32i +// CHECK: %[[T12:.*]] = cir.atomic.xchg(%[[ptr]] : !cir.ptr, %[[T11]] : !s32i, release) : !s32i +// CHECK: cir.store %[[T12]], %[[result]] : !s32i, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 4) { +// CHECK: %[[T11:.*]] = cir.load %[[temp_var]] : !cir.ptr, !s32i +// CHECK: %[[T12:.*]] = cir.atomic.xchg(%[[ptr]] : !cir.ptr, %[[T11]] : !s32i, acq_rel) : !s32i +// CHECK: cir.store %[[T12]], %[[result]] : !s32i, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 5) { +// CHECK: %[[T11:.*]] = cir.load %[[temp_var]] : !cir.ptr, !s32i +// CHECK: %[[T12:.*]] = cir.atomic.xchg(%[[ptr]] : !cir.ptr, %[[T11]] : !s32i, seq_cst) : !s32i +// CHECK: cir.store %[[T12]], %[[result]] : !s32i, !cir.ptr +// CHECK: cir.break +// CHECK: } +// CHECK: ] + +bool atomic_compare_exchange_n(int* ptr, int* expected, + int desired, int success, int failure) { + return __atomic_compare_exchange_n(ptr, expected, desired, false, + success, failure); +} + +// CHECK: %[[ptr:.*]] = cir.load %[[T0:.*]] : !cir.ptr>, !cir.ptr +// CHECK: %[[success:.*]] = cir.load %[[T3:.*]] : !cir.ptr, !s32i +// CHECK: %[[expected_addr:.*]] = cir.load %[[T1:.*]] : !cir.ptr>, !cir.ptr +// CHECK: %[[T11:.*]] = cir.load %[[T2:.*]] : !cir.ptr, !s32i +// CHECK: cir.store %[[T11]], %[[desired_var:.*]] : !s32i, !cir.ptr +// CHECK: %[[failure:.*]] = cir.load %[[T4:.*]] : !cir.ptr, !s32i +// CHECK: %[[T13:.*]] = cir.const #false +// CHECK: cir.switch (%[[success]] : !s32i) [ +// CHECK: case (default) { +// CHECK: cir.switch (%[[failure]] : !s32i) [ +// CHECK: case (default) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = relaxed, failure = relaxed) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var:.*]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (anyof, [1, 2] : !s32i) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = relaxed, failure = acquire) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 5) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = relaxed, failure = seq_cst) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: } +// CHECK: ] +// CHECK: cir.break +// CHECK: }, +// CHECK: case (anyof, [1, 2] : !s32i) { +// CHECK: cir.switch (%[[failure]] : !s32i) [ +// CHECK: case (default) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = acquire, failure = relaxed) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (anyof, [1, 2] : !s32i) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = acquire, failure = acquire) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 5) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = acquire, failure = seq_cst) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: } +// CHECK: ] +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 3) { +// CHECK: cir.switch (%[[failure]] : !s32i) [ +// CHECK: case (default) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = release, failure = relaxed) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (anyof, [1, 2] : !s32i) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = release, failure = acquire) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 5) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = release, failure = seq_cst) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: } +// CHECK: ] +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 4) { +// CHECK: cir.switch (%[[failure]] : !s32i) [ +// CHECK: case (default) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = acq_rel, failure = relaxed) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (anyof, [1, 2] : !s32i) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = acq_rel, failure = acquire) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 5) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = acq_rel, failure = seq_cst) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: } +// CHECK: ] +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 5) { +// CHECK: cir.switch (%[[failure]] : !s32i) [ +// CHECK: case (default) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = seq_cst, failure = relaxed) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (anyof, [1, 2] : !s32i) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = seq_cst, failure = acquire) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: }, +// CHECK: case (equal, 5) { +// CHECK: %[[expected:.*]] = cir.load %[[expected_addr]] : !cir.ptr, !s32i +// CHECK: %[[desired:.*]] = cir.load %[[desired_var]] : !cir.ptr, !s32i +// CHECK: %old, %cmp = cir.atomic.cmp_xchg(%[[ptr]] : !cir.ptr, %[[expected]] : !s32i, %[[desired]] : !s32i, success = seq_cst, failure = seq_cst) : (!s32i, !cir.bool) +// CHECK: %[[succeeded:.*]] = cir.unary(not, %cmp) : !cir.bool, !cir.bool +// CHECK: cir.if %[[succeeded]] { +// CHECK: cir.store %old, %[[expected_addr]] : !s32i, !cir.ptr +// CHECK: } +// CHECK: cir.store %cmp, %[[result_var]] : !cir.bool, !cir.ptr +// CHECK: cir.break +// CHECK: } +// CHECK: ] +// CHECK: cir.break +// CHECK: } +// CHECK: ] + diff --git a/clang/test/CIR/Lowering/atomic-runtime.cpp b/clang/test/CIR/Lowering/atomic-runtime.cpp new file mode 100644 index 000000000000..411a08dc5af2 --- /dev/null +++ b/clang/test/CIR/Lowering/atomic-runtime.cpp @@ -0,0 +1,37 @@ +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t.ll +// RUN: FileCheck --input-file=%t.ll %s + +// Test __atomic_* built-ins that have a memory order parameter with a runtime +// value. This requires generating a switch statement, so the amount of +// generated code is surprisingly large. +// +// This is just a quick smoke test. Only atomic_load_n is tested. + +int runtime_load(int *ptr, int order) { + return __atomic_load_n(ptr, order); +} + +// CHECK: %[[T7:[0-9]+]] = load ptr, ptr %[[T3:[0-9]+]], align 8 +// CHECK: %[[T8:[0-9]+]] = load i32, ptr %[[T4:[0-9]+]], align 4 +// CHECK: switch i32 %[[T8]], label %[[L9:[0-9]+]] [ +// CHECK: i32 1, label %[[L11:[0-9]+]] +// CHECK: i32 2, label %[[L11]] +// CHECK: i32 5, label %[[L13:[0-9]+]] +// CHECK: ] +// CHECK: [[L9]]: +// CHECK: %[[T10:[0-9]+]] = load atomic i32, ptr %[[T7]] monotonic, align 4 +// CHECK: store i32 %[[T10]], ptr %[[T6:[0-9]+]], align 4 +// CHECK: br label %[[L15:[0-9]+]] +// CHECK: [[L11]]: +// CHECK: %[[T12:[0-9]+]] = load atomic i32, ptr %[[T7]] acquire, align 4 +// CHECK: store i32 %[[T12]], ptr %[[T6]], align 4 +// CHECK: br label %[[L15]] +// CHECK: [[L13]]: +// CHECK: %[[T14:[0-9]+]] = load atomic i32, ptr %[[T7]] seq_cst, align 4 +// CHECK: store i32 %[[T14]], ptr %[[T6]], align 4 +// CHECK: br label %[[L15]] +// CHECK: [[L15]]: +// CHECK: %[[T16:[0-9]+]] = load i32, ptr %[[T6]], align 4 +// CHECK: store i32 %[[T16]], ptr %[[T5:[0-9]+]], align 4 +// CHECK: %[[T17:[0-9]+]] = load i32, ptr %[[T5]], align 4 +// CHECK: ret i32 %[[T17]]