diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td --- a/clang/include/clang/Basic/riscv_vector.td +++ b/clang/include/clang/Basic/riscv_vector.td @@ -812,6 +812,47 @@ } defvar NFList = [2, 3, 4, 5, 6, 7, 8]; +/* +Segment load builtin has different variants. + +Segment unit-stride load builtin, +(Address0, ..., Address{NF - 1}, mask, Ptr, VL) +(Address0, ..., Address{NF - 1}, mask, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, VL) +Segment unit-stride fault-only-first load builtin, +(Address0, ..., Address{NF - 1}, mask, Ptr, NewVL, VL) +(Address0, ..., Address{NF - 1}, mask, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, NewVL, VL) +Segment strided load builtin, +(Address0, ..., Address{NF - 1}, mask, Ptr, Stride, VL) +(Address0, ..., Address{NF - 1}, mask, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, Stride, VL) +Segment indexed load builtin, +(Address0, ..., Address{NF - 1}, mask, Ptr, Idx, VL) +(Address0, ..., Address{NF - 1}, mask, Maskedoff0, ..., Maskedoff{NF - 1}, + Ptr, Idx, VL) + +The Maskedoff parameter(s) exist when at least one of the policy behaviors +(vta or vma) implies undisturbed. + +Segment load intrinsics has different variants similar to their builtins. + +Segment unit-stride load intrinsic, + Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Mask, VL, Policy) + Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, VL) +Segment unit-stride fault-only-first load intrinsic, + Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Mask, VL, Policy) + Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, VL) +Segment strided load intrinsic, + Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Stride, Mask, VL, Policy) + Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, Stride, VL) +Segment indexed load intrinsic, + Masked: (Vector0, ..., Vector{NF - 1}, Ptr, Index, Mask, VL, Policy) + Unmasked: (Vector0, ..., Vector{NF - 1}, Ptr, Index, VL) + +The Vector(s) is poison when the policy behavior allows us to not care +about any masked-off elements. +*/ class PVString { string S = @@ -842,36 +883,30 @@ { ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); IntrinsicTypes = {ResultType, Ops.back()->getType()}; - SmallVector Operands; - if (IsMasked) { - // TAMA builtin: (val0 address, ..., mask, ptr, vl) - // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, vl) - // intrinsic: (maskedoff0, ..., ptr, mask, vl) - if (PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) { - Operands.append(NF, llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 2]); - } else { + SmallVector Operands; + + // Please refer to comment under 'defvar NFList' in this file + if ((IsMasked && PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) || + (!IsMasked && PolicyAttrs == TAIL_AGNOSTIC)) + Operands.append(NF, llvm::PoisonValue::get(ResultType)); + else { + if (IsMasked) Operands.append(Ops.begin() + NF + 1, Ops.begin() + 2 * NF + 1); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[2 * NF + 2]); - } - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - assert(Operands.size() == NF + 4); - } else { - // TA builtin: (val0 address, val1 address, ..., ptr, vl) - // TU builtin: (val0 address, ..., passthru0, ..., ptr, vl) - // intrinsic: (passthru0, passthru1, ..., ptr, vl) - if (PolicyAttrs == TAIL_AGNOSTIC) { - Operands.append(NF, llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 1]); - } else { - Operands.append(Ops.begin() + NF, Ops.begin() + 2 * NF + 2); - } + else // Unmasked + Operands.append(Ops.begin() + NF, Ops.begin() + 2 * NF); } + unsigned PtrOperandIdx = IsMasked ? + (PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) ? NF + 1 : 2 * NF + 1 : + (PolicyAttrs == TAIL_AGNOSTIC) ? NF : 2 * NF; + Value *PtrOperand = Ops[PtrOperandIdx]; + Value *VLOperand = Ops[PtrOperandIdx + 1]; + Operands.push_back(PtrOperand); + if (IsMasked) + Operands.push_back(Ops[NF]); + Operands.push_back(VLOperand); + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); clang::CharUnits Align = @@ -914,43 +949,30 @@ ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); IntrinsicTypes = {ResultType, Ops.back()->getType()}; SmallVector Operands; - Value *NewVL; - if (IsMasked) { - // TAMA builtin: (val0 address, ..., mask, ptr, new_vl, vl) - // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, new_vl, vl) - // intrinsic: (maskedoff0, ..., ptr, mask, vl) - if (PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) { - Operands.append(NF, llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 3]); - NewVL = Ops[NF + 2]; - } else { + // Please refer to comment under 'defvar NFList' in this file + if ((IsMasked && PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) || + (!IsMasked && PolicyAttrs == TAIL_AGNOSTIC)) + Operands.append(NF, llvm::PoisonValue::get(ResultType)); + else { + if (IsMasked) Operands.append(Ops.begin() + NF + 1, Ops.begin() + 2 * NF + 1); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[2 * NF + 3]); - NewVL = Ops[2 * NF + 2]; - } - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - assert(Operands.size() == NF + 4); - } else { - // TA builtin: (val0 address, val1 address, ..., ptr, new_vl, vl) - // TU builtin: (val0 address, ..., passthru0, ..., ptr, new_vl, vl) - // intrinsic: (passthru0, passthru1, ..., ptr, vl) - if (PolicyAttrs == TAIL_AGNOSTIC) { - Operands.append(NF, llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 2]); - NewVL = Ops[NF + 1]; - } else { + else // Unmasked Operands.append(Ops.begin() + NF, Ops.begin() + 2 * NF); - Operands.push_back(Ops[2 * NF]); - Operands.push_back(Ops[2 * NF + 2]); - NewVL = Ops[2 * NF + 1]; - } } + unsigned PtrOperandIdx = IsMasked ? + (PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) ? NF + 1 : 2 * NF + 1 : + (PolicyAttrs == TAIL_AGNOSTIC) ? NF : 2 * NF; + Value *PtrOperand = Ops[PtrOperandIdx]; + Value *NewVLOperand = Ops[PtrOperandIdx + 1]; + Value *VLOperand = Ops[PtrOperandIdx + 2]; + Operands.push_back(PtrOperand); + if (IsMasked) + Operands.push_back(Ops[NF]); + Operands.push_back(VLOperand); + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); clang::CharUnits Align = @@ -961,7 +983,7 @@ } // Store new_vl. llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {NF}); - return Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); + return Builder.CreateStore(Val, Address(NewVLOperand, Val->getType(), Align)); } }] in { defvar PV = PVString.S; @@ -995,41 +1017,30 @@ IntrinsicTypes = {ResultType, Ops.back()->getType()}; SmallVector Operands; - if (IsMasked) { - // TAMA builtin: (val0 address, ..., mask, ptr, stride, vl) - // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, stride, vl) - // intrinsic: (maskedoff0, ..., ptr, stride, mask, vl) - if (PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) { - Operands.append(NF, llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF + 2]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 3]); - } else { + // Please refer to comment under 'defvar NFList' in this file + if ((IsMasked && PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) || + (!IsMasked && PolicyAttrs == TAIL_AGNOSTIC)) + Operands.append(NF, llvm::PoisonValue::get(ResultType)); + else { + if (IsMasked) Operands.append(Ops.begin() + NF + 1, Ops.begin() + 2 * NF + 1); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[2 * NF + 2]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[2 * NF + 3]); - } - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - assert(Operands.size() == NF + 5); - } else { - // TA builtin: (val0 address, val1 address, ..., ptr, stride, vl) - // TU builtin: (val0 address, ..., passthru0, ..., ptr, stride, vl) - // intrinsic: (passthru0, passthru1, ..., ptr, stride, vl) - if (PolicyAttrs == TAIL_AGNOSTIC) { - Operands.append(NF, llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF + 2]); - } else { + else // Unmasked Operands.append(Ops.begin() + NF, Ops.begin() + 2 * NF); - Operands.push_back(Ops[2 * NF]); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[2 * NF + 2]); - } } + unsigned PtrOperandIdx = IsMasked ? + (PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) ? NF + 1 : 2 * NF + 1 : + (PolicyAttrs == TAIL_AGNOSTIC) ? NF : 2 * NF; + Value *PtrOperand = Ops[PtrOperandIdx]; + Value *StrideOperand = Ops[PtrOperandIdx + 1]; + Value *VLOperand = Ops[PtrOperandIdx + 2]; + Operands.push_back(PtrOperand); + Operands.push_back(StrideOperand); + if (IsMasked) + Operands.push_back(Ops[NF]); + Operands.push_back(VLOperand); + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); clang::CharUnits Align = @@ -1067,45 +1078,32 @@ { ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); SmallVector Operands; - if (IsMasked) { - // TAMA builtin: (val0 address, ..., mask, ptr, index, vl) - // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, index, vl) - // intrinsic: (maskedoff0, ..., ptr, index, mask, vl) - if (PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) { - Operands.append(NF, llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF + 2]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 3]); - IntrinsicTypes = {ResultType, Ops[NF + 2]->getType(), Ops.back()->getType()}; - } else { + + // Please refer to comment under 'defvar NFList' in this file + if ((IsMasked && PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) || + (!IsMasked && PolicyAttrs == TAIL_AGNOSTIC)) + Operands.append(NF, llvm::PoisonValue::get(ResultType)); + else { + if (IsMasked) Operands.append(Ops.begin() + NF + 1, Ops.begin() + 2 * NF + 1); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[2 * NF + 2]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[2 * NF + 3]); - IntrinsicTypes = {ResultType, Ops[2 * NF + 2]->getType(), Ops.back()->getType()}; - } - Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); - assert(Operands.size() == NF + 5); - } else { - // TA builtin: (val0 address, val1 address, ..., ptr, index, vl) - // TU builtin: (val0 address, ..., passthru0, ..., ptr, index, vl) - // intrinsic: (passthru0, passthru1, ..., ptr, index, vl) - if (PolicyAttrs == TAIL_AGNOSTIC) { - Operands.append(NF, llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF + 2]); - IntrinsicTypes = {ResultType, Ops[NF + 1]->getType(), Ops.back()->getType()}; - } else { + else // Unmasked Operands.append(Ops.begin() + NF, Ops.begin() + 2 * NF); - Operands.push_back(Ops[2 * NF]); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[2 * NF + 2]); - IntrinsicTypes = {ResultType, Ops[2 * NF + 1]->getType(), Ops.back()->getType()}; - } } + unsigned PtrOperandIdx = IsMasked ? + (PolicyAttrs == TAIL_AGNOSTIC_MASK_AGNOSTIC) ? NF + 1 : 2 * NF + 1 : + (PolicyAttrs == TAIL_AGNOSTIC) ? NF : 2 * NF; + Value *PtrOperand = Ops[PtrOperandIdx]; + Value *IndexOperand = Ops[PtrOperandIdx + 1]; + Value *VLOperand = Ops[PtrOperandIdx + 2]; + Operands.push_back(PtrOperand); + Operands.push_back(IndexOperand); + if (IsMasked) + Operands.push_back(Ops[NF]); + Operands.push_back(VLOperand); + if (IsMasked) + Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs)); + IntrinsicTypes = {ResultType, IndexOperand->getType(), Ops.back()->getType()}; + llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); clang::CharUnits Align =