diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -392,6 +392,14 @@ 0x1C0000000000, // OriginBase }; +// s390x Linux +static const MemoryMapParams Linux_S390X_MemoryMapParams = { + 0xC00000000000, // AndMask + 0, // XorMask (not used) + 0x080000000000, // ShadowBase + 0x1C0000000000, // OriginBase +}; + // aarch64 Linux static const MemoryMapParams Linux_AArch64_MemoryMapParams = { 0, // AndMask (not used) @@ -439,6 +447,11 @@ &Linux_PowerPC64_MemoryMapParams, }; +static const PlatformMemoryMapParams Linux_S390_MemoryMapParams = { + nullptr, + &Linux_S390X_MemoryMapParams, +}; + static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = { nullptr, &Linux_AArch64_MemoryMapParams, @@ -484,6 +497,7 @@ friend struct VarArgMIPS64Helper; friend struct VarArgAArch64Helper; friend struct VarArgPowerPC64Helper; + friend struct VarArgSystemZHelper; void initializeModule(Module &M); void initializeCallbacks(Module &M); @@ -796,14 +810,25 @@ AccessSizeIndex++) { unsigned AccessSize = 1 << AccessSizeIndex; std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize); + SmallVector, 2> MaybeWarningFnAttrs; + MaybeWarningFnAttrs.push_back(std::make_pair( + AttributeList::FirstArgIndex, Attribute::get(*C, Attribute::ZExt))); + MaybeWarningFnAttrs.push_back(std::make_pair( + AttributeList::FirstArgIndex + 1, Attribute::get(*C, Attribute::ZExt))); MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction( - FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), - IRB.getInt32Ty()); + FunctionName, AttributeList::get(*C, MaybeWarningFnAttrs), + IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty()); FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize); + SmallVector, 2> MaybeStoreOriginFnAttrs; + MaybeStoreOriginFnAttrs.push_back(std::make_pair( + AttributeList::FirstArgIndex, Attribute::get(*C, Attribute::ZExt))); + MaybeStoreOriginFnAttrs.push_back(std::make_pair( + AttributeList::FirstArgIndex + 2, Attribute::get(*C, Attribute::ZExt))); MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction( - FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), - IRB.getInt8PtrTy(), IRB.getInt32Ty()); + FunctionName, AttributeList::get(*C, MaybeStoreOriginFnAttrs), + IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt8PtrTy(), + IRB.getInt32Ty()); } MsanSetAllocaOrigin4Fn = M.getOrInsertFunction( @@ -924,6 +949,9 @@ case Triple::ppc64le: MapParams = Linux_PowerPC_MemoryMapParams.bits64; break; + case Triple::systemz: + MapParams = Linux_S390_MemoryMapParams.bits64; + break; case Triple::aarch64: case Triple::aarch64_be: MapParams = Linux_ARM_MemoryMapParams.bits64; @@ -4600,6 +4628,251 @@ } }; +/// SystemZ-specific implementation of VarArgHelper. +struct VarArgSystemZHelper : public VarArgHelper { + static const unsigned SystemZGpOffset = 16; + static const unsigned SystemZGpEndOffset = 56; + static const unsigned SystemZFpOffset = 128; + static const unsigned SystemZFpEndOffset = 160; + static const unsigned SysteZRegSaveAreaSize = 160; + static const unsigned SystemZOverflowOffset = 160; + static const unsigned SystemZVAListTagSize = 32; + static const unsigned SystemZOverflowArgAreaPtrOffset = 16; + static const unsigned SystemZRegSaveAreaPtrOffset = 24; + + Function &F; + MemorySanitizer &MS; + MemorySanitizerVisitor &MSV; + Value *VAArgTLSCopy = nullptr; + Value *VAArgTLSOriginCopy = nullptr; + Value *VAArgOverflowSize = nullptr; + + SmallVector VAStartInstrumentationList; + + enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory }; + + VarArgSystemZHelper(Function &F, MemorySanitizer &MS, + MemorySanitizerVisitor &MSV) + : F(F), MS(MS), MSV(MSV) {} + + ArgKind classifyArgument(Value *arg) { + Type *T = arg->getType(); + // T is a SystemZABIInfo output, and there are only a few possibilities + // of what it can be. In particular, single element structs and large + // types have already been taken care of. + if (T->isFloatingPointTy()) + return AK_FloatingPoint; + if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64) + return AK_GeneralPurpose; + if (T->isPointerTy()) + return AK_GeneralPurpose; + return AK_Memory; + } + + void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override { + unsigned GpOffset = SystemZGpOffset; + unsigned FpOffset = SystemZFpOffset; + unsigned OverflowOffset = SystemZOverflowOffset; + const DataLayout &DL = F.getParent()->getDataLayout(); + for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end(); + ArgIt != End; ++ArgIt) { + Value *A = *ArgIt; + unsigned ArgNo = CS.getArgumentNo(ArgIt); + bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams(); + // SystemZABIInfo does not produce ByVal parameters. + assert(!CS.paramHasAttr(ArgNo, Attribute::ByVal)); + ArgKind AK = classifyArgument(A); + if (AK == AK_GeneralPurpose && GpOffset >= SystemZGpEndOffset) + AK = AK_Memory; + if (AK == AK_FloatingPoint && FpOffset >= SystemZFpEndOffset) + AK = AK_Memory; + Value *ShadowBase = nullptr, *OriginBase = nullptr; + switch (AK) { + case AK_GeneralPurpose: { + // Always keep track of GpOffset, but store shadow only for varargs. + uint64_t ArgSize = 8; + if (GpOffset + ArgSize <= kParamTLSSize) { + if (!IsFixed) { + uint64_t ArgAllocSize = DL.getTypeAllocSize(A->getType()); + ShadowBase = getShadowPtrForVAArgument( + A->getType(), IRB, GpOffset + (ArgSize - ArgAllocSize)); + if (MS.TrackOrigins) + OriginBase = getOriginPtrForVAArgument( + A->getType(), IRB, GpOffset + (ArgSize - ArgAllocSize)); + } + GpOffset += ArgSize; + } else { + GpOffset = kParamTLSSize; + } + break; + } + case AK_FloatingPoint: { + // Always keep track of FpOffset, but store shadow only for varargs. + uint64_t ArgSize = 8; + if (FpOffset + ArgSize <= kParamTLSSize) { + if (!IsFixed) { + uint64_t ArgAllocSize = DL.getTypeAllocSize(A->getType()); + ShadowBase = getShadowPtrForVAArgument( + A->getType(), IRB, FpOffset + (ArgSize - ArgAllocSize)); + if (MS.TrackOrigins) + OriginBase = getOriginPtrForVAArgument( + A->getType(), IRB, FpOffset + (ArgSize - ArgAllocSize)); + } + FpOffset += ArgSize; + } else { + FpOffset = kParamTLSSize; + } + break; + } + case AK_Memory: { + // Keep track of OverflowOffset and store shadow only for varargs. + uint64_t ArgAllocSize = DL.getTypeAllocSize(A->getType()); + uint64_t ArgSize = alignTo(ArgAllocSize, 8); + if (!IsFixed) { + if (OverflowOffset + ArgSize <= kParamTLSSize) { + ShadowBase = getShadowPtrForVAArgument( + A->getType(), IRB, OverflowOffset + (ArgSize - ArgAllocSize)); + if (MS.TrackOrigins) + OriginBase = getOriginPtrForVAArgument( + A->getType(), IRB, OverflowOffset + (ArgSize - ArgAllocSize)); + OverflowOffset += ArgSize; + } else { + OverflowOffset = kParamTLSSize; + } + } + break; + } + } + if (!ShadowBase) + continue; + Value *Shadow = MSV.getShadow(A); + IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment); + if (MS.TrackOrigins) { + Value *Origin = MSV.getOrigin(A); + unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType()); + MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize, + std::max(kShadowTLSAlignment, kMinOriginAlignment)); + } + } + Constant *OverflowSize = ConstantInt::get( + IRB.getInt64Ty(), OverflowOffset - SystemZOverflowOffset); + IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS); + } + + Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, + unsigned ArgOffset) { + Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); + Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); + return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), + "_msarg_va_s"); + } + + Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) { + Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy); + Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); + return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0), + "_msarg_va_o"); + } + + void unpoisonVAListTagForInst(IntrinsicInst &I) { + IRBuilder<> IRB(&I); + Value *VAListTag = I.getArgOperand(0); + Value *ShadowPtr, *OriginPtr; + const Align Alignment = Align(8); + std::tie(ShadowPtr, OriginPtr) = + MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment, + /*isStore*/ true); + IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), + SystemZVAListTagSize, Alignment, false); + } + + void visitVAStartInst(VAStartInst &I) override { + VAStartInstrumentationList.push_back(&I); + unpoisonVAListTagForInst(I); + } + + void visitVACopyInst(VACopyInst &I) override { unpoisonVAListTagForInst(I); } + + void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) { + Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C); + Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr( + IRB.CreateAdd( + IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), + ConstantInt::get(MS.IntptrTy, SystemZRegSaveAreaPtrOffset)), + PointerType::get(RegSaveAreaPtrTy, 0)); + Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); + Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr; + const Align Alignment = Align(8); + std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) = + MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment, + /*isStore*/ true); + // TODO(iii): copy only fragments filled by visitCallSite() + IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment, + SysteZRegSaveAreaSize); + if (MS.TrackOrigins) + IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy, + Alignment, SysteZRegSaveAreaSize); + } + + void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) { + Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C); + Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr( + IRB.CreateAdd( + IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), + ConstantInt::get(MS.IntptrTy, SystemZOverflowArgAreaPtrOffset)), + PointerType::get(OverflowArgAreaPtrTy, 0)); + Value *OverflowArgAreaPtr = + IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr); + Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr; + const Align Alignment = Align(8); + std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) = + MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(), + Alignment, /*isStore*/ true); + Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy, + SystemZOverflowOffset); + IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment, + VAArgOverflowSize); + if (MS.TrackOrigins) { + SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy, + SystemZOverflowOffset); + IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment, + VAArgOverflowSize); + } + } + + void finalizeInstrumentation() override { + assert(!VAArgOverflowSize && !VAArgTLSCopy && + "finalizeInstrumentation called twice"); + if (!VAStartInstrumentationList.empty()) { + // If there is a va_start in this function, make a backup copy of + // va_arg_tls somewhere in the function entry block. + IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); + VAArgOverflowSize = + IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); + Value *CopySize = + IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset), + VAArgOverflowSize); + VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); + IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize); + if (MS.TrackOrigins) { + VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); + IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS, + Align(8), CopySize); + } + } + + // Instrument va_start. + // Copy va_list shadow from the backup copy of the TLS contents. + for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) { + CallInst *OrigInst = VAStartInstrumentationList[i]; + IRBuilder<> IRB(OrigInst->getNextNode()); + Value *VAListTag = OrigInst->getArgOperand(0); + copyRegSaveArea(IRB, VAListTag); + copyOverflowArea(IRB, VAListTag); + } + } +}; + /// A no-op implementation of VarArgHelper. struct VarArgNoOpHelper : public VarArgHelper { VarArgNoOpHelper(Function &F, MemorySanitizer &MS, @@ -4630,6 +4903,8 @@ else if (TargetTriple.getArch() == Triple::ppc64 || TargetTriple.getArch() == Triple::ppc64le) return new VarArgPowerPC64Helper(Func, Msan, Visitor); + else if (TargetTriple.getArch() == Triple::systemz) + return new VarArgSystemZHelper(Func, Msan, Visitor); else return new VarArgNoOpHelper(Func, Msan, Visitor); } diff --git a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll @@ -0,0 +1,98 @@ +; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s +; RUN: opt < %s -msan -S | FileCheck %s + +target triple = "s390x-unknown-linux-gnu" + +%struct.__va_list = type { i64, i64, i8*, i8* } + +define i64 @foo(i64 %guard, ...) { + %vl = alloca %struct.__va_list, align 8 + %1 = bitcast %struct.__va_list* %vl to i8* + call void @llvm.lifetime.start.p0i8(i64 32, i8* %1) + call void @llvm.va_start(i8* %1) + call void @llvm.va_end(i8* %1) + call void @llvm.lifetime.end.p0i8(i64 32, i8* %1) + ret i64 0 +} + +; First check if the variadic shadow values are saved in stack with correct +; size (which is 160 - size of the register save area). + +; CHECK-LABEL: @foo +; CHECK: [[A:%.*]] = load {{.*}} @__msan_va_arg_overflow_size_tls +; CHECK: [[B:%.*]] = add i64 160, [[A]] +; CHECK: alloca {{.*}} [[B]] + +; We expect two memcpy operations: one for the register save area, and one for +; the overflow arg area. + +; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 {{%.*}}, i8* align 8 {{%.*}}, i64 160, i1 false) +; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 {{%.*}}, i8* align 8 {{%.*}}, i64 [[A]], i1 false) + +declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1 +declare void @llvm.va_start(i8*) #2 +declare void @llvm.va_end(i8*) #2 +declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1 + +define i64 @bar() { + %1 = call i64 (i64, ...) @foo(i64 0, i64 1, i64 2, double 3.000000e+00, + double 4.000000e+00, i64 5, i64 6, + double 7.000000e+00, i64 8, i64 9, i64 10, i64 11) + ret i64 %1 +} + +; Save the incoming shadow values from the varargs in the __msan_va_arg_tls +; array at the offsets equal to those defined by the ABI for the corresponding +; registers in the register save area, and for the corresponding overflow args +; in the overflow arg area: +; - r2@16 is skipped, because it's fixed +; - r3@24 == 1 +; - r4@32 == 2 +; - f0@128 == 3.0 +; - f1@136 == 4.0 +; - r5@40 == 5 +; - r6@48 == 6 +; - f2@144 == 7.0 +; - overflow@[160::8] == [8, 9, 10, 11] +; Overflow arg area size is therefore 32. + +; CHECK-LABEL: @bar +; CHECK: store {{.*}} @__msan_va_arg_tls {{.*}} 24 +; CHECK: store {{.*}} @__msan_va_arg_tls {{.*}} 32 +; CHECK: store {{.*}} @__msan_va_arg_tls {{.*}} 128 +; CHECK: store {{.*}} @__msan_va_arg_tls {{.*}} 136 +; CHECK: store {{.*}} @__msan_va_arg_tls {{.*}} 40 +; CHECK: store {{.*}} @__msan_va_arg_tls {{.*}} 48 +; CHECK: store {{.*}} @__msan_va_arg_tls {{.*}} 144 +; CHECK: store {{.*}} @__msan_va_arg_tls {{.*}} 160 +; CHECK: store {{.*}} @__msan_va_arg_tls {{.*}} 168 +; CHECK: store {{.*}} @__msan_va_arg_tls {{.*}} 176 +; CHECK: store {{.*}} @__msan_va_arg_tls {{.*}} 184 +; CHECK: store {{.*}} 32, {{.*}} @__msan_va_arg_overflow_size_tls + +; Test that MSan doesn't generate code overflowing __msan_va_arg_tls when too many arguments are +; passed to a variadic function. + +define dso_local i64 @many_args() { +entry: + %ret = call i64 (i64, ...) @sum(i64 120, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, + i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1 + ) + ret i64 %ret +} + +; If the size of __msan_va_arg_tls changes the second argument of `add` must also be changed. +; CHECK: i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 792) +; CHECK-NOT: i64 add (i64 ptrtoint ([100 x i64]* @__msan_va_arg_tls to i64), i64 800) +declare i64 @sum(i64 %n, ...)