diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp --- a/clang/lib/Driver/ToolChains/Linux.cpp +++ b/clang/lib/Driver/ToolChains/Linux.cpp @@ -769,7 +769,7 @@ if (IsX86_64 || IsMIPS64 || IsAArch64 || IsPowerPC64 || IsSystemZ || IsLoongArch64) Res |= SanitizerKind::Thread; - if (IsX86_64) + if (IsX86_64 || IsSystemZ) Res |= SanitizerKind::KernelMemory; if (IsX86 || IsX86_64) Res |= SanitizerKind::Function; diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -122,6 +122,10 @@ /// Arbitrary sized accesses are handled with: /// __msan_metadata_ptr_for_load_n(ptr, size) /// __msan_metadata_ptr_for_store_n(ptr, size); +/// Note that the sanitizer code has to deal with how shadow/origin pairs +/// returned by the these functions are represented in different ABIs. In +/// the X86_64 ABI they are returned in RDX:RAX, and in the SystemZ ABI they +/// are written to memory pointed to by a hidden parameter. /// - TLS variables are stored in a single per-task struct. A call to a /// function __msan_get_context_state() returning a pointer to that struct /// is inserted into every instrumented function before the entry block; @@ -135,7 +139,7 @@ /// Also, KMSAN currently ignores uninitialized memory passed into inline asm /// calls, making sure we're on the safe side wrt. possible false positives. /// -/// KernelMemorySanitizer only supports X86_64 at the moment. +/// KernelMemorySanitizer only supports X86_64 and SystemZ at the moment. /// // // FIXME: This sanitizer does not yet handle scalable vectors @@ -543,6 +547,10 @@ void createKernelApi(Module &M, const TargetLibraryInfo &TLI); void createUserspaceApi(Module &M, const TargetLibraryInfo &TLI); + template + FunctionCallee getOrInsertMsanMetadataFunction(Module &M, StringRef Name, + ArgsTy... Args); + /// True if we're compiling the Linux kernel. bool CompileKernel; /// Track origins (allocation points) of uninitialized values. @@ -550,6 +558,7 @@ bool Recover; bool EagerChecks; + Triple TargetTriple; LLVMContext *C; Type *IntptrTy; Type *OriginTy; @@ -620,13 +629,18 @@ /// Functions for poisoning/unpoisoning local variables FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn; - /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin - /// pointers. + /// Pair of shadow/origin pointers. + Type *MsanMetadata; + + /// Each of the MsanMetadataPtrXxx functions returns a MsanMetadata. FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN; FunctionCallee MsanMetadataPtrForLoad_1_8[4]; FunctionCallee MsanMetadataPtrForStore_1_8[4]; FunctionCallee MsanInstrumentAsmStoreFn; + /// Storage for return values of the MsanMetadataPtrXxx functions. + Value *MsanMetadataAlloca; + /// Helper to choose between different MsanMetadataPtrXxx(). FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size); @@ -729,6 +743,21 @@ GlobalValue::PrivateLinkage, StrConst, ""); } +template +FunctionCallee +MemorySanitizer::getOrInsertMsanMetadataFunction(Module &M, StringRef Name, + ArgsTy... Args) { + if (TargetTriple.getArch() == Triple::systemz) { + // SystemZ ABI: shadow/origin pair is returned via a hidden parameter. + return M.getOrInsertFunction(Name, Type::getVoidTy(*C), + PointerType::get(MsanMetadata, 0), + std::forward(Args)...); + } + + return M.getOrInsertFunction(Name, MsanMetadata, + std::forward(Args)...); +} + /// Create KMSAN API callbacks. void MemorySanitizer::createKernelApi(Module &M, const TargetLibraryInfo &TLI) { IRBuilder<> IRB(*C); @@ -758,25 +787,25 @@ MsanGetContextStateFn = M.getOrInsertFunction( "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0)); - Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0), - PointerType::get(IRB.getInt32Ty(), 0)); + MsanMetadata = StructType::get(PointerType::get(IRB.getInt8Ty(), 0), + PointerType::get(IRB.getInt32Ty(), 0)); for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) { std::string name_load = "__msan_metadata_ptr_for_load_" + std::to_string(size); std::string name_store = "__msan_metadata_ptr_for_store_" + std::to_string(size); - MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction( - name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0)); - MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction( - name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0)); + MsanMetadataPtrForLoad_1_8[ind] = getOrInsertMsanMetadataFunction( + M, name_load, PointerType::get(IRB.getInt8Ty(), 0)); + MsanMetadataPtrForStore_1_8[ind] = getOrInsertMsanMetadataFunction( + M, name_store, PointerType::get(IRB.getInt8Ty(), 0)); } - MsanMetadataPtrForLoadN = M.getOrInsertFunction( - "__msan_metadata_ptr_for_load_n", RetTy, - PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty()); - MsanMetadataPtrForStoreN = M.getOrInsertFunction( - "__msan_metadata_ptr_for_store_n", RetTy, + MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction( + M, "__msan_metadata_ptr_for_load_n", PointerType::get(IRB.getInt8Ty(), 0), + IRB.getInt64Ty()); + MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction( + M, "__msan_metadata_ptr_for_store_n", PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty()); // Functions for poisoning and unpoisoning memory. @@ -927,6 +956,8 @@ void MemorySanitizer::initializeModule(Module &M) { auto &DL = M.getDataLayout(); + TargetTriple = Triple(M.getTargetTriple()); + bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0; bool OriginPassed = ClOriginBase.getNumOccurrences() > 0; // Check the overrides first @@ -937,7 +968,6 @@ CustomMapParams.OriginBase = ClOriginBase; MapParams = &CustomMapParams; } else { - Triple TargetTriple(M.getTargetTriple()); switch (TargetTriple.getOS()) { case Triple::FreeBSD: switch (TargetTriple.getArch()) { @@ -1464,6 +1494,8 @@ MS.RetvalOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, {Zero, IRB.getInt32(6)}, "retval_origin"); + if (MS.TargetTriple.getArch() == Triple::systemz) + MS.MsanMetadataAlloca = IRB.CreateAlloca(MS.MsanMetadata, (unsigned)0); } /// Add MemorySanitizer instrumentation to a function. @@ -1696,6 +1728,18 @@ return std::make_pair(ShadowPtr, OriginPtr); } + template + Value *createMetadataCall(IRBuilder<> &IRB, FunctionCallee Callee, + ArgsTy... Args) { + if (MS.TargetTriple.getArch() == Triple::systemz) { + IRB.CreateCall(Callee, + {MS.MsanMetadataAlloca, std::forward(Args)...}); + return IRB.CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca); + } + + return IRB.CreateCall(Callee, {std::forward(Args)...}); + } + std::pair getShadowOriginPtrKernelNoVec(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy, @@ -1708,12 +1752,13 @@ Value *AddrCast = IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0)); if (Getter) { - ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast); + ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast); } else { Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size); - ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN - : MS.MsanMetadataPtrForLoadN, - {AddrCast, SizeVal}); + ShadowOriginPtrs = createMetadataCall( + IRB, + isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN, + AddrCast, SizeVal); } Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0); ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0)); diff --git a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/basic-kernel.ll b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/basic-kernel.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/basic-kernel.ll @@ -0,0 +1,169 @@ +; RUN: opt < %s -S -mcpu=z13 -msan-kernel=1 -float-abi=soft -passes=msan 2>&1 | FileCheck %s + +target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-a:8:16-n32:64" +target triple = "s390x-unknown-linux-gnu" + +define void @Store1(ptr %p, i8 %x) sanitize_memory { +entry: + store i8 %x, ptr %p + ret void +} + +; CHECK-LABEL: @Store1 +; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }, align 8 +; CHECK: call void @__msan_metadata_ptr_for_store_1(ptr [[META_PTR]], ptr %p) +; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]], align 8 +; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0 +; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1 +; CHECK: store i8 {{.+}}, ptr [[SHADOW]], align 1 +; CHECK: ret void + +define void @Store2(ptr %p, i16 %x) sanitize_memory { +entry: + store i16 %x, ptr %p + ret void +} + +; CHECK-LABEL: @Store2 +; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }, align 8 +; CHECK: call void @__msan_metadata_ptr_for_store_2(ptr [[META_PTR]], ptr %p) +; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]], align 8 +; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0 +; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1 +; CHECK: store i16 {{.+}}, ptr [[SHADOW]], align 2 +; CHECK: ret void + +define void @Store4(ptr %p, i32 %x) sanitize_memory { +entry: + store i32 %x, ptr %p + ret void +} + +; CHECK-LABEL: @Store4 +; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }, align 8 +; CHECK: call void @__msan_metadata_ptr_for_store_4(ptr [[META_PTR]], ptr %p) +; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]], align 8 +; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0 +; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1 +; CHECK: store i32 {{.+}}, ptr [[SHADOW]], align 4 +; CHECK: ret void + +define void @Store8(ptr %p, i64 %x) sanitize_memory { +entry: + store i64 %x, ptr %p + ret void +} + +; CHECK-LABEL: @Store8 +; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }, align 8 +; CHECK: call void @__msan_metadata_ptr_for_store_8(ptr [[META_PTR]], ptr %p) +; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]], align 8 +; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0 +; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1 +; CHECK: store i64 {{.+}}, ptr [[SHADOW]], align 8 +; CHECK: ret void + +define void @Store16(ptr %p, i128 %x) sanitize_memory { +entry: + store i128 %x, ptr %p + ret void +} + +; CHECK-LABEL: @Store16 +; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }, align 8 +; CHECK: call void @__msan_metadata_ptr_for_store_n(ptr [[META_PTR]], ptr %p, i64 16) +; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]], align 8 +; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0 +; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1 +; CHECK: store i128 {{.+}}, ptr [[SHADOW]], align 8 +; CHECK: ret void + +define i8 @Load1(ptr %p) sanitize_memory { +entry: + %0 = load i8, ptr %p + ret i8 %0 +} + +; CHECK-LABEL: @Load1 +; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }, align 8 +; CHECK: call void @__msan_metadata_ptr_for_load_1(ptr [[META_PTR]], ptr %p) +; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]], align 8 +; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0 +; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1 +; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i8, ptr [[SHADOW]], align 1 +; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]], align 4 +; CHECK: store i8 [[SHADOW_VAL]], ptr %retval_shadow, align 8 +; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin, align 4 +; CHECK: ret i8 {{.+}} + +define i16 @Load2(ptr %p) sanitize_memory { +entry: + %0 = load i16, ptr %p + ret i16 %0 +} + +; CHECK-LABEL: @Load2 +; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }, align 8 +; CHECK: call void @__msan_metadata_ptr_for_load_2(ptr [[META_PTR]], ptr %p) +; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]], align 8 +; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0 +; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1 +; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i16, ptr [[SHADOW]], align 2 +; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]], align 4 +; CHECK: store i16 [[SHADOW_VAL]], ptr %retval_shadow, align 8 +; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin, align 4 +; CHECK: ret i16 {{.+}} + +define i32 @Load4(ptr %p) sanitize_memory { +entry: + %0 = load i32, ptr %p + ret i32 %0 +} + +; CHECK-LABEL: @Load4 +; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }, align 8 +; CHECK: call void @__msan_metadata_ptr_for_load_4(ptr [[META_PTR]], ptr %p) +; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]], align 8 +; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0 +; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1 +; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i32, ptr [[SHADOW]], align 4 +; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]], align 4 +; CHECK: store i32 [[SHADOW_VAL]], ptr %retval_shadow, align 8 +; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin, align 4 +; CHECK: ret i32 {{.+}} + +define i64 @Load8(ptr %p) sanitize_memory { +entry: + %0 = load i64, ptr %p + ret i64 %0 +} + +; CHECK-LABEL: @Load8 +; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }, align 8 +; CHECK: call void @__msan_metadata_ptr_for_load_8(ptr [[META_PTR]], ptr %p) +; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]], align 8 +; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0 +; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1 +; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i64, ptr [[SHADOW]], align 8 +; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]], align 8 +; CHECK: store i64 [[SHADOW_VAL]], ptr %retval_shadow, align 8 +; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin, align 4 +; CHECK: ret i64 {{.+}} + +define i128 @Load16(ptr %p) sanitize_memory { +entry: + %0 = load i128, ptr %p + ret i128 %0 +} + +; CHECK-LABEL: @Load16 +; CHECK: [[META_PTR:%[a-z0-9_]+]] = alloca { ptr, ptr }, align 8 +; CHECK: call void @__msan_metadata_ptr_for_load_n(ptr [[META_PTR]], ptr %p, i64 16) +; CHECK: [[META:%[a-z0-9_]+]] = load { ptr, ptr }, ptr [[META_PTR]], align 8 +; CHECK: [[SHADOW:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 0 +; CHECK: [[ORIGIN:%[a-z0-9_]+]] = extractvalue { ptr, ptr } [[META]], 1 +; CHECK: [[SHADOW_VAL:%[a-z0-9_]+]] = load i128, ptr [[SHADOW]], align 8 +; CHECK: [[ORIGIN_VAL:%[a-z0-9_]+]] = load i32, ptr [[ORIGIN]], align 8 +; CHECK: store i128 [[SHADOW_VAL]], ptr %retval_shadow, align 8 +; CHECK: store i32 [[ORIGIN_VAL]], ptr %retval_origin, align 4 +; CHECK: ret i128 {{.+}}