Index: Transforms/Instrumentation/MemorySanitizer.cpp =================================================================== --- Transforms/Instrumentation/MemorySanitizer.cpp +++ Transforms/Instrumentation/MemorySanitizer.cpp @@ -125,7 +125,6 @@ // These constants must be kept in sync with the ones in msan.h. static const unsigned kParamTLSSize = 800; -static const unsigned kRetvalTLSSize = 800; // Accesses sizes are powers of two: 1, 2, 4, 8. static const size_t kNumberOfAccessSizes = 4; @@ -324,6 +323,17 @@ bool doInitialization(Module &M) override; static char ID; // Pass identification, replacement for typeid. + Value* getTLSStruct(IRBuilder<> &IRB); + Value* getTLSStructEntry(IRBuilder<> &IRB, unsigned long offset); + + Value* getParamTLS(IRBuilder<> &IRB); + Value* getRetvalTLS(IRBuilder<> &IRB); + Value* getParamOriginTLS(IRBuilder<> &IRB); + Value* getRetvalOriginTLS(IRBuilder<> &IRB); + Value* getVAArgTLS(IRBuilder<> &IRB); + Value* getVAArgOverflowSizeTLS(IRBuilder<> &IRB); + Value* getOriginTLS(IRBuilder<> &IRB); + private: void initializeCallbacks(Module &M); @@ -333,23 +343,10 @@ LLVMContext *C; Type *IntptrTy; Type *OriginTy; - /// \brief Thread-local shadow storage for function parameters. - GlobalVariable *ParamTLS; - /// \brief Thread-local origin storage for function parameters. - GlobalVariable *ParamOriginTLS; - /// \brief Thread-local shadow storage for function return value. - GlobalVariable *RetvalTLS; - /// \brief Thread-local origin storage for function return value. - GlobalVariable *RetvalOriginTLS; - /// \brief Thread-local shadow storage for in-register va_arg function - /// parameters (x86_64-specific). - GlobalVariable *VAArgTLS; - /// \brief Thread-local shadow storage for va_arg overflow area - /// (x86_64-specific). - GlobalVariable *VAArgOverflowSizeTLS; - /// \brief Thread-local space used to pass origin value to the UMR reporting - /// function. - GlobalVariable *OriginTLS; + + bool isAndroid; + // \brief Defined as 'static THREADLOCAL MsanTLSSlot' for non-Android. + Value *MsanTLSSlot; /// \brief The run-time callback to print a warning. Value *WarningFn; @@ -457,35 +454,15 @@ "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy, nullptr); - // Create globals. - RetvalTLS = new GlobalVariable( - M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false, - GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr, - GlobalVariable::InitialExecTLSModel); - RetvalOriginTLS = new GlobalVariable( - M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr, - "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel); - - ParamTLS = new GlobalVariable( - M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false, - GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr, - GlobalVariable::InitialExecTLSModel); - ParamOriginTLS = new GlobalVariable( - M, ArrayType::get(OriginTy, kParamTLSSize / 4), false, - GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls", - nullptr, GlobalVariable::InitialExecTLSModel); - - VAArgTLS = new GlobalVariable( - M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false, - GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr, - GlobalVariable::InitialExecTLSModel); - VAArgOverflowSizeTLS = new GlobalVariable( - M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr, - "__msan_va_arg_overflow_size_tls", nullptr, - GlobalVariable::InitialExecTLSModel); - OriginTLS = new GlobalVariable( - M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr, - "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel); + if (!isAndroid) { + // Statically asserted in MsanThread::GetCurrentThread() + const int MsanTLSSlotSize = 3248; + + MsanTLSSlot = new GlobalVariable( + M, ArrayType::get(IntptrTy, MsanTLSSlotSize / 8), false, + GlobalVariable::ExternalLinkage, nullptr, "__tls0", nullptr, + GlobalVariable::InitialExecTLSModel); + } // We insert an empty inline asm after __msan_report* to avoid callback merge. EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false), @@ -493,6 +470,67 @@ /*hasSideEffects=*/true); } + +// Emits code equivalent to GetMsanTls() +// i.e., __get_tls()[TLS_SLOT_TSAN] on Android +// &__tls otherwise +Value* MemorySanitizer::getTLSStruct(IRBuilder<> &IRB) { + if (isAndroid) { + // Based on AArch64TargetLowering::getIRStackGuard + // TLS_SLOT_TSAN = 8 + const unsigned TlsOffset = 0x40; + Module *M = IRB.GetInsertBlock()->getParent()->getParent(); + Function *ThreadPointerFunc = + Intrinsic::getDeclaration(M, Intrinsic::thread_pointer); + return IRB.CreateLoad( + IRB.CreatePointerCast( + IRB.CreateConstGEP1_32(IRB.CreateCall(ThreadPointerFunc), TlsOffset), + Type::getInt8PtrTy(IRB.getContext())->getPointerTo(0))); + } else { + return MsanTLSSlot; + } +} + +// Emits code equivalent to ((char*)GetMsanTls() + offset) +// i.e., (char*)(__get_tls()[TLS_SLOT_TSAN])+offset on Android +// (char*)(&__tls)+offset otherwise +Value* MemorySanitizer::getTLSStructEntry(IRBuilder<> &IRB, unsigned long offset) { + Value *Base = IRB.CreatePointerCast(getTLSStruct(IRB), IntptrTy); + Base = IRB.CreateAdd(Base, ConstantInt::get(IntptrTy, offset)); + return IRB.CreateIntToPtr(Base, PointerType::get(IntptrTy, 0)); +} + +// These offsets are statically asserted in msan_linux.cc +// (MsanThread *GetCurrentThread()) +Value* MemorySanitizer::getParamTLS(IRBuilder<> &IRB) { + return getTLSStructEntry (IRB, 0); +} + +Value* MemorySanitizer::getRetvalTLS(IRBuilder<> &IRB) { + return getTLSStructEntry (IRB, 1600); +} + +Value* MemorySanitizer::getParamOriginTLS(IRBuilder<> &IRB) { + return getTLSStructEntry (IRB, 800); +} + +Value* MemorySanitizer::getRetvalOriginTLS(IRBuilder<> &IRB) { + return getTLSStructEntry (IRB, 2424); +} + +Value* MemorySanitizer::getVAArgTLS(IRBuilder<> &IRB) { + return getTLSStructEntry (IRB, 2448); +} + +Value* MemorySanitizer::getVAArgOverflowSizeTLS(IRBuilder<> &IRB) { + return getTLSStructEntry (IRB, 2432); +} + +Value* MemorySanitizer::getOriginTLS(IRBuilder<> &IRB) { + return getTLSStructEntry (IRB, 2428); +} + + /// \brief Module-level initialization. /// /// inserts a call to __msan_init to the module's constructor list. @@ -540,6 +578,7 @@ default: report_fatal_error("unsupported operating system"); } + isAndroid = TargetTriple.isAndroid(); C = &(M.getContext()); IRBuilder<> IRB(*C); @@ -788,7 +827,7 @@ if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) { if (MS.TrackOrigins) { IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0), - MS.OriginTLS); + MS.getOriginTLS(IRB)); } IRB.CreateCall(MS.WarningFn, {}); IRB.CreateCall(MS.EmptyAsm, {}); @@ -820,7 +859,7 @@ IRB.SetInsertPoint(CheckTerm); if (MS.TrackOrigins) { IRB.CreateStore(Origin ? (Value *)Origin : (Value *)IRB.getInt32(0), - MS.OriginTLS); + MS.getOriginTLS(IRB)); } IRB.CreateCall(MS.WarningFn, {}); IRB.CreateCall(MS.EmptyAsm, {}); @@ -992,7 +1031,7 @@ /// Shadow = ParamTLS+ArgOffset. Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB, int ArgOffset) { - Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy); + Value *Base = IRB.CreatePointerCast(MS.getParamTLS(IRB), MS.IntptrTy); Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0), "_msarg"); @@ -1002,7 +1041,7 @@ Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB, int ArgOffset) { if (!MS.TrackOrigins) return nullptr; - Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy); + Value *Base = IRB.CreatePointerCast(MS.getParamOriginTLS(IRB), MS.IntptrTy); Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0), "_msarg_o"); @@ -1010,7 +1049,7 @@ /// \brief Compute the shadow address for a retval. Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) { - Value *Base = IRB.CreatePointerCast(MS.RetvalTLS, MS.IntptrTy); + Value *Base = IRB.CreatePointerCast(MS.getRetvalTLS(IRB), MS.IntptrTy); return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0), "_msret"); } @@ -1018,7 +1057,7 @@ /// \brief Compute the origin address for a retval. Value *getOriginPtrForRetval(IRBuilder<> &IRB) { // We keep a single origin for the entire retval. Might be too optimistic. - return MS.RetvalOriginTLS; + return MS.getRetvalOriginTLS(IRB); } /// \brief Set SV to be the shadow value for V. @@ -2948,13 +2987,13 @@ } Constant *OverflowSize = ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset); - IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS); + IRB.CreateStore(OverflowSize, MS.getVAArgOverflowSizeTLS(IRB)); } /// \brief Compute the shadow address for a given va_arg. Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) { - Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); + Value *Base = IRB.CreatePointerCast(MS.getVAArgTLS(IRB), MS.IntptrTy); Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), "_msarg"); @@ -2994,12 +3033,12 @@ // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); - VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); + VAArgOverflowSize = IRB.CreateLoad(MS.getVAArgOverflowSizeTLS(IRB)); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize); VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); - IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8); + IRB.CreateMemCpy(VAArgTLSCopy, MS.getVAArgTLS(IRB), CopySize, 8); } // Instrument va_start. @@ -3075,13 +3114,13 @@ Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset); // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of // a new class member i.e. it is the total size of all VarArgs. - IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS); + IRB.CreateStore(TotalVAArgSize, MS.getVAArgOverflowSizeTLS(IRB)); } /// \brief Compute the shadow address for a given va_arg. Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) { - Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); + Value *Base = IRB.CreatePointerCast(MS.getVAArgTLS(IRB), MS.IntptrTy); Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), "_msarg"); @@ -3110,7 +3149,7 @@ assert(!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"); IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); - VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); + VAArgSize = IRB.CreateLoad(MS.getVAArgOverflowSizeTLS(IRB)); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize); @@ -3118,7 +3157,7 @@ // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); - IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8); + IRB.CreateMemCpy(VAArgTLSCopy, MS.getVAArgTLS(IRB), CopySize, 8); } // Instrument va_start. @@ -3230,13 +3269,13 @@ } Constant *OverflowSize = ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset); - IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS); + IRB.CreateStore(OverflowSize, MS.getVAArgOverflowSizeTLS(IRB)); } /// Compute the shadow address for a given va_arg. Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) { - Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); + Value *Base = IRB.CreatePointerCast(MS.getVAArgTLS(IRB), MS.IntptrTy); Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), "_msarg"); @@ -3291,12 +3330,12 @@ // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); - VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); + VAArgOverflowSize = IRB.CreateLoad(MS.getVAArgOverflowSizeTLS(IRB)); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize); VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); - IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8); + IRB.CreateMemCpy(VAArgTLSCopy, MS.getVAArgTLS(IRB), CopySize, 8); } Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize); @@ -3478,13 +3517,13 @@ VAArgOffset - VAArgBase); // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of // a new class member i.e. it is the total size of all VarArgs. - IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS); + IRB.CreateStore(TotalVAArgSize, MS.getVAArgOverflowSizeTLS(IRB)); } /// \brief Compute the shadow address for a given va_arg. Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) { - Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); + Value *Base = IRB.CreatePointerCast(MS.getVAArgTLS(IRB), MS.IntptrTy); Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), "_msarg"); @@ -3513,7 +3552,7 @@ assert(!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"); IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); - VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS); + VAArgSize = IRB.CreateLoad(MS.getVAArgOverflowSizeTLS(IRB)); Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize); @@ -3521,7 +3560,7 @@ // If there is a va_start in this function, make a backup copy of // va_arg_tls somewhere in the function entry block. VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); - IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8); + IRB.CreateMemCpy(VAArgTLSCopy, MS.getVAArgTLS(IRB), CopySize, 8); } // Instrument va_start.