diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3894,6 +3894,11 @@
   HelpText<"Add -rpath with architecture-specific resource directory to the linker flags">;
 def fno_rtlib_add_rpath: Flag<["-"], "fno-rtlib-add-rpath">, Flags<[NoArgumentUnused]>,
   HelpText<"Do not add -rpath with architecture-specific resource directory to the linker flags">;
+defm openmp_implicit_rpath: BoolFOption<"openmp-implicit-rpath",
+  LangOpts<"OpenMP">,
+  DefaultTrue,
+  PosFlag<SetTrue, [], "Set rpath on OpenMP executables">,
+  NegFlag<SetFalse>>;
 def r : Flag<["-"], "r">, Flags<[LinkerInput,NoArgumentUnused]>,
         Group<Link_Group>;
 def save_temps_EQ : Joined<["-", "--"], "save-temps=">, Flags<[CC1Option, NoXarchOption]>,
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.h b/clang/lib/Driver/ToolChains/CommonArgs.h
--- a/clang/lib/Driver/ToolChains/CommonArgs.h
+++ b/clang/lib/Driver/ToolChains/CommonArgs.h
@@ -106,6 +106,9 @@
                       const llvm::opt::ArgList &Args,
                       llvm::opt::ArgStringList &CmdArgs);
 
+void addOpenMPRuntimeSpecificRPath(const ToolChain &TC,
+                                   const llvm::opt::ArgList &Args,
+                                   llvm::opt::ArgStringList &CmdArgs);
 void addArchSpecificRPath(const ToolChain &TC, const llvm::opt::ArgList &Args,
                           llvm::opt::ArgStringList &CmdArgs);
 /// Returns true, if an OpenMP runtime has been added.
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -645,6 +645,22 @@
                          /*IsLTO=*/true);
 }
 
+void tools::addOpenMPRuntimeSpecificRPath(const ToolChain &TC,
+                                          const ArgList &Args,
+                                          ArgStringList &CmdArgs) {
+
+  if (Args.hasFlag(options::OPT_fopenmp_implicit_rpath,
+                   options::OPT_fno_openmp_implicit_rpath, true)) {
+    // Default to clang lib / lib64 folder, i.e. the same location as device
+    // runtime
+    SmallString<256> DefaultLibPath =
+        llvm::sys::path::parent_path(TC.getDriver().Dir);
+    llvm::sys::path::append(DefaultLibPath, Twine("lib") + CLANG_LIBDIR_SUFFIX);
+    CmdArgs.push_back("-rpath");
+    CmdArgs.push_back(Args.MakeArgString(DefaultLibPath));
+  }
+}
+
 void tools::addArchSpecificRPath(const ToolChain &TC, const ArgList &Args,
                                  ArgStringList &CmdArgs) {
   // Enable -frtlib-add-rpath by default for the case of VE.
@@ -702,6 +718,9 @@
 
   addArchSpecificRPath(TC, Args, CmdArgs);
 
+  if (RTKind == Driver::OMPRT_OMP)
+    addOpenMPRuntimeSpecificRPath(TC, Args, CmdArgs);
+
   return true;
 }
 
diff --git a/clang/test/OpenMP/Inputs/libomp.a b/clang/test/OpenMP/Inputs/libomp.a
new file mode 100644
--- /dev/null
+++ b/clang/test/OpenMP/Inputs/libomp.a
@@ -0,0 +1 @@
+!<arch>
diff --git a/clang/test/OpenMP/implicit_rpath.c b/clang/test/OpenMP/implicit_rpath.c
new file mode 100644
--- /dev/null
+++ b/clang/test/OpenMP/implicit_rpath.c
@@ -0,0 +1,29 @@
+// RUN: %clang -fopenmp %s -L%S/Inputs -o %t && llvm-readelf --dynamic-table %t | FileCheck %s --check-prefixes=CHECK-DEFAULT
+// RUN: %clang -fopenmp -fopenmp-implicit-rpath %s -L%S/Inputs -o %t && llvm-readelf --dynamic-table %t | FileCheck %s --check-prefixes=CHECK-EXPLICIT
+// RUN: %clang -fopenmp -fno-openmp-implicit-rpath %s -L%S/Inputs -o %t && llvm-readelf --dynamic-table %t | FileCheck %s --check-prefixes=CHECK-DISABLED
+
+// RUN: %clang -fopenmp -Wl,--disable-new-dtags %s -L%S/Inputs -o %t && llvm-readelf --dynamic-table %t | FileCheck %s --check-prefixes=CHECK-DEFAULT-RPATH
+// RUN: %clang -fopenmp -fopenmp-implicit-rpath -Wl,--disable-new-dtags %s -L%S/Inputs -o %t && llvm-readelf --dynamic-table %t | FileCheck %s --check-prefixes=CHECK-EXPLICIT-RPATH
+// RUN: %clang -fopenmp -fno-openmp-implicit-rpath -Wl,--disable-new-dtags %s -L%S/Inputs -o %t && llvm-readelf --dynamic-table %t | FileCheck %s --check-prefixes=CHECK-DISABLED-RPATH
+
+// RUN: %clang -fopenmp -Wl,--enable-new-dtags %s -L%S/Inputs -o %t && llvm-readelf --dynamic-table %t | FileCheck %s --check-prefixes=CHECK-DEFAULT-RUNPATH
+// RUN: %clang -fopenmp -fopenmp-implicit-rpath -Wl,--enable-new-dtags %s -L%S/Inputs -o %t && llvm-readelf --dynamic-table %t | FileCheck %s --check-prefixes=CHECK-EXPLICIT-RUNPATH
+// RUN: %clang -fopenmp -fno-openmp-implicit-rpath -Wl,--enable-new-dtags %s -L%S/Inputs -o %t && llvm-readelf --dynamic-table %t | FileCheck %s --check-prefixes=CHECK-DISABLED-RUNPATH
+
+// RUN: %clang -Wl,-rpath=early -fopenmp %s -L%S/Inputs -o %t -Wl,-rpath=late && llvm-readelf --dynamic-table %t | FileCheck %s --check-prefixes=CHECK-COMPOSABLE
+
+// CHECK-DEFAULT:      ({{R|RUN}}PATH) Library {{r|run}}path: [{{.*}}llvm{{.*}}lib{{.*}}]
+// CHECK-EXPLICIT:     ({{R|RUN}}PATH) Library {{r|run}}path: [{{.*}}llvm{{.*}}lib{{.*}}]
+// CHECK-DISABLED-NOT: ({{R|RUN}}PATH)
+
+// CHECK-DEFAULT-RPATH:      (RPATH) Library rpath: [{{.*}}llvm{{.*}}lib{{.*}}]
+// CHECK-EXPLICIT-RPATH:     (RPATH) Library rpath: [{{.*}}llvm{{.*}}lib{{.*}}]
+// CHECK-DISABLED-RPATH-NOT: (RPATH)
+
+// CHECK-DEFAULT-RUNPATH:      (RUNPATH) Library runpath: [{{.*}}llvm{{.*}}lib{{.*}}]
+// CHECK-EXPLICIT-RUNPATH:     (RUNPATH) Library runpath: [{{.*}}llvm{{.*}}lib{{.*}}]
+// CHECK-DISABLED-RUNPATH-NOT: (RUNPATH)
+
+// CHECK-COMPOSABLE: ({{R|RUN}}PATH) Library {{r|run}}path: [early:late:{{.*}}llvm{{.*}}lib{{.*}}]
+
+int main() {}
diff --git a/openmp/libomptarget/test/lit.cfg b/openmp/libomptarget/test/lit.cfg
--- a/openmp/libomptarget/test/lit.cfg
+++ b/openmp/libomptarget/test/lit.cfg
@@ -84,6 +84,8 @@
    config.available_features.add('unified_shared_memory')
 
 # Setup environment to find dynamic library at runtime
+# Disable the implicit path to increase certainty over which library is picked up
+config.test_flags += " -fno-openmp-implicit-rpath"
 if config.operating_system == 'Windows':
     append_dynamic_library_path('PATH', config.library_dir, ";")
     append_dynamic_library_path('PATH', config.omp_host_rtl_directory, ";")