diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -291,6 +291,7 @@
                      StringRef ABIName);
 
 ABI getTargetABI(StringRef ABIName);
+StringRef getABIName(ABI ABI);
 
 // Returns the register used to hold the stack pointer after realignment.
 MCRegister getBPReg();
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp
@@ -75,6 +75,27 @@
   return TargetABI;
 }
 
+StringRef getABIName(ABI ABI) {
+  switch (ABI) {
+  default:
+    llvm_unreachable("Unknown ABI!");
+  case ABI_ILP32:
+    return "ilp32";
+  case ABI_ILP32F:
+    return "ilp32f";
+  case ABI_ILP32D:
+    return "ilp32d";
+  case ABI_ILP32E:
+    return "ilp32e";
+  case ABI_LP64:
+    return "lp64";
+  case ABI_LP64F:
+    return "lp64f";
+  case ABI_LP64D:
+    return "lp64d";
+  }
+}
+
 // To avoid the BP value clobbered by a function call, we need to choose a
 // callee saved register to save the value. RV32E only has X8 and X9 as callee
 // saved registers and X8 will be used as fp. So we choose X9 as bp.
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -84,24 +84,33 @@
       TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;
   std::string FS =
       FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
-  std::string Key = CPU + TuneCPU + FS;
+  StringRef OptionABI = Options.MCOptions.getABIName();
+  std::string Key = CPU + TuneCPU + FS + OptionABI.str();
   auto &I = SubtargetMap[Key];
   if (!I) {
     // This needs to be done before we create a new subtarget since any
     // creation will depend on the TM and the code generation flags on the
     // function that reside in TargetOptions.
     resetTargetOptions(F);
-    auto ABIName = Options.MCOptions.getABIName();
-    if (const MDString *ModuleTargetABI = dyn_cast_or_null<MDString>(
+
+    I = std::make_unique<RISCVSubtarget>(TargetTriple, CPU, TuneCPU, FS,
+                                         OptionABI, *this);
+    // The ABI should equal target-abi module flag because the different ABI
+    // has its corresponding datalayout and IR alignment information. In
+    // addition, the IR parser need to have datalayout information before it
+    // gets the ABI info from module. It does not make sense to update the
+    // module's datalayout and alignment information after IR parsing.
+    if (const MDString *MDModuleABI = dyn_cast_or_null<MDString>(
             F.getParent()->getModuleFlag("target-abi"))) {
-      auto TargetABI = RISCVABI::getTargetABI(ABIName);
-      if (TargetABI != RISCVABI::ABI_Unknown &&
-          ModuleTargetABI->getString() != ABIName) {
-        report_fatal_error("-target-abi option != target-abi module flag");
-      }
-      ABIName = ModuleTargetABI->getString();
+      StringRef ModuleABIName = MDModuleABI->getString();
+      if (ModuleABIName.empty())
+        F.getContext().emitError("target-abi module flag is empty!");
+      StringRef OptionABIName = RISCVABI::getABIName(I->getTargetABI());
+      if (OptionABIName != ModuleABIName)
+        F.getContext().emitError(
+            "Mismatched ABIs. Current ABI is '" + OptionABIName +
+            "', but IR target-abi module flag is '" + ModuleABIName + "'.");
     }
-    I = std::make_unique<RISCVSubtarget>(TargetTriple, CPU, TuneCPU, FS, ABIName, *this);
   }
   return I.get();
 }
diff --git a/llvm/test/CodeGen/RISCV/module-target-abi-tests.ll b/llvm/test/CodeGen/RISCV/module-target-abi-tests.ll
new file mode 100644
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/module-target-abi-tests.ll
@@ -0,0 +1,35 @@
+; This test is designed to run with different target-abi module flags.
+;
+; Default RV32 ABI is ilp32 for empty target-abi option and module flag, so the result is correct.
+; RUN: cat %s > %t.emptyabi
+; RUN: echo '!0 = !{i32 1, !"target-abi", !""}' >> %t.emptyabi
+; RUN: llc -mtriple=riscv32 < %t.emptyabi -o /dev/null
+; RUN: llc -mtriple=riscv32 -target-abi ilp32 < %t.emptyabi -o /dev/null
+; RUN: not llc -mtriple=riscv32 -target-abi ilp32f < %t.emptyabi -o /dev/null 2>&1 \
+; RUN:   | FileCheck -check-prefix=DEFAULT-RV32-ILP32F %t.emptyabi
+;
+; Default RV32 ABI is ilp32 which equals target-abi module flag, so the result is correct.
+; RUN: cat %s > %t.ilp32abi
+; RUN: echo '!0 = !{i32 1, !"target-abi", !"ilp32"}' >> %t.ilp32abi
+; RUN: llc -mtriple=riscv32 < %t.ilp32abi -o /dev/null
+; RUN: not llc -mtriple=riscv32 -target-abi ilp32f < %t.ilp32abi -o /dev/null 2>&1 \
+; RUN:   | FileCheck -check-prefix=DEFAULT-RV32-ILP32F %t.ilp32abi
+
+; DEFAULT-RV32-ILP32F: error: Mismatched ABIs. Current ABI is 'ilp32f', but IR target-abi module flag is 'ilp32'.
+
+; Default RV32 ABI is ilp32 which mismatch with target-abi module flag ilp32f.
+; RUN: cat %s > %t.ilp32fabi
+; RUN: echo '!0 = !{i32 1, !"target-abi", !"ilp32f"}' >> %t.ilp32fabi
+; RUN: not llc -mtriple=riscv32 < %t.ilp32fabi -o /dev/null
+; RUN: not llc -mtriple=riscv32 -target-abi ilp32 < %t.ilp32fabi -o /dev/null 2>&1 \
+; RUN:   | FileCheck -check-prefix=RV32-ILP32F %t.ilp32fabi
+
+; RV32-ILP32F: Mismatched ABIs. Current ABI is 'ilp32', but IR target-abi module flag is 'ilp32f'.
+
+define float @foo(i32 %a) nounwind #0 {
+  %conv = sitofp i32 %a to float
+  ret float %conv
+}
+
+attributes #0 = { "target-features"="+f"}
+!llvm.module.flags = !{!0}
diff --git a/llvm/test/CodeGen/RISCV/module-target-abi.ll b/llvm/test/CodeGen/RISCV/module-target-abi.ll
deleted file mode 100644
--- a/llvm/test/CodeGen/RISCV/module-target-abi.ll
+++ /dev/null
@@ -1,24 +0,0 @@
-; RUN: llc -mtriple=riscv32 < %s 2>&1 \
-; RUN:   | FileCheck -check-prefix=DEFAULT %s
-; RUN: llc -mtriple=riscv32 -target-abi ilp32 < %s 2>&1 \
-; RUN:   | FileCheck -check-prefix=RV32IF-ILP32 %s
-; RUN: not --crash llc -mtriple=riscv32 -target-abi ilp32f < %s 2>&1 \
-; RUN:   | FileCheck -check-prefix=RV32IF-ILP32F %s
-; RUN: llc -mtriple=riscv32 -filetype=obj < %s | llvm-readelf -h - | FileCheck -check-prefixes=FLAGS %s
-
-; RV32IF-ILP32F: -target-abi option != target-abi module flag
-
-; FLAGS: Flags: 0x0
-
-define float @foo(i32 %a) nounwind #0 {
-; DEFAULT: # %bb.0:
-; DEFAULT: fmv.x.w a0, ft0
-; RV32IF-ILP32: # %bb.0:
-; RV32IF-ILP32: fmv.x.w a0, ft0
-  %conv = sitofp i32 %a to float
-  ret float %conv
-}
-
-attributes #0 = { "target-features"="+f"}
-!llvm.module.flags = !{!0}
-!0 = !{i32 1, !"target-abi", !"ilp32"}
diff --git a/llvm/test/CodeGen/RISCV/module-target-abi2.ll b/llvm/test/CodeGen/RISCV/module-target-abi2.ll
deleted file mode 100644
--- a/llvm/test/CodeGen/RISCV/module-target-abi2.ll
+++ /dev/null
@@ -1,27 +0,0 @@
-; RUN: llc -mtriple=riscv32 < %s 2>&1 \
-; RUN:   | FileCheck -check-prefix=DEFAULT %s
-; RUN: not --crash llc -mtriple=riscv32 -target-abi ilp32 < %s 2>&1 \
-; RUN:   | FileCheck -check-prefix=RV32IF-ILP32 %s
-; RUN: llc -mtriple=riscv32 -target-abi ilp32f < %s 2>&1 \
-; RUN:   | FileCheck -check-prefix=RV32IF-ILP32F %s
-; RUN: llc -mtriple=riscv32 -filetype=obj < %s | llvm-readelf -h - | FileCheck -check-prefixes=FLAGS %s
-
-; RV32IF-ILP32: -target-abi option != target-abi module flag
-
-; FLAGS: Flags: 0x0
-; // this should be "Flags :0x2, single-float ABI", it will be fixed later.
-
-define float @foo(i32 %a) nounwind #0 {
-; DEFAULT: # %bb.0:
-; DEFAULT-NEXT: fcvt.s.w fa0, a0
-; DEFAULT-NEXT: ret
-; RV32IF-ILP32F: # %bb.0:
-; RV32IF-ILP32F: fcvt.s.w fa0, a0
-; RV32IF-ILP32F: ret
-  %conv = sitofp i32 %a to float
-  ret float %conv
-}
-
-attributes #0 = { "target-features"="+f"}
-!llvm.module.flags = !{!0}
-!0 = !{i32 1, !"target-abi", !"ilp32f"}