[compiler-rt][hwasan] Factor out CanUseTaggingAbi() and EnableTaggingAbi()

[glider]

Summary

Simplify InitializeOsSupport() by separating code for detecting and
enabling the tagged address ABI.

Also drop the unnecessary errno checks (regardless of errno value, we
cannot assume that tagging works if the system call failed) and ensure
prctl(PR_SET_TAGGED_ADDR_CTRL) is only called on Android, not on x86
(where arch_prctl(ARCH_ENABLE_TAGGED_ADDR, kTagBits) is used).

Depends on D132544

[kstoimenov]

compiler-rt/lib/hwasan/hwasan_linux.cpp

146

Is the '-' in front of EINVAL intentional?

147–148

Looks like the previous version of the code checked for 'local_errno == EINVAL' and this one doesn't. Is that expected?

[glider]

compiler-rt/lib/hwasan/hwasan_linux.cpp

146

Yes, see https://lore.kernel.org/all/[e-mail]/
Syscalls usually return negative errno values.

147–148

Right, this is intentional, see the comments here: https://reviews.llvm.org/D132467

[vitalybuka]

compiler-rt/lib/hwasan/hwasan_linux.cpp

142

checking actual ARCH_GET_UNTAG_MASK like line 151 would be nice
separate patch is find

147–148

yes, we disscussed that on another review
also do we need to check error at all if we have line: 151

[kstoimenov]

LTGM

[glider]

rebase

compiler-rt/lib/hwasan/hwasan_linux.cpp

 }
 
 #  define PR_SET_TAGGED_ADDR_CTRL 55
 #  define PR_GET_TAGGED_ADDR_CTRL 56
 #  define PR_TAGGED_ADDR_ENABLE (1UL << 0)
 #  define ARCH_GET_UNTAG_MASK 0x4001
 #  define ARCH_ENABLE_TAGGED_ADDR 0x4002
 
-void InitializeOsSupport() {
-  // Check we're running on a kernel that can use the tagged address ABI.
-  int local_errno = 0;
-  bool has_abi;
+static bool CanUseTaggingAbi() {
 #  if defined(__x86_64__)
-  has_abi = (internal_iserror(internal_arch_prctl(ARCH_GET_UNTAG_MASK, 0),
-                              &local_errno) &&
-             local_errno == EINVAL);
+  // Check for x86 LAM support. This API is based on a currently unsubmitted
+  // patch to the Linux kernel (as of July 2022) and is thus subject to
+  // change. Patch is here:
+  // https://lore.kernel.org/linux-mm/[e-mail]/
+  return !internal_iserror(internal_arch_prctl(ARCH_GET_UNTAG_MASK, 0));
 #  else
-  has_abi =
-      (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0),
-                        &local_errno) &&
-       local_errno == EINVAL);
-#  endif
-  if (has_abi) {
+  // Check for ARM TBI support.
+  return !internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0));
+#  endif // __x86_64__
+}
+
+static bool EnableTaggingAbi() {
+#  if defined(__x86_64__)
+  // Enable x86 LAM tagging for the process.
+  return !internal_iserror(internal_arch_prctl(ARCH_ENABLE_TAGGED_ADDR, kTagBits));
+#  else
+  // Enable ARM TBI tagging for the process. If for some reason tagging is not
+  // supported, prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE) returns
+  // -EINVAL.
+  if (internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
+                                      PR_TAGGED_ADDR_ENABLE, 0, 0, 0)))
+    return false;
+  // Ensure that TBI is enabled.
+  if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) !=
+      PR_TAGGED_ADDR_ENABLE)
+    return false;
+  return true;
+#  endif // __x86_64__
+}
+
+void InitializeOsSupport() {
+  // Check we're running on a kernel that can use the tagged address ABI.
+  bool has_abi = CanUseTaggingAbi();
+
+  if (!has_abi) {
 #  if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE)
     // Some older Android kernels have the tagged pointer ABI on
     // unconditionally, and hence don't have the tagged-addr prctl while still
     // allow the ABI.
     // If targeting Android and the prctl is not around we assume this is the
     // case.
     return;
 #  else
     MaybeDieIfNoTaggingAbi(
         "HWAddressSanitizer requires a kernel with tagged address ABI.");
 #  endif
   }
 
-  // Turn on the tagged address ABI.
-  if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
-                                       PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) ||
-       !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) {
-#  if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
-    // Try the new prctl API for Intel LAM.  The API is based on a currently
-    // unsubmitted patch to the Linux kernel (as of July 2022) and is thus
-    // subject to change.  Patch is here:
-    // https://lore.kernel.org/linux-mm/[e-mail]/
-    if (!internal_iserror(
-            internal_arch_prctl(ARCH_ENABLE_TAGGED_ADDR, kTagBits))) {
-      return;
-    }
-#  endif  // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
+  if (EnableTaggingAbi())
 
 #  if SANITIZER_ANDROID
   MaybeDieIfNoTaggingAbi(
       "HWAddressSanitizer failed to enable tagged address syscall ABI.\n"
       "Check the `sysctl abi.tagged_addr_disabled` configuration.");
 #  else
   MaybeDieIfNoTaggingAbi(
       "HWAddressSanitizer failed to enable tagged address syscall ABI.\n");
 #  endif
-  }
-}
 
 bool InitShadow() {
   // Define the entire memory range.
   kHighMemEnd = GetHighMemEnd();
 
   // Determine shadow memory base offset.
   InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd));