From 99f06f28615042bade669777fca3896f92946fda Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Mon, 18 Aug 2025 16:36:46 -0700
Subject: [PATCH 01/41] [LFI] [AArch64] Add aarch64_lfi architecture

Adds the `aarch64_lfi`, `aarch64_lfi_stores` and `aarch64_lfi_jumps`
subarches for AArch64. Reserves the necessary registers and disables
compressed jump tables when targeting LFI.
---
 llvm/include/llvm/TargetParser/Triple.h       | 30 +++++++++++++++++++
 .../Target/AArch64/AArch64RegisterInfo.cpp    | 22 ++++++++++++++
 llvm/lib/Target/AArch64/AArch64Subtarget.h    |  1 +
 .../Target/AArch64/AArch64TargetMachine.cpp   |  5 ++++
 llvm/lib/TargetParser/Triple.cpp              | 18 +++++++++++
 5 files changed, 76 insertions(+)

diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 7d67966d17256..afecf2c39bf74 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -148,6 +148,10 @@ class Triple {
     AArch64SubArch_arm64e,
     AArch64SubArch_arm64ec,
 
+    AArch64SubArch_lfi,
+    AArch64SubArch_lfi_stores,
+    AArch64SubArch_lfi_jumps,
+
     KalimbaSubArch_v3,
     KalimbaSubArch_v4,
     KalimbaSubArch_v5,
@@ -676,6 +680,32 @@ class Triple {
            getSubArch() == Triple::AArch64SubArch_arm64ec;
   }
 
+  /// Checks if we're targeting any subarch of AArch64 LFI.
+  bool isAArch64LFI() const {
+    return getArch() == Triple::aarch64 &&
+           (getSubArch() == Triple::AArch64SubArch_lfi ||
+            getSubArch() == Triple::AArch64SubArch_lfi_stores ||
+            getSubArch() == Triple::AArch64SubArch_lfi_jumps);
+  }
+
+  /// Checks if we're targeting AArch64 LFI.
+  bool isAArch64LFIFull() const {
+    return getArch() == Triple::aarch64 &&
+           getSubArch() == Triple::AArch64SubArch_lfi;
+  }
+
+  /// Checks if we're targeting AArch64 LFI stores-only.
+  bool isAArch64LFIStores() const {
+    return getArch() == Triple::aarch64 &&
+           getSubArch() == Triple::AArch64SubArch_lfi_stores;
+  }
+
+  /// Checks if we're targeting AArch64 LFI jumps-only.
+  bool isAArch64LFIJumps() const {
+    return getArch() == Triple::aarch64 &&
+           getSubArch() == Triple::AArch64SubArch_lfi_jumps;
+  }
+
   bool isWindowsCoreCLREnvironment() const {
     return isOSWindows() && getEnvironment() == Triple::CoreCLR;
   }
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 367f6b626b420..22f845e1fa798 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -403,6 +403,18 @@ AArch64RegisterInfo::explainReservedReg(const MachineFunction &MF,
   if (hasBasePointer(MF) && MCRegisterInfo::regsOverlap(PhysReg, AArch64::X19))
     return std::string("X19 is used as the frame base pointer register.");
 
+  if (MF.getSubtarget<AArch64Subtarget>().isAArch64LFI()) {
+    bool warn = false;
+    if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X28) ||
+        MCRegisterInfo::regsOverlap(PhysReg, AArch64::X27) ||
+        MCRegisterInfo::regsOverlap(PhysReg, AArch64::X26))
+      warn = true;
+
+    if (warn)
+      return std::string(AArch64InstPrinter::getRegisterName(PhysReg)) +
+             " is used as a reserved register by LFI.";
+  }
+
   if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {
     bool warn = false;
     if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X13) ||
@@ -448,6 +460,16 @@ AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
       markSuperRegs(Reserved, i);
   }
 
+  if (MF.getSubtarget<AArch64Subtarget>().isAArch64LFI()) {
+    markSuperRegs(Reserved, AArch64::W28);
+    markSuperRegs(Reserved, AArch64::W27);
+    markSuperRegs(Reserved, AArch64::W26);
+    if (!MF.getProperties().hasProperty(MachineFunctionProperties::Property::NoVRegs)) {
+      markSuperRegs(Reserved, AArch64::LR);
+      markSuperRegs(Reserved, AArch64::W30);
+    }
+  }
+
   for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
     if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))
       markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i));
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index c6eb77e3bc3ba..cbc60a2e52567 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -288,6 +288,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   bool isTargetAndroid() const { return TargetTriple.isAndroid(); }
   bool isTargetFuchsia() const { return TargetTriple.isOSFuchsia(); }
   bool isWindowsArm64EC() const { return TargetTriple.isWindowsArm64EC(); }
+  bool isAArch64LFI() const { return TargetTriple.isAArch64LFI(); }
 
   bool isTargetCOFF() const { return TargetTriple.isOSBinFormatCOFF(); }
   bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); }
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 07f072446081a..846892b79cd64 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -369,6 +369,11 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT,
     this->Options.NoTrapAfterNoreturn = true;
   }
 
+  // Disable jump table compression since it may cause assembler errors after
+  // LFI instrumentation is applied.
+  if (TT.isAArch64LFI())
+    EnableCompressJumpTables = false;
+
   if (getMCAsmInfo()->usesWindowsCFI()) {
     // Unwinding can get confused if the last instruction in an
     // exception-handling region (function, funclet, try block, etc.)
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index e9e6f130f757c..6a18790ca6e83 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -112,6 +112,12 @@ StringRef Triple::getArchName(ArchType Kind, SubArchType SubArch) {
       return "arm64ec";
     if (SubArch == AArch64SubArch_arm64e)
       return "arm64e";
+    if (SubArch == AArch64SubArch_lfi)
+      return "aarch64_lfi";
+    if (SubArch == AArch64SubArch_lfi_stores)
+      return "aarch64_lfi_stores";
+    if (SubArch == AArch64SubArch_lfi_jumps)
+      return "aarch64_lfi_jumps";
     break;
   case Triple::spirv:
     switch (SubArch) {
@@ -575,6 +581,9 @@ static Triple::ArchType parseArch(StringRef ArchName) {
           .Case("arm64_32", Triple::aarch64_32)
           .Case("arm64e", Triple::aarch64)
           .Case("arm64ec", Triple::aarch64)
+          .Case("aarch64_lfi", Triple::aarch64)
+          .Case("aarch64_lfi_stores", Triple::aarch64)
+          .Case("aarch64_lfi_jumps", Triple::aarch64)
           .Case("arm", Triple::arm)
           .Case("armeb", Triple::armeb)
           .Case("thumb", Triple::thumb)
@@ -797,6 +806,15 @@ static Triple::SubArchType parseSubArch(StringRef SubArchName) {
   if (SubArchName == "arm64ec")
     return Triple::AArch64SubArch_arm64ec;
 
+  if (SubArchName == "aarch64_lfi")
+    return Triple::AArch64SubArch_lfi;
+
+  if (SubArchName == "aarch64_lfi_stores")
+    return Triple::AArch64SubArch_lfi_stores;
+
+  if (SubArchName == "aarch64_lfi_jumps")
+    return Triple::AArch64SubArch_lfi_jumps;
+
   if (SubArchName.starts_with("spirv"))
     return StringSwitch<Triple::SubArchType>(SubArchName)
         .EndsWith("v1.0", Triple::SPIRVSubArch_v10)

From 08019fe93d816556041fbf153cc2a50e6848644a Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 12:25:46 -0700
Subject: [PATCH 02/41] [LFI] Add LFILinux Clang toolchain

---
 clang/lib/Basic/Targets/AArch64.cpp      |  4 +++
 clang/lib/Driver/CMakeLists.txt          |  1 +
 clang/lib/Driver/Driver.cpp              |  3 ++
 clang/lib/Driver/ToolChains/LFILinux.cpp | 29 +++++++++++++++++++
 clang/lib/Driver/ToolChains/LFILinux.h   | 36 ++++++++++++++++++++++++
 llvm/include/llvm/TargetParser/Triple.h  |  4 +++
 6 files changed, 77 insertions(+)
 create mode 100644 clang/lib/Driver/ToolChains/LFILinux.cpp
 create mode 100644 clang/lib/Driver/ToolChains/LFILinux.h

diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index cabda0a1323a3..a3464a9b71d9d 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -413,6 +413,10 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__aarch64__");
   }
 
+  if (getTriple().isAArch64LFI()) {
+    Builder.defineMacro("__LFI__");
+  }
+
   // Inline assembly supports AArch64 flag outputs.
   Builder.defineMacro("__GCC_ASM_FLAG_OUTPUTS__");
 
diff --git a/clang/lib/Driver/CMakeLists.txt b/clang/lib/Driver/CMakeLists.txt
index 5bdb6614389cf..9fd7beb4b29a7 100644
--- a/clang/lib/Driver/CMakeLists.txt
+++ b/clang/lib/Driver/CMakeLists.txt
@@ -64,6 +64,7 @@ add_clang_library(clangDriver
   ToolChains/Hexagon.cpp
   ToolChains/HLSL.cpp
   ToolChains/Hurd.cpp
+  ToolChains/LFILinux.cpp
   ToolChains/Linux.cpp
   ToolChains/MipsLinux.cpp
   ToolChains/MinGW.cpp
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 87855fdb79971..6183798031d96 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -28,6 +28,7 @@
 #include "ToolChains/Haiku.h"
 #include "ToolChains/Hexagon.h"
 #include "ToolChains/Hurd.h"
+#include "ToolChains/LFILinux.h"
 #include "ToolChains/Lanai.h"
 #include "ToolChains/Linux.h"
 #include "ToolChains/MSP430.h"
@@ -6654,6 +6655,8 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
         TC = std::make_unique<toolchains::VEToolChain>(*this, Target, Args);
       else if (Target.isOHOSFamily())
         TC = std::make_unique<toolchains::OHOS>(*this, Target, Args);
+      else if (Target.isLFI())
+        TC = std::make_unique<toolchains::LFILinuxToolChain>(*this, Target, Args);
       else
         TC = std::make_unique<toolchains::Linux>(*this, Target, Args);
       break;
diff --git a/clang/lib/Driver/ToolChains/LFILinux.cpp b/clang/lib/Driver/ToolChains/LFILinux.cpp
new file mode 100644
index 0000000000000..37510c596b303
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/LFILinux.cpp
@@ -0,0 +1,29 @@
+//===-- LFILinux.cpp - LFI ToolChain Implementations ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "LFILinux.h"
+#include "clang/Driver/Driver.h"
+#include "clang/Driver/Options.h"
+
+using namespace clang::driver::toolchains;
+using namespace llvm::opt;
+
+void LFILinuxToolChain::AddCXXStdlibLibArgs(
+    const ArgList &Args, ArgStringList &CmdArgs) const {
+  switch (GetCXXStdlibType(Args)) {
+  case ToolChain::CST_Libcxx:
+    CmdArgs.push_back("-lc++");
+    if (Args.hasArg(options::OPT_fexperimental_library))
+      CmdArgs.push_back("-lc++experimental");
+    CmdArgs.push_back("-lc++abi");
+    break;
+  case ToolChain::CST_Libstdcxx:
+    CmdArgs.push_back("-lstdc++");
+    break;
+  }
+}
diff --git a/clang/lib/Driver/ToolChains/LFILinux.h b/clang/lib/Driver/ToolChains/LFILinux.h
new file mode 100644
index 0000000000000..4ef7b7334e8b0
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/LFILinux.h
@@ -0,0 +1,36 @@
+//===--- LFILinux.h - LFI ToolChain Implementations -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_LFI_LINUX_H
+#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_LFI_LINUX_H
+
+#include "Linux.h"
+
+namespace clang {
+namespace driver {
+namespace toolchains {
+
+class LLVM_LIBRARY_VISIBILITY LFILinuxToolChain : public Linux {
+public:
+  LFILinuxToolChain(const Driver &D, const llvm::Triple &Triple,
+                    const llvm::opt::ArgList &Args)
+      : Linux(D, Triple, Args) {
+        ExtraOpts.push_back("-z");
+        ExtraOpts.push_back("separate-code");
+      }
+
+  void
+  AddCXXStdlibLibArgs(const llvm::opt::ArgList &Args,
+                            llvm::opt::ArgStringList &CmdArgs) const override;
+};
+
+} // end namespace toolchains
+} // end namespace driver
+} // end namespace clang
+
+#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_LFI_LINUX_H
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index afecf2c39bf74..642c9e7a04436 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -680,6 +680,10 @@ class Triple {
            getSubArch() == Triple::AArch64SubArch_arm64ec;
   }
 
+  bool isLFI() const {
+    return isAArch64LFI();
+  }
+
   /// Checks if we're targeting any subarch of AArch64 LFI.
   bool isAArch64LFI() const {
     return getArch() == Triple::aarch64 &&

From cb92ceb0996152ff173b4b98376cfd02a13f42dd Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 14:28:12 -0700
Subject: [PATCH 03/41] [LFI] [AArch64] Machinery for LFI MC rewriter

This patch establishes the scaffolding for the LFI MC rewriter, based on
the NaCl auto-sandboxing assembler. There is a generic MCLFIExpander,
plus an AArch64-specific subclass where the rewriter will be implemented
in a future patch.
---
 llvm/include/llvm/MC/MCLFI.h                  |  22 ++++
 llvm/include/llvm/MC/MCLFIExpander.h          |  74 ++++++++++++
 llvm/include/llvm/MC/MCStreamer.h             |   7 ++
 llvm/include/llvm/MC/TargetRegistry.h         |  21 ++++
 llvm/lib/MC/CMakeLists.txt                    |   2 +
 llvm/lib/MC/MCAsmStreamer.cpp                 |   4 +
 llvm/lib/MC/MCLFI.cpp                         |  79 +++++++++++++
 llvm/lib/MC/MCLFIExpander.cpp                 | 107 ++++++++++++++++++
 llvm/lib/MC/MCObjectStreamer.cpp              |   6 +-
 llvm/lib/MC/MCParser/AsmParser.cpp            |   8 +-
 llvm/lib/MC/MCParser/CMakeLists.txt           |   1 +
 llvm/lib/MC/MCParser/LFIAsmParser.cpp         |  84 ++++++++++++++
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp |   5 +
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     |  79 +++++++++++++
 .../MCTargetDesc/AArch64MCLFIExpander.h       |  72 ++++++++++++
 .../MCTargetDesc/AArch64MCTargetDesc.cpp      |  13 +++
 .../AArch64/MCTargetDesc/CMakeLists.txt       |   1 +
 17 files changed, 583 insertions(+), 2 deletions(-)
 create mode 100644 llvm/include/llvm/MC/MCLFI.h
 create mode 100644 llvm/include/llvm/MC/MCLFIExpander.h
 create mode 100644 llvm/lib/MC/MCLFI.cpp
 create mode 100644 llvm/lib/MC/MCLFIExpander.cpp
 create mode 100644 llvm/lib/MC/MCParser/LFIAsmParser.cpp
 create mode 100644 llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
 create mode 100644 llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h

diff --git a/llvm/include/llvm/MC/MCLFI.h b/llvm/include/llvm/MC/MCLFI.h
new file mode 100644
index 0000000000000..a50a228d13643
--- /dev/null
+++ b/llvm/include/llvm/MC/MCLFI.h
@@ -0,0 +1,22 @@
+//===- MCLFI.h - LFI-specific code for MC -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/CommandLine.h"
+
+namespace llvm {
+
+class MCContext;
+class MCStreamer;
+class Triple;
+
+void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
+                             const Triple &TheTriple);
+
+}
diff --git a/llvm/include/llvm/MC/MCLFIExpander.h b/llvm/include/llvm/MC/MCLFIExpander.h
new file mode 100644
index 0000000000000..195e298c42e84
--- /dev/null
+++ b/llvm/include/llvm/MC/MCLFIExpander.h
@@ -0,0 +1,74 @@
+//===- llvm/MC/MCLFIExpander.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the MCLFIExpander class. This is an abstract
+// class that encapsulates the expansion logic for MCInsts, and holds
+// state such as available scratch registers.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_MC_MCLFIEXPANDER_H
+#define LLVM_MC_MCLFIEXPANDER_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+
+namespace llvm {
+class MCInst;
+class MCSubtargetInfo;
+class MCStreamer;
+
+class MCLFIExpander {
+private:
+  SmallVector<MCRegister, 2> ScratchRegs;
+  MCContext &Ctx;
+
+protected:
+  std::unique_ptr<MCInstrInfo> InstInfo;
+  std::unique_ptr<MCRegisterInfo> RegInfo;
+  void invalidateScratchRegs(const MCInst &Inst);
+  MCRegister getScratchReg(int index);
+  unsigned numScratchRegs() const;
+  virtual bool isValidScratchRegister(MCRegister Reg) const = 0;
+
+public:
+  MCLFIExpander(MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
+                 std::unique_ptr<MCInstrInfo> &&II)
+      : Ctx(Ctx), InstInfo(std::move(II)), RegInfo(std::move(RI)) {}
+
+  void Error(const MCInst &Inst, const char msg[]);
+
+  bool addScratchReg(MCRegister Reg);
+  void clearScratchRegs();
+
+  bool isPseudo(const MCInst &Inst) const;
+
+  bool mayAffectControlFlow(const MCInst &Inst) const;
+  bool isCall(const MCInst &Inst) const;
+  bool isBranch(const MCInst &Inst) const;
+  bool isIndirectBranch(const MCInst &Inst) const;
+  bool isReturn(const MCInst &Inst) const;
+  bool isVariadic(const MCInst &Inst) const;
+
+  bool mayLoad(const MCInst &Inst) const;
+  bool mayStore(const MCInst &Inst) const;
+
+  bool mayModifyRegister(const MCInst &Inst, MCRegister Reg) const;
+  bool explicitlyModifiesRegister(const MCInst &Inst, MCRegister Reg) const;
+
+  virtual ~MCLFIExpander() = default;
+  virtual bool expandInst(const MCInst &Inst, MCStreamer &Out,
+                          const MCSubtargetInfo &STI) = 0;
+};
+
+}
+#endif
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 558b14cebfd3d..96b6aff836932 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -21,6 +21,7 @@
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCLinkerOptimizationHint.h"
+#include "llvm/MC/MCLFIExpander.h"
 #include "llvm/MC/MCPseudoProbe.h"
 #include "llvm/MC/MCWinEH.h"
 #include "llvm/Support/Error.h"
@@ -276,6 +277,8 @@ class MCStreamer {
   /// Returns true if the .cv_loc directive is in the right section.
   bool checkCVLocSection(unsigned FuncId, unsigned FileNo, SMLoc Loc);
 
+  std::unique_ptr<MCLFIExpander> LFIExpander;
+
 public:
   MCStreamer(const MCStreamer &) = delete;
   MCStreamer &operator=(const MCStreamer &) = delete;
@@ -293,6 +296,10 @@ class MCStreamer {
     return StartTokLocPtr ? *StartTokLocPtr : SMLoc();
   }
 
+  void setLFIExpander(MCLFIExpander *Exp) { LFIExpander.reset(Exp); }
+
+  MCLFIExpander *getLFIExpander() { return LFIExpander.get(); }
+
   /// State management
   ///
   virtual void reset();
diff --git a/llvm/include/llvm/MC/TargetRegistry.h b/llvm/include/llvm/MC/TargetRegistry.h
index 42d510c17bce3..abeb1d4405be7 100644
--- a/llvm/include/llvm/MC/TargetRegistry.h
+++ b/llvm/include/llvm/MC/TargetRegistry.h
@@ -46,6 +46,7 @@ class MCDisassembler;
 class MCInstPrinter;
 class MCInstrAnalysis;
 class MCInstrInfo;
+class MCLFIExpander;
 class MCObjectWriter;
 class MCRegisterInfo;
 class MCRelocationInfo;
@@ -230,6 +231,10 @@ class Target {
       mca::InstrumentManager *(*)(const MCSubtargetInfo &STI,
                                   const MCInstrInfo &MCII);
 
+  using MCLFIExpanderCtorTy = MCLFIExpander *(*)(
+      MCStreamer &S, std::unique_ptr<MCRegisterInfo> &&RegInfo,
+      std::unique_ptr<MCInstrInfo> &&InstInfo);
+
 private:
   /// Next - The next registered target in the linked list, maintained by the
   /// TargetRegistry.
@@ -340,6 +345,10 @@ class Target {
   /// InstrumentManager, if registered (default = nullptr).
   InstrumentManagerCtorTy InstrumentManagerCtorFn = nullptr;
 
+  // MCLFIExpanderCtorFn - Construction function for this target's
+  // MCLFIExpander, if registered.
+  MCLFIExpanderCtorTy MCLFIExpanderCtorFn;
+
 public:
   Target() = default;
 
@@ -538,6 +547,13 @@ class Target {
                                      const MCSubtargetInfo &STI, bool, bool,
                                      bool) const;
 
+  void createMCLFIExpander(MCStreamer &S,
+                           std::unique_ptr<MCRegisterInfo> &&RegInfo,
+                           std::unique_ptr<MCInstrInfo> &&InstInfo) const {
+    if (MCLFIExpanderCtorFn)
+      MCLFIExpanderCtorFn(S, std::move(RegInfo), std::move(InstInfo));
+  }
+
   MCStreamer *createAsmStreamer(MCContext &Ctx,
                                 std::unique_ptr<formatted_raw_ostream> OS,
                                 MCInstPrinter *IP,
@@ -1008,6 +1024,11 @@ struct TargetRegistry {
     T.InstrumentManagerCtorFn = Fn;
   }
 
+  static void RegisterMCLFIExpander(Target &T,
+                                    Target::MCLFIExpanderCtorTy Fn) {
+    T.MCLFIExpanderCtorFn = Fn;
+  }
+
   /// @}
 };
 
diff --git a/llvm/lib/MC/CMakeLists.txt b/llvm/lib/MC/CMakeLists.txt
index e1d19196c8766..0456768b9cfaf 100644
--- a/llvm/lib/MC/CMakeLists.txt
+++ b/llvm/lib/MC/CMakeLists.txt
@@ -30,6 +30,8 @@ add_llvm_component_library(LLVMMC
   MCInstrAnalysis.cpp
   MCInstrDesc.cpp
   MCInstrInfo.cpp
+  MCLFI.cpp
+  MCLFIExpander.cpp
   MCLabel.cpp
   MCLinkerOptimizationHint.cpp
   MCMachOStreamer.cpp
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index dd8058c6d5cd8..38d8976965c81 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -20,6 +20,7 @@
 #include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
+#include "llvm/MC/MCLFIExpander.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCPseudoProbe.h"
@@ -2453,6 +2454,9 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
 
 void MCAsmStreamer::emitInstruction(const MCInst &Inst,
                                     const MCSubtargetInfo &STI) {
+  if (LFIExpander && LFIExpander->expandInst(Inst, *this, STI))
+    return;
+
   if (MAI->isAIX() && CurFrag)
     // Now that a machine instruction has been assembled into this section, make
     // a line entry for any .loc directive that has been seen.
diff --git a/llvm/lib/MC/MCLFI.cpp b/llvm/lib/MC/MCLFI.cpp
new file mode 100644
index 0000000000000..3cad2e4eff452
--- /dev/null
+++ b/llvm/lib/MC/MCLFI.cpp
@@ -0,0 +1,79 @@
+//===- lib/MC/MCLFI.cpp - LFI-specific MC implementation ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCLFI.h"
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/TargetParser/Triple.h"
+
+static const char NoteNamespace[] = "LFI";
+
+namespace llvm {
+
+cl::opt<bool> FlagEnableAutoSandboxing("nacl-enable-autosandboxing",
+                                cl::desc("Don't use the autosandboxing"
+                                         " assembler for the NaCl SFI."),
+                                cl::init(true));
+
+void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
+                              const Triple &TheTriple) {
+  assert(TheTriple.isLFI());
+  const char *NoteName;
+  const char *NoteArch;
+  switch (TheTriple.getArch()) {
+    case Triple::aarch64:
+      NoteName = ".note.LFI.ABI.aarch64";
+      NoteArch = "aarch64";
+      break;
+    default:
+      report_fatal_error("Unsupported architecture for LFI");
+  }
+
+  std::string Error; //empty
+  const Target *TheTarget =
+    TargetRegistry::lookupTarget(TheTriple.getTriple(), Error);
+
+  // Create the Target specific MCLFIExpander
+  assert(TheTarget != nullptr);
+  if (FlagEnableAutoSandboxing) {
+    TheTarget->createMCLFIExpander(
+      Streamer,
+      std::unique_ptr<MCRegisterInfo>(
+          TheTarget->createMCRegInfo(TheTriple.getTriple())),
+      std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
+  }
+
+  // Emit an ELF Note section in its own COMDAT group which identifies LFI
+  // object files.
+  MCSectionELF* Note = Ctx.getELFSection(NoteName, ELF::SHT_NOTE,
+                                         ELF::SHF_ALLOC | ELF::SHF_GROUP,
+                                         0, NoteName, /*IsComdat=*/true);
+
+  Streamer.pushSection();
+  Streamer.switchSection(Note);
+  Streamer.emitIntValue(strlen(NoteNamespace) + 1, 4);
+  Streamer.emitIntValue(strlen(NoteArch) + 1, 4);
+  Streamer.emitIntValue(ELF::NT_VERSION, 4);
+  Streamer.emitBytes(NoteNamespace);
+  Streamer.emitIntValue(0, 1); // NUL terminator
+  Streamer.emitValueToAlignment(Align(4));
+  Streamer.emitBytes(NoteArch);
+  Streamer.emitIntValue(0, 1); // NUL terminator
+  Streamer.emitValueToAlignment(Align(4));
+  Streamer.popSection();
+}
+
+} // namespace llvm
diff --git a/llvm/lib/MC/MCLFIExpander.cpp b/llvm/lib/MC/MCLFIExpander.cpp
new file mode 100644
index 0000000000000..900e1ec17418d
--- /dev/null
+++ b/llvm/lib/MC/MCLFIExpander.cpp
@@ -0,0 +1,107 @@
+//===- MCLFIExpander.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the MCLFIExpander class. This is a base
+// class that encapsulates the expansion logic for MCInsts, and holds
+// state such as available scratch registers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+
+namespace llvm {
+
+void MCLFIExpander::Error(const MCInst &Inst, const char msg[]) {
+  Ctx.reportError(Inst.getLoc(), msg);
+}
+
+bool MCLFIExpander::addScratchReg(MCRegister Reg) {
+  if (!isValidScratchRegister(Reg))
+    return true;
+  ScratchRegs.push_back(Reg);
+  return false;
+}
+
+void MCLFIExpander::invalidateScratchRegs(const MCInst &Inst) {
+  // TODO: There are arch-specific special cases where this fails, e.g.
+  // xchg/cmpxchg
+  // TODO(zyedidia): investigate whether there are cases for AArch64.
+  const MCInstrDesc &Desc = InstInfo->get(Inst.getOpcode());
+  for (auto I = ScratchRegs.begin(), E = ScratchRegs.end(); I != E; ++I) {
+    if (Desc.hasDefOfPhysReg(Inst, *I, *RegInfo))
+      I = ScratchRegs.erase(I);
+  }
+}
+
+void MCLFIExpander::clearScratchRegs() {
+  ScratchRegs.clear();
+}
+
+MCRegister MCLFIExpander::getScratchReg(int index) {
+  assert(index >= 0 && static_cast<unsigned>(index) < numScratchRegs());
+  return ScratchRegs[numScratchRegs()  - index - 1];
+}
+
+unsigned MCLFIExpander::numScratchRegs() const { return ScratchRegs.size(); }
+
+bool MCLFIExpander::isPseudo(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isPseudo();
+}
+
+bool MCLFIExpander::mayAffectControlFlow(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).mayAffectControlFlow(Inst, *RegInfo);
+}
+
+bool MCLFIExpander::isCall(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isCall();
+}
+
+bool MCLFIExpander::isBranch(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isBranch();
+}
+
+bool MCLFIExpander::isIndirectBranch(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isIndirectBranch();
+}
+
+bool MCLFIExpander::isReturn(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isReturn();
+}
+
+bool MCLFIExpander::isVariadic(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).isVariadic();
+}
+
+bool MCLFIExpander::mayLoad(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).mayLoad();
+}
+
+bool MCLFIExpander::mayStore(const MCInst &Inst) const {
+  return InstInfo->get(Inst.getOpcode()).mayStore();
+}
+
+bool MCLFIExpander::mayModifyRegister(const MCInst &Inst, MCRegister Reg) const {
+  return InstInfo->get(Inst.getOpcode()).hasDefOfPhysReg(Inst, Reg, *RegInfo);
+}
+
+bool MCLFIExpander::explicitlyModifiesRegister(const MCInst &Inst,
+                                                MCRegister Reg) const {
+  const MCInstrDesc &Desc = InstInfo->get(Inst.getOpcode());
+  for (int i = 0; i < Desc.NumDefs; ++i) {
+    if (Desc.operands()[i].OperandType == MCOI::OPERAND_REGISTER &&
+        RegInfo->isSubRegisterEq(Reg, Inst.getOperand(i).getReg()))
+      return true;
+  }
+  return false;
+}
+}
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index fff30955b2576..75c3f00d64ed2 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -15,6 +15,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCLFIExpander.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
@@ -349,9 +350,12 @@ void MCObjectStreamer::emitInstruction(const MCInst &Inst,
 
 void MCObjectStreamer::emitInstructionImpl(const MCInst &Inst,
                                            const MCSubtargetInfo &STI) {
+  MCSection *Sec = getCurrentSectionOnly();
+  if (LFIExpander && LFIExpander->expandInst(Inst, *this, STI))
+    return;
+
   MCStreamer::emitInstruction(Inst, STI);
 
-  MCSection *Sec = getCurrentSectionOnly();
   Sec->setHasInstructions(true);
 
   // Now that a machine instruction has been assembled into this section, make
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index bf952df1b2418..4f4d6fb9a7065 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -125,6 +125,7 @@ class AsmParser : public MCAsmParser {
   SourceMgr::DiagHandlerTy SavedDiagHandler;
   void *SavedDiagContext;
   std::unique_ptr<MCAsmParserExtension> PlatformParser;
+  std::unique_ptr<MCAsmParserExtension> LFIParser;
   SMLoc StartTokLoc;
   std::optional<SMLoc> CFIStartProcLoc;
 
@@ -769,6 +770,7 @@ extern MCAsmParserExtension *createCOFFAsmParser();
 extern MCAsmParserExtension *createGOFFAsmParser();
 extern MCAsmParserExtension *createXCOFFAsmParser();
 extern MCAsmParserExtension *createWasmAsmParser();
+extern MCAsmParserExtension *createLFIAsmParser(MCLFIExpander *Exp);
 
 } // end namespace llvm
 
@@ -776,7 +778,7 @@ enum { DEFAULT_ADDRSPACE = 0 };
 
 AsmParser::AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out,
                      const MCAsmInfo &MAI, unsigned CB = 0)
-    : Lexer(MAI), Ctx(Ctx), Out(Out), MAI(MAI), SrcMgr(SM),
+    : Lexer(MAI), Ctx(Ctx), Out(Out), MAI(MAI), SrcMgr(SM), LFIParser(nullptr),
       CurBuffer(CB ? CB : SM.getMainFileID()), MacrosEnabledFlag(true) {
   HadError = false;
   // Save the old handler.
@@ -819,6 +821,10 @@ AsmParser::AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out,
   }
 
   PlatformParser->Initialize(*this);
+  if (Out.getLFIExpander()) {
+    LFIParser.reset(createLFIAsmParser(Out.getLFIExpander()));
+    LFIParser->Initialize(*this);
+  }
   initializeDirectiveKindMap();
   initializeCVDefRangeTypeMap();
 
diff --git a/llvm/lib/MC/MCParser/CMakeLists.txt b/llvm/lib/MC/MCParser/CMakeLists.txt
index d3fa2675a255e..93f0ff4ca0140 100644
--- a/llvm/lib/MC/MCParser/CMakeLists.txt
+++ b/llvm/lib/MC/MCParser/CMakeLists.txt
@@ -6,6 +6,7 @@ add_llvm_component_library(LLVMMCParser
   GOFFAsmParser.cpp
   DarwinAsmParser.cpp
   ELFAsmParser.cpp
+  LFIAsmParser.cpp
   MCAsmLexer.cpp
   MCAsmParser.cpp
   MCAsmParserExtension.cpp
diff --git a/llvm/lib/MC/MCParser/LFIAsmParser.cpp b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
new file mode 100644
index 0000000000000..cf9c172bd423b
--- /dev/null
+++ b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
@@ -0,0 +1,84 @@
+//===- LFIAsmParser.cpp - LFI Assembly Parser -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCParser/MCAsmParserExtension.h"
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCParser/MCAsmLexer.h"
+#include "llvm/MC/MCParser/MCTargetAsmParser.h"
+#include "llvm/MC/MCRegister.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+class LFIAsmParser : public MCAsmParserExtension {
+  MCLFIExpander *Expander;
+  template<bool (LFIAsmParser::*HandlerMethod)(StringRef, SMLoc)>
+  void addDirectiveHandler(StringRef Directive) {
+    MCAsmParser::ExtensionDirectiveHandler Handler = std::make_pair(
+        this, HandleDirective<LFIAsmParser, HandlerMethod>);
+
+    getParser().addDirectiveHandler(Directive, Handler);
+  }
+
+ public:
+  LFIAsmParser(MCLFIExpander *Exp) : Expander(Exp) {}
+  void Initialize(MCAsmParser &Parser) override {
+    // Call the base implementation.
+    MCAsmParserExtension::Initialize(Parser);
+    addDirectiveHandler<&LFIAsmParser::ParseScratch>(".scratch");
+    addDirectiveHandler<&LFIAsmParser::ParseUnscratch>(".scratch_clear");
+  }
+
+  /// ::= {.scratch} reg
+  bool ParseScratch(StringRef Directive, SMLoc Loc) {
+    getParser().checkForValidSection();
+    MCRegister RegNo;
+    const char *kInvalidOptionError =
+        "expected register name after '.scratch' directive";
+
+    if (getLexer().isNot(AsmToken::EndOfStatement)) {
+      if (getParser().getTargetParser().parseRegister(RegNo, Loc, Loc))
+        return Error(Loc, kInvalidOptionError);
+
+      else if (getLexer().isNot(AsmToken::EndOfStatement))
+        return Error(Loc, kInvalidOptionError);
+    }
+    else {
+      return Error(Loc, kInvalidOptionError);
+    }
+    Lex();
+
+    if(Expander->addScratchReg(RegNo))
+      return Error(Loc, "Register can't be used as a scratch register");
+    return false;
+  }
+
+  /// ::= {.unscratch}
+  bool ParseUnscratch(StringRef Directive, SMLoc Loc) {
+    getParser().checkForValidSection();
+    if (getLexer().isNot(AsmToken::EndOfStatement))
+      return TokError("unexpected token in '.scratch_clear' directive");
+    Lex();
+
+    Expander->clearScratchRegs();
+
+    return false;
+  }
+};
+
+namespace llvm {
+MCAsmParserExtension *createLFIAsmParser(MCLFIExpander *Exp) {
+  return new LFIAsmParser(Exp);
+}
+}
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index c6b4a219d201f..b899c73d529ad 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -50,6 +50,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstBuilder.h"
+#include "llvm/MC/MCLFI.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCStreamer.h"
@@ -394,6 +395,10 @@ void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
 
   // Emit AArch64 Build Attributes
   emitAttributes(BAFlags, PAuthABIPlatform, PAuthABIVersion, TS);
+
+  if (TT.isLFI())
+    initializeLFIMCStreamer(*OutStreamer.get(), OutContext, TT);
+
   // Emit a .note.gnu.property section with the flags.
   TS->emitNoteSection(GNUFlags, PAuthABIPlatform, PAuthABIVersion);
 }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
new file mode 100644
index 0000000000000..a2decbe6db0e1
--- /dev/null
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -0,0 +1,79 @@
+//===- AArch64MCLFIExpander.cpp ---------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the AArch64MCLFIExpander class, the AArch64 specific
+// subclass of MCLFIExpander.
+//
+//===----------------------------------------------------------------------===//
+#include "AArch64MCLFIExpander.h"
+#include "AArch64AddressingModes.h"
+#include "Utils/AArch64BaseInfo.h"
+
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrDesc.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "lfi"
+
+bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
+  return false;
+}
+
+void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
+                                                  MCStreamer &Out,
+                                                  const MCSubtargetInfo &STI,
+                                                  bool isCall) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &Out,
+                                        const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
+                                          const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandControlFlow(const MCInst &Inst,
+                                               MCStreamer &Out,
+                                               const MCSubtargetInfo &STI) {
+
+}
+
+bool AArch64::AArch64MCLFIExpander::mayModifyStack(const MCInst &Inst) {
+  return false;
+}
+
+void AArch64::AArch64MCLFIExpander::expandStackManipulation(
+    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandPrefetch(const MCInst &Inst, MCStreamer &Out,
+                                            const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
+                                             MCStreamer &Out,
+                                             const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
+                                          const MCSubtargetInfo &STI) {
+}
+
+bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
+                                        const MCSubtargetInfo &STI) {
+  return false;
+}
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
new file mode 100644
index 0000000000000..194fa36eb9c2b
--- /dev/null
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -0,0 +1,72 @@
+//===- AArch64MCLFIExpander.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This file was written by the Native Client authors, modified for LFI.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the AArch64MCLFIExpander class, the AArch64 specific
+// subclass of MCLFIExpander.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_MC_AARCH64MCLFIEXPANDER_H
+#define LLVM_MC_AARCH64MCLFIEXPANDER_H
+
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCRegisterInfo.h"
+
+namespace llvm {
+class MCContext;
+class MCInst;
+class MCStreamer;
+class MCSubtargetInfo;
+
+namespace AArch64 {
+class AArch64MCLFIExpander : public MCLFIExpander {
+public:
+  AArch64MCLFIExpander(MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
+                    std::unique_ptr<MCInstrInfo> &&II)
+      : MCLFIExpander(Ctx, std::move(RI), std::move(II)) {}
+
+  bool expandInst(const MCInst &Inst, MCStreamer &Out,
+                  const MCSubtargetInfo &STI) override;
+
+protected:
+  bool isValidScratchRegister(MCRegister Reg) const override;
+
+private:
+  bool Guard = false; // recursion guard
+
+  void expandIndirectBranch(const MCInst &Inst, MCStreamer &Out,
+                            const MCSubtargetInfo &STI, bool isCall);
+
+  void expandCall(const MCInst &Inst, MCStreamer &Out,
+                  const MCSubtargetInfo &STI);
+
+  void expandReturn(const MCInst &Inst, MCStreamer &Out,
+                    const MCSubtargetInfo &STI);
+
+  bool mayModifyStack(const MCInst &Inst);
+
+  void expandControlFlow(const MCInst &Inst, MCStreamer &Out,
+                         const MCSubtargetInfo &STI);
+
+  void expandStackManipulation(const MCInst &Inst, MCStreamer &Out,
+                               const MCSubtargetInfo &STI);
+
+  void expandPrefetch(const MCInst &Inst, MCStreamer &Out,
+                      const MCSubtargetInfo &STI);
+
+  void expandLoadStore(const MCInst &Inst, MCStreamer &Out,
+                       const MCSubtargetInfo &STI);
+
+  void doExpandInst(const MCInst &Inst, MCStreamer &Out,
+                    const MCSubtargetInfo &STI);
+};
+} // namespace AArch64
+} // namespace llvm
+#endif
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
index c7f44ec018f5a..c7a044346baab 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
@@ -13,6 +13,7 @@
 #include "AArch64MCTargetDesc.h"
 #include "AArch64ELFStreamer.h"
 #include "AArch64MCAsmInfo.h"
+#include "AArch64MCLFIExpander.h"
 #include "AArch64WinCOFFStreamer.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "MCTargetDesc/AArch64InstPrinter.h"
@@ -518,6 +519,16 @@ static MCInstrAnalysis *createAArch64InstrAnalysis(const MCInstrInfo *Info) {
   return new AArch64MCInstrAnalysis(Info);
 }
 
+static MCLFIExpander *createAArch64MCLFIExpander(MCStreamer &S,
+    std::unique_ptr<MCRegisterInfo> &&RegInfo,
+    std::unique_ptr<MCInstrInfo> &&InstInfo) {
+  auto *Exp = new AArch64::AArch64MCLFIExpander(S.getContext(),
+                                                std::move(RegInfo),
+                                                std::move(InstInfo));
+  S.setLFIExpander(Exp);
+  return Exp;
+}
+
 // Force static initialization.
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64TargetMC() {
   for (Target *T : {&getTheAArch64leTarget(), &getTheAArch64beTarget(),
@@ -546,6 +557,8 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64TargetMC() {
     TargetRegistry::RegisterMachOStreamer(*T, createMachOStreamer);
     TargetRegistry::RegisterCOFFStreamer(*T, createWinCOFFStreamer);
 
+    TargetRegistry::RegisterMCLFIExpander(*T, createAArch64MCLFIExpander);
+
     // Register the obj target streamer.
     TargetRegistry::RegisterObjectTargetStreamer(
         *T, createAArch64ObjectTargetStreamer);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt
index 7f220657e45f8..bd144eaddc0f4 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/CMakeLists.txt
@@ -6,6 +6,7 @@ add_llvm_component_library(LLVMAArch64Desc
   AArch64MCAsmInfo.cpp
   AArch64MCCodeEmitter.cpp
   AArch64MCExpr.cpp
+  AArch64MCLFIExpander.cpp
   AArch64MCTargetDesc.cpp
   AArch64MachObjectWriter.cpp
   AArch64TargetStreamer.cpp

From 8c37241040c79c73b311d6b11d1305b96e952564 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 15:03:40 -0700
Subject: [PATCH 04/41] [LFI] Apply LFI rewriter to .s files in clang
 cc1as/llvm-mc

When assembling a .s file using clang or llvm-mc, automatically run the
LFI rewriter if targeting LFI.
---
 clang/tools/driver/cc1as_main.cpp |  3 +++
 llvm/tools/llvm-mc/llvm-mc.cpp    | 14 ++++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index 7fe97cc6e6ace..cb2f6bdb9c096 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -27,6 +27,7 @@
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCLFI.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
@@ -577,6 +578,8 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
     Str.reset(TheTarget->createMCObjectStreamer(
         T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI));
     Str.get()->initSections(Opts.NoExecStack, *STI);
+    if (T.isLFI())
+      initializeLFIMCStreamer(*Str.get(), Ctx, T);
     if (T.isOSBinFormatMachO() && T.isOSDarwin()) {
       Triple *TVT = Opts.DarwinTargetVariantTriple
                         ? &*Opts.DarwinTargetVariantTriple
diff --git a/llvm/tools/llvm-mc/llvm-mc.cpp b/llvm/tools/llvm-mc/llvm-mc.cpp
index 70f92d09aded7..6de991c6d1bba 100644
--- a/llvm/tools/llvm-mc/llvm-mc.cpp
+++ b/llvm/tools/llvm-mc/llvm-mc.cpp
@@ -18,6 +18,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCLFI.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCParser/AsmLexer.h"
@@ -551,6 +552,12 @@ int main(int argc, char **argv) {
     Str.reset(TheTarget->createAsmStreamer(Ctx, std::move(FOut), IP,
                                            std::move(CE), std::move(MAB)));
 
+    Triple T(TripleName);
+    if (T.isLFI()) {
+      Str->initSections(NoExecStack, *STI);
+      initializeLFIMCStreamer(*Str.get(), Ctx, T);
+    }
+
   } else if (FileType == OFT_Null) {
     Str.reset(TheTarget->createNullStreamer(Ctx));
   } else {
@@ -570,6 +577,13 @@ int main(int argc, char **argv) {
         std::unique_ptr<MCCodeEmitter>(CE), *STI));
     if (NoExecStack)
       Str->switchSection(Ctx.getAsmInfo()->getNonexecutableStackSection(Ctx));
+
+    Triple T(TripleName);
+    if (T.isLFI()) {
+      Str->initSections(NoExecStack, *STI);
+      initializeLFIMCStreamer(*Str.get(), Ctx, T);
+    }
+
     Str->emitVersionForTarget(TheTriple, VersionTuple(), nullptr,
                               VersionTuple());
   }

From 6f62734b8689b82f79ceaf35a637d486038aab86 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 15:24:08 -0700
Subject: [PATCH 05/41] [LFI] [AArch64] LFI rewrites for indirect branches

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 68 ++++++++++++++++++-
 llvm/test/MC/AArch64/LFI/branch.s             | 13 ++++
 2 files changed, 78 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/branch.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index a2decbe6db0e1..91588a4b46df3 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -28,18 +28,51 @@ using namespace llvm;
 
 #define DEBUG_TYPE "lfi"
 
+static MCRegister LFIAddrReg = AArch64::X28;
+static MCRegister LFIBaseReg = AArch64::X27;
+
 bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
-  return false;
+  return Reg != AArch64::SP;
+}
+
+static void emitAddMask(MCRegister Dest, MCRegister Src, MCStreamer &Out,
+                        const MCSubtargetInfo &STI) {
+  MCInst Add;
+  Add.setOpcode(AArch64::ADDXrx);
+  Add.addOperand(MCOperand::createReg(Dest));
+  Add.addOperand(MCOperand::createReg(LFIBaseReg));
+  Add.addOperand(MCOperand::createReg(getWRegFromXReg(Src)));
+  Add.addOperand(MCOperand::createImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, 0)));
+  Out.emitInstruction(Add, STI);;
+}
+
+static void emitBranch(unsigned int Opcode, MCRegister Target, MCStreamer &Out,
+                       const MCSubtargetInfo &STI) {
+  MCInst Branch;
+  Branch.setOpcode(Opcode);
+  Branch.addOperand(MCOperand::createReg(Target));
+  Out.emitInstruction(Branch, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
                                                   MCStreamer &Out,
                                                   const MCSubtargetInfo &STI,
-                                                  bool isCall) {
+                                                  bool IsCall) {
+  (void) IsCall;
+
+  assert(Inst.getOperand(0).isReg());
+  MCRegister BranchReg = Inst.getOperand(0).getReg();
+
+  emitAddMask(LFIAddrReg, BranchReg, Out, STI);
+  emitBranch(Inst.getOpcode(), LFIAddrReg, Out, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &Out,
                                         const MCSubtargetInfo &STI) {
+  if (Inst.getOperand(0).isReg())
+    expandIndirectBranch(Inst, Out, STI, true);
+  else
+    Out.emitInstruction(Inst, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
@@ -71,9 +104,38 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
 
 void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
                                           const MCSubtargetInfo &STI) {
+  if (isReturn(Inst))
+    return expandReturn(Inst, Out, STI);
+
+  if (isIndirectBranch(Inst))
+    return expandIndirectBranch(Inst, Out, STI, false);
+
+  if (isCall(Inst))
+    return expandCall(Inst, Out, STI);
+
+  if (isBranch(Inst))
+    return Out.emitInstruction(Inst, STI);
+
+  if (mayAffectControlFlow(Inst))
+    return expandControlFlow(Inst, Out, STI);
+
+  if (mayModifyStack(Inst))
+    return expandStackManipulation(Inst, Out, STI);
+
+  if (mayLoad(Inst) || mayStore(Inst))
+    return expandLoadStore(Inst, Out, STI);
+
+  return Out.emitInstruction(Inst, STI);
 }
 
 bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
                                         const MCSubtargetInfo &STI) {
-  return false;
+  if (Guard)
+    return false;
+  Guard = true;
+
+  doExpandInst(Inst, Out, STI);
+
+  Guard = false;
+  return true;
 }
diff --git a/llvm/test/MC/AArch64/LFI/branch.s b/llvm/test/MC/AArch64/LFI/branch.s
new file mode 100644
index 0000000000000..33e469e51aa20
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/branch.s
@@ -0,0 +1,13 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+foo:
+  b foo
+// CHECK: b foo
+
+  br x0
+// CHECK:      add x28, x27, w0, uxtw
+// CHECK-NEXT: br x28
+
+  blr x0
+// CHECK:      add x28, x27, w0, uxtw
+// CHECK-NEXT: blr x28

From 5151a9a8ece16dc7c9a6e1c9492eda8075b6bc56 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 16:48:30 -0700
Subject: [PATCH 06/41] [LFI] [AArch64] Rewrites for stack manipulation and
 returns

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 55 +++++++++++--------
 .../MCTargetDesc/AArch64MCLFIExpander.h       |  2 +
 llvm/test/MC/AArch64/LFI/branch.s             |  7 +++
 llvm/test/MC/AArch64/LFI/stack.s              | 37 +++++++++++++
 4 files changed, 79 insertions(+), 22 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/stack.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 91588a4b46df3..a79611dbe0bc8 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -30,6 +30,7 @@ using namespace llvm;
 
 static MCRegister LFIAddrReg = AArch64::X28;
 static MCRegister LFIBaseReg = AArch64::X27;
+static MCRegister LFIScratchReg = AArch64::X26;
 
 bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
   return Reg != AArch64::SP;
@@ -55,9 +56,9 @@ static void emitBranch(unsigned int Opcode, MCRegister Target, MCStreamer &Out,
 }
 
 void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
-                                                  MCStreamer &Out,
-                                                  const MCSubtargetInfo &STI,
-                                                  bool IsCall) {
+                                                         MCStreamer &Out,
+                                                         const MCSubtargetInfo &STI,
+                                                         bool IsCall) {
   (void) IsCall;
 
   assert(Inst.getOperand(0).isReg());
@@ -68,7 +69,7 @@ void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
 }
 
 void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &Out,
-                                        const MCSubtargetInfo &STI) {
+                                               const MCSubtargetInfo &STI) {
   if (Inst.getOperand(0).isReg())
     expandIndirectBranch(Inst, Out, STI, true);
   else
@@ -76,34 +77,47 @@ void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &O
 }
 
 void AArch64::AArch64MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
-                                          const MCSubtargetInfo &STI) {
+                                                 const MCSubtargetInfo &STI) {
+  assert(Inst.getOperand(0).isReg());
+  if (Inst.getOperand(0).getReg() != AArch64::LR)
+    expandIndirectBranch(Inst, Out, STI, false);
+  else
+    Out.emitInstruction(Inst, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandControlFlow(const MCInst &Inst,
-                                               MCStreamer &Out,
-                                               const MCSubtargetInfo &STI) {
-
+bool AArch64::AArch64MCLFIExpander::mayModifyStack(const MCInst &Inst) {
+  return mayModifyRegister(Inst, AArch64::SP);
 }
 
-bool AArch64::AArch64MCLFIExpander::mayModifyStack(const MCInst &Inst) {
-  return false;
+bool AArch64::AArch64MCLFIExpander::mayModifyReserved(const MCInst &Inst) {
+  return mayModifyRegister(Inst, LFIAddrReg) || mayModifyRegister(Inst, LFIBaseReg);
 }
 
 void AArch64::AArch64MCLFIExpander::expandStackManipulation(
     const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
-}
-
-void AArch64::AArch64MCLFIExpander::expandPrefetch(const MCInst &Inst, MCStreamer &Out,
-                                            const MCSubtargetInfo &STI) {
+  if (mayLoad(Inst) || mayStore(Inst)) {
+    return Out.emitInstruction(Inst, STI);
+  }
+
+  MCInst ModInst;
+  MCRegister Scratch = LFIScratchReg;
+  assert(Inst.getOperand(0).isReg() && Inst.getOperand(0).getReg() == AArch64::SP);
+  ModInst.setOpcode(Inst.getOpcode());
+  ModInst.addOperand(MCOperand::createReg(Scratch));
+  for (unsigned I = 1, E = Inst.getNumOperands(); I != E; ++I) {
+    ModInst.addOperand(Inst.getOperand(I));
+  }
+  Out.emitInstruction(ModInst, STI);
+  emitAddMask(AArch64::SP, Scratch, Out, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
-                                             MCStreamer &Out,
-                                             const MCSubtargetInfo &STI) {
+                                                    MCStreamer &Out,
+                                                    const MCSubtargetInfo &STI) {
 }
 
 void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
-                                          const MCSubtargetInfo &STI) {
+                                                 const MCSubtargetInfo &STI) {
   if (isReturn(Inst))
     return expandReturn(Inst, Out, STI);
 
@@ -116,9 +130,6 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
   if (isBranch(Inst))
     return Out.emitInstruction(Inst, STI);
 
-  if (mayAffectControlFlow(Inst))
-    return expandControlFlow(Inst, Out, STI);
-
   if (mayModifyStack(Inst))
     return expandStackManipulation(Inst, Out, STI);
 
@@ -129,7 +140,7 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
 }
 
 bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
-                                        const MCSubtargetInfo &STI) {
+                                               const MCSubtargetInfo &STI) {
   if (Guard)
     return false;
   Guard = true;
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index 194fa36eb9c2b..00e547c9b3925 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -52,6 +52,8 @@ class AArch64MCLFIExpander : public MCLFIExpander {
 
   bool mayModifyStack(const MCInst &Inst);
 
+  bool mayModifyReserved(const MCInst &Inst);
+
   void expandControlFlow(const MCInst &Inst, MCStreamer &Out,
                          const MCSubtargetInfo &STI);
 
diff --git a/llvm/test/MC/AArch64/LFI/branch.s b/llvm/test/MC/AArch64/LFI/branch.s
index 33e469e51aa20..e2c126cf24db7 100644
--- a/llvm/test/MC/AArch64/LFI/branch.s
+++ b/llvm/test/MC/AArch64/LFI/branch.s
@@ -11,3 +11,10 @@ foo:
   blr x0
 // CHECK:      add x28, x27, w0, uxtw
 // CHECK-NEXT: blr x28
+
+  ret
+// CHECK: ret
+
+  ret x0
+// CHECK:      add x28, x27, w0, uxtw
+// CHECK-NEXT: ret x28
diff --git a/llvm/test/MC/AArch64/LFI/stack.s b/llvm/test/MC/AArch64/LFI/stack.s
new file mode 100644
index 0000000000000..349a85677cee5
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/stack.s
@@ -0,0 +1,37 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+ldr x0, [sp, #16]!
+// CHECK: ldr x0, [sp, #16]!
+
+ldr x0, [sp], #16
+// CHECK: ldr x0, [sp], #16
+
+str x0, [sp, #16]!
+// CHECK: str x0, [sp, #16]!
+
+str x0, [sp], #16
+// CHECK: str x0, [sp], #16
+
+mov sp, x0
+// CHECK:      add x26, x0, #0
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+add sp, sp, #8
+// CHECK:      add x26, sp, #8
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+sub sp, sp, #8
+// CHECK:      sub x26, sp, #8
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+add sp, sp, x0
+// CHECK:      add x26, sp, x0
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+sub sp, sp, x0
+// CHECK:      sub x26, sp, x0
+// CHECK-NEXT: add sp, x27, w26, uxtw
+
+sub sp, sp, #1, lsl #12
+// CHECK:      sub x26, sp, #1, lsl #12
+// CHECK-NEXT: add sp, x27, w26, uxtw

From 003d56da02e2e311a844bb755da41b257465f402 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Tue, 19 Aug 2025 23:04:10 -0700
Subject: [PATCH 07/41] [LFI] [AArch64] Rewrites for syscalls and TLS

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 128 +++++++++++++++++-
 .../MCTargetDesc/AArch64MCLFIExpander.h       |  16 ++-
 llvm/test/MC/AArch64/LFI/sys.s                |  41 ++++++
 3 files changed, 182 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/sys.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index a79611dbe0bc8..95c787ecae606 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -93,9 +93,20 @@ bool AArch64::AArch64MCLFIExpander::mayModifyReserved(const MCInst &Inst) {
   return mayModifyRegister(Inst, LFIAddrReg) || mayModifyRegister(Inst, LFIBaseReg);
 }
 
-void AArch64::AArch64MCLFIExpander::expandStackManipulation(
+bool AArch64::AArch64MCLFIExpander::mayModifyLR(const MCInst &Inst) {
+  return mayModifyRegister(Inst, AArch64::LR);
+}
+
+// Rewrites for modifications of "special" registers: x28, x27, lr.
+void AArch64::AArch64MCLFIExpander::expandSpecialModification(
+    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
+}
+
+void AArch64::AArch64MCLFIExpander::expandStackModification(
     const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
   if (mayLoad(Inst) || mayStore(Inst)) {
+    if (mayModifyReserved(Inst) || mayModifyLR(Inst))
+      return expandSpecialModification(Inst, Out, STI);
     return Out.emitInstruction(Inst, STI);
   }
 
@@ -116,8 +127,118 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     const MCSubtargetInfo &STI) {
 }
 
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs,
+    int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rs));
+  Inst.addOperand(MCOperand::createImm(Imm));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
+    MCRegister Rt2, int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rt1));
+  Inst.addOperand(MCOperand::createReg(Rt2));
+  Inst.addOperand(MCOperand::createImm(Imm));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emit(unsigned int Op, MCRegister Reg, MCStreamer &Out,
+    const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emit(AArch64::ORRXrs, Dest, AArch64::XZR, Src, 0, Out, STI);
+}
+
+enum LFICallType {
+  LFISyscall,
+  LFITLSRead,
+  LFITLSWrite,
+};
+
+static void emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emitMov(LFIScratchReg, AArch64::LR, Out, STI);
+  unsigned Offset;
+  switch (CallType) {
+    case LFISyscall: Offset = 0; break;
+    case LFITLSRead: Offset = 1; break;
+    case LFITLSWrite: Offset = 2; break;
+  }
+  emit(AArch64::LDRXui, AArch64::LR, LFIBaseReg, Offset, Out, STI);
+  emit(AArch64::BLR, AArch64::LR, Out, STI);
+  emitAddMask(AArch64::LR, LFIScratchReg, Out, STI);
+}
+
+void AArch64::AArch64MCLFIExpander::expandSyscall(const MCInst &Inst, MCStreamer &Out,
+                   const MCSubtargetInfo &STI) {
+  emitLFICall(LFISyscall, Out, STI);
+}
+
+static void emitSwap(MCRegister Reg1, MCRegister Reg2, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emit(AArch64::EORXrs, Reg1, Reg1, Reg2, 0, Out, STI);
+  emit(AArch64::EORXrs, Reg2, Reg1, Reg2, 0, Out, STI);
+  emit(AArch64::EORXrs, Reg1, Reg1, Reg2, 0, Out, STI);
+}
+
+void AArch64::AArch64MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer &Out,
+                   const MCSubtargetInfo &STI) {
+  MCRegister Reg = Inst.getOperand(0).getReg();
+  if (Reg == AArch64::X0) {
+    emitLFICall(LFITLSRead, Out, STI);
+  } else {
+    emitMov(Reg, AArch64::X0, Out, STI);
+    emitLFICall(LFITLSRead, Out, STI);
+    emitSwap(AArch64::X0, Reg, Out, STI);
+  }
+}
+
+void AArch64::AArch64MCLFIExpander::expandTLSWrite(const MCInst &Inst, MCStreamer &Out,
+                    const MCSubtargetInfo &STI) {
+  MCRegister Reg = Inst.getOperand(1).getReg();
+  if (Reg == AArch64::X0) {
+    emitLFICall(LFITLSWrite, Out, STI);
+  } else {
+    emitSwap(Reg, AArch64::X0, Out, STI);
+    emitLFICall(LFITLSWrite, Out, STI);
+    emitSwap(AArch64::X0, Reg, Out, STI);
+  }
+}
+
+static bool isSyscall(const MCInst &Inst) {
+  return Inst.getOpcode() == AArch64::SVC;
+}
+
+static bool isTLSRead(const MCInst &Inst) {
+  return Inst.getOpcode() == AArch64::MRS &&
+    Inst.getOperand(1).getReg() == AArch64SysReg::TPIDR_EL0;
+}
+
+static bool isTLSWrite(const MCInst &Inst) {
+  return Inst.getOpcode() == AArch64::MSR &&
+    Inst.getOperand(0).getReg() == AArch64SysReg::TPIDR_EL0;
+}
+
 void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
                                                  const MCSubtargetInfo &STI) {
+  if (isSyscall(Inst))
+    return expandSyscall(Inst, Out, STI);
+
+  if (isTLSRead(Inst))
+    return expandTLSRead(Inst, Out, STI);
+
+  if (isTLSWrite(Inst))
+    return expandTLSWrite(Inst, Out, STI);
+
   if (isReturn(Inst))
     return expandReturn(Inst, Out, STI);
 
@@ -131,7 +252,10 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
     return Out.emitInstruction(Inst, STI);
 
   if (mayModifyStack(Inst))
-    return expandStackManipulation(Inst, Out, STI);
+    return expandStackModification(Inst, Out, STI);
+
+  if (mayModifyReserved(Inst) || mayModifyLR(Inst))
+    return expandSpecialModification(Inst, Out, STI);
 
   if (mayLoad(Inst) || mayStore(Inst))
     return expandLoadStore(Inst, Out, STI);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index 00e547c9b3925..1d2869c6095ac 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -54,18 +54,32 @@ class AArch64MCLFIExpander : public MCLFIExpander {
 
   bool mayModifyReserved(const MCInst &Inst);
 
+  bool mayModifyLR(const MCInst &Inst);
+
   void expandControlFlow(const MCInst &Inst, MCStreamer &Out,
                          const MCSubtargetInfo &STI);
 
-  void expandStackManipulation(const MCInst &Inst, MCStreamer &Out,
+  void expandStackModification(const MCInst &Inst, MCStreamer &Out,
                                const MCSubtargetInfo &STI);
 
+  void expandSpecialModification(const MCInst &Inst, MCStreamer &Out,
+                                 const MCSubtargetInfo &STI);
+
   void expandPrefetch(const MCInst &Inst, MCStreamer &Out,
                       const MCSubtargetInfo &STI);
 
   void expandLoadStore(const MCInst &Inst, MCStreamer &Out,
                        const MCSubtargetInfo &STI);
 
+  void expandSyscall(const MCInst &Inst, MCStreamer &Out,
+                     const MCSubtargetInfo &STI);
+
+  void expandTLSRead(const MCInst &Inst, MCStreamer &Out,
+                     const MCSubtargetInfo &STI);
+
+  void expandTLSWrite(const MCInst &Inst, MCStreamer &Out,
+                      const MCSubtargetInfo &STI);
+
   void doExpandInst(const MCInst &Inst, MCStreamer &Out,
                     const MCSubtargetInfo &STI);
 };
diff --git a/llvm/test/MC/AArch64/LFI/sys.s b/llvm/test/MC/AArch64/LFI/sys.s
new file mode 100644
index 0000000000000..679f1512191e0
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/sys.s
@@ -0,0 +1,41 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+svc #0
+// CHECK:      mov x26, x30
+// CHECK-NEXT: ldr x30, [x27]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+msr tpidr_el0, x0
+// CHECK:      mov x26, x30
+// CHECK-NEXT: ldr x30, [x27, #16]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+msr tpidr_el0, x1
+// CHECK:      eor x1, x1, x0
+// CHECK-NEXT: eor x0, x1, x0
+// CHECK-NEXT: eor x1, x1, x0
+// CHECK-NEXT: mov x26, x30
+// CHECK-NEXT: ldr x30, [x27, #16]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+// CHECK-NEXT: eor x0, x0, x1
+// CHECK-NEXT: eor x1, x0, x1
+// CHECK-NEXT: eor x0, x0, x1
+
+mrs x0, tpidr_el0
+// CHECK:      mov x26, x30
+// CHECK-NEXT: ldr x30, [x27, #8]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+mrs x1, tpidr_el0
+// CHECK:      mov x1, x0
+// CHECK-NEXT: mov x26, x30
+// CHECK-NEXT: ldr x30, [x27, #8]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+// CHECK-NEXT: eor x0, x0, x1
+// CHECK-NEXT: eor x1, x0, x1
+// CHECK-NEXT: eor x0, x0, x1

From a3c01617a1c04ad4129e6de71030a680cd2de760 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Wed, 20 Aug 2025 11:56:01 -0700
Subject: [PATCH 08/41] [LFI] [AArch64] Use .scratch register if available

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 101 +++++++++---------
 .../MCTargetDesc/AArch64MCLFIExpander.h       |  11 ++
 llvm/test/MC/AArch64/LFI/stack.s              |   6 ++
 llvm/test/MC/AArch64/LFI/sys.s                |   8 ++
 4 files changed, 76 insertions(+), 50 deletions(-)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 95c787ecae606..9ab22122de3fb 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -36,15 +36,54 @@ bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const
   return Reg != AArch64::SP;
 }
 
+MCRegister AArch64::AArch64MCLFIExpander::getScratch() {
+  if (numScratchRegs() == 0) {
+    return LFIScratchReg;
+  }
+  return getScratchReg(0);
+}
+
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs,
+    int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rs));
+  Inst.addOperand(MCOperand::createImm(Imm));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
+    MCRegister Rt2, int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rt1));
+  Inst.addOperand(MCOperand::createReg(Rt2));
+  Inst.addOperand(MCOperand::createImm(Imm));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emit(unsigned int Op, MCRegister Reg, MCStreamer &Out,
+    const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Reg));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emit(AArch64::ORRXrs, Dest, AArch64::XZR, Src, 0, Out, STI);
+}
+
 static void emitAddMask(MCRegister Dest, MCRegister Src, MCStreamer &Out,
                         const MCSubtargetInfo &STI) {
-  MCInst Add;
-  Add.setOpcode(AArch64::ADDXrx);
-  Add.addOperand(MCOperand::createReg(Dest));
-  Add.addOperand(MCOperand::createReg(LFIBaseReg));
-  Add.addOperand(MCOperand::createReg(getWRegFromXReg(Src)));
-  Add.addOperand(MCOperand::createImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, 0)));
-  Out.emitInstruction(Add, STI);;
+  emit(AArch64::ADDXrx,
+      Dest,
+      LFIBaseReg,
+      getWRegFromXReg(Src),
+      AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, 0),
+      Out, STI);
 }
 
 static void emitBranch(unsigned int Opcode, MCRegister Target, MCStreamer &Out,
@@ -111,7 +150,7 @@ void AArch64::AArch64MCLFIExpander::expandStackModification(
   }
 
   MCInst ModInst;
-  MCRegister Scratch = LFIScratchReg;
+  MCRegister Scratch = getScratch();
   assert(Inst.getOperand(0).isReg() && Inst.getOperand(0).getReg() == AArch64::SP);
   ModInst.setOpcode(Inst.getOpcode());
   ModInst.addOperand(MCOperand::createReg(Scratch));
@@ -127,47 +166,9 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     const MCSubtargetInfo &STI) {
 }
 
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs,
-    int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
-  MCInst Inst;
-  Inst.setOpcode(Op);
-  Inst.addOperand(MCOperand::createReg(Rd));
-  Inst.addOperand(MCOperand::createReg(Rs));
-  Inst.addOperand(MCOperand::createImm(Imm));
-  Out.emitInstruction(Inst, STI);
-}
-
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
-    MCRegister Rt2, int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
-  MCInst Inst;
-  Inst.setOpcode(Op);
-  Inst.addOperand(MCOperand::createReg(Rd));
-  Inst.addOperand(MCOperand::createReg(Rt1));
-  Inst.addOperand(MCOperand::createReg(Rt2));
-  Inst.addOperand(MCOperand::createImm(Imm));
-  Out.emitInstruction(Inst, STI);
-}
-
-static void emit(unsigned int Op, MCRegister Reg, MCStreamer &Out,
-    const MCSubtargetInfo &STI) {
-  MCInst Inst;
-  Inst.setOpcode(Op);
-  Inst.addOperand(MCOperand::createReg(Reg));
-  Out.emitInstruction(Inst, STI);
-}
-
-static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out, const MCSubtargetInfo &STI) {
-  emit(AArch64::ORRXrs, Dest, AArch64::XZR, Src, 0, Out, STI);
-}
-
-enum LFICallType {
-  LFISyscall,
-  LFITLSRead,
-  LFITLSWrite,
-};
-
-static void emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
-  emitMov(LFIScratchReg, AArch64::LR, Out, STI);
+void AArch64::AArch64MCLFIExpander::emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCRegister Scratch = getScratch();
+  emitMov(Scratch, AArch64::LR, Out, STI);
   unsigned Offset;
   switch (CallType) {
     case LFISyscall: Offset = 0; break;
@@ -176,7 +177,7 @@ static void emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtarget
   }
   emit(AArch64::LDRXui, AArch64::LR, LFIBaseReg, Offset, Out, STI);
   emit(AArch64::BLR, AArch64::LR, Out, STI);
-  emitAddMask(AArch64::LR, LFIScratchReg, Out, STI);
+  emitAddMask(AArch64::LR, Scratch, Out, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandSyscall(const MCInst &Inst, MCStreamer &Out,
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index 1d2869c6095ac..e5529f3a58ee8 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -41,6 +41,8 @@ class AArch64MCLFIExpander : public MCLFIExpander {
 private:
   bool Guard = false; // recursion guard
 
+  MCRegister getScratch();
+
   void expandIndirectBranch(const MCInst &Inst, MCStreamer &Out,
                             const MCSubtargetInfo &STI, bool isCall);
 
@@ -82,6 +84,15 @@ class AArch64MCLFIExpander : public MCLFIExpander {
 
   void doExpandInst(const MCInst &Inst, MCStreamer &Out,
                     const MCSubtargetInfo &STI);
+
+  enum LFICallType {
+    LFISyscall,
+    LFITLSRead,
+    LFITLSWrite,
+  };
+
+  void emitLFICall(LFICallType CallType, MCStreamer &Out,
+                   const MCSubtargetInfo &STI);
 };
 } // namespace AArch64
 } // namespace llvm
diff --git a/llvm/test/MC/AArch64/LFI/stack.s b/llvm/test/MC/AArch64/LFI/stack.s
index 349a85677cee5..bae875071dc72 100644
--- a/llvm/test/MC/AArch64/LFI/stack.s
+++ b/llvm/test/MC/AArch64/LFI/stack.s
@@ -35,3 +35,9 @@ sub sp, sp, x0
 sub sp, sp, #1, lsl #12
 // CHECK:      sub x26, sp, #1, lsl #12
 // CHECK-NEXT: add sp, x27, w26, uxtw
+
+.scratch x10
+add sp, sp, x0
+// CHECK:      add x10, sp, x0
+// CHECK-NEXT: add sp, x27, w10, uxtw
+.scratch_clear
diff --git a/llvm/test/MC/AArch64/LFI/sys.s b/llvm/test/MC/AArch64/LFI/sys.s
index 679f1512191e0..6f54fcad754d6 100644
--- a/llvm/test/MC/AArch64/LFI/sys.s
+++ b/llvm/test/MC/AArch64/LFI/sys.s
@@ -6,6 +6,14 @@ svc #0
 // CHECK-NEXT: blr x30
 // CHECK-NEXT: add x30, x27, w26, uxtw
 
+.scratch x10
+svc #0
+// CHECK:      mov x10, x30
+// CHECK-NEXT: ldr x30, [x27]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w10, uxtw
+.scratch_clear
+
 msr tpidr_el0, x0
 // CHECK:      mov x26, x30
 // CHECK-NEXT: ldr x30, [x27, #16]

From 006dfb698da71ffcb9007a1483168799ca71bbd1 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Wed, 20 Aug 2025 13:24:36 -0700
Subject: [PATCH 09/41] [LFI] [AArch64] Rewrites for LR modifications

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 38 ++++++++++++++++---
 .../MCTargetDesc/AArch64MCLFIExpander.h       |  2 +-
 llvm/test/MC/AArch64/LFI/reserved.s           | 12 ++++++
 llvm/test/MC/AArch64/LFI/return.s             | 19 ++++++++++
 4 files changed, 64 insertions(+), 7 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/reserved.s
 create mode 100644 llvm/test/MC/AArch64/LFI/return.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 9ab22122de3fb..c81d1fa340bc9 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -136,16 +136,32 @@ bool AArch64::AArch64MCLFIExpander::mayModifyLR(const MCInst &Inst) {
   return mayModifyRegister(Inst, AArch64::LR);
 }
 
-// Rewrites for modifications of "special" registers: x28, x27, lr.
-void AArch64::AArch64MCLFIExpander::expandSpecialModification(
+void AArch64::AArch64MCLFIExpander::expandLRModification(
     const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCRegister Scratch = getScratch();
+  MCInst New;
+  New.setLoc(Inst.getLoc());
+  New.setOpcode(Inst.getOpcode());
+  for (unsigned I = 0; I < Inst.getNumOperands(); ++I) {
+    const MCOperand &Op = Inst.getOperand(I);
+    if (Op.isReg() && Op.getReg() == AArch64::LR) {
+      New.addOperand(MCOperand::createReg(Scratch));
+    } else {
+      New.addOperand(Op);
+    }
+  }
+  if (mayLoad(New) || mayStore(New))
+    expandLoadStore(New, Out, STI);
+  else
+    Out.emitInstruction(New, STI);
+  emitAddMask(AArch64::LR, Scratch, Out, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::expandStackModification(
     const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
   if (mayLoad(Inst) || mayStore(Inst)) {
-    if (mayModifyReserved(Inst) || mayModifyLR(Inst))
-      return expandSpecialModification(Inst, Out, STI);
+    if (mayModifyLR(Inst))
+      return expandLRModification(Inst, Out, STI);
     return Out.emitInstruction(Inst, STI);
   }
 
@@ -153,6 +169,7 @@ void AArch64::AArch64MCLFIExpander::expandStackModification(
   MCRegister Scratch = getScratch();
   assert(Inst.getOperand(0).isReg() && Inst.getOperand(0).getReg() == AArch64::SP);
   ModInst.setOpcode(Inst.getOpcode());
+  ModInst.setLoc(Inst.getLoc());
   ModInst.addOperand(MCOperand::createReg(Scratch));
   for (unsigned I = 1, E = Inst.getNumOperands(); I != E; ++I) {
     ModInst.addOperand(Inst.getOperand(I));
@@ -164,6 +181,9 @@ void AArch64::AArch64MCLFIExpander::expandStackModification(
 void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     MCStreamer &Out,
                                                     const MCSubtargetInfo &STI) {
+  Out.getContext().reportWarning(
+      Inst.getLoc(), "TODO: expandLoadStore");
+  Out.emitInstruction(Inst, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
@@ -252,11 +272,17 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
   if (isBranch(Inst))
     return Out.emitInstruction(Inst, STI);
 
+  // Bail out with an error. In the future, we could consider automatically
+  // rewriting uses of reserved LFI registers.
+  if (mayModifyReserved(Inst))
+    return Out.getContext().reportError(
+        Inst.getLoc(), "illegal modification of reserved LFI register");
+
   if (mayModifyStack(Inst))
     return expandStackModification(Inst, Out, STI);
 
-  if (mayModifyReserved(Inst) || mayModifyLR(Inst))
-    return expandSpecialModification(Inst, Out, STI);
+  if (mayModifyLR(Inst))
+    return expandLRModification(Inst, Out, STI);
 
   if (mayLoad(Inst) || mayStore(Inst))
     return expandLoadStore(Inst, Out, STI);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index e5529f3a58ee8..0ca366566df57 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -64,7 +64,7 @@ class AArch64MCLFIExpander : public MCLFIExpander {
   void expandStackModification(const MCInst &Inst, MCStreamer &Out,
                                const MCSubtargetInfo &STI);
 
-  void expandSpecialModification(const MCInst &Inst, MCStreamer &Out,
+  void expandLRModification(const MCInst &Inst, MCStreamer &Out,
                                  const MCSubtargetInfo &STI);
 
   void expandPrefetch(const MCInst &Inst, MCStreamer &Out,
diff --git a/llvm/test/MC/AArch64/LFI/reserved.s b/llvm/test/MC/AArch64/LFI/reserved.s
new file mode 100644
index 0000000000000..7b9ffb83474c0
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/reserved.s
@@ -0,0 +1,12 @@
+// RUN: not llvm-mc -filetype asm -triple aarch64_lfi %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-ERROR %s <%t
+
+mov x28, x0
+// CHECK-ERROR: error: illegal modification of reserved LFI register
+// CHECK-ERROR:        mov x28, x0
+// CHECK-ERROR:        ^
+
+ldr x27, [sp]
+// CHECK-ERROR: error: illegal modification of reserved LFI register
+// CHECK-ERROR:        ldr x27, [sp]
+// CHECK-ERROR:        ^
diff --git a/llvm/test/MC/AArch64/LFI/return.s b/llvm/test/MC/AArch64/LFI/return.s
new file mode 100644
index 0000000000000..1295626134969
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/return.s
@@ -0,0 +1,19 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+mov x30, x0
+// CHECK:      mov x26, x0
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+ldr x30, [sp]
+// CHECK:      ldr x26, [sp]
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+ldp x29, x30, [sp]
+// CHECK:      ldp x29, x26, [sp]
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+.scratch x10
+ldr x30, [sp]
+// CHECK:      ldr x10, [sp]
+// CHECK-NEXT: add x30, x27, w10, uxtw
+.scratch_clear

From 5e4cc704cf0b8fc873bd8e1ad8972e9e5c6969d8 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Wed, 20 Aug 2025 15:52:52 -0700
Subject: [PATCH 10/41] [LFI] [AArch64] Rewrites for loads/stores

Doesn't handle basic pre/post-indexed loads/stores yet.
---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 458 +++++++++-
 .../MCTargetDesc/AArch64MCLFIExpander.h       |   7 +
 .../Target/AArch64/Utils/AArch64BaseInfo.h    | 811 ++++++++++++++++++
 llvm/test/MC/AArch64/LFI/mem.s                | 106 +++
 4 files changed, 1379 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/mem.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index c81d1fa340bc9..dc8ad0039ebad 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -64,6 +64,31 @@ static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
   Out.emitInstruction(Inst, STI);
 }
 
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
+    MCRegister Rt2, int64_t Imm1, int64_t Imm2,
+    MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rt1));
+  Inst.addOperand(MCOperand::createReg(Rt2));
+  Inst.addOperand(MCOperand::createImm(Imm1));
+  Inst.addOperand(MCOperand::createImm(Imm2));
+  Out.emitInstruction(Inst, STI);
+}
+
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt,
+    int64_t Imm1, int64_t Imm2,
+    MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst Inst;
+  Inst.setOpcode(Op);
+  Inst.addOperand(MCOperand::createReg(Rd));
+  Inst.addOperand(MCOperand::createReg(Rt));
+  Inst.addOperand(MCOperand::createImm(Imm1));
+  Inst.addOperand(MCOperand::createImm(Imm2));
+  Out.emitInstruction(Inst, STI);
+}
+
 static void emit(unsigned int Op, MCRegister Reg, MCStreamer &Out,
     const MCSubtargetInfo &STI) {
   MCInst Inst;
@@ -76,6 +101,7 @@ static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out, const MCSu
   emit(AArch64::ORRXrs, Dest, AArch64::XZR, Src, 0, Out, STI);
 }
 
+// Emit 'add Dest, LFIBaseReg, W(Src), uxtw'
 static void emitAddMask(MCRegister Dest, MCRegister Src, MCStreamer &Out,
                         const MCSubtargetInfo &STI) {
   emit(AArch64::ADDXrx,
@@ -86,6 +112,11 @@ static void emitAddMask(MCRegister Dest, MCRegister Src, MCStreamer &Out,
       Out, STI);
 }
 
+// Emit 'Op(ld/st) Dest, [LFIBaseReg, W(Target), uxtw]'
+static void emitMemMask(unsigned Op, MCRegister Dest, MCRegister Target, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emit(Op, Dest, LFIBaseReg, getWRegFromXReg(Target), 0, 0, Out, STI);
+}
+
 static void emitBranch(unsigned int Opcode, MCRegister Target, MCStreamer &Out,
                        const MCSubtargetInfo &STI) {
   MCInst Branch;
@@ -178,12 +209,124 @@ void AArch64::AArch64MCLFIExpander::expandStackModification(
   emitAddMask(AArch64::SP, Scratch, Out, STI);
 }
 
+static bool canConvertToRoW(unsigned Op);
+static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift);
+static unsigned convertRoWToRoW(unsigned Op, unsigned &Shift);
+static unsigned convertUiToRoW(unsigned Op);
+static unsigned convertPreToRoW(unsigned Op);
+static unsigned convertPostToRoW(unsigned Op);
+
+static void emitSafeLoadStore(const MCInst &Inst, unsigned N, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst LoadStore;
+  LoadStore.setOpcode(Inst.getOpcode());
+  for (unsigned I = 0; I < N; ++I)
+    LoadStore.addOperand(Inst.getOperand(I));
+  LoadStore.addOperand(MCOperand::createReg(LFIAddrReg));
+  for (unsigned I = N + 1; I < Inst.getNumOperands(); ++I)
+    LoadStore.addOperand(Inst.getOperand(I));
+  Out.emitInstruction(LoadStore, STI);
+}
+
+void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, MemInstInfo &MII,
+    MCStreamer &Out, const MCSubtargetInfo &STI) {
+  emitAddMask(LFIAddrReg, Inst.getOperand(MII.BaseRegIdx).getReg(), Out, STI);
+
+  if (MII.IsPrePost) {
+    assert(OffsetIdx != -1 && "Pre/Post must have valid OffsetIdx");
+    return Out.getContext().reportWarning(
+        Inst.getLoc(), "TODO: pre/post index");
+  }
+
+  return emitSafeLoadStore(Inst, MII.BaseRegIdx, Out, STI);
+}
+
+void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemInstInfo &MII,
+    MCStreamer &Out, const MCSubtargetInfo &STI) {
+  unsigned MemOp;
+  unsigned Op = Inst.getOpcode();
+  if ((MemOp = convertUiToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) {
+    auto OffsetMCO = Inst.getOperand(2);
+    if (OffsetMCO.isImm() && OffsetMCO.getImm() == 0)
+      return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Inst.getOperand(1).getReg(), Out, STI);
+    return expandLoadStoreBasic(Inst, MII, Out, STI);
+  }
+
+  if ((MemOp = convertPreToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) {
+    MCRegister Reg = Inst.getOperand(2).getReg();
+    int64_t Imm = Inst.getOperand(3).getImm();
+    if (Imm >= 0)
+      emit(AArch64::ADDXri, Reg, Reg, Imm, 0, Out, STI);
+    else
+      emit(AArch64::SUBXri, Reg, Reg, -Imm, 0, Out, STI);
+    return emitMemMask(MemOp, Inst.getOperand(1).getReg(), Reg, Out, STI);
+  }
+
+  if ((MemOp = convertPostToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) {
+    MCRegister Reg = Inst.getOperand(2).getReg();
+    emitMemMask(MemOp, Inst.getOperand(1).getReg(), Reg, Out, STI);
+    int64_t Imm = Inst.getOperand(3).getImm();
+    if (Imm >= 0)
+      emit(AArch64::ADDXri, Reg, Reg, Imm, 0, Out, STI);
+    else
+      emit(AArch64::SUBXri, Reg, Reg, -Imm, 0, Out, STI);
+    return;
+  }
+
+  unsigned Shift;
+  if ((MemOp = convertRoXToRoW(Op, Shift)) != AArch64::INSTRUCTION_LIST_END) {
+    MCRegister Reg1 = Inst.getOperand(1).getReg();
+    MCRegister Reg2 = Inst.getOperand(2).getReg();
+    int64_t Extend = Inst.getOperand(3).getImm();
+    int64_t IsShift = Inst.getOperand(4).getImm();
+    MCRegister Scratch = getScratch();
+    if (!IsShift)
+      Shift = 0;
+    if (Extend)
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::SXTX, Shift), Out, STI);
+    else
+      emit(AArch64::ADDXrs, Scratch, Reg1, Reg2, AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift), Out, STI);
+    return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Scratch, Out, STI);
+  }
+
+  if ((MemOp = convertRoWToRoW(Op, Shift)) != AArch64::INSTRUCTION_LIST_END) {
+    MCRegister Reg1 = Inst.getOperand(1).getReg();
+    MCRegister Reg2 = Inst.getOperand(2).getReg();
+    int64_t S = Inst.getOperand(3).getImm();
+    int64_t IsShift = Inst.getOperand(4).getImm();
+    MCRegister Scratch = getScratch();
+    if (!IsShift)
+      Shift = 0;
+    if (S)
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::SXTW, Shift), Out, STI);
+    else
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, Shift), Out, STI);
+    return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Scratch, Out, STI);
+  }
+}
+
 void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     MCStreamer &Out,
                                                     const MCSubtargetInfo &STI) {
-  Out.getContext().reportWarning(
-      Inst.getLoc(), "TODO: expandLoadStore");
-  Out.emitInstruction(Inst, STI);
+  auto MII = getLoadInfo(Inst);
+  if (!MII.has_value()) {
+    MII = getStoreInfo(Inst);
+    if (!MII.has_value())
+      return Out.getContext().reportError(
+          Inst.getLoc(), "this load/store is not supported by LFI");
+  }
+
+  // Stack accesses without a register offset don't need rewriting.
+  if (Inst.getOperand(MII->BaseRegIdx).getReg() == AArch64::SP) {
+    if (MII->BaseRegIdx == (int) Inst.getNumOperands() - 1 ||
+        !Inst.getOperand(MII->BaseRegIdx + 1).isReg())
+      return Out.emitInstruction(Inst, STI);
+  }
+
+  // Try to convert to RoW if we can, otherwise use fallback.
+  if (canConvertToRoW(Inst.getOpcode()))
+    expandLoadStoreRoW(Inst, MII.value(), Out, STI);
+  else
+    expandLoadStoreBasic(Inst, MII.value(), Out, STI);
 }
 
 void AArch64::AArch64MCLFIExpander::emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
@@ -301,3 +444,312 @@ bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &O
   Guard = false;
   return true;
 }
+
+static unsigned convertRoXToRoW(unsigned Op, unsigned &Shift) {
+  Shift = 0;
+  switch (Op) {
+  case AArch64::LDRBBroX:
+    return AArch64::LDRBBroW;
+  case AArch64::LDRBroX:
+    return AArch64::LDRBroW;
+  case AArch64::LDRDroX:
+    Shift = 3;
+    return AArch64::LDRDroW;
+  case AArch64::LDRHHroX:
+    Shift = 1;
+    return AArch64::LDRHHroW;
+  case AArch64::LDRHroX:
+    Shift = 1;
+    return AArch64::LDRHroW;
+  case AArch64::LDRQroX:
+    Shift = 4;
+    return AArch64::LDRQroW;
+  case AArch64::LDRSBWroX:
+    Shift = 1;
+    return AArch64::LDRSBWroW;
+  case AArch64::LDRSBXroX:
+    Shift = 1;
+    return AArch64::LDRSBXroW;
+  case AArch64::LDRSHWroX:
+    Shift = 1;
+    return AArch64::LDRSHWroW;
+  case AArch64::LDRSHXroX:
+    Shift = 1;
+    return AArch64::LDRSHXroW;
+  case AArch64::LDRSWroX:
+    Shift = 2;
+    return AArch64::LDRSWroW;
+  case AArch64::LDRSroX:
+    Shift = 2;
+    return AArch64::LDRSroW;
+  case AArch64::LDRWroX:
+    Shift = 2;
+    return AArch64::LDRWroW;
+  case AArch64::LDRXroX:
+    Shift = 3;
+    return AArch64::LDRXroW;
+  case AArch64::STRBBroX:
+    return AArch64::STRBBroW;
+  case AArch64::STRBroX:
+    return AArch64::STRBroW;
+  case AArch64::STRDroX:
+    Shift = 3;
+    return AArch64::STRDroW;
+  case AArch64::STRHHroX:
+    Shift = 1;
+    return AArch64::STRHHroW;
+  case AArch64::STRHroX:
+    Shift = 1;
+    return AArch64::STRHroW;
+  case AArch64::STRQroX:
+    Shift = 4;
+    return AArch64::STRQroW;
+  case AArch64::STRSroX:
+    Shift = 2;
+    return AArch64::STRSroW;
+  case AArch64::STRWroX:
+    Shift = 2;
+    return AArch64::STRWroW;
+  case AArch64::STRXroX:
+    Shift = 3;
+    return AArch64::STRXroW;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static unsigned convertRoWToRoW(unsigned Op, unsigned &Shift) {
+  Shift = 0;
+  switch (Op) {
+  case AArch64::LDRBBroW:
+    return AArch64::LDRBBroW;
+  case AArch64::LDRBroW:
+    return AArch64::LDRBroW;
+  case AArch64::LDRDroW:
+    Shift = 3;
+    return AArch64::LDRDroW;
+  case AArch64::LDRHHroW:
+    Shift = 1;
+    return AArch64::LDRHHroW;
+  case AArch64::LDRHroW:
+    Shift = 1;
+    return AArch64::LDRHroW;
+  case AArch64::LDRQroW:
+    Shift = 4;
+    return AArch64::LDRQroW;
+  case AArch64::LDRSBWroW:
+    Shift = 1;
+    return AArch64::LDRSBWroW;
+  case AArch64::LDRSBXroW:
+    Shift = 1;
+    return AArch64::LDRSBXroW;
+  case AArch64::LDRSHWroW:
+    Shift = 1;
+    return AArch64::LDRSHWroW;
+  case AArch64::LDRSHXroW:
+    Shift = 1;
+    return AArch64::LDRSHXroW;
+  case AArch64::LDRSWroW:
+    Shift = 2;
+    return AArch64::LDRSWroW;
+  case AArch64::LDRSroW:
+    Shift = 2;
+    return AArch64::LDRSroW;
+  case AArch64::LDRWroW:
+    Shift = 2;
+    return AArch64::LDRWroW;
+  case AArch64::LDRXroW:
+    Shift = 3;
+    return AArch64::LDRXroW;
+  case AArch64::STRBBroW:
+    return AArch64::STRBBroW;
+  case AArch64::STRBroW:
+    return AArch64::STRBroW;
+  case AArch64::STRDroW:
+    Shift = 3;
+    return AArch64::STRDroW;
+  case AArch64::STRHHroW:
+    Shift = 1;
+    return AArch64::STRHHroW;
+  case AArch64::STRHroW:
+    Shift = 1;
+    return AArch64::STRHroW;
+  case AArch64::STRQroW:
+    Shift = 4;
+    return AArch64::STRQroW;
+  case AArch64::STRSroW:
+    Shift = 2;
+    return AArch64::STRSroW;
+  case AArch64::STRWroW:
+    Shift = 2;
+    return AArch64::STRWroW;
+  case AArch64::STRXroW:
+    Shift = 3;
+    return AArch64::STRXroW;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static unsigned convertUiToRoW(unsigned Op) {
+  switch (Op) {
+  case AArch64::LDRBBui:
+    return AArch64::LDRBBroW;
+  case AArch64::LDRBui:
+    return AArch64::LDRBroW;
+  case AArch64::LDRDui:
+    return AArch64::LDRDroW;
+  case AArch64::LDRHHui:
+    return AArch64::LDRHHroW;
+  case AArch64::LDRHui:
+    return AArch64::LDRHroW;
+  case AArch64::LDRQui:
+    return AArch64::LDRQroW;
+  case AArch64::LDRSBWui:
+    return AArch64::LDRSBWroW;
+  case AArch64::LDRSBXui:
+    return AArch64::LDRSBXroW;
+  case AArch64::LDRSHWui:
+    return AArch64::LDRSHWroW;
+  case AArch64::LDRSHXui:
+    return AArch64::LDRSHXroW;
+  case AArch64::LDRSWui:
+    return AArch64::LDRSWroW;
+  case AArch64::LDRSui:
+    return AArch64::LDRSroW;
+  case AArch64::LDRWui:
+    return AArch64::LDRWroW;
+  case AArch64::LDRXui:
+    return AArch64::LDRXroW;
+  case AArch64::STRBBui:
+    return AArch64::STRBBroW;
+  case AArch64::STRBui:
+    return AArch64::STRBroW;
+  case AArch64::STRDui:
+    return AArch64::STRDroW;
+  case AArch64::STRHHui:
+    return AArch64::STRHHroW;
+  case AArch64::STRHui:
+    return AArch64::STRHroW;
+  case AArch64::STRQui:
+    return AArch64::STRQroW;
+  case AArch64::STRSui:
+    return AArch64::STRSroW;
+  case AArch64::STRWui:
+    return AArch64::STRWroW;
+  case AArch64::STRXui:
+    return AArch64::STRXroW;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static unsigned convertPreToRoW(unsigned Op) {
+  switch (Op) {
+  case AArch64::LDRBBpre:
+    return AArch64::LDRBBroW;
+  case AArch64::LDRBpre:
+    return AArch64::LDRBroW;
+  case AArch64::LDRDpre:
+    return AArch64::LDRDroW;
+  case AArch64::LDRHHpre:
+    return AArch64::LDRHHroW;
+  case AArch64::LDRHpre:
+    return AArch64::LDRHroW;
+  case AArch64::LDRQpre:
+    return AArch64::LDRQroW;
+  case AArch64::LDRSBWpre:
+    return AArch64::LDRSBWroW;
+  case AArch64::LDRSBXpre:
+    return AArch64::LDRSBXroW;
+  case AArch64::LDRSHWpre:
+    return AArch64::LDRSHWroW;
+  case AArch64::LDRSHXpre:
+    return AArch64::LDRSHXroW;
+  case AArch64::LDRSWpre:
+    return AArch64::LDRSWroW;
+  case AArch64::LDRSpre:
+    return AArch64::LDRSroW;
+  case AArch64::LDRWpre:
+    return AArch64::LDRWroW;
+  case AArch64::LDRXpre:
+    return AArch64::LDRXroW;
+  case AArch64::STRBBpre:
+    return AArch64::STRBBroW;
+  case AArch64::STRBpre:
+    return AArch64::STRBroW;
+  case AArch64::STRDpre:
+    return AArch64::STRDroW;
+  case AArch64::STRHHpre:
+    return AArch64::STRHHroW;
+  case AArch64::STRHpre:
+    return AArch64::STRHroW;
+  case AArch64::STRQpre:
+    return AArch64::STRQroW;
+  case AArch64::STRSpre:
+    return AArch64::STRSroW;
+  case AArch64::STRWpre:
+    return AArch64::STRWroW;
+  case AArch64::STRXpre:
+    return AArch64::STRXroW;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static unsigned convertPostToRoW(unsigned Op) {
+  switch (Op) {
+  case AArch64::LDRBBpost:
+    return AArch64::LDRBBroW;
+  case AArch64::LDRBpost:
+    return AArch64::LDRBroW;
+  case AArch64::LDRDpost:
+    return AArch64::LDRDroW;
+  case AArch64::LDRHHpost:
+    return AArch64::LDRHHroW;
+  case AArch64::LDRHpost:
+    return AArch64::LDRHroW;
+  case AArch64::LDRQpost:
+    return AArch64::LDRQroW;
+  case AArch64::LDRSBWpost:
+    return AArch64::LDRSBWroW;
+  case AArch64::LDRSBXpost:
+    return AArch64::LDRSBXroW;
+  case AArch64::LDRSHWpost:
+    return AArch64::LDRSHWroW;
+  case AArch64::LDRSHXpost:
+    return AArch64::LDRSHXroW;
+  case AArch64::LDRSWpost:
+    return AArch64::LDRSWroW;
+  case AArch64::LDRSpost:
+    return AArch64::LDRSroW;
+  case AArch64::LDRWpost:
+    return AArch64::LDRWroW;
+  case AArch64::LDRXpost:
+    return AArch64::LDRXroW;
+  case AArch64::STRBBpost:
+    return AArch64::STRBBroW;
+  case AArch64::STRBpost:
+    return AArch64::STRBroW;
+  case AArch64::STRDpost:
+    return AArch64::STRDroW;
+  case AArch64::STRHHpost:
+    return AArch64::STRHHroW;
+  case AArch64::STRHpost:
+    return AArch64::STRHroW;
+  case AArch64::STRQpost:
+    return AArch64::STRQroW;
+  case AArch64::STRSpost:
+    return AArch64::STRSroW;
+  case AArch64::STRWpost:
+    return AArch64::STRWroW;
+  case AArch64::STRXpost:
+    return AArch64::STRXroW;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static bool canConvertToRoW(unsigned Op) {
+  unsigned Shift;
+  return convertUiToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
+    convertPreToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
+    convertPostToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
+    convertRoXToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END ||
+    convertRoWToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END;
+}
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index 0ca366566df57..2bfb66955f47c 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_MC_AARCH64MCLFIEXPANDER_H
 #define LLVM_MC_AARCH64MCLFIEXPANDER_H
 
+#include "Utils/AArch64BaseInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCLFIExpander.h"
 #include "llvm/MC/MCRegisterInfo.h"
@@ -73,6 +74,12 @@ class AArch64MCLFIExpander : public MCLFIExpander {
   void expandLoadStore(const MCInst &Inst, MCStreamer &Out,
                        const MCSubtargetInfo &STI);
 
+  void expandLoadStoreBasic(const MCInst &Inst, MemInstInfo &InstInfo,
+      MCStreamer &Out, const MCSubtargetInfo &STI);
+
+  void expandLoadStoreRoW(const MCInst &Inst, MemInstInfo &InstInfo,
+      MCStreamer &Out, const MCSubtargetInfo &STI);
+
   void expandSyscall(const MCInst &Inst, MCStreamer &Out,
                      const MCSubtargetInfo &STI);
 
diff --git a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
index 9671fa3b3d92f..a3dea22335d21 100644
--- a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
+++ b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
@@ -22,6 +22,7 @@
 #include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/TargetParser/SubtargetFeature.h"
 
@@ -247,6 +248,816 @@ static inline bool atomicBarrierDroppedOnZero(unsigned Opcode) {
   return false;
 }
 
+struct MemInstInfo {
+  MemInstInfo() = default;
+
+  int DestRegIdx;
+  int BaseRegIdx;
+  int OffsetIdx;
+  bool IsPrePost = false;
+  bool IsPair = false;
+};
+
+static inline std::optional<MemInstInfo> getLoadInfo(const MCInst &Inst) {
+  int DestRegIdx;
+  int BaseRegIdx;
+  int OffsetIdx;
+  bool IsPrePost = false;
+  bool IsPair = false;
+  switch (Inst.getOpcode()) {
+  default:
+    return std::nullopt;
+
+  case AArch64::LD1i64:
+  case AArch64::LD2i64:
+    DestRegIdx = 0;
+    BaseRegIdx = 3;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::LD1i8:
+  case AArch64::LD1i16:
+  case AArch64::LD1i32:
+  case AArch64::LD2i8:
+  case AArch64::LD2i16:
+  case AArch64::LD2i32:
+  case AArch64::LD3i8:
+  case AArch64::LD3i16:
+  case AArch64::LD3i32:
+  case AArch64::LD3i64:
+  case AArch64::LD4i8:
+  case AArch64::LD4i16:
+  case AArch64::LD4i32:
+  case AArch64::LD4i64:
+    DestRegIdx = -1;
+    BaseRegIdx = 3;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::LD1Onev1d:
+  case AArch64::LD1Onev2s:
+  case AArch64::LD1Onev4h:
+  case AArch64::LD1Onev8b:
+  case AArch64::LD1Onev2d:
+  case AArch64::LD1Onev4s:
+  case AArch64::LD1Onev8h:
+  case AArch64::LD1Onev16b:
+  case AArch64::LD1Rv1d:
+  case AArch64::LD1Rv2s:
+  case AArch64::LD1Rv4h:
+  case AArch64::LD1Rv8b:
+  case AArch64::LD1Rv2d:
+  case AArch64::LD1Rv4s:
+  case AArch64::LD1Rv8h:
+  case AArch64::LD1Rv16b:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::LD1Twov1d:
+  case AArch64::LD1Twov2s:
+  case AArch64::LD1Twov4h:
+  case AArch64::LD1Twov8b:
+  case AArch64::LD1Twov2d:
+  case AArch64::LD1Twov4s:
+  case AArch64::LD1Twov8h:
+  case AArch64::LD1Twov16b:
+  case AArch64::LD1Threev1d:
+  case AArch64::LD1Threev2s:
+  case AArch64::LD1Threev4h:
+  case AArch64::LD1Threev8b:
+  case AArch64::LD1Threev2d:
+  case AArch64::LD1Threev4s:
+  case AArch64::LD1Threev8h:
+  case AArch64::LD1Threev16b:
+  case AArch64::LD1Fourv1d:
+  case AArch64::LD1Fourv2s:
+  case AArch64::LD1Fourv4h:
+  case AArch64::LD1Fourv8b:
+  case AArch64::LD1Fourv2d:
+  case AArch64::LD1Fourv4s:
+  case AArch64::LD1Fourv8h:
+  case AArch64::LD1Fourv16b:
+  case AArch64::LD2Twov2s:
+  case AArch64::LD2Twov4s:
+  case AArch64::LD2Twov8b:
+  case AArch64::LD2Twov2d:
+  case AArch64::LD2Twov4h:
+  case AArch64::LD2Twov8h:
+  case AArch64::LD2Twov16b:
+  case AArch64::LD2Rv1d:
+  case AArch64::LD2Rv2s:
+  case AArch64::LD2Rv4s:
+  case AArch64::LD2Rv8b:
+  case AArch64::LD2Rv2d:
+  case AArch64::LD2Rv4h:
+  case AArch64::LD2Rv8h:
+  case AArch64::LD2Rv16b:
+  case AArch64::LD3Threev2s:
+  case AArch64::LD3Threev4h:
+  case AArch64::LD3Threev8b:
+  case AArch64::LD3Threev2d:
+  case AArch64::LD3Threev4s:
+  case AArch64::LD3Threev8h:
+  case AArch64::LD3Threev16b:
+  case AArch64::LD3Rv1d:
+  case AArch64::LD3Rv2s:
+  case AArch64::LD3Rv4h:
+  case AArch64::LD3Rv8b:
+  case AArch64::LD3Rv2d:
+  case AArch64::LD3Rv4s:
+  case AArch64::LD3Rv8h:
+  case AArch64::LD3Rv16b:
+  case AArch64::LD4Fourv2s:
+  case AArch64::LD4Fourv4h:
+  case AArch64::LD4Fourv8b:
+  case AArch64::LD4Fourv2d:
+  case AArch64::LD4Fourv4s:
+  case AArch64::LD4Fourv8h:
+  case AArch64::LD4Fourv16b:
+  case AArch64::LD4Rv1d:
+  case AArch64::LD4Rv2s:
+  case AArch64::LD4Rv4h:
+  case AArch64::LD4Rv8b:
+  case AArch64::LD4Rv2d:
+  case AArch64::LD4Rv4s:
+  case AArch64::LD4Rv8h:
+  case AArch64::LD4Rv16b:
+    DestRegIdx = -1;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::LD1i64_POST:
+  case AArch64::LD2i64_POST:
+    DestRegIdx = 1;
+    BaseRegIdx = 4;
+    OffsetIdx = 5;
+    IsPrePost = true;
+    break;
+
+  case AArch64::LD1i8_POST:
+  case AArch64::LD1i16_POST:
+  case AArch64::LD1i32_POST:
+  case AArch64::LD2i8_POST:
+  case AArch64::LD2i16_POST:
+  case AArch64::LD2i32_POST:
+  case AArch64::LD3i8_POST:
+  case AArch64::LD3i16_POST:
+  case AArch64::LD3i32_POST:
+  case AArch64::LD3i64_POST:
+  case AArch64::LD4i8_POST:
+  case AArch64::LD4i16_POST:
+  case AArch64::LD4i32_POST:
+  case AArch64::LD4i64_POST:
+    DestRegIdx = -1;
+    BaseRegIdx = 4;
+    OffsetIdx = 5;
+    IsPrePost = true;
+    break;
+
+  case AArch64::LD1Onev1d_POST:
+  case AArch64::LD1Onev2s_POST:
+  case AArch64::LD1Onev4h_POST:
+  case AArch64::LD1Onev8b_POST:
+  case AArch64::LD1Onev2d_POST:
+  case AArch64::LD1Onev4s_POST:
+  case AArch64::LD1Onev8h_POST:
+  case AArch64::LD1Onev16b_POST:
+  case AArch64::LD1Rv1d_POST:
+  case AArch64::LD1Rv2s_POST:
+  case AArch64::LD1Rv4h_POST:
+  case AArch64::LD1Rv8b_POST:
+  case AArch64::LD1Rv2d_POST:
+  case AArch64::LD1Rv4s_POST:
+  case AArch64::LD1Rv8h_POST:
+  case AArch64::LD1Rv16b_POST:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::LD1Twov1d_POST:
+  case AArch64::LD1Twov2s_POST:
+  case AArch64::LD1Twov4h_POST:
+  case AArch64::LD1Twov8b_POST:
+  case AArch64::LD1Twov2d_POST:
+  case AArch64::LD1Twov4s_POST:
+  case AArch64::LD1Twov8h_POST:
+  case AArch64::LD1Twov16b_POST:
+  case AArch64::LD1Threev1d_POST:
+  case AArch64::LD1Threev2s_POST:
+  case AArch64::LD1Threev4h_POST:
+  case AArch64::LD1Threev8b_POST:
+  case AArch64::LD1Threev2d_POST:
+  case AArch64::LD1Threev4s_POST:
+  case AArch64::LD1Threev8h_POST:
+  case AArch64::LD1Threev16b_POST:
+  case AArch64::LD1Fourv1d_POST:
+  case AArch64::LD1Fourv2s_POST:
+  case AArch64::LD1Fourv4h_POST:
+  case AArch64::LD1Fourv8b_POST:
+  case AArch64::LD1Fourv2d_POST:
+  case AArch64::LD1Fourv4s_POST:
+  case AArch64::LD1Fourv8h_POST:
+  case AArch64::LD1Fourv16b_POST:
+  case AArch64::LD2Twov2s_POST:
+  case AArch64::LD2Twov4s_POST:
+  case AArch64::LD2Twov8b_POST:
+  case AArch64::LD2Twov2d_POST:
+  case AArch64::LD2Twov4h_POST:
+  case AArch64::LD2Twov8h_POST:
+  case AArch64::LD2Twov16b_POST:
+  case AArch64::LD2Rv1d_POST:
+  case AArch64::LD2Rv2s_POST:
+  case AArch64::LD2Rv4s_POST:
+  case AArch64::LD2Rv8b_POST:
+  case AArch64::LD2Rv2d_POST:
+  case AArch64::LD2Rv4h_POST:
+  case AArch64::LD2Rv8h_POST:
+  case AArch64::LD2Rv16b_POST:
+  case AArch64::LD3Threev2s_POST:
+  case AArch64::LD3Threev4h_POST:
+  case AArch64::LD3Threev8b_POST:
+  case AArch64::LD3Threev2d_POST:
+  case AArch64::LD3Threev4s_POST:
+  case AArch64::LD3Threev8h_POST:
+  case AArch64::LD3Threev16b_POST:
+  case AArch64::LD3Rv1d_POST:
+  case AArch64::LD3Rv2s_POST:
+  case AArch64::LD3Rv4h_POST:
+  case AArch64::LD3Rv8b_POST:
+  case AArch64::LD3Rv2d_POST:
+  case AArch64::LD3Rv4s_POST:
+  case AArch64::LD3Rv8h_POST:
+  case AArch64::LD3Rv16b_POST:
+  case AArch64::LD4Fourv2s_POST:
+  case AArch64::LD4Fourv4h_POST:
+  case AArch64::LD4Fourv8b_POST:
+  case AArch64::LD4Fourv2d_POST:
+  case AArch64::LD4Fourv4s_POST:
+  case AArch64::LD4Fourv8h_POST:
+  case AArch64::LD4Fourv16b_POST:
+  case AArch64::LD4Rv1d_POST:
+  case AArch64::LD4Rv2s_POST:
+  case AArch64::LD4Rv4h_POST:
+  case AArch64::LD4Rv8b_POST:
+  case AArch64::LD4Rv2d_POST:
+  case AArch64::LD4Rv4s_POST:
+  case AArch64::LD4Rv8h_POST:
+  case AArch64::LD4Rv16b_POST:
+    DestRegIdx = -1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::LDRBBroW:
+  case AArch64::LDRBBroX:
+  case AArch64::LDRBBui:
+  case AArch64::LDRBroW:
+  case AArch64::LDRBroX:
+  case AArch64::LDRBui:
+  case AArch64::LDRDl:
+  case AArch64::LDRDroW:
+  case AArch64::LDRDroX:
+  case AArch64::LDRDui:
+  case AArch64::LDRHHroW:
+  case AArch64::LDRHHroX:
+  case AArch64::LDRHHui:
+  case AArch64::LDRHroW:
+  case AArch64::LDRHroX:
+  case AArch64::LDRHui:
+  case AArch64::LDRQl:
+  case AArch64::LDRQroW:
+  case AArch64::LDRQroX:
+  case AArch64::LDRQui:
+  case AArch64::LDRSBWroW:
+  case AArch64::LDRSBWroX:
+  case AArch64::LDRSBWui:
+  case AArch64::LDRSBXroW:
+  case AArch64::LDRSBXroX:
+  case AArch64::LDRSBXui:
+  case AArch64::LDRSHWroW:
+  case AArch64::LDRSHWroX:
+  case AArch64::LDRSHWui:
+  case AArch64::LDRSHXroW:
+  case AArch64::LDRSHXroX:
+  case AArch64::LDRSHXui:
+  case AArch64::LDRSWl:
+  case AArch64::LDRSWroW:
+  case AArch64::LDRSWroX:
+  case AArch64::LDRSWui:
+  case AArch64::LDRSl:
+  case AArch64::LDRSroW:
+  case AArch64::LDRSroX:
+  case AArch64::LDRSui:
+  case AArch64::LDRWl:
+  case AArch64::LDRWroW:
+  case AArch64::LDRWroX:
+  case AArch64::LDRWui:
+  case AArch64::LDRXl:
+  case AArch64::LDRXroW:
+  case AArch64::LDRXroX:
+  case AArch64::LDRXui:
+  case AArch64::LDURBBi:
+  case AArch64::LDURBi:
+  case AArch64::LDURDi:
+  case AArch64::LDURHHi:
+  case AArch64::LDURHi:
+  case AArch64::LDURQi:
+  case AArch64::LDURSBWi:
+  case AArch64::LDURSBXi:
+  case AArch64::LDURSHWi:
+  case AArch64::LDURSHXi:
+  case AArch64::LDURSWi:
+  case AArch64::LDURSi:
+  case AArch64::LDURWi:
+  case AArch64::LDURXi:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = 2;
+    IsPrePost = false;
+    break;
+
+  case AArch64::LDRBBpost:
+  case AArch64::LDRBBpre:
+  case AArch64::LDRBpost:
+  case AArch64::LDRBpre:
+  case AArch64::LDRDpost:
+  case AArch64::LDRDpre:
+  case AArch64::LDRHHpost:
+  case AArch64::LDRHHpre:
+  case AArch64::LDRHpost:
+  case AArch64::LDRHpre:
+  case AArch64::LDRQpost:
+  case AArch64::LDRQpre:
+  case AArch64::LDRSBWpost:
+  case AArch64::LDRSBWpre:
+  case AArch64::LDRSBXpost:
+  case AArch64::LDRSBXpre:
+  case AArch64::LDRSHWpost:
+  case AArch64::LDRSHWpre:
+  case AArch64::LDRSHXpost:
+  case AArch64::LDRSHXpre:
+  case AArch64::LDRSWpost:
+  case AArch64::LDRSWpre:
+  case AArch64::LDRSpost:
+  case AArch64::LDRSpre:
+  case AArch64::LDRWpost:
+  case AArch64::LDRWpre:
+  case AArch64::LDRXpost:
+  case AArch64::LDRXpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::LDNPDi:
+  case AArch64::LDNPQi:
+  case AArch64::LDNPSi:
+  case AArch64::LDPQi:
+  case AArch64::LDPDi:
+  case AArch64::LDPSi:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+
+  case AArch64::LDPSWi:
+  case AArch64::LDPWi:
+  case AArch64::LDPXi:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+
+  case AArch64::LDPQpost:
+  case AArch64::LDPQpre:
+  case AArch64::LDPDpost:
+  case AArch64::LDPDpre:
+  case AArch64::LDPSpost:
+  case AArch64::LDPSpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 3;
+    OffsetIdx = 4;
+    IsPrePost = true;
+    IsPair = true;
+    break;
+
+  case AArch64::LDPSWpost:
+  case AArch64::LDPSWpre:
+  case AArch64::LDPWpost:
+  case AArch64::LDPWpre:
+  case AArch64::LDPXpost:
+  case AArch64::LDPXpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 3;
+    OffsetIdx = 4;
+    IsPrePost = true;
+    IsPair = true;
+    break;
+
+  // LD(A)?(X)?R(B|H|W|X)
+  case AArch64::LDXRB:
+  case AArch64::LDXRH:
+  case AArch64::LDXRW:
+  case AArch64::LDXRX:
+  case AArch64::LDAXRB:
+  case AArch64::LDAXRH:
+  case AArch64::LDAXRW:
+  case AArch64::LDAXRX:
+  case AArch64::LDARB:
+  case AArch64::LDARH:
+  case AArch64::LDARW:
+  case AArch64::LDARX:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  // LD(A|X)P(W|X)
+  case AArch64::LDXPW:
+  case AArch64::LDXPX:
+  case AArch64::LDAXPW:
+  case AArch64::LDAXPX:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+
+  }
+
+  return MemInstInfo { DestRegIdx, BaseRegIdx, OffsetIdx, IsPrePost, IsPair };
+}
+
+static inline std::optional<MemInstInfo> getStoreInfo(const MCInst &Inst) {
+  int DestRegIdx;
+  int BaseRegIdx;
+  int OffsetIdx;
+  bool IsPrePost = false;
+  bool IsPair = false;
+  switch (Inst.getOpcode()) {
+  default:
+    return std::nullopt;
+
+  case AArch64::ST1i8:
+  case AArch64::ST1i16:
+  case AArch64::ST1i32:
+  case AArch64::ST1i64:
+  case AArch64::ST2i8:
+  case AArch64::ST2i16:
+  case AArch64::ST2i32:
+  case AArch64::ST2i64:
+  case AArch64::ST3i8:
+  case AArch64::ST3i16:
+  case AArch64::ST3i32:
+  case AArch64::ST3i64:
+  case AArch64::ST4i8:
+  case AArch64::ST4i16:
+  case AArch64::ST4i32:
+  case AArch64::ST4i64:
+    DestRegIdx = -1;
+    BaseRegIdx = 2;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::ST1Onev1d:
+  case AArch64::ST1Onev2s:
+  case AArch64::ST1Onev4h:
+  case AArch64::ST1Onev8b:
+  case AArch64::ST1Onev2d:
+  case AArch64::ST1Onev4s:
+  case AArch64::ST1Onev8h:
+  case AArch64::ST1Onev16b:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::ST1Twov1d:
+  case AArch64::ST1Twov2s:
+  case AArch64::ST1Twov4h:
+  case AArch64::ST1Twov8b:
+  case AArch64::ST1Twov2d:
+  case AArch64::ST1Twov4s:
+  case AArch64::ST1Twov8h:
+  case AArch64::ST1Twov16b:
+  case AArch64::ST1Threev1d:
+  case AArch64::ST1Threev2s:
+  case AArch64::ST1Threev4h:
+  case AArch64::ST1Threev8b:
+  case AArch64::ST1Threev2d:
+  case AArch64::ST1Threev4s:
+  case AArch64::ST1Threev8h:
+  case AArch64::ST1Threev16b:
+  case AArch64::ST1Fourv1d:
+  case AArch64::ST1Fourv2s:
+  case AArch64::ST1Fourv4h:
+  case AArch64::ST1Fourv8b:
+  case AArch64::ST1Fourv2d:
+  case AArch64::ST1Fourv4s:
+  case AArch64::ST1Fourv8h:
+  case AArch64::ST1Fourv16b:
+  case AArch64::ST2Twov2s:
+  case AArch64::ST2Twov4s:
+  case AArch64::ST2Twov8b:
+  case AArch64::ST2Twov2d:
+  case AArch64::ST2Twov4h:
+  case AArch64::ST2Twov8h:
+  case AArch64::ST2Twov16b:
+  case AArch64::ST3Threev2s:
+  case AArch64::ST3Threev4h:
+  case AArch64::ST3Threev8b:
+  case AArch64::ST3Threev2d:
+  case AArch64::ST3Threev4s:
+  case AArch64::ST3Threev8h:
+  case AArch64::ST3Threev16b:
+  case AArch64::ST4Fourv2s:
+  case AArch64::ST4Fourv4h:
+  case AArch64::ST4Fourv8b:
+  case AArch64::ST4Fourv2d:
+  case AArch64::ST4Fourv4s:
+  case AArch64::ST4Fourv8h:
+  case AArch64::ST4Fourv16b:
+    DestRegIdx = -1;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::ST1i64_POST:
+  case AArch64::ST2i64_POST:
+    DestRegIdx = 1;
+    BaseRegIdx = 4;
+    OffsetIdx = 5;
+    IsPrePost = true;
+    break;
+
+  // SIMDStSingleS
+  case AArch64::ST1i8_POST:
+  case AArch64::ST1i16_POST:
+  case AArch64::ST1i32_POST:
+  case AArch64::ST2i8_POST:
+  case AArch64::ST2i16_POST:
+  case AArch64::ST2i32_POST:
+  case AArch64::ST3i8_POST:
+  case AArch64::ST3i16_POST:
+  case AArch64::ST3i32_POST:
+  case AArch64::ST3i64_POST:
+  case AArch64::ST4i8_POST:
+  case AArch64::ST4i16_POST:
+  case AArch64::ST4i32_POST:
+  case AArch64::ST4i64_POST:
+    DestRegIdx = -1;
+    BaseRegIdx = 3;
+    OffsetIdx = 4;
+    IsPrePost = true;
+    break;
+
+  case AArch64::ST1Onev1d_POST:
+  case AArch64::ST1Onev2s_POST:
+  case AArch64::ST1Onev4h_POST:
+  case AArch64::ST1Onev8b_POST:
+  case AArch64::ST1Onev2d_POST:
+  case AArch64::ST1Onev4s_POST:
+  case AArch64::ST1Onev8h_POST:
+  case AArch64::ST1Onev16b_POST:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::ST1Twov1d_POST:
+  case AArch64::ST1Twov2s_POST:
+  case AArch64::ST1Twov4h_POST:
+  case AArch64::ST1Twov8b_POST:
+  case AArch64::ST1Twov2d_POST:
+  case AArch64::ST1Twov4s_POST:
+  case AArch64::ST1Twov8h_POST:
+  case AArch64::ST1Twov16b_POST:
+  case AArch64::ST1Threev1d_POST:
+  case AArch64::ST1Threev2s_POST:
+  case AArch64::ST1Threev4h_POST:
+  case AArch64::ST1Threev8b_POST:
+  case AArch64::ST1Threev2d_POST:
+  case AArch64::ST1Threev4s_POST:
+  case AArch64::ST1Threev8h_POST:
+  case AArch64::ST1Threev16b_POST:
+  case AArch64::ST1Fourv1d_POST:
+  case AArch64::ST1Fourv2s_POST:
+  case AArch64::ST1Fourv4h_POST:
+  case AArch64::ST1Fourv8b_POST:
+  case AArch64::ST1Fourv2d_POST:
+  case AArch64::ST1Fourv4s_POST:
+  case AArch64::ST1Fourv8h_POST:
+  case AArch64::ST1Fourv16b_POST:
+  case AArch64::ST2Twov2s_POST:
+  case AArch64::ST2Twov4s_POST:
+  case AArch64::ST2Twov8b_POST:
+  case AArch64::ST2Twov2d_POST:
+  case AArch64::ST2Twov4h_POST:
+  case AArch64::ST2Twov8h_POST:
+  case AArch64::ST2Twov16b_POST:
+  case AArch64::ST3Threev2s_POST:
+  case AArch64::ST3Threev4h_POST:
+  case AArch64::ST3Threev8b_POST:
+  case AArch64::ST3Threev2d_POST:
+  case AArch64::ST3Threev4s_POST:
+  case AArch64::ST3Threev8h_POST:
+  case AArch64::ST3Threev16b_POST:
+  case AArch64::ST4Fourv2s_POST:
+  case AArch64::ST4Fourv4h_POST:
+  case AArch64::ST4Fourv8b_POST:
+  case AArch64::ST4Fourv2d_POST:
+  case AArch64::ST4Fourv4s_POST:
+  case AArch64::ST4Fourv8h_POST:
+  case AArch64::ST4Fourv16b_POST:
+    DestRegIdx = -1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::STRBBroW:
+  case AArch64::STRBBroX:
+  case AArch64::STRBBui:
+  case AArch64::STRBroW:
+  case AArch64::STRBroX:
+  case AArch64::STRBui:
+  case AArch64::STRDroW:
+  case AArch64::STRDroX:
+  case AArch64::STRDui:
+  case AArch64::STRHHroW:
+  case AArch64::STRHHroX:
+  case AArch64::STRHHui:
+  case AArch64::STRHroW:
+  case AArch64::STRHroX:
+  case AArch64::STRHui:
+  case AArch64::STRQroW:
+  case AArch64::STRQroX:
+  case AArch64::STRQui:
+  case AArch64::STRSroW:
+  case AArch64::STRSroX:
+  case AArch64::STRSui:
+  case AArch64::STRWroW:
+  case AArch64::STRWroX:
+  case AArch64::STRWui:
+  case AArch64::STRXroW:
+  case AArch64::STRXroX:
+  case AArch64::STRXui:
+  case AArch64::STURBBi:
+  case AArch64::STURBi:
+  case AArch64::STURDi:
+  case AArch64::STURHHi:
+  case AArch64::STURHi:
+  case AArch64::STURQi:
+  case AArch64::STURSi:
+  case AArch64::STURWi:
+  case AArch64::STURXi:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = 2;
+    IsPrePost = false;
+    break;
+
+  case AArch64::STRBBpost:
+  case AArch64::STRBBpre:
+  case AArch64::STRBpost:
+  case AArch64::STRBpre:
+  case AArch64::STRDpost:
+  case AArch64::STRDpre:
+  case AArch64::STRHHpost:
+  case AArch64::STRHHpre:
+  case AArch64::STRHpost:
+  case AArch64::STRHpre:
+  case AArch64::STRQpost:
+  case AArch64::STRQpre:
+  case AArch64::STRSpost:
+  case AArch64::STRSpre:
+  case AArch64::STRWpost:
+  case AArch64::STRWpre:
+  case AArch64::STRXpost:
+  case AArch64::STRXpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = true;
+    break;
+
+  case AArch64::STNPDi:
+  case AArch64::STNPQi:
+  case AArch64::STNPSi:
+  case AArch64::STPQi:
+  case AArch64::STPDi:
+  case AArch64::STPSi:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+
+  case AArch64::STPWi:
+  case AArch64::STPXi:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = 3;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+
+  case AArch64::STPQpost:
+  case AArch64::STPQpre:
+  case AArch64::STPDpost:
+  case AArch64::STPDpre:
+  case AArch64::STPSpost:
+  case AArch64::STPSpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 3;
+    OffsetIdx = 4;
+    IsPrePost = true;
+    IsPair = true;
+    break;
+
+  case AArch64::STPWpost:
+  case AArch64::STPWpre:
+  case AArch64::STPXpost:
+  case AArch64::STPXpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 3;
+    OffsetIdx = 4;
+    IsPrePost = true;
+    IsPair = true;
+    break;
+
+  // op Dst, [Base]
+  case AArch64::STLRB:
+  case AArch64::STLRH:
+  case AArch64::STLRW:
+  case AArch64::STLRX:
+    DestRegIdx = 0;
+    BaseRegIdx = 1;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  // Base = op Dst, [Base, (-4|-8)]!
+  case AArch64::STLRWpre:
+  case AArch64::STLRXpre:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    OffsetIdx = -1; // constant offset
+    IsPrePost = true;
+    break;
+
+  case AArch64::STXRB:
+  case AArch64::STXRH:
+  case AArch64::STXRW:
+  case AArch64::STXRX:
+  case AArch64::STLXRB:
+  case AArch64::STLXRH:
+  case AArch64::STLXRW:
+  case AArch64::STLXRX:
+    DestRegIdx = 0;
+    BaseRegIdx = 2;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    break;
+
+  case AArch64::STXPW:
+  case AArch64::STXPX:
+  case AArch64::STLXPW:
+  case AArch64::STLXPX:
+    DestRegIdx = 1; // return the status result at 0
+    BaseRegIdx = 3;
+    OffsetIdx = -1;
+    IsPrePost = false;
+    IsPair = true;
+    break;
+  }
+
+  return MemInstInfo { DestRegIdx, BaseRegIdx, OffsetIdx, IsPrePost, IsPair };
+}
+
+
 namespace AArch64CC {
 
 // The CondCodes constants map directly to the 4-bit encoding of the condition
diff --git a/llvm/test/MC/AArch64/LFI/mem.s b/llvm/test/MC/AArch64/LFI/mem.s
new file mode 100644
index 0000000000000..65be8cd1197d4
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/mem.s
@@ -0,0 +1,106 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+ldr x0, [sp]
+// CHECK: ldr x0, [sp]
+
+ldr x0, [sp, #8]
+// CHECK: ldr x0, [sp, #8]
+
+ldp x0, x1, [sp, #8]
+// CHECK: ldp x0, x1, [sp, #8]
+
+str x0, [sp]
+// CHECK: str x0, [sp]
+
+str x0, [sp, #8]
+// CHECK: str x0, [sp, #8]
+
+stp x0, x1, [sp, #8]
+// CHECK: stp x0, x1, [sp, #8]
+
+ldur x0, [x1]
+// CHECK:      add x28, x27, w1, uxtw
+// CHECK-NEXT: ldur x0, [x28]
+
+stur x0, [x1]
+// CHECK:      add x28, x27, w1, uxtw
+// CHECK-NEXT: stur x0, [x28]
+
+ldp x0, x1, [x2]
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28]
+
+stp x0, x1, [x2]
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: stp x0, x1, [x28]
+
+ldr x0, [x1]
+// CHECK: ldr x0, [x27, w1, uxtw]
+
+ldr x0, [x1, #8]
+// CHECK:      add x28, x27, w1, uxtw
+// CHECK-NEXT: ldr x0, [x28, #8]
+
+ldr x0, [x1, #8]!
+// CHECK:      add x1, x1, #8
+// CHECK-NEXT: ldr x0, [x27, w1, uxtw]
+
+str x0, [x1, #8]!
+// CHECK:      add x1, x1, #8
+// CHECK-NEXT: str x0, [x27, w1, uxtw]
+
+ldr x0, [x1, #-8]!
+// CHECK:      sub x1, x1, #8
+// CHECK-NEXT: ldr x0, [x27, w1, uxtw]
+
+str x0, [x1, #-8]!
+// CHECK:      sub x1, x1, #8
+// CHECK-NEXT: str x0, [x27, w1, uxtw]
+
+ldr x0, [x1], #8
+// CHECK:      ldr x0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #8
+
+str x0, [x1], #8
+// CHECK:      str x0, [x27, w1, uxtw]
+// CHECK-NEXT: add x1, x1, #8
+
+ldr x0, [x1], #-8
+// CHECK:      ldr x0, [x27, w1, uxtw]
+// CHECK-NEXT: sub x1, x1, #8
+
+str x0, [x1], #-8
+// CHECK:      str x0, [x27, w1, uxtw]
+// CHECK-NEXT: sub x1, x1, #8
+
+ldr x0, [x1, x2]
+// CHECK:      add x26, x1, x2
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, x2, lsl #3]
+// CHECK:      add x26, x1, x2, lsl #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, x2, sxtx #0]
+// CHECK:      add x26, x1, x2, sxtx
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, x2, sxtx #3]
+// CHECK:      add x26, x1, x2, sxtx #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, uxtw]
+// CHECK:      add x26, x1, w2, uxtw
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, uxtw #3]
+// CHECK:      add x26, x1, w2, uxtw #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, sxtw]
+// CHECK:      add x26, x1, w2, sxtw
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldr x0, [x1, w2, sxtw #3]
+// CHECK:      add x26, x1, w2, sxtw #3
+// CHECK-NEXT: ldr x0, [x27, w26, uxtw]

From 309c571891e44b14991eebb69da63f92615448cc Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Wed, 20 Aug 2025 16:30:38 -0700
Subject: [PATCH 11/41] [LFI] [AArch64] Rewrites for basic pre/post indexed
 load/stores

There is a remaining corner case for certain instructions with
hard-coded immediate offset values.
---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 667 +++++++++++++++++-
 llvm/test/MC/AArch64/LFI/mem.s                |  28 +
 2 files changed, 693 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index dc8ad0039ebad..f3de76761ecbe 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -216,6 +216,24 @@ static unsigned convertUiToRoW(unsigned Op);
 static unsigned convertPreToRoW(unsigned Op);
 static unsigned convertPostToRoW(unsigned Op);
 
+static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, bool &IsBaseNoOffset);
+static unsigned getPrePostScale(unsigned Op);
+
+static void emitSafeLoadStoreDemoted(const MCInst &Inst, unsigned N, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCInst LoadStore;
+  bool IsPre, IsBaseNoOffset;
+  auto NewOpCode = convertPrePostToBase(Inst.getOpcode(), IsPre, IsBaseNoOffset);
+  LoadStore.setOpcode(NewOpCode);
+  for (unsigned I = 1; I < N; I++)
+    LoadStore.addOperand(Inst.getOperand(I));
+  LoadStore.addOperand(MCOperand::createReg(LFIAddrReg));
+  if (IsPre)
+    LoadStore.addOperand(Inst.getOperand(N + 1));
+  else if (!IsBaseNoOffset)
+    LoadStore.addOperand(MCOperand::createImm(0));
+  Out.emitInstruction(LoadStore, STI);
+}
+
 static void emitSafeLoadStore(const MCInst &Inst, unsigned N, MCStreamer &Out, const MCSubtargetInfo &STI) {
   MCInst LoadStore;
   LoadStore.setOpcode(Inst.getOpcode());
@@ -233,8 +251,33 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, Mem
 
   if (MII.IsPrePost) {
     assert(OffsetIdx != -1 && "Pre/Post must have valid OffsetIdx");
-    return Out.getContext().reportWarning(
-        Inst.getLoc(), "TODO: pre/post index");
+
+    emitSafeLoadStoreDemoted(Inst, MII.BaseRegIdx, Out, STI);
+    MCRegister Base = Inst.getOperand(MII.BaseRegIdx).getReg();
+    MCOperand OffsetMO = Inst.getOperand(MII.OffsetIdx);
+    if (OffsetMO.isReg()) {
+      // The immediate offset of post-indexed addressing NEON Instrs has a fixed value, and
+      // it is encoded as a post-index addressing with XZR register operand.
+      // e.g., LD3Threev3d_POST can only have #48 as its operand and its offset
+      // MachineOperand holds XZR, which is a *Register* kind, not Imm.
+
+      // TODO: handle this case: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+      // MCRegister OffReg = OffsetMO.getReg();
+      // if (OffReg == AArch64::XZR) {
+      //   const LdStNInstrDesc *Info = getLdStNInstrDesc(Inst.getOpcode());
+      //   assert(Info && Info->NaturalOffset >= 0);
+      //   return emit(AArch64::ADDXri, Base, Base, Info->NaturalOffset, 0, Out, STI);
+      // } else {
+      //   assert(OffReg != AArch64::WZR);
+      //   return emit(AArch64::ADDXrs, Base, Base, OffsetMO.getReg(), 0, Out, STI);
+      // }
+      return;
+    } else {
+      auto Offset = Inst.getOperand(MII.OffsetIdx).getImm() * getPrePostScale(Inst.getOpcode());
+      if (Offset >= 0)
+        return emit(AArch64::ADDXri, Base, Base, Offset, 0, Out, STI);
+      return emit(AArch64::SUBXri, Base, Base, -Offset, 0, Out, STI);
+    }
   }
 
   return emitSafeLoadStore(Inst, MII.BaseRegIdx, Out, STI);
@@ -753,3 +796,623 @@ static bool canConvertToRoW(unsigned Op) {
     convertRoXToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END ||
     convertRoWToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END;
 }
+
+static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, bool &IsBaseNoOffset) {
+  IsPre = false;
+  IsBaseNoOffset = false;
+  switch (Op) {
+  case AArch64::LDPDpost:
+    return AArch64::LDPDi;
+  case AArch64::LDPDpre:
+    IsPre = true;
+    return AArch64::LDPDi;
+  case AArch64::LDPQpost:
+    return AArch64::LDPQi;
+  case AArch64::LDPQpre:
+    IsPre = true;
+    return AArch64::LDPQi;
+  case AArch64::LDPSWpost:
+    return AArch64::LDPSWi;
+  case AArch64::LDPSWpre:
+    IsPre = true;
+    return AArch64::LDPSWi;
+  case AArch64::LDPSpost:
+    return AArch64::LDPSi;
+  case AArch64::LDPSpre:
+    IsPre = true;
+    return AArch64::LDPSi;
+  case AArch64::LDPWpost:
+    return AArch64::LDPWi;
+  case AArch64::LDPWpre:
+    IsPre = true;
+    return AArch64::LDPWi;
+  case AArch64::LDPXpost:
+    return AArch64::LDPXi;
+  case AArch64::LDPXpre:
+    IsPre = true;
+    return AArch64::LDPXi;
+  case AArch64::STPDpost:
+    return AArch64::STPDi;
+  case AArch64::STPDpre:
+    IsPre = true;
+    return AArch64::STPDi;
+  case AArch64::STPQpost:
+    return AArch64::STPQi;
+  case AArch64::STPQpre:
+    IsPre = true;
+    return AArch64::STPQi;
+  case AArch64::STPSpost:
+    return AArch64::STPSi;
+  case AArch64::STPSpre:
+    IsPre = true;
+    return AArch64::STPSi;
+  case AArch64::STPWpost:
+    return AArch64::STPWi;
+  case AArch64::STPWpre:
+    IsPre = true;
+    return AArch64::STPWi;
+  case AArch64::STPXpost:
+    return AArch64::STPXi;
+  case AArch64::STPXpre:
+    IsPre = true;
+    return AArch64::STPXi;
+  // case AArch64::STLRWpre:
+  //   IsPre = true;
+  //   IsBaseNoOffset = true;
+  //   return AArch64::STPWi;
+  // case AArch64::STLRXpre:
+  //   IsPre = true;
+  //   IsBaseNoOffset = true;
+    // return AArch64::STLRX;
+  case AArch64::LD1i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1i64;
+  case AArch64::LD2i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2i64;
+  case AArch64::LD1i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1i8;
+  case AArch64::LD1i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1i16;
+  case AArch64::LD1i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1i32;
+  case AArch64::LD2i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2i8;
+  case AArch64::LD2i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2i16;
+  case AArch64::LD2i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2i32;
+  case AArch64::LD3i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3i8;
+  case AArch64::LD3i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3i16;
+  case AArch64::LD3i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3i32;
+  case AArch64::LD3i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3i64;
+  case AArch64::LD4i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4i8;
+  case AArch64::LD4i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4i16;
+  case AArch64::LD4i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4i32;
+  case AArch64::LD4i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4i64;
+  case AArch64::LD1Onev1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev1d;
+  case AArch64::LD1Onev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev2s;
+  case AArch64::LD1Onev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev4h;
+  case AArch64::LD1Onev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev8b;
+  case AArch64::LD1Onev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev2d;
+  case AArch64::LD1Onev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev4s;
+  case AArch64::LD1Onev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev8h;
+  case AArch64::LD1Onev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Onev16b;
+  case AArch64::LD1Rv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv1d;
+  case AArch64::LD1Rv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv2s;
+  case AArch64::LD1Rv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv4h;
+  case AArch64::LD1Rv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv8b;
+  case AArch64::LD1Rv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv2d;
+  case AArch64::LD1Rv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv4s;
+  case AArch64::LD1Rv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv8h;
+  case AArch64::LD1Rv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Rv16b;
+  case AArch64::LD1Twov1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov1d;
+  case AArch64::LD1Twov2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov2s;
+  case AArch64::LD1Twov4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov4h;
+  case AArch64::LD1Twov8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov8b;
+  case AArch64::LD1Twov2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov2d;
+  case AArch64::LD1Twov4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov4s;
+  case AArch64::LD1Twov8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov8h;
+  case AArch64::LD1Twov16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Twov16b;
+  case AArch64::LD1Threev1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev1d;
+  case AArch64::LD1Threev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev2s;
+  case AArch64::LD1Threev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev4h;
+  case AArch64::LD1Threev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev8b;
+  case AArch64::LD1Threev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev2d;
+  case AArch64::LD1Threev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev4s;
+  case AArch64::LD1Threev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev8h;
+  case AArch64::LD1Threev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Threev16b;
+  case AArch64::LD1Fourv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv1d;
+  case AArch64::LD1Fourv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv2s;
+  case AArch64::LD1Fourv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv4h;
+  case AArch64::LD1Fourv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv8b;
+  case AArch64::LD1Fourv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv2d;
+  case AArch64::LD1Fourv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv4s;
+  case AArch64::LD1Fourv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv8h;
+  case AArch64::LD1Fourv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD1Fourv16b;
+  case AArch64::LD2Twov2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov2s;
+  case AArch64::LD2Twov4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov4s;
+  case AArch64::LD2Twov8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov8b;
+  case AArch64::LD2Twov2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov2d;
+  case AArch64::LD2Twov4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov4h;
+  case AArch64::LD2Twov8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov8h;
+  case AArch64::LD2Twov16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Twov16b;
+  case AArch64::LD2Rv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv1d;
+  case AArch64::LD2Rv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv2s;
+  case AArch64::LD2Rv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv4s;
+  case AArch64::LD2Rv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv8b;
+  case AArch64::LD2Rv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv2d;
+  case AArch64::LD2Rv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv4h;
+  case AArch64::LD2Rv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv8h;
+  case AArch64::LD2Rv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD2Rv16b;
+  case AArch64::LD3Threev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev2s;
+  case AArch64::LD3Threev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev4h;
+  case AArch64::LD3Threev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev8b;
+  case AArch64::LD3Threev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev2d;
+  case AArch64::LD3Threev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev4s;
+  case AArch64::LD3Threev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev8h;
+  case AArch64::LD3Threev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Threev16b;
+  case AArch64::LD3Rv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv1d;
+  case AArch64::LD3Rv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv2s;
+  case AArch64::LD3Rv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv4h;
+  case AArch64::LD3Rv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv8b;
+  case AArch64::LD3Rv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv2d;
+  case AArch64::LD3Rv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv4s;
+  case AArch64::LD3Rv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv8h;
+  case AArch64::LD3Rv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD3Rv16b;
+  case AArch64::LD4Fourv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv2s;
+  case AArch64::LD4Fourv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv4h;
+  case AArch64::LD4Fourv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv8b;
+  case AArch64::LD4Fourv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv2d;
+  case AArch64::LD4Fourv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv4s;
+  case AArch64::LD4Fourv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv8h;
+  case AArch64::LD4Fourv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Fourv16b;
+  case AArch64::LD4Rv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv1d;
+  case AArch64::LD4Rv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv2s;
+  case AArch64::LD4Rv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv4h;
+  case AArch64::LD4Rv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv8b;
+  case AArch64::LD4Rv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv2d;
+  case AArch64::LD4Rv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv4s;
+  case AArch64::LD4Rv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv8h;
+  case AArch64::LD4Rv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::LD4Rv16b;
+  case AArch64::ST1i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1i64;
+  case AArch64::ST2i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2i64;
+  case AArch64::ST1i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1i8;
+  case AArch64::ST1i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1i16;
+  case AArch64::ST1i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1i32;
+  case AArch64::ST2i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2i8;
+  case AArch64::ST2i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2i16;
+  case AArch64::ST2i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2i32;
+  case AArch64::ST3i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3i8;
+  case AArch64::ST3i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3i16;
+  case AArch64::ST3i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3i32;
+  case AArch64::ST3i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3i64;
+  case AArch64::ST4i8_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4i8;
+  case AArch64::ST4i16_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4i16;
+  case AArch64::ST4i32_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4i32;
+  case AArch64::ST4i64_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4i64;
+  case AArch64::ST1Onev1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev1d;
+  case AArch64::ST1Onev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev2s;
+  case AArch64::ST1Onev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev4h;
+  case AArch64::ST1Onev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev8b;
+  case AArch64::ST1Onev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev2d;
+  case AArch64::ST1Onev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev4s;
+  case AArch64::ST1Onev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev8h;
+  case AArch64::ST1Onev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Onev16b;
+  case AArch64::ST1Twov1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov1d;
+  case AArch64::ST1Twov2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov2s;
+  case AArch64::ST1Twov4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov4h;
+  case AArch64::ST1Twov8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov8b;
+  case AArch64::ST1Twov2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov2d;
+  case AArch64::ST1Twov4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov4s;
+  case AArch64::ST1Twov8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov8h;
+  case AArch64::ST1Twov16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Twov16b;
+  case AArch64::ST1Threev1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev1d;
+  case AArch64::ST1Threev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev2s;
+  case AArch64::ST1Threev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev4h;
+  case AArch64::ST1Threev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev8b;
+  case AArch64::ST1Threev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev2d;
+  case AArch64::ST1Threev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev4s;
+  case AArch64::ST1Threev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev8h;
+  case AArch64::ST1Threev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Threev16b;
+  case AArch64::ST1Fourv1d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv1d;
+  case AArch64::ST1Fourv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv2s;
+  case AArch64::ST1Fourv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv4h;
+  case AArch64::ST1Fourv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv8b;
+  case AArch64::ST1Fourv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv2d;
+  case AArch64::ST1Fourv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv4s;
+  case AArch64::ST1Fourv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv8h;
+  case AArch64::ST1Fourv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST1Fourv16b;
+  case AArch64::ST2Twov2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov2s;
+  case AArch64::ST2Twov4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov4s;
+  case AArch64::ST2Twov8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov8b;
+  case AArch64::ST2Twov2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov2d;
+  case AArch64::ST2Twov4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov4h;
+  case AArch64::ST2Twov8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov8h;
+  case AArch64::ST2Twov16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST2Twov16b;
+  case AArch64::ST3Threev2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev2s;
+  case AArch64::ST3Threev4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev4h;
+  case AArch64::ST3Threev8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev8b;
+  case AArch64::ST3Threev2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev2d;
+  case AArch64::ST3Threev4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev4s;
+  case AArch64::ST3Threev8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev8h;
+  case AArch64::ST3Threev16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST3Threev16b;
+  case AArch64::ST4Fourv2s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv2s;
+  case AArch64::ST4Fourv4h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv4h;
+  case AArch64::ST4Fourv8b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv8b;
+  case AArch64::ST4Fourv2d_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv2d;
+  case AArch64::ST4Fourv4s_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv4s;
+  case AArch64::ST4Fourv8h_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv8h;
+  case AArch64::ST4Fourv16b_POST:
+    IsBaseNoOffset = true;
+    return AArch64::ST4Fourv16b;
+  }
+  return AArch64::INSTRUCTION_LIST_END;
+}
+
+static unsigned getPrePostScale(unsigned Op) {
+  switch (Op) {
+  case AArch64::LDPDpost:
+  case AArch64::LDPDpre:
+    return 8;
+  case AArch64::LDPQpost:
+  case AArch64::LDPQpre:
+    return 16;
+  case AArch64::LDPSWpost:
+  case AArch64::LDPSWpre:
+    return 4;
+  case AArch64::LDPSpost:
+  case AArch64::LDPSpre:
+    return 4;
+  case AArch64::LDPWpost:
+  case AArch64::LDPWpre:
+    return 4;
+  case AArch64::LDPXpost:
+  case AArch64::LDPXpre:
+    return 8;
+  case AArch64::STPDpost:
+  case AArch64::STPDpre:
+    return 8;
+  case AArch64::STPQpost:
+  case AArch64::STPQpre:
+    return 16;
+  case AArch64::STPSpost:
+  case AArch64::STPSpre:
+    return 4;
+  case AArch64::STPWpost:
+  case AArch64::STPWpre:
+    return 4;
+  case AArch64::STPXpost:
+  case AArch64::STPXpre:
+    return 8;
+  }
+  return 0;
+}
diff --git a/llvm/test/MC/AArch64/LFI/mem.s b/llvm/test/MC/AArch64/LFI/mem.s
index 65be8cd1197d4..03dfd42b30454 100644
--- a/llvm/test/MC/AArch64/LFI/mem.s
+++ b/llvm/test/MC/AArch64/LFI/mem.s
@@ -104,3 +104,31 @@ ldr x0, [x1, w2, sxtw]
 ldr x0, [x1, w2, sxtw #3]
 // CHECK:      add x26, x1, w2, sxtw #3
 // CHECK-NEXT: ldr x0, [x27, w26, uxtw]
+
+ldp x0, x1, [sp], #8
+// CHECK: ldp x0, x1, [sp], #8
+
+ldp x0, x1, [x2], #8
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28]
+// CHECK-NEXT: add x2, x2, #8
+
+ldp x0, x1, [x2, #8]!
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28, #8]
+// CHECK-NEXT: add x2, x2, #8
+
+ldp x0, x1, [x2], #-8
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28]
+// CHECK-NEXT: sub x2, x2, #8
+
+ldp x0, x1, [x2, #-8]!
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28, #-8]
+// CHECK-NEXT: sub x2, x2, #8
+
+stp x0, x1, [x2, #-8]!
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: stp x0, x1, [x28, #-8]
+// CHECK-NEXT: sub x2, x2, #8

From 905fd5a6eb1373125310a9120a40a47e3044ef72 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 11:01:01 -0700
Subject: [PATCH 12/41] [LFI] [AArch64] Assertion fixes for MRS/MSR

---
 .../Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index f3de76761ecbe..b74dcccc8279c 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -250,7 +250,7 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, Mem
   emitAddMask(LFIAddrReg, Inst.getOperand(MII.BaseRegIdx).getReg(), Out, STI);
 
   if (MII.IsPrePost) {
-    assert(OffsetIdx != -1 && "Pre/Post must have valid OffsetIdx");
+    assert(MII.OffsetIdx != -1 && "Pre/Post must have valid OffsetIdx");
 
     emitSafeLoadStoreDemoted(Inst, MII.BaseRegIdx, Out, STI);
     MCRegister Base = Inst.getOperand(MII.BaseRegIdx).getReg();
@@ -354,8 +354,7 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
   if (!MII.has_value()) {
     MII = getStoreInfo(Inst);
     if (!MII.has_value())
-      return Out.getContext().reportError(
-          Inst.getLoc(), "this load/store is not supported by LFI");
+      return Out.emitInstruction(Inst, STI);
   }
 
   // Stack accesses without a register offset don't need rewriting.
@@ -427,12 +426,12 @@ static bool isSyscall(const MCInst &Inst) {
 
 static bool isTLSRead(const MCInst &Inst) {
   return Inst.getOpcode() == AArch64::MRS &&
-    Inst.getOperand(1).getReg() == AArch64SysReg::TPIDR_EL0;
+    Inst.getOperand(1).getImm() == AArch64SysReg::TPIDR_EL0;
 }
 
 static bool isTLSWrite(const MCInst &Inst) {
   return Inst.getOpcode() == AArch64::MSR &&
-    Inst.getOperand(0).getReg() == AArch64SysReg::TPIDR_EL0;
+    Inst.getOperand(0).getImm() == AArch64SysReg::TPIDR_EL0;
 }
 
 void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,

From 2b4b00bf2104e4e61e9f12b8620c04af118009d7 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 11:32:59 -0700
Subject: [PATCH 13/41] [LFI] [Compiler-RT] Update CMake for aarch64_lfi
 support

---
 compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake | 2 +-
 compiler-rt/cmake/builtin-config-ix.cmake            | 2 +-
 compiler-rt/lib/builtins/CMakeLists.txt              | 1 +
 libunwind/src/UnwindRegistersRestore.S               | 5 +++++
 4 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
index ee5be276f3df7..f19b0b7961a9e 100644
--- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
+++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
@@ -1,4 +1,4 @@
-set(ARM64 aarch64)
+set(ARM64 aarch64 aarch64_lfi)
 set(ARM32 arm armhf)
 set(HEXAGON hexagon)
 set(X86 i386)
diff --git a/compiler-rt/cmake/builtin-config-ix.cmake b/compiler-rt/cmake/builtin-config-ix.cmake
index b1bde47ec8555..8e043ba24c985 100644
--- a/compiler-rt/cmake/builtin-config-ix.cmake
+++ b/compiler-rt/cmake/builtin-config-ix.cmake
@@ -58,7 +58,7 @@ else()
 endif()
 
 set(AMDGPU amdgcn)
-set(ARM64 aarch64)
+set(ARM64 aarch64 aarch64_lfi)
 set(ARM32 arm armhf armv4t armv5te armv6 armv6m armv7m armv7em armv7 armv7s armv7k armv8m.base armv8m.main armv8.1m.main)
 set(AVR avr)
 set(HEXAGON hexagon)
diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 19316c52d12ce..d059277e6f006 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -649,6 +649,7 @@ set(armv7k_SOURCES ${arm_SOURCES})
 set(arm64_SOURCES ${aarch64_SOURCES})
 set(arm64e_SOURCES ${aarch64_SOURCES})
 set(arm64_32_SOURCES ${aarch64_SOURCES})
+set(aarch64_lfi_SOURCES ${aarch64_SOURCES})
 
 # macho_embedded archs
 set(armv6m_SOURCES ${thumb1_SOURCES})
diff --git a/libunwind/src/UnwindRegistersRestore.S b/libunwind/src/UnwindRegistersRestore.S
index 1702d016c368b..bc3fff38a13dc 100644
--- a/libunwind/src/UnwindRegistersRestore.S
+++ b/libunwind/src/UnwindRegistersRestore.S
@@ -655,8 +655,13 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_jumpto)
   ldp    x20,x21, [x0, #0x0A0]
   ldp    x22,x23, [x0, #0x0B0]
   ldp    x24,x25, [x0, #0x0C0]
+#ifdef __LFI__
+  ldp    x26,xzr, [x0, #0x0D0]
+  ldp    xzr,x29, [x0, #0x0E0]
+#else
   ldp    x26,x27, [x0, #0x0D0]
   ldp    x28,x29, [x0, #0x0E0]
+#endif
   ldr    x30,     [x0, #0x100]  // restore pc into lr
 #if defined(__ARM_FP) && __ARM_FP != 0
   ldp    d0, d1,  [x0, #0x110]

From c76e3b390206b621e62b04e08353d1e7f484062c Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 13:01:54 -0700
Subject: [PATCH 14/41] [LFI] Add .no_expand/.expand directives

These directives make it possible to disable the LFI rewriter in GNU
assembly.
---
 llvm/include/llvm/MC/MCLFIExpander.h  |  5 +++++
 llvm/lib/MC/MCAsmStreamer.cpp         |  2 +-
 llvm/lib/MC/MCLFIExpander.cpp         | 12 ++++++++++++
 llvm/lib/MC/MCObjectStreamer.cpp      |  2 +-
 llvm/lib/MC/MCParser/LFIAsmParser.cpp | 28 ++++++++++++++++++++++++++-
 llvm/test/MC/AArch64/LFI/other.s      |  6 ++++++
 6 files changed, 52 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/other.s

diff --git a/llvm/include/llvm/MC/MCLFIExpander.h b/llvm/include/llvm/MC/MCLFIExpander.h
index 195e298c42e84..d77383c2576f5 100644
--- a/llvm/include/llvm/MC/MCLFIExpander.h
+++ b/llvm/include/llvm/MC/MCLFIExpander.h
@@ -31,6 +31,7 @@ class MCLFIExpander {
 private:
   SmallVector<MCRegister, 2> ScratchRegs;
   MCContext &Ctx;
+  bool Enabled;
 
 protected:
   std::unique_ptr<MCInstrInfo> InstInfo;
@@ -50,6 +51,10 @@ class MCLFIExpander {
   bool addScratchReg(MCRegister Reg);
   void clearScratchRegs();
 
+  void disable();
+  void enable();
+  bool isEnabled();
+
   bool isPseudo(const MCInst &Inst) const;
 
   bool mayAffectControlFlow(const MCInst &Inst) const;
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 38d8976965c81..00f49fd90ca3e 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -2454,7 +2454,7 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
 
 void MCAsmStreamer::emitInstruction(const MCInst &Inst,
                                     const MCSubtargetInfo &STI) {
-  if (LFIExpander && LFIExpander->expandInst(Inst, *this, STI))
+  if (LFIExpander && LFIExpander->isEnabled() && LFIExpander->expandInst(Inst, *this, STI))
     return;
 
   if (MAI->isAIX() && CurFrag)
diff --git a/llvm/lib/MC/MCLFIExpander.cpp b/llvm/lib/MC/MCLFIExpander.cpp
index 900e1ec17418d..9ed2286d61ee9 100644
--- a/llvm/lib/MC/MCLFIExpander.cpp
+++ b/llvm/lib/MC/MCLFIExpander.cpp
@@ -47,6 +47,18 @@ void MCLFIExpander::clearScratchRegs() {
   ScratchRegs.clear();
 }
 
+void MCLFIExpander::disable() {
+  Enabled = false;
+}
+
+void MCLFIExpander::enable() {
+  Enabled = true;
+}
+
+bool MCLFIExpander::isEnabled() {
+  return Enabled;
+}
+
 MCRegister MCLFIExpander::getScratchReg(int index) {
   assert(index >= 0 && static_cast<unsigned>(index) < numScratchRegs());
   return ScratchRegs[numScratchRegs()  - index - 1];
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index 75c3f00d64ed2..cc18cccf779fd 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -351,7 +351,7 @@ void MCObjectStreamer::emitInstruction(const MCInst &Inst,
 void MCObjectStreamer::emitInstructionImpl(const MCInst &Inst,
                                            const MCSubtargetInfo &STI) {
   MCSection *Sec = getCurrentSectionOnly();
-  if (LFIExpander && LFIExpander->expandInst(Inst, *this, STI))
+  if (LFIExpander && LFIExpander->isEnabled() && LFIExpander->expandInst(Inst, *this, STI))
     return;
 
   MCStreamer::emitInstruction(Inst, STI);
diff --git a/llvm/lib/MC/MCParser/LFIAsmParser.cpp b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
index cf9c172bd423b..f0932a0df6b6b 100644
--- a/llvm/lib/MC/MCParser/LFIAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
@@ -38,6 +38,8 @@ class LFIAsmParser : public MCAsmParserExtension {
     MCAsmParserExtension::Initialize(Parser);
     addDirectiveHandler<&LFIAsmParser::ParseScratch>(".scratch");
     addDirectiveHandler<&LFIAsmParser::ParseUnscratch>(".scratch_clear");
+    addDirectiveHandler<&LFIAsmParser::ParseExpandDisable>(".no_expand");
+    addDirectiveHandler<&LFIAsmParser::ParseExpandEnable>(".expand");
   }
 
   /// ::= {.scratch} reg
@@ -64,7 +66,7 @@ class LFIAsmParser : public MCAsmParserExtension {
     return false;
   }
 
-  /// ::= {.unscratch}
+  /// ::= {.scratch_clear}
   bool ParseUnscratch(StringRef Directive, SMLoc Loc) {
     getParser().checkForValidSection();
     if (getLexer().isNot(AsmToken::EndOfStatement))
@@ -75,6 +77,30 @@ class LFIAsmParser : public MCAsmParserExtension {
 
     return false;
   }
+
+  /// ::= {.no_expand}
+  bool ParseExpandDisable(StringRef Directive, SMLoc Loc) {
+    getParser().checkForValidSection();
+    if (getLexer().isNot(AsmToken::EndOfStatement))
+      return TokError("unexpected token in '.no_expand' directive");
+    Lex();
+
+    Expander->disable();
+
+    return false;
+  }
+
+  /// ::= {.expand}
+  bool ParseExpandEnable(StringRef Directive, SMLoc Loc) {
+    getParser().checkForValidSection();
+    if (getLexer().isNot(AsmToken::EndOfStatement))
+      return TokError("unexpected token in '.expand' directive");
+    Lex();
+
+    Expander->enable();
+
+    return false;
+  }
 };
 
 namespace llvm {
diff --git a/llvm/test/MC/AArch64/LFI/other.s b/llvm/test/MC/AArch64/LFI/other.s
new file mode 100644
index 0000000000000..c63ee1cd5d88f
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/other.s
@@ -0,0 +1,6 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+.no_expand
+ldr x0, [x1]
+// CHECK: ldr x0, [x1]
+.expand

From 284413019bf53960846bc73700dcc9b73c4c4579 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 13:19:05 -0700
Subject: [PATCH 15/41] [LFI] Make sure rewriter is enabled by default

---
 llvm/include/llvm/MC/MCLFIExpander.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/include/llvm/MC/MCLFIExpander.h b/llvm/include/llvm/MC/MCLFIExpander.h
index d77383c2576f5..7ccb5963e8439 100644
--- a/llvm/include/llvm/MC/MCLFIExpander.h
+++ b/llvm/include/llvm/MC/MCLFIExpander.h
@@ -31,7 +31,7 @@ class MCLFIExpander {
 private:
   SmallVector<MCRegister, 2> ScratchRegs;
   MCContext &Ctx;
-  bool Enabled;
+  bool Enabled = true;
 
 protected:
   std::unique_ptr<MCInstrInfo> InstInfo;

From 0c514ef819a0c18a61193c6e99e73f87b20945d4 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 13:36:41 -0700
Subject: [PATCH 16/41] [LFI] [Compiler-RT] Disable use of LFI reserved regs

---
 compiler-rt/lib/hwasan/hwasan_interceptors.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
index c10b5c158548e..fbe9f04db3669 100644
--- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
@@ -382,7 +382,9 @@ InternalLongjmp(__hw_register_buf env, int retval) {
       "ldp	x21, x22, [%0, #2<<3];"
       "ldp	x23, x24, [%0, #4<<3];"
       "ldp	x25, x26, [%0, #6<<3];"
+#ifndef __LFI__
       "ldp	x27, x28, [%0, #8<<3];"
+#endif
       "ldp	x29, x30, [%0, #10<<3];"
       "ldp	 d8,  d9, [%0, #14<<3];"
       "ldp	d10, d11, [%0, #16<<3];"

From 91c1f4586a829936da5010081ccaf16d77d84853 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 14:24:56 -0700
Subject: [PATCH 17/41] [LFI] [AArch64] Delete uses of reserved regs by default

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 40 ++++++++++++++-----
 llvm/test/MC/AArch64/LFI/reserved.s           | 19 ++++++---
 2 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index b74dcccc8279c..e5e3dce2d46c4 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -22,12 +22,17 @@
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "lfi"
 
+static cl::opt<bool> AArch64LFIErrorReserved("aarch64-lfi-error-reserved",
+    cl::Hidden, cl::init(false),
+    cl::desc("Produce errors for uses of LFI reserved registers"));
+
 static MCRegister LFIAddrReg = AArch64::X28;
 static MCRegister LFIBaseReg = AArch64::X27;
 static MCRegister LFIScratchReg = AArch64::X26;
@@ -167,20 +172,26 @@ bool AArch64::AArch64MCLFIExpander::mayModifyLR(const MCInst &Inst) {
   return mayModifyRegister(Inst, AArch64::LR);
 }
 
-void AArch64::AArch64MCLFIExpander::expandLRModification(
-    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
-  MCRegister Scratch = getScratch();
+static MCInst replaceReg(const MCInst &Inst,
+                         MCRegister Dest, MCRegister Src) {
   MCInst New;
-  New.setLoc(Inst.getLoc());
   New.setOpcode(Inst.getOpcode());
+  New.setLoc(Inst.getLoc());
   for (unsigned I = 0; I < Inst.getNumOperands(); ++I) {
     const MCOperand &Op = Inst.getOperand(I);
-    if (Op.isReg() && Op.getReg() == AArch64::LR) {
-      New.addOperand(MCOperand::createReg(Scratch));
+    if (Op.isReg() && Op.getReg() == Src) {
+      New.addOperand(MCOperand::createReg(Dest));
     } else {
       New.addOperand(Op);
     }
   }
+  return New;
+}
+
+void AArch64::AArch64MCLFIExpander::expandLRModification(
+    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
+  MCRegister Scratch = getScratch();
+  MCInst New = replaceReg(Inst, Scratch, AArch64::LR);
   if (mayLoad(New) || mayStore(New))
     expandLoadStore(New, Out, STI);
   else
@@ -459,9 +470,20 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
 
   // Bail out with an error. In the future, we could consider automatically
   // rewriting uses of reserved LFI registers.
-  if (mayModifyReserved(Inst))
-    return Out.getContext().reportError(
-        Inst.getLoc(), "illegal modification of reserved LFI register");
+  if (mayModifyReserved(Inst)) {
+    if (AArch64LFIErrorReserved)
+      return Out.getContext().reportError(
+          Inst.getLoc(), "illegal modification of reserved LFI register");
+    Out.getContext().reportWarning(
+        Inst.getLoc(), "deleting modification of reserved LFI register");
+    MCInst New = replaceReg(Inst, AArch64::XZR, LFIBaseReg);
+    if (mayModifyReserved(New)) {
+      MCRegister Scratch = getScratch();
+      New = replaceReg(New, Scratch, LFIAddrReg);
+    }
+    assert(!mayModifyReserved(New));
+    return doExpandInst(New, Out, STI);
+  }
 
   if (mayModifyStack(Inst))
     return expandStackModification(Inst, Out, STI);
diff --git a/llvm/test/MC/AArch64/LFI/reserved.s b/llvm/test/MC/AArch64/LFI/reserved.s
index 7b9ffb83474c0..e387ec5ec21e2 100644
--- a/llvm/test/MC/AArch64/LFI/reserved.s
+++ b/llvm/test/MC/AArch64/LFI/reserved.s
@@ -1,12 +1,21 @@
-// RUN: not llvm-mc -filetype asm -triple aarch64_lfi %s 2> %t
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s 2> %t
 // RUN: FileCheck --check-prefix=CHECK-ERROR %s <%t
 
 mov x28, x0
-// CHECK-ERROR: error: illegal modification of reserved LFI register
+// CHECK:       mov x26, x0
+// CHECK-ERROR: warning: deleting modification of reserved LFI register
 // CHECK-ERROR:        mov x28, x0
 // CHECK-ERROR:        ^
 
-ldr x27, [sp]
-// CHECK-ERROR: error: illegal modification of reserved LFI register
-// CHECK-ERROR:        ldr x27, [sp]
+ldr x27, [x0]
+// CHECK:       ldr xzr, [x27, w0, uxtw]
+// CHECK-ERROR: warning: deleting modification of reserved LFI register
+// CHECK-ERROR:        ldr x27, [x0]
+// CHECK-ERROR:        ^
+
+ldp x27, x28, [x0]
+// CHECK:       add x28, x27, w0, uxtw
+// CHECK-NEXT:  ldp xzr, x26, [x28]
+// CHECK-ERROR: warning: deleting modification of reserved LFI register
+// CHECK-ERROR:        ldp x27, x28, [x0]
 // CHECK-ERROR:        ^

From 455ddb425600f610747423602270b22d30162e64 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 15:06:00 -0700
Subject: [PATCH 18/41] [LFI] Switch to +lfi-stores and +lfi-jumps feature

Removes the aarch64_lfi_stores and aarch64_lfi_jumps target triples and
replaces them with +lfi-stores and +lfi-jumps subtarget features for
AArch64.
---
 llvm/include/llvm/TargetParser/Triple.h       | 24 +------------------
 llvm/lib/Target/AArch64/AArch64Features.td    | 10 ++++++++
 llvm/lib/Target/AArch64/AArch64Subtarget.h    |  2 ++
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 17 +++++++++++++
 llvm/lib/TargetParser/Triple.cpp              | 12 ----------
 5 files changed, 30 insertions(+), 35 deletions(-)

diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 642c9e7a04436..7806b780fc5f1 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -149,8 +149,6 @@ class Triple {
     AArch64SubArch_arm64ec,
 
     AArch64SubArch_lfi,
-    AArch64SubArch_lfi_stores,
-    AArch64SubArch_lfi_jumps,
 
     KalimbaSubArch_v3,
     KalimbaSubArch_v4,
@@ -684,32 +682,12 @@ class Triple {
     return isAArch64LFI();
   }
 
-  /// Checks if we're targeting any subarch of AArch64 LFI.
+  /// Checks if we're targeting the AArch64 LFI subarch.
   bool isAArch64LFI() const {
-    return getArch() == Triple::aarch64 &&
-           (getSubArch() == Triple::AArch64SubArch_lfi ||
-            getSubArch() == Triple::AArch64SubArch_lfi_stores ||
-            getSubArch() == Triple::AArch64SubArch_lfi_jumps);
-  }
-
-  /// Checks if we're targeting AArch64 LFI.
-  bool isAArch64LFIFull() const {
     return getArch() == Triple::aarch64 &&
            getSubArch() == Triple::AArch64SubArch_lfi;
   }
 
-  /// Checks if we're targeting AArch64 LFI stores-only.
-  bool isAArch64LFIStores() const {
-    return getArch() == Triple::aarch64 &&
-           getSubArch() == Triple::AArch64SubArch_lfi_stores;
-  }
-
-  /// Checks if we're targeting AArch64 LFI jumps-only.
-  bool isAArch64LFIJumps() const {
-    return getArch() == Triple::aarch64 &&
-           getSubArch() == Triple::AArch64SubArch_lfi_jumps;
-  }
-
   bool isWindowsCoreCLREnvironment() const {
     return isOSWindows() && getEnvironment() == Triple::CoreCLR;
   }
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index d47dcfe92ff19..8364ba46fa53d 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -948,3 +948,13 @@ def FeatureHardenSlsNoComdat : SubtargetFeature<"harden-sls-nocomdat",
 // Only intended to be used by disassemblers.
 def FeatureAll
     : SubtargetFeature<"all", "IsAll", "true", "Enable all instructions">;
+
+//===----------------------------------------------------------------------===//
+// LFI
+//===----------------------------------------------------------------------===//
+
+def FeatureLFIStores : SubtargetFeature<"lfi-stores", "LFIStores", "true",
+                                        "Isolate only store+jump instructions">;
+
+def FeatureLFIJumps : SubtargetFeature<"lfi-jumps", "LFIJumps", "true",
+                                       "Isolate only jump instructions">;
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index cbc60a2e52567..f7875ebdfb6a4 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -289,6 +289,8 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   bool isTargetFuchsia() const { return TargetTriple.isOSFuchsia(); }
   bool isWindowsArm64EC() const { return TargetTriple.isWindowsArm64EC(); }
   bool isAArch64LFI() const { return TargetTriple.isAArch64LFI(); }
+  bool isLFIStores() const { return LFIStores; }
+  bool isLFIJumps() const { return LFIJumps; }
 
   bool isTargetCOFF() const { return TargetTriple.isOSBinFormatCOFF(); }
   bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index e5e3dce2d46c4..4da8a4bbc7b68 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -22,6 +22,7 @@
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
@@ -37,6 +38,17 @@ static MCRegister LFIAddrReg = AArch64::X28;
 static MCRegister LFIBaseReg = AArch64::X27;
 static MCRegister LFIScratchReg = AArch64::X26;
 
+static bool featureStores(const MCSubtargetInfo &STI) {
+  const auto Stores = FeatureBitset({AArch64::FeatureLFIStores});
+  return (STI.getFeatureBits() & Stores) == Stores;
+}
+
+static bool featureJumps(const MCSubtargetInfo &STI) {
+  const auto Jumps = FeatureBitset({AArch64::FeatureLFIJumps});
+  return (STI.getFeatureBits() & Jumps) == Jumps;
+}
+
+
 bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
   return Reg != AArch64::SP;
 }
@@ -361,6 +373,11 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemIn
 void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     MCStreamer &Out,
                                                     const MCSubtargetInfo &STI) {
+  if (featureJumps(STI))
+    return Out.emitInstruction(Inst, STI);
+  if (featureStores(STI) && !mayStore(Inst))
+    return Out.emitInstruction(Inst, STI);
+
   auto MII = getLoadInfo(Inst);
   if (!MII.has_value()) {
     MII = getStoreInfo(Inst);
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index 6a18790ca6e83..b5d5889363801 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -114,10 +114,6 @@ StringRef Triple::getArchName(ArchType Kind, SubArchType SubArch) {
       return "arm64e";
     if (SubArch == AArch64SubArch_lfi)
       return "aarch64_lfi";
-    if (SubArch == AArch64SubArch_lfi_stores)
-      return "aarch64_lfi_stores";
-    if (SubArch == AArch64SubArch_lfi_jumps)
-      return "aarch64_lfi_jumps";
     break;
   case Triple::spirv:
     switch (SubArch) {
@@ -582,8 +578,6 @@ static Triple::ArchType parseArch(StringRef ArchName) {
           .Case("arm64e", Triple::aarch64)
           .Case("arm64ec", Triple::aarch64)
           .Case("aarch64_lfi", Triple::aarch64)
-          .Case("aarch64_lfi_stores", Triple::aarch64)
-          .Case("aarch64_lfi_jumps", Triple::aarch64)
           .Case("arm", Triple::arm)
           .Case("armeb", Triple::armeb)
           .Case("thumb", Triple::thumb)
@@ -809,12 +803,6 @@ static Triple::SubArchType parseSubArch(StringRef SubArchName) {
   if (SubArchName == "aarch64_lfi")
     return Triple::AArch64SubArch_lfi;
 
-  if (SubArchName == "aarch64_lfi_stores")
-    return Triple::AArch64SubArch_lfi_stores;
-
-  if (SubArchName == "aarch64_lfi_jumps")
-    return Triple::AArch64SubArch_lfi_jumps;
-
   if (SubArchName.starts_with("spirv"))
     return StringSwitch<Triple::SubArchType>(SubArchName)
         .EndsWith("v1.0", Triple::SPIRVSubArch_v10)

From d4933b462c5e9e8897d644097acd66954f13ebb6 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 21 Aug 2025 17:15:37 -0700
Subject: [PATCH 19/41] [LFI] [AArch64] Add +lfi-tls-reg feature

---
 llvm/lib/Target/AArch64/AArch64Features.td    |  3 +++
 .../Target/AArch64/AArch64RegisterInfo.cpp    |  3 +++
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 20 +++++++++----------
 llvm/test/MC/AArch64/LFI/tls-reg.s            | 13 ++++++++++++
 4 files changed, 28 insertions(+), 11 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/tls-reg.s

diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 8364ba46fa53d..e909152b20035 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -958,3 +958,6 @@ def FeatureLFIStores : SubtargetFeature<"lfi-stores", "LFIStores", "true",
 
 def FeatureLFIJumps : SubtargetFeature<"lfi-jumps", "LFIJumps", "true",
                                        "Isolate only jump instructions">;
+
+def FeatureLFITLSReg : SubtargetFeature<"lfi-tls-reg", "LFITLSReg", "true",
+                                        "Reserve a register for thread-local LFI data">;
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 22f845e1fa798..a52465c07840a 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -464,6 +464,9 @@ AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
     markSuperRegs(Reserved, AArch64::W28);
     markSuperRegs(Reserved, AArch64::W27);
     markSuperRegs(Reserved, AArch64::W26);
+    auto TLSReg = FeatureBitset({AArch64::FeatureLFITLSReg});
+    if ((MF.getSubtarget<AArch64Subtarget>().getFeatureBits() & TLSReg) == TLSReg)
+      markSuperRegs(Reserved, AArch64::W25);
     if (!MF.getProperties().hasProperty(MachineFunctionProperties::Property::NoVRegs)) {
       markSuperRegs(Reserved, AArch64::LR);
       markSuperRegs(Reserved, AArch64::W30);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 4da8a4bbc7b68..fdb25f3f0ead0 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -37,18 +37,12 @@ static cl::opt<bool> AArch64LFIErrorReserved("aarch64-lfi-error-reserved",
 static MCRegister LFIAddrReg = AArch64::X28;
 static MCRegister LFIBaseReg = AArch64::X27;
 static MCRegister LFIScratchReg = AArch64::X26;
+static MCRegister LFITLSReg = AArch64::X25;
 
-static bool featureStores(const MCSubtargetInfo &STI) {
-  const auto Stores = FeatureBitset({AArch64::FeatureLFIStores});
-  return (STI.getFeatureBits() & Stores) == Stores;
+static bool hasFeature(const FeatureBitset Feature, const MCSubtargetInfo &STI) {
+  return (STI.getFeatureBits() & Feature) == Feature;
 }
 
-static bool featureJumps(const MCSubtargetInfo &STI) {
-  const auto Jumps = FeatureBitset({AArch64::FeatureLFIJumps});
-  return (STI.getFeatureBits() & Jumps) == Jumps;
-}
-
-
 bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
   return Reg != AArch64::SP;
 }
@@ -373,9 +367,9 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemIn
 void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                                     MCStreamer &Out,
                                                     const MCSubtargetInfo &STI) {
-  if (featureJumps(STI))
+  if (hasFeature(FeatureBitset({AArch64::FeatureLFIJumps}), STI))
     return Out.emitInstruction(Inst, STI);
-  if (featureStores(STI) && !mayStore(Inst))
+  if (hasFeature(FeatureBitset({AArch64::FeatureLFIStores}), STI) && !mayStore(Inst))
     return Out.emitInstruction(Inst, STI);
 
   auto MII = getLoadInfo(Inst);
@@ -427,6 +421,10 @@ static void emitSwap(MCRegister Reg1, MCRegister Reg2, MCStreamer &Out, const MC
 void AArch64::AArch64MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer &Out,
                    const MCSubtargetInfo &STI) {
   MCRegister Reg = Inst.getOperand(0).getReg();
+
+  if (hasFeature(FeatureBitset({AArch64::FeatureLFITLSReg}), STI))
+    return emit(AArch64::LDRXui, Reg, LFITLSReg, 0, Out, STI);
+
   if (Reg == AArch64::X0) {
     emitLFICall(LFITLSRead, Out, STI);
   } else {
diff --git a/llvm/test/MC/AArch64/LFI/tls-reg.s b/llvm/test/MC/AArch64/LFI/tls-reg.s
new file mode 100644
index 0000000000000..b079f8316f89e
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/tls-reg.s
@@ -0,0 +1,13 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi -mattr +lfi-tls-reg %s | FileCheck %s
+
+msr tpidr_el0, x0
+// CHECK:      mov x26, x30
+// CHECK-NEXT: ldr x30, [x27, #16]
+// CHECK-NEXT: blr x30
+// CHECK-NEXT: add x30, x27, w26, uxtw
+
+mrs x0, tpidr_el0
+// CHECK:      ldr x0, [x25]
+
+mrs x1, tpidr_el0
+// CHECK:      ldr x1, [x25]

From 684253eb4e77d89b8d98677c95684d4459dfb170 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Fri, 22 Aug 2025 15:42:14 -0700
Subject: [PATCH 20/41] [LFI] [AArch64] Handle special case pre/post index

Also changes the instruction size estimate to include an additional
guard instruction for loads/stores so that branch relaxation correctly
resizes branches even before the expander is run.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  |  17 +-
 .../MCTargetDesc/AArch64InstPrinter.cpp       | 360 ------------------
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     |  20 +-
 .../Target/AArch64/Utils/AArch64BaseInfo.cpp  | 352 +++++++++++++++++
 .../Target/AArch64/Utils/AArch64BaseInfo.h    |  10 +
 llvm/test/MC/AArch64/LFI/mem.s                |  14 +
 6 files changed, 398 insertions(+), 375 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 0f2b969fba35c..b6f5e410e32d8 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -34,6 +34,7 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/StackMaps.h"
+#include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DebugInfoMetadata.h"
@@ -106,6 +107,7 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   unsigned NumBytes = 0;
   const MCInstrDesc &Desc = MI.getDesc();
 
+  const auto &STI = MF->getSubtarget<AArch64Subtarget>();
   if (!MI.isBundle() && isTailCallReturnInst(MI)) {
     NumBytes = Desc.getSize() ? Desc.getSize() : 4;
 
@@ -113,7 +115,6 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
     if (!MFI->shouldSignReturnAddress(MF))
       return NumBytes;
 
-    const auto &STI = MF->getSubtarget<AArch64Subtarget>();
     auto Method = STI.getAuthenticatedLRCheckMethod(*MF);
     NumBytes += AArch64PAuth::getCheckerSizeInBytes(Method);
     return NumBytes;
@@ -124,8 +125,10 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   // Specific cases handle instructions of variable sizes
   switch (Desc.getOpcode()) {
   default:
-    if (Desc.getSize())
-      return Desc.getSize();
+    if (Desc.getSize()) {
+      NumBytes = Desc.getSize();
+      break;
+    }
 
     // Anything not explicitly designated otherwise (i.e. pseudo-instructions
     // with fixed constant size but not specified in .td file) is a normal
@@ -174,6 +177,14 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
     break;
   }
 
+  if (STI.getTargetTriple().isAArch64LFI()) {
+    // Loads and stores are frequent and may be expanded to include an
+    // additional guard instruction, so we overestimate the size here to allow
+    // things like branch relaxation to be more accurate.
+    if (Desc.mayLoad() || Desc.mayStore())
+      NumBytes += 4;
+  }
+
   return NumBytes;
 }
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
index 875b505549f0a..e6dd82fcea387 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
@@ -420,366 +420,6 @@ static bool isTblTbxInstruction(unsigned Opcode, StringRef &Layout,
   }
 }
 
-struct LdStNInstrDesc {
-  unsigned Opcode;
-  const char *Mnemonic;
-  const char *Layout;
-  int ListOperand;
-  bool HasLane;
-  int NaturalOffset;
-};
-
-static const LdStNInstrDesc LdStNInstInfo[] = {
-  { AArch64::LD1i8,             "ld1",  ".b",     1, true,  0  },
-  { AArch64::LD1i16,            "ld1",  ".h",     1, true,  0  },
-  { AArch64::LD1i32,            "ld1",  ".s",     1, true,  0  },
-  { AArch64::LD1i64,            "ld1",  ".d",     1, true,  0  },
-  { AArch64::LD1i8_POST,        "ld1",  ".b",     2, true,  1  },
-  { AArch64::LD1i16_POST,       "ld1",  ".h",     2, true,  2  },
-  { AArch64::LD1i32_POST,       "ld1",  ".s",     2, true,  4  },
-  { AArch64::LD1i64_POST,       "ld1",  ".d",     2, true,  8  },
-  { AArch64::LD1Rv16b,          "ld1r", ".16b",   0, false, 0  },
-  { AArch64::LD1Rv8h,           "ld1r", ".8h",    0, false, 0  },
-  { AArch64::LD1Rv4s,           "ld1r", ".4s",    0, false, 0  },
-  { AArch64::LD1Rv2d,           "ld1r", ".2d",    0, false, 0  },
-  { AArch64::LD1Rv8b,           "ld1r", ".8b",    0, false, 0  },
-  { AArch64::LD1Rv4h,           "ld1r", ".4h",    0, false, 0  },
-  { AArch64::LD1Rv2s,           "ld1r", ".2s",    0, false, 0  },
-  { AArch64::LD1Rv1d,           "ld1r", ".1d",    0, false, 0  },
-  { AArch64::LD1Rv16b_POST,     "ld1r", ".16b",   1, false, 1  },
-  { AArch64::LD1Rv8h_POST,      "ld1r", ".8h",    1, false, 2  },
-  { AArch64::LD1Rv4s_POST,      "ld1r", ".4s",    1, false, 4  },
-  { AArch64::LD1Rv2d_POST,      "ld1r", ".2d",    1, false, 8  },
-  { AArch64::LD1Rv8b_POST,      "ld1r", ".8b",    1, false, 1  },
-  { AArch64::LD1Rv4h_POST,      "ld1r", ".4h",    1, false, 2  },
-  { AArch64::LD1Rv2s_POST,      "ld1r", ".2s",    1, false, 4  },
-  { AArch64::LD1Rv1d_POST,      "ld1r", ".1d",    1, false, 8  },
-  { AArch64::LD1Onev16b,        "ld1",  ".16b",   0, false, 0  },
-  { AArch64::LD1Onev8h,         "ld1",  ".8h",    0, false, 0  },
-  { AArch64::LD1Onev4s,         "ld1",  ".4s",    0, false, 0  },
-  { AArch64::LD1Onev2d,         "ld1",  ".2d",    0, false, 0  },
-  { AArch64::LD1Onev8b,         "ld1",  ".8b",    0, false, 0  },
-  { AArch64::LD1Onev4h,         "ld1",  ".4h",    0, false, 0  },
-  { AArch64::LD1Onev2s,         "ld1",  ".2s",    0, false, 0  },
-  { AArch64::LD1Onev1d,         "ld1",  ".1d",    0, false, 0  },
-  { AArch64::LD1Onev16b_POST,   "ld1",  ".16b",   1, false, 16 },
-  { AArch64::LD1Onev8h_POST,    "ld1",  ".8h",    1, false, 16 },
-  { AArch64::LD1Onev4s_POST,    "ld1",  ".4s",    1, false, 16 },
-  { AArch64::LD1Onev2d_POST,    "ld1",  ".2d",    1, false, 16 },
-  { AArch64::LD1Onev8b_POST,    "ld1",  ".8b",    1, false, 8  },
-  { AArch64::LD1Onev4h_POST,    "ld1",  ".4h",    1, false, 8  },
-  { AArch64::LD1Onev2s_POST,    "ld1",  ".2s",    1, false, 8  },
-  { AArch64::LD1Onev1d_POST,    "ld1",  ".1d",    1, false, 8  },
-  { AArch64::LD1Twov16b,        "ld1",  ".16b",   0, false, 0  },
-  { AArch64::LD1Twov8h,         "ld1",  ".8h",    0, false, 0  },
-  { AArch64::LD1Twov4s,         "ld1",  ".4s",    0, false, 0  },
-  { AArch64::LD1Twov2d,         "ld1",  ".2d",    0, false, 0  },
-  { AArch64::LD1Twov8b,         "ld1",  ".8b",    0, false, 0  },
-  { AArch64::LD1Twov4h,         "ld1",  ".4h",    0, false, 0  },
-  { AArch64::LD1Twov2s,         "ld1",  ".2s",    0, false, 0  },
-  { AArch64::LD1Twov1d,         "ld1",  ".1d",    0, false, 0  },
-  { AArch64::LD1Twov16b_POST,   "ld1",  ".16b",   1, false, 32 },
-  { AArch64::LD1Twov8h_POST,    "ld1",  ".8h",    1, false, 32 },
-  { AArch64::LD1Twov4s_POST,    "ld1",  ".4s",    1, false, 32 },
-  { AArch64::LD1Twov2d_POST,    "ld1",  ".2d",    1, false, 32 },
-  { AArch64::LD1Twov8b_POST,    "ld1",  ".8b",    1, false, 16 },
-  { AArch64::LD1Twov4h_POST,    "ld1",  ".4h",    1, false, 16 },
-  { AArch64::LD1Twov2s_POST,    "ld1",  ".2s",    1, false, 16 },
-  { AArch64::LD1Twov1d_POST,    "ld1",  ".1d",    1, false, 16 },
-  { AArch64::LD1Threev16b,      "ld1",  ".16b",   0, false, 0  },
-  { AArch64::LD1Threev8h,       "ld1",  ".8h",    0, false, 0  },
-  { AArch64::LD1Threev4s,       "ld1",  ".4s",    0, false, 0  },
-  { AArch64::LD1Threev2d,       "ld1",  ".2d",    0, false, 0  },
-  { AArch64::LD1Threev8b,       "ld1",  ".8b",    0, false, 0  },
-  { AArch64::LD1Threev4h,       "ld1",  ".4h",    0, false, 0  },
-  { AArch64::LD1Threev2s,       "ld1",  ".2s",    0, false, 0  },
-  { AArch64::LD1Threev1d,       "ld1",  ".1d",    0, false, 0  },
-  { AArch64::LD1Threev16b_POST, "ld1",  ".16b",   1, false, 48 },
-  { AArch64::LD1Threev8h_POST,  "ld1",  ".8h",    1, false, 48 },
-  { AArch64::LD1Threev4s_POST,  "ld1",  ".4s",    1, false, 48 },
-  { AArch64::LD1Threev2d_POST,  "ld1",  ".2d",    1, false, 48 },
-  { AArch64::LD1Threev8b_POST,  "ld1",  ".8b",    1, false, 24 },
-  { AArch64::LD1Threev4h_POST,  "ld1",  ".4h",    1, false, 24 },
-  { AArch64::LD1Threev2s_POST,  "ld1",  ".2s",    1, false, 24 },
-  { AArch64::LD1Threev1d_POST,  "ld1",  ".1d",    1, false, 24 },
-  { AArch64::LD1Fourv16b,       "ld1",  ".16b",   0, false, 0  },
-  { AArch64::LD1Fourv8h,        "ld1",  ".8h",    0, false, 0  },
-  { AArch64::LD1Fourv4s,        "ld1",  ".4s",    0, false, 0  },
-  { AArch64::LD1Fourv2d,        "ld1",  ".2d",    0, false, 0  },
-  { AArch64::LD1Fourv8b,        "ld1",  ".8b",    0, false, 0  },
-  { AArch64::LD1Fourv4h,        "ld1",  ".4h",    0, false, 0  },
-  { AArch64::LD1Fourv2s,        "ld1",  ".2s",    0, false, 0  },
-  { AArch64::LD1Fourv1d,        "ld1",  ".1d",    0, false, 0  },
-  { AArch64::LD1Fourv16b_POST,  "ld1",  ".16b",   1, false, 64 },
-  { AArch64::LD1Fourv8h_POST,   "ld1",  ".8h",    1, false, 64 },
-  { AArch64::LD1Fourv4s_POST,   "ld1",  ".4s",    1, false, 64 },
-  { AArch64::LD1Fourv2d_POST,   "ld1",  ".2d",    1, false, 64 },
-  { AArch64::LD1Fourv8b_POST,   "ld1",  ".8b",    1, false, 32 },
-  { AArch64::LD1Fourv4h_POST,   "ld1",  ".4h",    1, false, 32 },
-  { AArch64::LD1Fourv2s_POST,   "ld1",  ".2s",    1, false, 32 },
-  { AArch64::LD1Fourv1d_POST,   "ld1",  ".1d",    1, false, 32 },
-  { AArch64::LD2i8,             "ld2",  ".b",     1, true,  0  },
-  { AArch64::LD2i16,            "ld2",  ".h",     1, true,  0  },
-  { AArch64::LD2i32,            "ld2",  ".s",     1, true,  0  },
-  { AArch64::LD2i64,            "ld2",  ".d",     1, true,  0  },
-  { AArch64::LD2i8_POST,        "ld2",  ".b",     2, true,  2  },
-  { AArch64::LD2i16_POST,       "ld2",  ".h",     2, true,  4  },
-  { AArch64::LD2i32_POST,       "ld2",  ".s",     2, true,  8  },
-  { AArch64::LD2i64_POST,       "ld2",  ".d",     2, true,  16  },
-  { AArch64::LD2Rv16b,          "ld2r", ".16b",   0, false, 0  },
-  { AArch64::LD2Rv8h,           "ld2r", ".8h",    0, false, 0  },
-  { AArch64::LD2Rv4s,           "ld2r", ".4s",    0, false, 0  },
-  { AArch64::LD2Rv2d,           "ld2r", ".2d",    0, false, 0  },
-  { AArch64::LD2Rv8b,           "ld2r", ".8b",    0, false, 0  },
-  { AArch64::LD2Rv4h,           "ld2r", ".4h",    0, false, 0  },
-  { AArch64::LD2Rv2s,           "ld2r", ".2s",    0, false, 0  },
-  { AArch64::LD2Rv1d,           "ld2r", ".1d",    0, false, 0  },
-  { AArch64::LD2Rv16b_POST,     "ld2r", ".16b",   1, false, 2  },
-  { AArch64::LD2Rv8h_POST,      "ld2r", ".8h",    1, false, 4  },
-  { AArch64::LD2Rv4s_POST,      "ld2r", ".4s",    1, false, 8  },
-  { AArch64::LD2Rv2d_POST,      "ld2r", ".2d",    1, false, 16 },
-  { AArch64::LD2Rv8b_POST,      "ld2r", ".8b",    1, false, 2  },
-  { AArch64::LD2Rv4h_POST,      "ld2r", ".4h",    1, false, 4  },
-  { AArch64::LD2Rv2s_POST,      "ld2r", ".2s",    1, false, 8  },
-  { AArch64::LD2Rv1d_POST,      "ld2r", ".1d",    1, false, 16 },
-  { AArch64::LD2Twov16b,        "ld2",  ".16b",   0, false, 0  },
-  { AArch64::LD2Twov8h,         "ld2",  ".8h",    0, false, 0  },
-  { AArch64::LD2Twov4s,         "ld2",  ".4s",    0, false, 0  },
-  { AArch64::LD2Twov2d,         "ld2",  ".2d",    0, false, 0  },
-  { AArch64::LD2Twov8b,         "ld2",  ".8b",    0, false, 0  },
-  { AArch64::LD2Twov4h,         "ld2",  ".4h",    0, false, 0  },
-  { AArch64::LD2Twov2s,         "ld2",  ".2s",    0, false, 0  },
-  { AArch64::LD2Twov16b_POST,   "ld2",  ".16b",   1, false, 32 },
-  { AArch64::LD2Twov8h_POST,    "ld2",  ".8h",    1, false, 32 },
-  { AArch64::LD2Twov4s_POST,    "ld2",  ".4s",    1, false, 32 },
-  { AArch64::LD2Twov2d_POST,    "ld2",  ".2d",    1, false, 32 },
-  { AArch64::LD2Twov8b_POST,    "ld2",  ".8b",    1, false, 16 },
-  { AArch64::LD2Twov4h_POST,    "ld2",  ".4h",    1, false, 16 },
-  { AArch64::LD2Twov2s_POST,    "ld2",  ".2s",    1, false, 16 },
-  { AArch64::LD3i8,             "ld3",  ".b",     1, true,  0  },
-  { AArch64::LD3i16,            "ld3",  ".h",     1, true,  0  },
-  { AArch64::LD3i32,            "ld3",  ".s",     1, true,  0  },
-  { AArch64::LD3i64,            "ld3",  ".d",     1, true,  0  },
-  { AArch64::LD3i8_POST,        "ld3",  ".b",     2, true,  3  },
-  { AArch64::LD3i16_POST,       "ld3",  ".h",     2, true,  6  },
-  { AArch64::LD3i32_POST,       "ld3",  ".s",     2, true,  12 },
-  { AArch64::LD3i64_POST,       "ld3",  ".d",     2, true,  24 },
-  { AArch64::LD3Rv16b,          "ld3r", ".16b",   0, false, 0  },
-  { AArch64::LD3Rv8h,           "ld3r", ".8h",    0, false, 0  },
-  { AArch64::LD3Rv4s,           "ld3r", ".4s",    0, false, 0  },
-  { AArch64::LD3Rv2d,           "ld3r", ".2d",    0, false, 0  },
-  { AArch64::LD3Rv8b,           "ld3r", ".8b",    0, false, 0  },
-  { AArch64::LD3Rv4h,           "ld3r", ".4h",    0, false, 0  },
-  { AArch64::LD3Rv2s,           "ld3r", ".2s",    0, false, 0  },
-  { AArch64::LD3Rv1d,           "ld3r", ".1d",    0, false, 0  },
-  { AArch64::LD3Rv16b_POST,     "ld3r", ".16b",   1, false, 3  },
-  { AArch64::LD3Rv8h_POST,      "ld3r", ".8h",    1, false, 6  },
-  { AArch64::LD3Rv4s_POST,      "ld3r", ".4s",    1, false, 12 },
-  { AArch64::LD3Rv2d_POST,      "ld3r", ".2d",    1, false, 24 },
-  { AArch64::LD3Rv8b_POST,      "ld3r", ".8b",    1, false, 3  },
-  { AArch64::LD3Rv4h_POST,      "ld3r", ".4h",    1, false, 6  },
-  { AArch64::LD3Rv2s_POST,      "ld3r", ".2s",    1, false, 12 },
-  { AArch64::LD3Rv1d_POST,      "ld3r", ".1d",    1, false, 24 },
-  { AArch64::LD3Threev16b,      "ld3",  ".16b",   0, false, 0  },
-  { AArch64::LD3Threev8h,       "ld3",  ".8h",    0, false, 0  },
-  { AArch64::LD3Threev4s,       "ld3",  ".4s",    0, false, 0  },
-  { AArch64::LD3Threev2d,       "ld3",  ".2d",    0, false, 0  },
-  { AArch64::LD3Threev8b,       "ld3",  ".8b",    0, false, 0  },
-  { AArch64::LD3Threev4h,       "ld3",  ".4h",    0, false, 0  },
-  { AArch64::LD3Threev2s,       "ld3",  ".2s",    0, false, 0  },
-  { AArch64::LD3Threev16b_POST, "ld3",  ".16b",   1, false, 48 },
-  { AArch64::LD3Threev8h_POST,  "ld3",  ".8h",    1, false, 48 },
-  { AArch64::LD3Threev4s_POST,  "ld3",  ".4s",    1, false, 48 },
-  { AArch64::LD3Threev2d_POST,  "ld3",  ".2d",    1, false, 48 },
-  { AArch64::LD3Threev8b_POST,  "ld3",  ".8b",    1, false, 24 },
-  { AArch64::LD3Threev4h_POST,  "ld3",  ".4h",    1, false, 24 },
-  { AArch64::LD3Threev2s_POST,  "ld3",  ".2s",    1, false, 24 },
-  { AArch64::LD4i8,             "ld4",  ".b",     1, true,  0  },
-  { AArch64::LD4i16,            "ld4",  ".h",     1, true,  0  },
-  { AArch64::LD4i32,            "ld4",  ".s",     1, true,  0  },
-  { AArch64::LD4i64,            "ld4",  ".d",     1, true,  0  },
-  { AArch64::LD4i8_POST,        "ld4",  ".b",     2, true,  4  },
-  { AArch64::LD4i16_POST,       "ld4",  ".h",     2, true,  8  },
-  { AArch64::LD4i32_POST,       "ld4",  ".s",     2, true,  16 },
-  { AArch64::LD4i64_POST,       "ld4",  ".d",     2, true,  32 },
-  { AArch64::LD4Rv16b,          "ld4r", ".16b",   0, false, 0  },
-  { AArch64::LD4Rv8h,           "ld4r", ".8h",    0, false, 0  },
-  { AArch64::LD4Rv4s,           "ld4r", ".4s",    0, false, 0  },
-  { AArch64::LD4Rv2d,           "ld4r", ".2d",    0, false, 0  },
-  { AArch64::LD4Rv8b,           "ld4r", ".8b",    0, false, 0  },
-  { AArch64::LD4Rv4h,           "ld4r", ".4h",    0, false, 0  },
-  { AArch64::LD4Rv2s,           "ld4r", ".2s",    0, false, 0  },
-  { AArch64::LD4Rv1d,           "ld4r", ".1d",    0, false, 0  },
-  { AArch64::LD4Rv16b_POST,     "ld4r", ".16b",   1, false, 4  },
-  { AArch64::LD4Rv8h_POST,      "ld4r", ".8h",    1, false, 8  },
-  { AArch64::LD4Rv4s_POST,      "ld4r", ".4s",    1, false, 16 },
-  { AArch64::LD4Rv2d_POST,      "ld4r", ".2d",    1, false, 32 },
-  { AArch64::LD4Rv8b_POST,      "ld4r", ".8b",    1, false, 4  },
-  { AArch64::LD4Rv4h_POST,      "ld4r", ".4h",    1, false, 8  },
-  { AArch64::LD4Rv2s_POST,      "ld4r", ".2s",    1, false, 16 },
-  { AArch64::LD4Rv1d_POST,      "ld4r", ".1d",    1, false, 32 },
-  { AArch64::LD4Fourv16b,       "ld4",  ".16b",   0, false, 0  },
-  { AArch64::LD4Fourv8h,        "ld4",  ".8h",    0, false, 0  },
-  { AArch64::LD4Fourv4s,        "ld4",  ".4s",    0, false, 0  },
-  { AArch64::LD4Fourv2d,        "ld4",  ".2d",    0, false, 0  },
-  { AArch64::LD4Fourv8b,        "ld4",  ".8b",    0, false, 0  },
-  { AArch64::LD4Fourv4h,        "ld4",  ".4h",    0, false, 0  },
-  { AArch64::LD4Fourv2s,        "ld4",  ".2s",    0, false, 0  },
-  { AArch64::LD4Fourv16b_POST,  "ld4",  ".16b",   1, false, 64 },
-  { AArch64::LD4Fourv8h_POST,   "ld4",  ".8h",    1, false, 64 },
-  { AArch64::LD4Fourv4s_POST,   "ld4",  ".4s",    1, false, 64 },
-  { AArch64::LD4Fourv2d_POST,   "ld4",  ".2d",    1, false, 64 },
-  { AArch64::LD4Fourv8b_POST,   "ld4",  ".8b",    1, false, 32 },
-  { AArch64::LD4Fourv4h_POST,   "ld4",  ".4h",    1, false, 32 },
-  { AArch64::LD4Fourv2s_POST,   "ld4",  ".2s",    1, false, 32 },
-  { AArch64::ST1i8,             "st1",  ".b",     0, true,  0  },
-  { AArch64::ST1i16,            "st1",  ".h",     0, true,  0  },
-  { AArch64::ST1i32,            "st1",  ".s",     0, true,  0  },
-  { AArch64::ST1i64,            "st1",  ".d",     0, true,  0  },
-  { AArch64::ST1i8_POST,        "st1",  ".b",     1, true,  1  },
-  { AArch64::ST1i16_POST,       "st1",  ".h",     1, true,  2  },
-  { AArch64::ST1i32_POST,       "st1",  ".s",     1, true,  4  },
-  { AArch64::ST1i64_POST,       "st1",  ".d",     1, true,  8  },
-  { AArch64::ST1Onev16b,        "st1",  ".16b",   0, false, 0  },
-  { AArch64::ST1Onev8h,         "st1",  ".8h",    0, false, 0  },
-  { AArch64::ST1Onev4s,         "st1",  ".4s",    0, false, 0  },
-  { AArch64::ST1Onev2d,         "st1",  ".2d",    0, false, 0  },
-  { AArch64::ST1Onev8b,         "st1",  ".8b",    0, false, 0  },
-  { AArch64::ST1Onev4h,         "st1",  ".4h",    0, false, 0  },
-  { AArch64::ST1Onev2s,         "st1",  ".2s",    0, false, 0  },
-  { AArch64::ST1Onev1d,         "st1",  ".1d",    0, false, 0  },
-  { AArch64::ST1Onev16b_POST,   "st1",  ".16b",   1, false, 16 },
-  { AArch64::ST1Onev8h_POST,    "st1",  ".8h",    1, false, 16 },
-  { AArch64::ST1Onev4s_POST,    "st1",  ".4s",    1, false, 16 },
-  { AArch64::ST1Onev2d_POST,    "st1",  ".2d",    1, false, 16 },
-  { AArch64::ST1Onev8b_POST,    "st1",  ".8b",    1, false, 8  },
-  { AArch64::ST1Onev4h_POST,    "st1",  ".4h",    1, false, 8  },
-  { AArch64::ST1Onev2s_POST,    "st1",  ".2s",    1, false, 8  },
-  { AArch64::ST1Onev1d_POST,    "st1",  ".1d",    1, false, 8  },
-  { AArch64::ST1Twov16b,        "st1",  ".16b",   0, false, 0  },
-  { AArch64::ST1Twov8h,         "st1",  ".8h",    0, false, 0  },
-  { AArch64::ST1Twov4s,         "st1",  ".4s",    0, false, 0  },
-  { AArch64::ST1Twov2d,         "st1",  ".2d",    0, false, 0  },
-  { AArch64::ST1Twov8b,         "st1",  ".8b",    0, false, 0  },
-  { AArch64::ST1Twov4h,         "st1",  ".4h",    0, false, 0  },
-  { AArch64::ST1Twov2s,         "st1",  ".2s",    0, false, 0  },
-  { AArch64::ST1Twov1d,         "st1",  ".1d",    0, false, 0  },
-  { AArch64::ST1Twov16b_POST,   "st1",  ".16b",   1, false, 32 },
-  { AArch64::ST1Twov8h_POST,    "st1",  ".8h",    1, false, 32 },
-  { AArch64::ST1Twov4s_POST,    "st1",  ".4s",    1, false, 32 },
-  { AArch64::ST1Twov2d_POST,    "st1",  ".2d",    1, false, 32 },
-  { AArch64::ST1Twov8b_POST,    "st1",  ".8b",    1, false, 16 },
-  { AArch64::ST1Twov4h_POST,    "st1",  ".4h",    1, false, 16 },
-  { AArch64::ST1Twov2s_POST,    "st1",  ".2s",    1, false, 16 },
-  { AArch64::ST1Twov1d_POST,    "st1",  ".1d",    1, false, 16 },
-  { AArch64::ST1Threev16b,      "st1",  ".16b",   0, false, 0  },
-  { AArch64::ST1Threev8h,       "st1",  ".8h",    0, false, 0  },
-  { AArch64::ST1Threev4s,       "st1",  ".4s",    0, false, 0  },
-  { AArch64::ST1Threev2d,       "st1",  ".2d",    0, false, 0  },
-  { AArch64::ST1Threev8b,       "st1",  ".8b",    0, false, 0  },
-  { AArch64::ST1Threev4h,       "st1",  ".4h",    0, false, 0  },
-  { AArch64::ST1Threev2s,       "st1",  ".2s",    0, false, 0  },
-  { AArch64::ST1Threev1d,       "st1",  ".1d",    0, false, 0  },
-  { AArch64::ST1Threev16b_POST, "st1",  ".16b",   1, false, 48 },
-  { AArch64::ST1Threev8h_POST,  "st1",  ".8h",    1, false, 48 },
-  { AArch64::ST1Threev4s_POST,  "st1",  ".4s",    1, false, 48 },
-  { AArch64::ST1Threev2d_POST,  "st1",  ".2d",    1, false, 48 },
-  { AArch64::ST1Threev8b_POST,  "st1",  ".8b",    1, false, 24 },
-  { AArch64::ST1Threev4h_POST,  "st1",  ".4h",    1, false, 24 },
-  { AArch64::ST1Threev2s_POST,  "st1",  ".2s",    1, false, 24 },
-  { AArch64::ST1Threev1d_POST,  "st1",  ".1d",    1, false, 24 },
-  { AArch64::ST1Fourv16b,       "st1",  ".16b",   0, false, 0  },
-  { AArch64::ST1Fourv8h,        "st1",  ".8h",    0, false, 0  },
-  { AArch64::ST1Fourv4s,        "st1",  ".4s",    0, false, 0  },
-  { AArch64::ST1Fourv2d,        "st1",  ".2d",    0, false, 0  },
-  { AArch64::ST1Fourv8b,        "st1",  ".8b",    0, false, 0  },
-  { AArch64::ST1Fourv4h,        "st1",  ".4h",    0, false, 0  },
-  { AArch64::ST1Fourv2s,        "st1",  ".2s",    0, false, 0  },
-  { AArch64::ST1Fourv1d,        "st1",  ".1d",    0, false, 0  },
-  { AArch64::ST1Fourv16b_POST,  "st1",  ".16b",   1, false, 64 },
-  { AArch64::ST1Fourv8h_POST,   "st1",  ".8h",    1, false, 64 },
-  { AArch64::ST1Fourv4s_POST,   "st1",  ".4s",    1, false, 64 },
-  { AArch64::ST1Fourv2d_POST,   "st1",  ".2d",    1, false, 64 },
-  { AArch64::ST1Fourv8b_POST,   "st1",  ".8b",    1, false, 32 },
-  { AArch64::ST1Fourv4h_POST,   "st1",  ".4h",    1, false, 32 },
-  { AArch64::ST1Fourv2s_POST,   "st1",  ".2s",    1, false, 32 },
-  { AArch64::ST1Fourv1d_POST,   "st1",  ".1d",    1, false, 32 },
-  { AArch64::ST2i8,             "st2",  ".b",     0, true,  0  },
-  { AArch64::ST2i16,            "st2",  ".h",     0, true,  0  },
-  { AArch64::ST2i32,            "st2",  ".s",     0, true,  0  },
-  { AArch64::ST2i64,            "st2",  ".d",     0, true,  0  },
-  { AArch64::ST2i8_POST,        "st2",  ".b",     1, true,  2  },
-  { AArch64::ST2i16_POST,       "st2",  ".h",     1, true,  4  },
-  { AArch64::ST2i32_POST,       "st2",  ".s",     1, true,  8  },
-  { AArch64::ST2i64_POST,       "st2",  ".d",     1, true,  16 },
-  { AArch64::ST2Twov16b,        "st2",  ".16b",   0, false, 0  },
-  { AArch64::ST2Twov8h,         "st2",  ".8h",    0, false, 0  },
-  { AArch64::ST2Twov4s,         "st2",  ".4s",    0, false, 0  },
-  { AArch64::ST2Twov2d,         "st2",  ".2d",    0, false, 0  },
-  { AArch64::ST2Twov8b,         "st2",  ".8b",    0, false, 0  },
-  { AArch64::ST2Twov4h,         "st2",  ".4h",    0, false, 0  },
-  { AArch64::ST2Twov2s,         "st2",  ".2s",    0, false, 0  },
-  { AArch64::ST2Twov16b_POST,   "st2",  ".16b",   1, false, 32 },
-  { AArch64::ST2Twov8h_POST,    "st2",  ".8h",    1, false, 32 },
-  { AArch64::ST2Twov4s_POST,    "st2",  ".4s",    1, false, 32 },
-  { AArch64::ST2Twov2d_POST,    "st2",  ".2d",    1, false, 32 },
-  { AArch64::ST2Twov8b_POST,    "st2",  ".8b",    1, false, 16 },
-  { AArch64::ST2Twov4h_POST,    "st2",  ".4h",    1, false, 16 },
-  { AArch64::ST2Twov2s_POST,    "st2",  ".2s",    1, false, 16 },
-  { AArch64::ST3i8,             "st3",  ".b",     0, true,  0  },
-  { AArch64::ST3i16,            "st3",  ".h",     0, true,  0  },
-  { AArch64::ST3i32,            "st3",  ".s",     0, true,  0  },
-  { AArch64::ST3i64,            "st3",  ".d",     0, true,  0  },
-  { AArch64::ST3i8_POST,        "st3",  ".b",     1, true,  3  },
-  { AArch64::ST3i16_POST,       "st3",  ".h",     1, true,  6  },
-  { AArch64::ST3i32_POST,       "st3",  ".s",     1, true,  12 },
-  { AArch64::ST3i64_POST,       "st3",  ".d",     1, true,  24 },
-  { AArch64::ST3Threev16b,      "st3",  ".16b",   0, false, 0  },
-  { AArch64::ST3Threev8h,       "st3",  ".8h",    0, false, 0  },
-  { AArch64::ST3Threev4s,       "st3",  ".4s",    0, false, 0  },
-  { AArch64::ST3Threev2d,       "st3",  ".2d",    0, false, 0  },
-  { AArch64::ST3Threev8b,       "st3",  ".8b",    0, false, 0  },
-  { AArch64::ST3Threev4h,       "st3",  ".4h",    0, false, 0  },
-  { AArch64::ST3Threev2s,       "st3",  ".2s",    0, false, 0  },
-  { AArch64::ST3Threev16b_POST, "st3",  ".16b",   1, false, 48 },
-  { AArch64::ST3Threev8h_POST,  "st3",  ".8h",    1, false, 48 },
-  { AArch64::ST3Threev4s_POST,  "st3",  ".4s",    1, false, 48 },
-  { AArch64::ST3Threev2d_POST,  "st3",  ".2d",    1, false, 48 },
-  { AArch64::ST3Threev8b_POST,  "st3",  ".8b",    1, false, 24 },
-  { AArch64::ST3Threev4h_POST,  "st3",  ".4h",    1, false, 24 },
-  { AArch64::ST3Threev2s_POST,  "st3",  ".2s",    1, false, 24 },
-  { AArch64::ST4i8,             "st4",  ".b",     0, true,  0  },
-  { AArch64::ST4i16,            "st4",  ".h",     0, true,  0  },
-  { AArch64::ST4i32,            "st4",  ".s",     0, true,  0  },
-  { AArch64::ST4i64,            "st4",  ".d",     0, true,  0  },
-  { AArch64::ST4i8_POST,        "st4",  ".b",     1, true,  4  },
-  { AArch64::ST4i16_POST,       "st4",  ".h",     1, true,  8  },
-  { AArch64::ST4i32_POST,       "st4",  ".s",     1, true,  16 },
-  { AArch64::ST4i64_POST,       "st4",  ".d",     1, true,  32 },
-  { AArch64::ST4Fourv16b,       "st4",  ".16b",   0, false, 0  },
-  { AArch64::ST4Fourv8h,        "st4",  ".8h",    0, false, 0  },
-  { AArch64::ST4Fourv4s,        "st4",  ".4s",    0, false, 0  },
-  { AArch64::ST4Fourv2d,        "st4",  ".2d",    0, false, 0  },
-  { AArch64::ST4Fourv8b,        "st4",  ".8b",    0, false, 0  },
-  { AArch64::ST4Fourv4h,        "st4",  ".4h",    0, false, 0  },
-  { AArch64::ST4Fourv2s,        "st4",  ".2s",    0, false, 0  },
-  { AArch64::ST4Fourv16b_POST,  "st4",  ".16b",   1, false, 64 },
-  { AArch64::ST4Fourv8h_POST,   "st4",  ".8h",    1, false, 64 },
-  { AArch64::ST4Fourv4s_POST,   "st4",  ".4s",    1, false, 64 },
-  { AArch64::ST4Fourv2d_POST,   "st4",  ".2d",    1, false, 64 },
-  { AArch64::ST4Fourv8b_POST,   "st4",  ".8b",    1, false, 32 },
-  { AArch64::ST4Fourv4h_POST,   "st4",  ".4h",    1, false, 32 },
-  { AArch64::ST4Fourv2s_POST,   "st4",  ".2s",    1, false, 32 },
-};
-
-static const LdStNInstrDesc *getLdStNInstrDesc(unsigned Opcode) {
-  for (const auto &Info : LdStNInstInfo)
-    if (Info.Opcode == Opcode)
-      return &Info;
-
-  return nullptr;
-}
-
 void AArch64AppleInstPrinter::printInst(const MCInst *MI, uint64_t Address,
                                         StringRef Annot,
                                         const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index fdb25f3f0ead0..58597f506e365 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -277,18 +277,14 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, Mem
       // it is encoded as a post-index addressing with XZR register operand.
       // e.g., LD3Threev3d_POST can only have #48 as its operand and its offset
       // MachineOperand holds XZR, which is a *Register* kind, not Imm.
-
-      // TODO: handle this case: ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
-      // MCRegister OffReg = OffsetMO.getReg();
-      // if (OffReg == AArch64::XZR) {
-      //   const LdStNInstrDesc *Info = getLdStNInstrDesc(Inst.getOpcode());
-      //   assert(Info && Info->NaturalOffset >= 0);
-      //   return emit(AArch64::ADDXri, Base, Base, Info->NaturalOffset, 0, Out, STI);
-      // } else {
-      //   assert(OffReg != AArch64::WZR);
-      //   return emit(AArch64::ADDXrs, Base, Base, OffsetMO.getReg(), 0, Out, STI);
-      // }
-      return;
+      MCRegister OffReg = OffsetMO.getReg();
+      if (OffReg == AArch64::XZR) {
+        const LdStNInstrDesc *Info = getLdStNInstrDesc(Inst.getOpcode());
+        assert(Info && Info->NaturalOffset >= 0);
+        return emit(AArch64::ADDXri, Base, Base, Info->NaturalOffset, 0, Out, STI);
+      }
+      assert(OffReg != AArch64::WZR);
+      return emit(AArch64::ADDXrs, Base, Base, OffsetMO.getReg(), 0, Out, STI);
     } else {
       auto Offset = Inst.getOperand(MII.OffsetIdx).getImm() * getPrePostScale(Inst.getOpcode());
       if (Offset >= 0)
diff --git a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
index 7767028c20bcc..756fdca32dcc6 100644
--- a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
+++ b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
@@ -190,4 +190,356 @@ namespace llvm {
 #define GET_SVCRsList_IMPL
 #include "AArch64GenSystemOperands.inc"
   }
+
+static const LdStNInstrDesc LdStNInstInfo[] = {
+  { AArch64::LD1i8,             "ld1",  ".b",     1, true,  0  },
+  { AArch64::LD1i16,            "ld1",  ".h",     1, true,  0  },
+  { AArch64::LD1i32,            "ld1",  ".s",     1, true,  0  },
+  { AArch64::LD1i64,            "ld1",  ".d",     1, true,  0  },
+  { AArch64::LD1i8_POST,        "ld1",  ".b",     2, true,  1  },
+  { AArch64::LD1i16_POST,       "ld1",  ".h",     2, true,  2  },
+  { AArch64::LD1i32_POST,       "ld1",  ".s",     2, true,  4  },
+  { AArch64::LD1i64_POST,       "ld1",  ".d",     2, true,  8  },
+  { AArch64::LD1Rv16b,          "ld1r", ".16b",   0, false, 0  },
+  { AArch64::LD1Rv8h,           "ld1r", ".8h",    0, false, 0  },
+  { AArch64::LD1Rv4s,           "ld1r", ".4s",    0, false, 0  },
+  { AArch64::LD1Rv2d,           "ld1r", ".2d",    0, false, 0  },
+  { AArch64::LD1Rv8b,           "ld1r", ".8b",    0, false, 0  },
+  { AArch64::LD1Rv4h,           "ld1r", ".4h",    0, false, 0  },
+  { AArch64::LD1Rv2s,           "ld1r", ".2s",    0, false, 0  },
+  { AArch64::LD1Rv1d,           "ld1r", ".1d",    0, false, 0  },
+  { AArch64::LD1Rv16b_POST,     "ld1r", ".16b",   1, false, 1  },
+  { AArch64::LD1Rv8h_POST,      "ld1r", ".8h",    1, false, 2  },
+  { AArch64::LD1Rv4s_POST,      "ld1r", ".4s",    1, false, 4  },
+  { AArch64::LD1Rv2d_POST,      "ld1r", ".2d",    1, false, 8  },
+  { AArch64::LD1Rv8b_POST,      "ld1r", ".8b",    1, false, 1  },
+  { AArch64::LD1Rv4h_POST,      "ld1r", ".4h",    1, false, 2  },
+  { AArch64::LD1Rv2s_POST,      "ld1r", ".2s",    1, false, 4  },
+  { AArch64::LD1Rv1d_POST,      "ld1r", ".1d",    1, false, 8  },
+  { AArch64::LD1Onev16b,        "ld1",  ".16b",   0, false, 0  },
+  { AArch64::LD1Onev8h,         "ld1",  ".8h",    0, false, 0  },
+  { AArch64::LD1Onev4s,         "ld1",  ".4s",    0, false, 0  },
+  { AArch64::LD1Onev2d,         "ld1",  ".2d",    0, false, 0  },
+  { AArch64::LD1Onev8b,         "ld1",  ".8b",    0, false, 0  },
+  { AArch64::LD1Onev4h,         "ld1",  ".4h",    0, false, 0  },
+  { AArch64::LD1Onev2s,         "ld1",  ".2s",    0, false, 0  },
+  { AArch64::LD1Onev1d,         "ld1",  ".1d",    0, false, 0  },
+  { AArch64::LD1Onev16b_POST,   "ld1",  ".16b",   1, false, 16 },
+  { AArch64::LD1Onev8h_POST,    "ld1",  ".8h",    1, false, 16 },
+  { AArch64::LD1Onev4s_POST,    "ld1",  ".4s",    1, false, 16 },
+  { AArch64::LD1Onev2d_POST,    "ld1",  ".2d",    1, false, 16 },
+  { AArch64::LD1Onev8b_POST,    "ld1",  ".8b",    1, false, 8  },
+  { AArch64::LD1Onev4h_POST,    "ld1",  ".4h",    1, false, 8  },
+  { AArch64::LD1Onev2s_POST,    "ld1",  ".2s",    1, false, 8  },
+  { AArch64::LD1Onev1d_POST,    "ld1",  ".1d",    1, false, 8  },
+  { AArch64::LD1Twov16b,        "ld1",  ".16b",   0, false, 0  },
+  { AArch64::LD1Twov8h,         "ld1",  ".8h",    0, false, 0  },
+  { AArch64::LD1Twov4s,         "ld1",  ".4s",    0, false, 0  },
+  { AArch64::LD1Twov2d,         "ld1",  ".2d",    0, false, 0  },
+  { AArch64::LD1Twov8b,         "ld1",  ".8b",    0, false, 0  },
+  { AArch64::LD1Twov4h,         "ld1",  ".4h",    0, false, 0  },
+  { AArch64::LD1Twov2s,         "ld1",  ".2s",    0, false, 0  },
+  { AArch64::LD1Twov1d,         "ld1",  ".1d",    0, false, 0  },
+  { AArch64::LD1Twov16b_POST,   "ld1",  ".16b",   1, false, 32 },
+  { AArch64::LD1Twov8h_POST,    "ld1",  ".8h",    1, false, 32 },
+  { AArch64::LD1Twov4s_POST,    "ld1",  ".4s",    1, false, 32 },
+  { AArch64::LD1Twov2d_POST,    "ld1",  ".2d",    1, false, 32 },
+  { AArch64::LD1Twov8b_POST,    "ld1",  ".8b",    1, false, 16 },
+  { AArch64::LD1Twov4h_POST,    "ld1",  ".4h",    1, false, 16 },
+  { AArch64::LD1Twov2s_POST,    "ld1",  ".2s",    1, false, 16 },
+  { AArch64::LD1Twov1d_POST,    "ld1",  ".1d",    1, false, 16 },
+  { AArch64::LD1Threev16b,      "ld1",  ".16b",   0, false, 0  },
+  { AArch64::LD1Threev8h,       "ld1",  ".8h",    0, false, 0  },
+  { AArch64::LD1Threev4s,       "ld1",  ".4s",    0, false, 0  },
+  { AArch64::LD1Threev2d,       "ld1",  ".2d",    0, false, 0  },
+  { AArch64::LD1Threev8b,       "ld1",  ".8b",    0, false, 0  },
+  { AArch64::LD1Threev4h,       "ld1",  ".4h",    0, false, 0  },
+  { AArch64::LD1Threev2s,       "ld1",  ".2s",    0, false, 0  },
+  { AArch64::LD1Threev1d,       "ld1",  ".1d",    0, false, 0  },
+  { AArch64::LD1Threev16b_POST, "ld1",  ".16b",   1, false, 48 },
+  { AArch64::LD1Threev8h_POST,  "ld1",  ".8h",    1, false, 48 },
+  { AArch64::LD1Threev4s_POST,  "ld1",  ".4s",    1, false, 48 },
+  { AArch64::LD1Threev2d_POST,  "ld1",  ".2d",    1, false, 48 },
+  { AArch64::LD1Threev8b_POST,  "ld1",  ".8b",    1, false, 24 },
+  { AArch64::LD1Threev4h_POST,  "ld1",  ".4h",    1, false, 24 },
+  { AArch64::LD1Threev2s_POST,  "ld1",  ".2s",    1, false, 24 },
+  { AArch64::LD1Threev1d_POST,  "ld1",  ".1d",    1, false, 24 },
+  { AArch64::LD1Fourv16b,       "ld1",  ".16b",   0, false, 0  },
+  { AArch64::LD1Fourv8h,        "ld1",  ".8h",    0, false, 0  },
+  { AArch64::LD1Fourv4s,        "ld1",  ".4s",    0, false, 0  },
+  { AArch64::LD1Fourv2d,        "ld1",  ".2d",    0, false, 0  },
+  { AArch64::LD1Fourv8b,        "ld1",  ".8b",    0, false, 0  },
+  { AArch64::LD1Fourv4h,        "ld1",  ".4h",    0, false, 0  },
+  { AArch64::LD1Fourv2s,        "ld1",  ".2s",    0, false, 0  },
+  { AArch64::LD1Fourv1d,        "ld1",  ".1d",    0, false, 0  },
+  { AArch64::LD1Fourv16b_POST,  "ld1",  ".16b",   1, false, 64 },
+  { AArch64::LD1Fourv8h_POST,   "ld1",  ".8h",    1, false, 64 },
+  { AArch64::LD1Fourv4s_POST,   "ld1",  ".4s",    1, false, 64 },
+  { AArch64::LD1Fourv2d_POST,   "ld1",  ".2d",    1, false, 64 },
+  { AArch64::LD1Fourv8b_POST,   "ld1",  ".8b",    1, false, 32 },
+  { AArch64::LD1Fourv4h_POST,   "ld1",  ".4h",    1, false, 32 },
+  { AArch64::LD1Fourv2s_POST,   "ld1",  ".2s",    1, false, 32 },
+  { AArch64::LD1Fourv1d_POST,   "ld1",  ".1d",    1, false, 32 },
+  { AArch64::LD2i8,             "ld2",  ".b",     1, true,  0  },
+  { AArch64::LD2i16,            "ld2",  ".h",     1, true,  0  },
+  { AArch64::LD2i32,            "ld2",  ".s",     1, true,  0  },
+  { AArch64::LD2i64,            "ld2",  ".d",     1, true,  0  },
+  { AArch64::LD2i8_POST,        "ld2",  ".b",     2, true,  2  },
+  { AArch64::LD2i16_POST,       "ld2",  ".h",     2, true,  4  },
+  { AArch64::LD2i32_POST,       "ld2",  ".s",     2, true,  8  },
+  { AArch64::LD2i64_POST,       "ld2",  ".d",     2, true,  16  },
+  { AArch64::LD2Rv16b,          "ld2r", ".16b",   0, false, 0  },
+  { AArch64::LD2Rv8h,           "ld2r", ".8h",    0, false, 0  },
+  { AArch64::LD2Rv4s,           "ld2r", ".4s",    0, false, 0  },
+  { AArch64::LD2Rv2d,           "ld2r", ".2d",    0, false, 0  },
+  { AArch64::LD2Rv8b,           "ld2r", ".8b",    0, false, 0  },
+  { AArch64::LD2Rv4h,           "ld2r", ".4h",    0, false, 0  },
+  { AArch64::LD2Rv2s,           "ld2r", ".2s",    0, false, 0  },
+  { AArch64::LD2Rv1d,           "ld2r", ".1d",    0, false, 0  },
+  { AArch64::LD2Rv16b_POST,     "ld2r", ".16b",   1, false, 2  },
+  { AArch64::LD2Rv8h_POST,      "ld2r", ".8h",    1, false, 4  },
+  { AArch64::LD2Rv4s_POST,      "ld2r", ".4s",    1, false, 8  },
+  { AArch64::LD2Rv2d_POST,      "ld2r", ".2d",    1, false, 16 },
+  { AArch64::LD2Rv8b_POST,      "ld2r", ".8b",    1, false, 2  },
+  { AArch64::LD2Rv4h_POST,      "ld2r", ".4h",    1, false, 4  },
+  { AArch64::LD2Rv2s_POST,      "ld2r", ".2s",    1, false, 8  },
+  { AArch64::LD2Rv1d_POST,      "ld2r", ".1d",    1, false, 16 },
+  { AArch64::LD2Twov16b,        "ld2",  ".16b",   0, false, 0  },
+  { AArch64::LD2Twov8h,         "ld2",  ".8h",    0, false, 0  },
+  { AArch64::LD2Twov4s,         "ld2",  ".4s",    0, false, 0  },
+  { AArch64::LD2Twov2d,         "ld2",  ".2d",    0, false, 0  },
+  { AArch64::LD2Twov8b,         "ld2",  ".8b",    0, false, 0  },
+  { AArch64::LD2Twov4h,         "ld2",  ".4h",    0, false, 0  },
+  { AArch64::LD2Twov2s,         "ld2",  ".2s",    0, false, 0  },
+  { AArch64::LD2Twov16b_POST,   "ld2",  ".16b",   1, false, 32 },
+  { AArch64::LD2Twov8h_POST,    "ld2",  ".8h",    1, false, 32 },
+  { AArch64::LD2Twov4s_POST,    "ld2",  ".4s",    1, false, 32 },
+  { AArch64::LD2Twov2d_POST,    "ld2",  ".2d",    1, false, 32 },
+  { AArch64::LD2Twov8b_POST,    "ld2",  ".8b",    1, false, 16 },
+  { AArch64::LD2Twov4h_POST,    "ld2",  ".4h",    1, false, 16 },
+  { AArch64::LD2Twov2s_POST,    "ld2",  ".2s",    1, false, 16 },
+  { AArch64::LD3i8,             "ld3",  ".b",     1, true,  0  },
+  { AArch64::LD3i16,            "ld3",  ".h",     1, true,  0  },
+  { AArch64::LD3i32,            "ld3",  ".s",     1, true,  0  },
+  { AArch64::LD3i64,            "ld3",  ".d",     1, true,  0  },
+  { AArch64::LD3i8_POST,        "ld3",  ".b",     2, true,  3  },
+  { AArch64::LD3i16_POST,       "ld3",  ".h",     2, true,  6  },
+  { AArch64::LD3i32_POST,       "ld3",  ".s",     2, true,  12 },
+  { AArch64::LD3i64_POST,       "ld3",  ".d",     2, true,  24 },
+  { AArch64::LD3Rv16b,          "ld3r", ".16b",   0, false, 0  },
+  { AArch64::LD3Rv8h,           "ld3r", ".8h",    0, false, 0  },
+  { AArch64::LD3Rv4s,           "ld3r", ".4s",    0, false, 0  },
+  { AArch64::LD3Rv2d,           "ld3r", ".2d",    0, false, 0  },
+  { AArch64::LD3Rv8b,           "ld3r", ".8b",    0, false, 0  },
+  { AArch64::LD3Rv4h,           "ld3r", ".4h",    0, false, 0  },
+  { AArch64::LD3Rv2s,           "ld3r", ".2s",    0, false, 0  },
+  { AArch64::LD3Rv1d,           "ld3r", ".1d",    0, false, 0  },
+  { AArch64::LD3Rv16b_POST,     "ld3r", ".16b",   1, false, 3  },
+  { AArch64::LD3Rv8h_POST,      "ld3r", ".8h",    1, false, 6  },
+  { AArch64::LD3Rv4s_POST,      "ld3r", ".4s",    1, false, 12 },
+  { AArch64::LD3Rv2d_POST,      "ld3r", ".2d",    1, false, 24 },
+  { AArch64::LD3Rv8b_POST,      "ld3r", ".8b",    1, false, 3  },
+  { AArch64::LD3Rv4h_POST,      "ld3r", ".4h",    1, false, 6  },
+  { AArch64::LD3Rv2s_POST,      "ld3r", ".2s",    1, false, 12 },
+  { AArch64::LD3Rv1d_POST,      "ld3r", ".1d",    1, false, 24 },
+  { AArch64::LD3Threev16b,      "ld3",  ".16b",   0, false, 0  },
+  { AArch64::LD3Threev8h,       "ld3",  ".8h",    0, false, 0  },
+  { AArch64::LD3Threev4s,       "ld3",  ".4s",    0, false, 0  },
+  { AArch64::LD3Threev2d,       "ld3",  ".2d",    0, false, 0  },
+  { AArch64::LD3Threev8b,       "ld3",  ".8b",    0, false, 0  },
+  { AArch64::LD3Threev4h,       "ld3",  ".4h",    0, false, 0  },
+  { AArch64::LD3Threev2s,       "ld3",  ".2s",    0, false, 0  },
+  { AArch64::LD3Threev16b_POST, "ld3",  ".16b",   1, false, 48 },
+  { AArch64::LD3Threev8h_POST,  "ld3",  ".8h",    1, false, 48 },
+  { AArch64::LD3Threev4s_POST,  "ld3",  ".4s",    1, false, 48 },
+  { AArch64::LD3Threev2d_POST,  "ld3",  ".2d",    1, false, 48 },
+  { AArch64::LD3Threev8b_POST,  "ld3",  ".8b",    1, false, 24 },
+  { AArch64::LD3Threev4h_POST,  "ld3",  ".4h",    1, false, 24 },
+  { AArch64::LD3Threev2s_POST,  "ld3",  ".2s",    1, false, 24 },
+  { AArch64::LD4i8,             "ld4",  ".b",     1, true,  0  },
+  { AArch64::LD4i16,            "ld4",  ".h",     1, true,  0  },
+  { AArch64::LD4i32,            "ld4",  ".s",     1, true,  0  },
+  { AArch64::LD4i64,            "ld4",  ".d",     1, true,  0  },
+  { AArch64::LD4i8_POST,        "ld4",  ".b",     2, true,  4  },
+  { AArch64::LD4i16_POST,       "ld4",  ".h",     2, true,  8  },
+  { AArch64::LD4i32_POST,       "ld4",  ".s",     2, true,  16 },
+  { AArch64::LD4i64_POST,       "ld4",  ".d",     2, true,  32 },
+  { AArch64::LD4Rv16b,          "ld4r", ".16b",   0, false, 0  },
+  { AArch64::LD4Rv8h,           "ld4r", ".8h",    0, false, 0  },
+  { AArch64::LD4Rv4s,           "ld4r", ".4s",    0, false, 0  },
+  { AArch64::LD4Rv2d,           "ld4r", ".2d",    0, false, 0  },
+  { AArch64::LD4Rv8b,           "ld4r", ".8b",    0, false, 0  },
+  { AArch64::LD4Rv4h,           "ld4r", ".4h",    0, false, 0  },
+  { AArch64::LD4Rv2s,           "ld4r", ".2s",    0, false, 0  },
+  { AArch64::LD4Rv1d,           "ld4r", ".1d",    0, false, 0  },
+  { AArch64::LD4Rv16b_POST,     "ld4r", ".16b",   1, false, 4  },
+  { AArch64::LD4Rv8h_POST,      "ld4r", ".8h",    1, false, 8  },
+  { AArch64::LD4Rv4s_POST,      "ld4r", ".4s",    1, false, 16 },
+  { AArch64::LD4Rv2d_POST,      "ld4r", ".2d",    1, false, 32 },
+  { AArch64::LD4Rv8b_POST,      "ld4r", ".8b",    1, false, 4  },
+  { AArch64::LD4Rv4h_POST,      "ld4r", ".4h",    1, false, 8  },
+  { AArch64::LD4Rv2s_POST,      "ld4r", ".2s",    1, false, 16 },
+  { AArch64::LD4Rv1d_POST,      "ld4r", ".1d",    1, false, 32 },
+  { AArch64::LD4Fourv16b,       "ld4",  ".16b",   0, false, 0  },
+  { AArch64::LD4Fourv8h,        "ld4",  ".8h",    0, false, 0  },
+  { AArch64::LD4Fourv4s,        "ld4",  ".4s",    0, false, 0  },
+  { AArch64::LD4Fourv2d,        "ld4",  ".2d",    0, false, 0  },
+  { AArch64::LD4Fourv8b,        "ld4",  ".8b",    0, false, 0  },
+  { AArch64::LD4Fourv4h,        "ld4",  ".4h",    0, false, 0  },
+  { AArch64::LD4Fourv2s,        "ld4",  ".2s",    0, false, 0  },
+  { AArch64::LD4Fourv16b_POST,  "ld4",  ".16b",   1, false, 64 },
+  { AArch64::LD4Fourv8h_POST,   "ld4",  ".8h",    1, false, 64 },
+  { AArch64::LD4Fourv4s_POST,   "ld4",  ".4s",    1, false, 64 },
+  { AArch64::LD4Fourv2d_POST,   "ld4",  ".2d",    1, false, 64 },
+  { AArch64::LD4Fourv8b_POST,   "ld4",  ".8b",    1, false, 32 },
+  { AArch64::LD4Fourv4h_POST,   "ld4",  ".4h",    1, false, 32 },
+  { AArch64::LD4Fourv2s_POST,   "ld4",  ".2s",    1, false, 32 },
+  { AArch64::ST1i8,             "st1",  ".b",     0, true,  0  },
+  { AArch64::ST1i16,            "st1",  ".h",     0, true,  0  },
+  { AArch64::ST1i32,            "st1",  ".s",     0, true,  0  },
+  { AArch64::ST1i64,            "st1",  ".d",     0, true,  0  },
+  { AArch64::ST1i8_POST,        "st1",  ".b",     1, true,  1  },
+  { AArch64::ST1i16_POST,       "st1",  ".h",     1, true,  2  },
+  { AArch64::ST1i32_POST,       "st1",  ".s",     1, true,  4  },
+  { AArch64::ST1i64_POST,       "st1",  ".d",     1, true,  8  },
+  { AArch64::ST1Onev16b,        "st1",  ".16b",   0, false, 0  },
+  { AArch64::ST1Onev8h,         "st1",  ".8h",    0, false, 0  },
+  { AArch64::ST1Onev4s,         "st1",  ".4s",    0, false, 0  },
+  { AArch64::ST1Onev2d,         "st1",  ".2d",    0, false, 0  },
+  { AArch64::ST1Onev8b,         "st1",  ".8b",    0, false, 0  },
+  { AArch64::ST1Onev4h,         "st1",  ".4h",    0, false, 0  },
+  { AArch64::ST1Onev2s,         "st1",  ".2s",    0, false, 0  },
+  { AArch64::ST1Onev1d,         "st1",  ".1d",    0, false, 0  },
+  { AArch64::ST1Onev16b_POST,   "st1",  ".16b",   1, false, 16 },
+  { AArch64::ST1Onev8h_POST,    "st1",  ".8h",    1, false, 16 },
+  { AArch64::ST1Onev4s_POST,    "st1",  ".4s",    1, false, 16 },
+  { AArch64::ST1Onev2d_POST,    "st1",  ".2d",    1, false, 16 },
+  { AArch64::ST1Onev8b_POST,    "st1",  ".8b",    1, false, 8  },
+  { AArch64::ST1Onev4h_POST,    "st1",  ".4h",    1, false, 8  },
+  { AArch64::ST1Onev2s_POST,    "st1",  ".2s",    1, false, 8  },
+  { AArch64::ST1Onev1d_POST,    "st1",  ".1d",    1, false, 8  },
+  { AArch64::ST1Twov16b,        "st1",  ".16b",   0, false, 0  },
+  { AArch64::ST1Twov8h,         "st1",  ".8h",    0, false, 0  },
+  { AArch64::ST1Twov4s,         "st1",  ".4s",    0, false, 0  },
+  { AArch64::ST1Twov2d,         "st1",  ".2d",    0, false, 0  },
+  { AArch64::ST1Twov8b,         "st1",  ".8b",    0, false, 0  },
+  { AArch64::ST1Twov4h,         "st1",  ".4h",    0, false, 0  },
+  { AArch64::ST1Twov2s,         "st1",  ".2s",    0, false, 0  },
+  { AArch64::ST1Twov1d,         "st1",  ".1d",    0, false, 0  },
+  { AArch64::ST1Twov16b_POST,   "st1",  ".16b",   1, false, 32 },
+  { AArch64::ST1Twov8h_POST,    "st1",  ".8h",    1, false, 32 },
+  { AArch64::ST1Twov4s_POST,    "st1",  ".4s",    1, false, 32 },
+  { AArch64::ST1Twov2d_POST,    "st1",  ".2d",    1, false, 32 },
+  { AArch64::ST1Twov8b_POST,    "st1",  ".8b",    1, false, 16 },
+  { AArch64::ST1Twov4h_POST,    "st1",  ".4h",    1, false, 16 },
+  { AArch64::ST1Twov2s_POST,    "st1",  ".2s",    1, false, 16 },
+  { AArch64::ST1Twov1d_POST,    "st1",  ".1d",    1, false, 16 },
+  { AArch64::ST1Threev16b,      "st1",  ".16b",   0, false, 0  },
+  { AArch64::ST1Threev8h,       "st1",  ".8h",    0, false, 0  },
+  { AArch64::ST1Threev4s,       "st1",  ".4s",    0, false, 0  },
+  { AArch64::ST1Threev2d,       "st1",  ".2d",    0, false, 0  },
+  { AArch64::ST1Threev8b,       "st1",  ".8b",    0, false, 0  },
+  { AArch64::ST1Threev4h,       "st1",  ".4h",    0, false, 0  },
+  { AArch64::ST1Threev2s,       "st1",  ".2s",    0, false, 0  },
+  { AArch64::ST1Threev1d,       "st1",  ".1d",    0, false, 0  },
+  { AArch64::ST1Threev16b_POST, "st1",  ".16b",   1, false, 48 },
+  { AArch64::ST1Threev8h_POST,  "st1",  ".8h",    1, false, 48 },
+  { AArch64::ST1Threev4s_POST,  "st1",  ".4s",    1, false, 48 },
+  { AArch64::ST1Threev2d_POST,  "st1",  ".2d",    1, false, 48 },
+  { AArch64::ST1Threev8b_POST,  "st1",  ".8b",    1, false, 24 },
+  { AArch64::ST1Threev4h_POST,  "st1",  ".4h",    1, false, 24 },
+  { AArch64::ST1Threev2s_POST,  "st1",  ".2s",    1, false, 24 },
+  { AArch64::ST1Threev1d_POST,  "st1",  ".1d",    1, false, 24 },
+  { AArch64::ST1Fourv16b,       "st1",  ".16b",   0, false, 0  },
+  { AArch64::ST1Fourv8h,        "st1",  ".8h",    0, false, 0  },
+  { AArch64::ST1Fourv4s,        "st1",  ".4s",    0, false, 0  },
+  { AArch64::ST1Fourv2d,        "st1",  ".2d",    0, false, 0  },
+  { AArch64::ST1Fourv8b,        "st1",  ".8b",    0, false, 0  },
+  { AArch64::ST1Fourv4h,        "st1",  ".4h",    0, false, 0  },
+  { AArch64::ST1Fourv2s,        "st1",  ".2s",    0, false, 0  },
+  { AArch64::ST1Fourv1d,        "st1",  ".1d",    0, false, 0  },
+  { AArch64::ST1Fourv16b_POST,  "st1",  ".16b",   1, false, 64 },
+  { AArch64::ST1Fourv8h_POST,   "st1",  ".8h",    1, false, 64 },
+  { AArch64::ST1Fourv4s_POST,   "st1",  ".4s",    1, false, 64 },
+  { AArch64::ST1Fourv2d_POST,   "st1",  ".2d",    1, false, 64 },
+  { AArch64::ST1Fourv8b_POST,   "st1",  ".8b",    1, false, 32 },
+  { AArch64::ST1Fourv4h_POST,   "st1",  ".4h",    1, false, 32 },
+  { AArch64::ST1Fourv2s_POST,   "st1",  ".2s",    1, false, 32 },
+  { AArch64::ST1Fourv1d_POST,   "st1",  ".1d",    1, false, 32 },
+  { AArch64::ST2i8,             "st2",  ".b",     0, true,  0  },
+  { AArch64::ST2i16,            "st2",  ".h",     0, true,  0  },
+  { AArch64::ST2i32,            "st2",  ".s",     0, true,  0  },
+  { AArch64::ST2i64,            "st2",  ".d",     0, true,  0  },
+  { AArch64::ST2i8_POST,        "st2",  ".b",     1, true,  2  },
+  { AArch64::ST2i16_POST,       "st2",  ".h",     1, true,  4  },
+  { AArch64::ST2i32_POST,       "st2",  ".s",     1, true,  8  },
+  { AArch64::ST2i64_POST,       "st2",  ".d",     1, true,  16 },
+  { AArch64::ST2Twov16b,        "st2",  ".16b",   0, false, 0  },
+  { AArch64::ST2Twov8h,         "st2",  ".8h",    0, false, 0  },
+  { AArch64::ST2Twov4s,         "st2",  ".4s",    0, false, 0  },
+  { AArch64::ST2Twov2d,         "st2",  ".2d",    0, false, 0  },
+  { AArch64::ST2Twov8b,         "st2",  ".8b",    0, false, 0  },
+  { AArch64::ST2Twov4h,         "st2",  ".4h",    0, false, 0  },
+  { AArch64::ST2Twov2s,         "st2",  ".2s",    0, false, 0  },
+  { AArch64::ST2Twov16b_POST,   "st2",  ".16b",   1, false, 32 },
+  { AArch64::ST2Twov8h_POST,    "st2",  ".8h",    1, false, 32 },
+  { AArch64::ST2Twov4s_POST,    "st2",  ".4s",    1, false, 32 },
+  { AArch64::ST2Twov2d_POST,    "st2",  ".2d",    1, false, 32 },
+  { AArch64::ST2Twov8b_POST,    "st2",  ".8b",    1, false, 16 },
+  { AArch64::ST2Twov4h_POST,    "st2",  ".4h",    1, false, 16 },
+  { AArch64::ST2Twov2s_POST,    "st2",  ".2s",    1, false, 16 },
+  { AArch64::ST3i8,             "st3",  ".b",     0, true,  0  },
+  { AArch64::ST3i16,            "st3",  ".h",     0, true,  0  },
+  { AArch64::ST3i32,            "st3",  ".s",     0, true,  0  },
+  { AArch64::ST3i64,            "st3",  ".d",     0, true,  0  },
+  { AArch64::ST3i8_POST,        "st3",  ".b",     1, true,  3  },
+  { AArch64::ST3i16_POST,       "st3",  ".h",     1, true,  6  },
+  { AArch64::ST3i32_POST,       "st3",  ".s",     1, true,  12 },
+  { AArch64::ST3i64_POST,       "st3",  ".d",     1, true,  24 },
+  { AArch64::ST3Threev16b,      "st3",  ".16b",   0, false, 0  },
+  { AArch64::ST3Threev8h,       "st3",  ".8h",    0, false, 0  },
+  { AArch64::ST3Threev4s,       "st3",  ".4s",    0, false, 0  },
+  { AArch64::ST3Threev2d,       "st3",  ".2d",    0, false, 0  },
+  { AArch64::ST3Threev8b,       "st3",  ".8b",    0, false, 0  },
+  { AArch64::ST3Threev4h,       "st3",  ".4h",    0, false, 0  },
+  { AArch64::ST3Threev2s,       "st3",  ".2s",    0, false, 0  },
+  { AArch64::ST3Threev16b_POST, "st3",  ".16b",   1, false, 48 },
+  { AArch64::ST3Threev8h_POST,  "st3",  ".8h",    1, false, 48 },
+  { AArch64::ST3Threev4s_POST,  "st3",  ".4s",    1, false, 48 },
+  { AArch64::ST3Threev2d_POST,  "st3",  ".2d",    1, false, 48 },
+  { AArch64::ST3Threev8b_POST,  "st3",  ".8b",    1, false, 24 },
+  { AArch64::ST3Threev4h_POST,  "st3",  ".4h",    1, false, 24 },
+  { AArch64::ST3Threev2s_POST,  "st3",  ".2s",    1, false, 24 },
+  { AArch64::ST4i8,             "st4",  ".b",     0, true,  0  },
+  { AArch64::ST4i16,            "st4",  ".h",     0, true,  0  },
+  { AArch64::ST4i32,            "st4",  ".s",     0, true,  0  },
+  { AArch64::ST4i64,            "st4",  ".d",     0, true,  0  },
+  { AArch64::ST4i8_POST,        "st4",  ".b",     1, true,  4  },
+  { AArch64::ST4i16_POST,       "st4",  ".h",     1, true,  8  },
+  { AArch64::ST4i32_POST,       "st4",  ".s",     1, true,  16 },
+  { AArch64::ST4i64_POST,       "st4",  ".d",     1, true,  32 },
+  { AArch64::ST4Fourv16b,       "st4",  ".16b",   0, false, 0  },
+  { AArch64::ST4Fourv8h,        "st4",  ".8h",    0, false, 0  },
+  { AArch64::ST4Fourv4s,        "st4",  ".4s",    0, false, 0  },
+  { AArch64::ST4Fourv2d,        "st4",  ".2d",    0, false, 0  },
+  { AArch64::ST4Fourv8b,        "st4",  ".8b",    0, false, 0  },
+  { AArch64::ST4Fourv4h,        "st4",  ".4h",    0, false, 0  },
+  { AArch64::ST4Fourv2s,        "st4",  ".2s",    0, false, 0  },
+  { AArch64::ST4Fourv16b_POST,  "st4",  ".16b",   1, false, 64 },
+  { AArch64::ST4Fourv8h_POST,   "st4",  ".8h",    1, false, 64 },
+  { AArch64::ST4Fourv4s_POST,   "st4",  ".4s",    1, false, 64 },
+  { AArch64::ST4Fourv2d_POST,   "st4",  ".2d",    1, false, 64 },
+  { AArch64::ST4Fourv8b_POST,   "st4",  ".8b",    1, false, 32 },
+  { AArch64::ST4Fourv4h_POST,   "st4",  ".4h",    1, false, 32 },
+  { AArch64::ST4Fourv2s_POST,   "st4",  ".2s",    1, false, 32 },
+};
+
+const LdStNInstrDesc *getLdStNInstrDesc(unsigned Opcode) {
+  for (const auto &Info : LdStNInstInfo)
+    if (Info.Opcode == Opcode)
+      return &Info;
+
+  return nullptr;
+}
+
 }
diff --git a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
index a3dea22335d21..ffbc69af0dffc 100644
--- a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
+++ b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.h
@@ -1057,6 +1057,16 @@ static inline std::optional<MemInstInfo> getStoreInfo(const MCInst &Inst) {
   return MemInstInfo { DestRegIdx, BaseRegIdx, OffsetIdx, IsPrePost, IsPair };
 }
 
+struct LdStNInstrDesc {
+  unsigned Opcode;
+  const char *Mnemonic;
+  const char *Layout;
+  int ListOperand;
+  bool HasLane;
+  int NaturalOffset;
+};
+
+const LdStNInstrDesc *getLdStNInstrDesc(unsigned Opcode);
 
 namespace AArch64CC {
 
diff --git a/llvm/test/MC/AArch64/LFI/mem.s b/llvm/test/MC/AArch64/LFI/mem.s
index 03dfd42b30454..469b068a786f2 100644
--- a/llvm/test/MC/AArch64/LFI/mem.s
+++ b/llvm/test/MC/AArch64/LFI/mem.s
@@ -132,3 +132,17 @@ stp x0, x1, [x2, #-8]!
 // CHECK:      add x28, x27, w2, uxtw
 // CHECK-NEXT: stp x0, x1, [x28, #-8]
 // CHECK-NEXT: sub x2, x2, #8
+
+ld3 { v0.4s, v1.4s, v2.4s }, [x0], #48
+// CHECK:      add x28, x27, w0, uxtw
+// CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x28]
+// CHECK-NEXT: add x0, x0, #48
+
+st2 { v1.8b, v2.8b }, [x14], #16
+// CHECK:      add x28, x27, w14, uxtw
+// CHECK-NEXT: st2 { v1.8b, v2.8b }, [x28]
+// CHECK-NEXT: add x14, x14, #16
+
+st2 { v1.8b, v2.8b }, [x14]
+// CHECK:      add x28, x27, w14, uxtw
+// CHECK-NEXT: st2 { v1.8b, v2.8b }, [x28]

From fac255b128f3f9f0aa81fa75e8a7fef1261b5834 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Sat, 23 Aug 2025 15:10:38 -0700
Subject: [PATCH 21/41] [LFI] Remove unnecessary auto-sandboxing flag

---
 llvm/lib/MC/MCLFI.cpp | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/MC/MCLFI.cpp b/llvm/lib/MC/MCLFI.cpp
index 3cad2e4eff452..615bdb2b8d776 100644
--- a/llvm/lib/MC/MCLFI.cpp
+++ b/llvm/lib/MC/MCLFI.cpp
@@ -23,11 +23,6 @@ static const char NoteNamespace[] = "LFI";
 
 namespace llvm {
 
-cl::opt<bool> FlagEnableAutoSandboxing("nacl-enable-autosandboxing",
-                                cl::desc("Don't use the autosandboxing"
-                                         " assembler for the NaCl SFI."),
-                                cl::init(true));
-
 void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
                               const Triple &TheTriple) {
   assert(TheTriple.isLFI());
@@ -48,13 +43,11 @@ void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
 
   // Create the Target specific MCLFIExpander
   assert(TheTarget != nullptr);
-  if (FlagEnableAutoSandboxing) {
-    TheTarget->createMCLFIExpander(
-      Streamer,
-      std::unique_ptr<MCRegisterInfo>(
-          TheTarget->createMCRegInfo(TheTriple.getTriple())),
-      std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
-  }
+  TheTarget->createMCLFIExpander(
+    Streamer,
+    std::unique_ptr<MCRegisterInfo>(
+        TheTarget->createMCRegInfo(TheTriple.getTriple())),
+    std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
 
   // Emit an ELF Note section in its own COMDAT group which identifies LFI
   // object files.

From 5e3ae1da139f495b39e7df033806f9de31f87039 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Sat, 23 Aug 2025 15:16:40 -0700
Subject: [PATCH 22/41] [LFI] Apply clang-format to new files

---
 clang/lib/Driver/ToolChains/LFILinux.cpp      |   4 +-
 clang/lib/Driver/ToolChains/LFILinux.h        |  11 +-
 llvm/include/llvm/MC/MCLFI.h                  |   2 +-
 llvm/include/llvm/MC/MCLFIExpander.h          |   4 +-
 llvm/lib/MC/MCLFI.cpp                         |  32 +--
 llvm/lib/MC/MCLFIExpander.cpp                 |  25 +--
 llvm/lib/MC/MCParser/LFIAsmParser.cpp         |  17 +-
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 189 ++++++++++--------
 .../MCTargetDesc/AArch64MCLFIExpander.h       |   8 +-
 9 files changed, 157 insertions(+), 135 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/LFILinux.cpp b/clang/lib/Driver/ToolChains/LFILinux.cpp
index 37510c596b303..7ab2372756af9 100644
--- a/clang/lib/Driver/ToolChains/LFILinux.cpp
+++ b/clang/lib/Driver/ToolChains/LFILinux.cpp
@@ -13,8 +13,8 @@
 using namespace clang::driver::toolchains;
 using namespace llvm::opt;
 
-void LFILinuxToolChain::AddCXXStdlibLibArgs(
-    const ArgList &Args, ArgStringList &CmdArgs) const {
+void LFILinuxToolChain::AddCXXStdlibLibArgs(const ArgList &Args,
+                                            ArgStringList &CmdArgs) const {
   switch (GetCXXStdlibType(Args)) {
   case ToolChain::CST_Libcxx:
     CmdArgs.push_back("-lc++");
diff --git a/clang/lib/Driver/ToolChains/LFILinux.h b/clang/lib/Driver/ToolChains/LFILinux.h
index 4ef7b7334e8b0..e85f74803c1ce 100644
--- a/clang/lib/Driver/ToolChains/LFILinux.h
+++ b/clang/lib/Driver/ToolChains/LFILinux.h
@@ -20,13 +20,12 @@ class LLVM_LIBRARY_VISIBILITY LFILinuxToolChain : public Linux {
   LFILinuxToolChain(const Driver &D, const llvm::Triple &Triple,
                     const llvm::opt::ArgList &Args)
       : Linux(D, Triple, Args) {
-        ExtraOpts.push_back("-z");
-        ExtraOpts.push_back("separate-code");
-      }
+    ExtraOpts.push_back("-z");
+    ExtraOpts.push_back("separate-code");
+  }
 
-  void
-  AddCXXStdlibLibArgs(const llvm::opt::ArgList &Args,
-                            llvm::opt::ArgStringList &CmdArgs) const override;
+  void AddCXXStdlibLibArgs(const llvm::opt::ArgList &Args,
+                           llvm::opt::ArgStringList &CmdArgs) const override;
 };
 
 } // end namespace toolchains
diff --git a/llvm/include/llvm/MC/MCLFI.h b/llvm/include/llvm/MC/MCLFI.h
index a50a228d13643..544bda4c7c8d6 100644
--- a/llvm/include/llvm/MC/MCLFI.h
+++ b/llvm/include/llvm/MC/MCLFI.h
@@ -19,4 +19,4 @@ class Triple;
 void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
                              const Triple &TheTriple);
 
-}
+} // namespace llvm
diff --git a/llvm/include/llvm/MC/MCLFIExpander.h b/llvm/include/llvm/MC/MCLFIExpander.h
index 7ccb5963e8439..db0b254785ac2 100644
--- a/llvm/include/llvm/MC/MCLFIExpander.h
+++ b/llvm/include/llvm/MC/MCLFIExpander.h
@@ -43,7 +43,7 @@ class MCLFIExpander {
 
 public:
   MCLFIExpander(MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
-                 std::unique_ptr<MCInstrInfo> &&II)
+                std::unique_ptr<MCInstrInfo> &&II)
       : Ctx(Ctx), InstInfo(std::move(II)), RegInfo(std::move(RI)) {}
 
   void Error(const MCInst &Inst, const char msg[]);
@@ -75,5 +75,5 @@ class MCLFIExpander {
                           const MCSubtargetInfo &STI) = 0;
 };
 
-}
+} // namespace llvm
 #endif
diff --git a/llvm/lib/MC/MCLFI.cpp b/llvm/lib/MC/MCLFI.cpp
index 615bdb2b8d776..34c187c67c3c7 100644
--- a/llvm/lib/MC/MCLFI.cpp
+++ b/llvm/lib/MC/MCLFI.cpp
@@ -24,36 +24,36 @@ static const char NoteNamespace[] = "LFI";
 namespace llvm {
 
 void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
-                              const Triple &TheTriple) {
+                             const Triple &TheTriple) {
   assert(TheTriple.isLFI());
   const char *NoteName;
   const char *NoteArch;
   switch (TheTriple.getArch()) {
-    case Triple::aarch64:
-      NoteName = ".note.LFI.ABI.aarch64";
-      NoteArch = "aarch64";
-      break;
-    default:
-      report_fatal_error("Unsupported architecture for LFI");
+  case Triple::aarch64:
+    NoteName = ".note.LFI.ABI.aarch64";
+    NoteArch = "aarch64";
+    break;
+  default:
+    report_fatal_error("Unsupported architecture for LFI");
   }
 
-  std::string Error; //empty
+  std::string Error; // empty
   const Target *TheTarget =
-    TargetRegistry::lookupTarget(TheTriple.getTriple(), Error);
+      TargetRegistry::lookupTarget(TheTriple.getTriple(), Error);
 
   // Create the Target specific MCLFIExpander
   assert(TheTarget != nullptr);
   TheTarget->createMCLFIExpander(
-    Streamer,
-    std::unique_ptr<MCRegisterInfo>(
-        TheTarget->createMCRegInfo(TheTriple.getTriple())),
-    std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
+      Streamer,
+      std::unique_ptr<MCRegisterInfo>(
+          TheTarget->createMCRegInfo(TheTriple.getTriple())),
+      std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
 
   // Emit an ELF Note section in its own COMDAT group which identifies LFI
   // object files.
-  MCSectionELF* Note = Ctx.getELFSection(NoteName, ELF::SHT_NOTE,
-                                         ELF::SHF_ALLOC | ELF::SHF_GROUP,
-                                         0, NoteName, /*IsComdat=*/true);
+  MCSectionELF *Note = Ctx.getELFSection(NoteName, ELF::SHT_NOTE,
+                                         ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
+                                         NoteName, /*IsComdat=*/true);
 
   Streamer.pushSection();
   Streamer.switchSection(Note);
diff --git a/llvm/lib/MC/MCLFIExpander.cpp b/llvm/lib/MC/MCLFIExpander.cpp
index 9ed2286d61ee9..539203c0b1476 100644
--- a/llvm/lib/MC/MCLFIExpander.cpp
+++ b/llvm/lib/MC/MCLFIExpander.cpp
@@ -43,25 +43,17 @@ void MCLFIExpander::invalidateScratchRegs(const MCInst &Inst) {
   }
 }
 
-void MCLFIExpander::clearScratchRegs() {
-  ScratchRegs.clear();
-}
+void MCLFIExpander::clearScratchRegs() { ScratchRegs.clear(); }
 
-void MCLFIExpander::disable() {
-  Enabled = false;
-}
+void MCLFIExpander::disable() { Enabled = false; }
 
-void MCLFIExpander::enable() {
-  Enabled = true;
-}
+void MCLFIExpander::enable() { Enabled = true; }
 
-bool MCLFIExpander::isEnabled() {
-  return Enabled;
-}
+bool MCLFIExpander::isEnabled() { return Enabled; }
 
 MCRegister MCLFIExpander::getScratchReg(int index) {
   assert(index >= 0 && static_cast<unsigned>(index) < numScratchRegs());
-  return ScratchRegs[numScratchRegs()  - index - 1];
+  return ScratchRegs[numScratchRegs() - index - 1];
 }
 
 unsigned MCLFIExpander::numScratchRegs() const { return ScratchRegs.size(); }
@@ -102,12 +94,13 @@ bool MCLFIExpander::mayStore(const MCInst &Inst) const {
   return InstInfo->get(Inst.getOpcode()).mayStore();
 }
 
-bool MCLFIExpander::mayModifyRegister(const MCInst &Inst, MCRegister Reg) const {
+bool MCLFIExpander::mayModifyRegister(const MCInst &Inst,
+                                      MCRegister Reg) const {
   return InstInfo->get(Inst.getOpcode()).hasDefOfPhysReg(Inst, Reg, *RegInfo);
 }
 
 bool MCLFIExpander::explicitlyModifiesRegister(const MCInst &Inst,
-                                                MCRegister Reg) const {
+                                               MCRegister Reg) const {
   const MCInstrDesc &Desc = InstInfo->get(Inst.getOpcode());
   for (int i = 0; i < Desc.NumDefs; ++i) {
     if (Desc.operands()[i].OperandType == MCOI::OPERAND_REGISTER &&
@@ -116,4 +109,4 @@ bool MCLFIExpander::explicitlyModifiesRegister(const MCInst &Inst,
   }
   return false;
 }
-}
+} // namespace llvm
diff --git a/llvm/lib/MC/MCParser/LFIAsmParser.cpp b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
index f0932a0df6b6b..bf91e3f6ae162 100644
--- a/llvm/lib/MC/MCParser/LFIAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/LFIAsmParser.cpp
@@ -10,9 +10,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/MC/MCParser/MCAsmParserExtension.h"
 #include "llvm/MC/MCLFIExpander.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
+#include "llvm/MC/MCParser/MCAsmParserExtension.h"
 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
 #include "llvm/MC/MCRegister.h"
 #include "llvm/MC/MCStreamer.h"
@@ -23,15 +23,15 @@ using namespace llvm;
 
 class LFIAsmParser : public MCAsmParserExtension {
   MCLFIExpander *Expander;
-  template<bool (LFIAsmParser::*HandlerMethod)(StringRef, SMLoc)>
+  template <bool (LFIAsmParser::*HandlerMethod)(StringRef, SMLoc)>
   void addDirectiveHandler(StringRef Directive) {
-    MCAsmParser::ExtensionDirectiveHandler Handler = std::make_pair(
-        this, HandleDirective<LFIAsmParser, HandlerMethod>);
+    MCAsmParser::ExtensionDirectiveHandler Handler =
+        std::make_pair(this, HandleDirective<LFIAsmParser, HandlerMethod>);
 
     getParser().addDirectiveHandler(Directive, Handler);
   }
 
- public:
+public:
   LFIAsmParser(MCLFIExpander *Exp) : Expander(Exp) {}
   void Initialize(MCAsmParser &Parser) override {
     // Call the base implementation.
@@ -55,13 +55,12 @@ class LFIAsmParser : public MCAsmParserExtension {
 
       else if (getLexer().isNot(AsmToken::EndOfStatement))
         return Error(Loc, kInvalidOptionError);
-    }
-    else {
+    } else {
       return Error(Loc, kInvalidOptionError);
     }
     Lex();
 
-    if(Expander->addScratchReg(RegNo))
+    if (Expander->addScratchReg(RegNo))
       return Error(Loc, "Register can't be used as a scratch register");
     return false;
   }
@@ -107,4 +106,4 @@ namespace llvm {
 MCAsmParserExtension *createLFIAsmParser(MCLFIExpander *Exp) {
   return new LFIAsmParser(Exp);
 }
-}
+} // namespace llvm
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 58597f506e365..7de5d80408e16 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -30,8 +30,8 @@ using namespace llvm;
 
 #define DEBUG_TYPE "lfi"
 
-static cl::opt<bool> AArch64LFIErrorReserved("aarch64-lfi-error-reserved",
-    cl::Hidden, cl::init(false),
+static cl::opt<bool> AArch64LFIErrorReserved(
+    "aarch64-lfi-error-reserved", cl::Hidden, cl::init(false),
     cl::desc("Produce errors for uses of LFI reserved registers"));
 
 static MCRegister LFIAddrReg = AArch64::X28;
@@ -39,11 +39,13 @@ static MCRegister LFIBaseReg = AArch64::X27;
 static MCRegister LFIScratchReg = AArch64::X26;
 static MCRegister LFITLSReg = AArch64::X25;
 
-static bool hasFeature(const FeatureBitset Feature, const MCSubtargetInfo &STI) {
+static bool hasFeature(const FeatureBitset Feature,
+                       const MCSubtargetInfo &STI) {
   return (STI.getFeatureBits() & Feature) == Feature;
 }
 
-bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
+bool AArch64::AArch64MCLFIExpander::isValidScratchRegister(
+    MCRegister Reg) const {
   return Reg != AArch64::SP;
 }
 
@@ -54,8 +56,8 @@ MCRegister AArch64::AArch64MCLFIExpander::getScratch() {
   return getScratchReg(0);
 }
 
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs,
-    int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs, int64_t Imm,
+                 MCStreamer &Out, const MCSubtargetInfo &STI) {
   MCInst Inst;
   Inst.setOpcode(Op);
   Inst.addOperand(MCOperand::createReg(Rd));
@@ -64,8 +66,8 @@ static void emit(unsigned int Op, MCRegister Rd, MCRegister Rs,
   Out.emitInstruction(Inst, STI);
 }
 
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
-    MCRegister Rt2, int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1, MCRegister Rt2,
+                 int64_t Imm, MCStreamer &Out, const MCSubtargetInfo &STI) {
   MCInst Inst;
   Inst.setOpcode(Op);
   Inst.addOperand(MCOperand::createReg(Rd));
@@ -75,9 +77,9 @@ static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
   Out.emitInstruction(Inst, STI);
 }
 
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
-    MCRegister Rt2, int64_t Imm1, int64_t Imm2,
-    MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1, MCRegister Rt2,
+                 int64_t Imm1, int64_t Imm2, MCStreamer &Out,
+                 const MCSubtargetInfo &STI) {
   MCInst Inst;
   Inst.setOpcode(Op);
   Inst.addOperand(MCOperand::createReg(Rd));
@@ -88,9 +90,8 @@ static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt1,
   Out.emitInstruction(Inst, STI);
 }
 
-static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt,
-    int64_t Imm1, int64_t Imm2,
-    MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt, int64_t Imm1,
+                 int64_t Imm2, MCStreamer &Out, const MCSubtargetInfo &STI) {
   MCInst Inst;
   Inst.setOpcode(Op);
   Inst.addOperand(MCOperand::createReg(Rd));
@@ -101,30 +102,28 @@ static void emit(unsigned int Op, MCRegister Rd, MCRegister Rt,
 }
 
 static void emit(unsigned int Op, MCRegister Reg, MCStreamer &Out,
-    const MCSubtargetInfo &STI) {
+                 const MCSubtargetInfo &STI) {
   MCInst Inst;
   Inst.setOpcode(Op);
   Inst.addOperand(MCOperand::createReg(Reg));
   Out.emitInstruction(Inst, STI);
 }
 
-static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out,
+                    const MCSubtargetInfo &STI) {
   emit(AArch64::ORRXrs, Dest, AArch64::XZR, Src, 0, Out, STI);
 }
 
 // Emit 'add Dest, LFIBaseReg, W(Src), uxtw'
 static void emitAddMask(MCRegister Dest, MCRegister Src, MCStreamer &Out,
                         const MCSubtargetInfo &STI) {
-  emit(AArch64::ADDXrx,
-      Dest,
-      LFIBaseReg,
-      getWRegFromXReg(Src),
-      AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, 0),
-      Out, STI);
+  emit(AArch64::ADDXrx, Dest, LFIBaseReg, getWRegFromXReg(Src),
+       AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, 0), Out, STI);
 }
 
 // Emit 'Op(ld/st) Dest, [LFIBaseReg, W(Target), uxtw]'
-static void emitMemMask(unsigned Op, MCRegister Dest, MCRegister Target, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emitMemMask(unsigned Op, MCRegister Dest, MCRegister Target,
+                        MCStreamer &Out, const MCSubtargetInfo &STI) {
   emit(Op, Dest, LFIBaseReg, getWRegFromXReg(Target), 0, 0, Out, STI);
 }
 
@@ -136,11 +135,10 @@ static void emitBranch(unsigned int Opcode, MCRegister Target, MCStreamer &Out,
   Out.emitInstruction(Branch, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
-                                                         MCStreamer &Out,
-                                                         const MCSubtargetInfo &STI,
-                                                         bool IsCall) {
-  (void) IsCall;
+void AArch64::AArch64MCLFIExpander::expandIndirectBranch(
+    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI,
+    bool IsCall) {
+  (void)IsCall;
 
   assert(Inst.getOperand(0).isReg());
   MCRegister BranchReg = Inst.getOperand(0).getReg();
@@ -149,7 +147,8 @@ void AArch64::AArch64MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
   emitBranch(Inst.getOpcode(), LFIAddrReg, Out, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &Out,
+void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst,
+                                               MCStreamer &Out,
                                                const MCSubtargetInfo &STI) {
   if (Inst.getOperand(0).isReg())
     expandIndirectBranch(Inst, Out, STI, true);
@@ -157,7 +156,8 @@ void AArch64::AArch64MCLFIExpander::expandCall(const MCInst &Inst, MCStreamer &O
     Out.emitInstruction(Inst, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
+void AArch64::AArch64MCLFIExpander::expandReturn(const MCInst &Inst,
+                                                 MCStreamer &Out,
                                                  const MCSubtargetInfo &STI) {
   assert(Inst.getOperand(0).isReg());
   if (Inst.getOperand(0).getReg() != AArch64::LR)
@@ -171,15 +171,15 @@ bool AArch64::AArch64MCLFIExpander::mayModifyStack(const MCInst &Inst) {
 }
 
 bool AArch64::AArch64MCLFIExpander::mayModifyReserved(const MCInst &Inst) {
-  return mayModifyRegister(Inst, LFIAddrReg) || mayModifyRegister(Inst, LFIBaseReg);
+  return mayModifyRegister(Inst, LFIAddrReg) ||
+         mayModifyRegister(Inst, LFIBaseReg);
 }
 
 bool AArch64::AArch64MCLFIExpander::mayModifyLR(const MCInst &Inst) {
   return mayModifyRegister(Inst, AArch64::LR);
 }
 
-static MCInst replaceReg(const MCInst &Inst,
-                         MCRegister Dest, MCRegister Src) {
+static MCInst replaceReg(const MCInst &Inst, MCRegister Dest, MCRegister Src) {
   MCInst New;
   New.setOpcode(Inst.getOpcode());
   New.setLoc(Inst.getLoc());
@@ -215,7 +215,8 @@ void AArch64::AArch64MCLFIExpander::expandStackModification(
 
   MCInst ModInst;
   MCRegister Scratch = getScratch();
-  assert(Inst.getOperand(0).isReg() && Inst.getOperand(0).getReg() == AArch64::SP);
+  assert(Inst.getOperand(0).isReg() &&
+         Inst.getOperand(0).getReg() == AArch64::SP);
   ModInst.setOpcode(Inst.getOpcode());
   ModInst.setLoc(Inst.getLoc());
   ModInst.addOperand(MCOperand::createReg(Scratch));
@@ -233,13 +234,17 @@ static unsigned convertUiToRoW(unsigned Op);
 static unsigned convertPreToRoW(unsigned Op);
 static unsigned convertPostToRoW(unsigned Op);
 
-static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, bool &IsBaseNoOffset);
+static unsigned convertPrePostToBase(unsigned Op, bool &IsPre,
+                                     bool &IsBaseNoOffset);
 static unsigned getPrePostScale(unsigned Op);
 
-static void emitSafeLoadStoreDemoted(const MCInst &Inst, unsigned N, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emitSafeLoadStoreDemoted(const MCInst &Inst, unsigned N,
+                                     MCStreamer &Out,
+                                     const MCSubtargetInfo &STI) {
   MCInst LoadStore;
   bool IsPre, IsBaseNoOffset;
-  auto NewOpCode = convertPrePostToBase(Inst.getOpcode(), IsPre, IsBaseNoOffset);
+  auto NewOpCode =
+      convertPrePostToBase(Inst.getOpcode(), IsPre, IsBaseNoOffset);
   LoadStore.setOpcode(NewOpCode);
   for (unsigned I = 1; I < N; I++)
     LoadStore.addOperand(Inst.getOperand(I));
@@ -251,7 +256,8 @@ static void emitSafeLoadStoreDemoted(const MCInst &Inst, unsigned N, MCStreamer
   Out.emitInstruction(LoadStore, STI);
 }
 
-static void emitSafeLoadStore(const MCInst &Inst, unsigned N, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emitSafeLoadStore(const MCInst &Inst, unsigned N, MCStreamer &Out,
+                              const MCSubtargetInfo &STI) {
   MCInst LoadStore;
   LoadStore.setOpcode(Inst.getOpcode());
   for (unsigned I = 0; I < N; ++I)
@@ -262,8 +268,9 @@ static void emitSafeLoadStore(const MCInst &Inst, unsigned N, MCStreamer &Out, c
   Out.emitInstruction(LoadStore, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, MemInstInfo &MII,
-    MCStreamer &Out, const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(
+    const MCInst &Inst, MemInstInfo &MII, MCStreamer &Out,
+    const MCSubtargetInfo &STI) {
   emitAddMask(LFIAddrReg, Inst.getOperand(MII.BaseRegIdx).getReg(), Out, STI);
 
   if (MII.IsPrePost) {
@@ -273,20 +280,23 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, Mem
     MCRegister Base = Inst.getOperand(MII.BaseRegIdx).getReg();
     MCOperand OffsetMO = Inst.getOperand(MII.OffsetIdx);
     if (OffsetMO.isReg()) {
-      // The immediate offset of post-indexed addressing NEON Instrs has a fixed value, and
-      // it is encoded as a post-index addressing with XZR register operand.
-      // e.g., LD3Threev3d_POST can only have #48 as its operand and its offset
-      // MachineOperand holds XZR, which is a *Register* kind, not Imm.
+      // The immediate offset of post-indexed addressing NEON Instrs has a fixed
+      // value, and it is encoded as a post-index addressing with XZR register
+      // operand. e.g., LD3Threev3d_POST can only have #48 as its operand and
+      // its offset MachineOperand holds XZR, which is a *Register* kind, not
+      // Imm.
       MCRegister OffReg = OffsetMO.getReg();
       if (OffReg == AArch64::XZR) {
         const LdStNInstrDesc *Info = getLdStNInstrDesc(Inst.getOpcode());
         assert(Info && Info->NaturalOffset >= 0);
-        return emit(AArch64::ADDXri, Base, Base, Info->NaturalOffset, 0, Out, STI);
+        return emit(AArch64::ADDXri, Base, Base, Info->NaturalOffset, 0, Out,
+                    STI);
       }
       assert(OffReg != AArch64::WZR);
       return emit(AArch64::ADDXrs, Base, Base, OffsetMO.getReg(), 0, Out, STI);
     } else {
-      auto Offset = Inst.getOperand(MII.OffsetIdx).getImm() * getPrePostScale(Inst.getOpcode());
+      auto Offset = Inst.getOperand(MII.OffsetIdx).getImm() *
+                    getPrePostScale(Inst.getOpcode());
       if (Offset >= 0)
         return emit(AArch64::ADDXri, Base, Base, Offset, 0, Out, STI);
       return emit(AArch64::SUBXri, Base, Base, -Offset, 0, Out, STI);
@@ -296,14 +306,16 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreBasic(const MCInst &Inst, Mem
   return emitSafeLoadStore(Inst, MII.BaseRegIdx, Out, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemInstInfo &MII,
-    MCStreamer &Out, const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(
+    const MCInst &Inst, MemInstInfo &MII, MCStreamer &Out,
+    const MCSubtargetInfo &STI) {
   unsigned MemOp;
   unsigned Op = Inst.getOpcode();
   if ((MemOp = convertUiToRoW(Op)) != AArch64::INSTRUCTION_LIST_END) {
     auto OffsetMCO = Inst.getOperand(2);
     if (OffsetMCO.isImm() && OffsetMCO.getImm() == 0)
-      return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Inst.getOperand(1).getReg(), Out, STI);
+      return emitMemMask(MemOp, Inst.getOperand(0).getReg(),
+                         Inst.getOperand(1).getReg(), Out, STI);
     return expandLoadStoreBasic(Inst, MII, Out, STI);
   }
 
@@ -338,9 +350,11 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemIn
     if (!IsShift)
       Shift = 0;
     if (Extend)
-      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::SXTX, Shift), Out, STI);
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2,
+           AArch64_AM::getArithExtendImm(AArch64_AM::SXTX, Shift), Out, STI);
     else
-      emit(AArch64::ADDXrs, Scratch, Reg1, Reg2, AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift), Out, STI);
+      emit(AArch64::ADDXrs, Scratch, Reg1, Reg2,
+           AArch64_AM::getShifterImm(AArch64_AM::LSL, Shift), Out, STI);
     return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Scratch, Out, STI);
   }
 
@@ -353,19 +367,21 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(const MCInst &Inst, MemIn
     if (!IsShift)
       Shift = 0;
     if (S)
-      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::SXTW, Shift), Out, STI);
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2,
+           AArch64_AM::getArithExtendImm(AArch64_AM::SXTW, Shift), Out, STI);
     else
-      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2, AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, Shift), Out, STI);
+      emit(AArch64::ADDXrx, Scratch, Reg1, Reg2,
+           AArch64_AM::getArithExtendImm(AArch64_AM::UXTW, Shift), Out, STI);
     return emitMemMask(MemOp, Inst.getOperand(0).getReg(), Scratch, Out, STI);
   }
 }
 
-void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
-                                                    MCStreamer &Out,
-                                                    const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandLoadStore(
+    const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
   if (hasFeature(FeatureBitset({AArch64::FeatureLFIJumps}), STI))
     return Out.emitInstruction(Inst, STI);
-  if (hasFeature(FeatureBitset({AArch64::FeatureLFIStores}), STI) && !mayStore(Inst))
+  if (hasFeature(FeatureBitset({AArch64::FeatureLFIStores}), STI) &&
+      !mayStore(Inst))
     return Out.emitInstruction(Inst, STI);
 
   auto MII = getLoadInfo(Inst);
@@ -377,7 +393,7 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
 
   // Stack accesses without a register offset don't need rewriting.
   if (Inst.getOperand(MII->BaseRegIdx).getReg() == AArch64::SP) {
-    if (MII->BaseRegIdx == (int) Inst.getNumOperands() - 1 ||
+    if (MII->BaseRegIdx == (int)Inst.getNumOperands() - 1 ||
         !Inst.getOperand(MII->BaseRegIdx + 1).isReg())
       return Out.emitInstruction(Inst, STI);
   }
@@ -389,33 +405,44 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(const MCInst &Inst,
     expandLoadStoreBasic(Inst, MII.value(), Out, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::emitLFICall(LFICallType CallType, MCStreamer &Out, const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::emitLFICall(LFICallType CallType,
+                                                MCStreamer &Out,
+                                                const MCSubtargetInfo &STI) {
   MCRegister Scratch = getScratch();
   emitMov(Scratch, AArch64::LR, Out, STI);
   unsigned Offset;
   switch (CallType) {
-    case LFISyscall: Offset = 0; break;
-    case LFITLSRead: Offset = 1; break;
-    case LFITLSWrite: Offset = 2; break;
+  case LFISyscall:
+    Offset = 0;
+    break;
+  case LFITLSRead:
+    Offset = 1;
+    break;
+  case LFITLSWrite:
+    Offset = 2;
+    break;
   }
   emit(AArch64::LDRXui, AArch64::LR, LFIBaseReg, Offset, Out, STI);
   emit(AArch64::BLR, AArch64::LR, Out, STI);
   emitAddMask(AArch64::LR, Scratch, Out, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandSyscall(const MCInst &Inst, MCStreamer &Out,
-                   const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandSyscall(const MCInst &Inst,
+                                                  MCStreamer &Out,
+                                                  const MCSubtargetInfo &STI) {
   emitLFICall(LFISyscall, Out, STI);
 }
 
-static void emitSwap(MCRegister Reg1, MCRegister Reg2, MCStreamer &Out, const MCSubtargetInfo &STI) {
+static void emitSwap(MCRegister Reg1, MCRegister Reg2, MCStreamer &Out,
+                     const MCSubtargetInfo &STI) {
   emit(AArch64::EORXrs, Reg1, Reg1, Reg2, 0, Out, STI);
   emit(AArch64::EORXrs, Reg2, Reg1, Reg2, 0, Out, STI);
   emit(AArch64::EORXrs, Reg1, Reg1, Reg2, 0, Out, STI);
 }
 
-void AArch64::AArch64MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer &Out,
-                   const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandTLSRead(const MCInst &Inst,
+                                                  MCStreamer &Out,
+                                                  const MCSubtargetInfo &STI) {
   MCRegister Reg = Inst.getOperand(0).getReg();
 
   if (hasFeature(FeatureBitset({AArch64::FeatureLFITLSReg}), STI))
@@ -430,8 +457,9 @@ void AArch64::AArch64MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer
   }
 }
 
-void AArch64::AArch64MCLFIExpander::expandTLSWrite(const MCInst &Inst, MCStreamer &Out,
-                    const MCSubtargetInfo &STI) {
+void AArch64::AArch64MCLFIExpander::expandTLSWrite(const MCInst &Inst,
+                                                   MCStreamer &Out,
+                                                   const MCSubtargetInfo &STI) {
   MCRegister Reg = Inst.getOperand(1).getReg();
   if (Reg == AArch64::X0) {
     emitLFICall(LFITLSWrite, Out, STI);
@@ -448,15 +476,16 @@ static bool isSyscall(const MCInst &Inst) {
 
 static bool isTLSRead(const MCInst &Inst) {
   return Inst.getOpcode() == AArch64::MRS &&
-    Inst.getOperand(1).getImm() == AArch64SysReg::TPIDR_EL0;
+         Inst.getOperand(1).getImm() == AArch64SysReg::TPIDR_EL0;
 }
 
 static bool isTLSWrite(const MCInst &Inst) {
   return Inst.getOpcode() == AArch64::MSR &&
-    Inst.getOperand(0).getImm() == AArch64SysReg::TPIDR_EL0;
+         Inst.getOperand(0).getImm() == AArch64SysReg::TPIDR_EL0;
 }
 
-void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
+void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst,
+                                                 MCStreamer &Out,
                                                  const MCSubtargetInfo &STI) {
   if (isSyscall(Inst))
     return expandSyscall(Inst, Out, STI);
@@ -508,7 +537,8 @@ void AArch64::AArch64MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer
   return Out.emitInstruction(Inst, STI);
 }
 
-bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
+bool AArch64::AArch64MCLFIExpander::expandInst(const MCInst &Inst,
+                                               MCStreamer &Out,
                                                const MCSubtargetInfo &STI) {
   if (Guard)
     return false;
@@ -823,13 +853,14 @@ static unsigned convertPostToRoW(unsigned Op) {
 static bool canConvertToRoW(unsigned Op) {
   unsigned Shift;
   return convertUiToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
-    convertPreToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
-    convertPostToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
-    convertRoXToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END ||
-    convertRoWToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END;
+         convertPreToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
+         convertPostToRoW(Op) != AArch64::INSTRUCTION_LIST_END ||
+         convertRoXToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END ||
+         convertRoWToRoW(Op, Shift) != AArch64::INSTRUCTION_LIST_END;
 }
 
-static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, bool &IsBaseNoOffset) {
+static unsigned convertPrePostToBase(unsigned Op, bool &IsPre,
+                                     bool &IsBaseNoOffset) {
   IsPre = false;
   IsBaseNoOffset = false;
   switch (Op) {
@@ -895,7 +926,7 @@ static unsigned convertPrePostToBase(unsigned Op, bool &IsPre, bool &IsBaseNoOff
   // case AArch64::STLRXpre:
   //   IsPre = true;
   //   IsBaseNoOffset = true;
-    // return AArch64::STLRX;
+  // return AArch64::STLRX;
   case AArch64::LD1i64_POST:
     IsBaseNoOffset = true;
     return AArch64::LD1i64;
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
index 2bfb66955f47c..d664ef9e37757 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.h
@@ -30,7 +30,7 @@ namespace AArch64 {
 class AArch64MCLFIExpander : public MCLFIExpander {
 public:
   AArch64MCLFIExpander(MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
-                    std::unique_ptr<MCInstrInfo> &&II)
+                       std::unique_ptr<MCInstrInfo> &&II)
       : MCLFIExpander(Ctx, std::move(RI), std::move(II)) {}
 
   bool expandInst(const MCInst &Inst, MCStreamer &Out,
@@ -66,7 +66,7 @@ class AArch64MCLFIExpander : public MCLFIExpander {
                                const MCSubtargetInfo &STI);
 
   void expandLRModification(const MCInst &Inst, MCStreamer &Out,
-                                 const MCSubtargetInfo &STI);
+                            const MCSubtargetInfo &STI);
 
   void expandPrefetch(const MCInst &Inst, MCStreamer &Out,
                       const MCSubtargetInfo &STI);
@@ -75,10 +75,10 @@ class AArch64MCLFIExpander : public MCLFIExpander {
                        const MCSubtargetInfo &STI);
 
   void expandLoadStoreBasic(const MCInst &Inst, MemInstInfo &InstInfo,
-      MCStreamer &Out, const MCSubtargetInfo &STI);
+                            MCStreamer &Out, const MCSubtargetInfo &STI);
 
   void expandLoadStoreRoW(const MCInst &Inst, MemInstInfo &InstInfo,
-      MCStreamer &Out, const MCSubtargetInfo &STI);
+                          MCStreamer &Out, const MCSubtargetInfo &STI);
 
   void expandSyscall(const MCInst &Inst, MCStreamer &Out,
                      const MCSubtargetInfo &STI);

From 7a00571ffdd91e657fdf7b39218b0f560adf6ff3 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Sun, 24 Aug 2025 14:32:28 -0700
Subject: [PATCH 23/41] [LFI] [AArch64] Add rewrites for LSE atomics

---
 .../MCTargetDesc/AArch64MCLFIExpander.cpp     | 176 +++++++++++++++++-
 llvm/test/MC/AArch64/LFI/lse.s                |  15 ++
 llvm/test/MC/AArch64/LFI/mem.s                |  33 ++++
 3 files changed, 218 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/MC/AArch64/LFI/lse.s

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index 7de5d80408e16..ebf11f7f3e8e8 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -376,6 +376,21 @@ void AArch64::AArch64MCLFIExpander::expandLoadStoreRoW(
   }
 }
 
+static std::optional<MemInstInfo> getAtomicLoadStoreInfo(const MCInst &Inst);
+
+static std::optional<MemInstInfo> getMemInstInfo(const MCInst &Inst) {
+  auto MII = getLoadInfo(Inst);
+  if (MII.has_value())
+    return MII;
+  MII = getStoreInfo(Inst);
+  if (MII.has_value())
+    return MII;
+  MII = getAtomicLoadStoreInfo(Inst);
+  if (MII.has_value())
+    return MII;
+  return std::nullopt;
+}
+
 void AArch64::AArch64MCLFIExpander::expandLoadStore(
     const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) {
   if (hasFeature(FeatureBitset({AArch64::FeatureLFIJumps}), STI))
@@ -384,12 +399,9 @@ void AArch64::AArch64MCLFIExpander::expandLoadStore(
       !mayStore(Inst))
     return Out.emitInstruction(Inst, STI);
 
-  auto MII = getLoadInfo(Inst);
-  if (!MII.has_value()) {
-    MII = getStoreInfo(Inst);
-    if (!MII.has_value())
-      return Out.emitInstruction(Inst, STI);
-  }
+  auto MII = getMemInstInfo(Inst);
+  if (!MII.has_value())
+    return Out.emitInstruction(Inst, STI);
 
   // Stack accesses without a register offset don't need rewriting.
   if (Inst.getOperand(MII->BaseRegIdx).getReg() == AArch64::SP) {
@@ -1479,3 +1491,155 @@ static unsigned getPrePostScale(unsigned Op) {
   }
   return 0;
 }
+
+static std::optional<MemInstInfo> getAtomicLoadStoreInfo(const MCInst &Inst) {
+  int DestRegIdx;
+  int BaseRegIdx;
+  const int OffsetIdx = -1;
+  const bool IsPrePost = false;
+  bool IsPair = false;
+  // (ST|LD)OPRegister: ld*, st*
+  // CompareAndSwap(Pair)?(Unprvileged)?
+  // Swap(LSUI)?
+  // StoreRelease
+  // LoadAcquire
+  switch (Inst.getOpcode()) {
+  // Compare-and-swap CAS(A)?(L)?(B|H|W|X)
+  // MIR: Rs = op Rs, Rt, [Xn]
+  case AArch64::CASB:
+  case AArch64::CASH:
+  case AArch64::CASW:
+  case AArch64::CASX:
+  case AArch64::CASAB:
+  case AArch64::CASAH:
+  case AArch64::CASAW:
+  case AArch64::CASAX:
+  case AArch64::CASALB:
+  case AArch64::CASALH:
+  case AArch64::CASALW:
+  case AArch64::CASALX:
+  case AArch64::CASLB:
+  case AArch64::CASLH:
+  case AArch64::CASLW:
+  case AArch64::CASLX:
+    DestRegIdx = 2;
+    BaseRegIdx = 3;
+    break;
+  // rs1_rs2 = op rs1_rs2, rt1_rt2, [Xn]
+  case AArch64::CASPW:
+  case AArch64::CASPX:
+  case AArch64::CASPAW:
+  case AArch64::CASPAX:
+  case AArch64::CASPALW:
+  case AArch64::CASPALX:
+  case AArch64::CASPLW:
+  case AArch64::CASPLX:
+    DestRegIdx = 2;
+    BaseRegIdx = 3;
+    IsPair = true;
+    break;
+  // swap SWP(A)?(L)?(B|H|W|X)
+  // MIR: op Rs, Rt, [Xn]
+  case AArch64::SWPB:
+  case AArch64::SWPH:
+  case AArch64::SWPW:
+  case AArch64::SWPX:
+  case AArch64::SWPAB:
+  case AArch64::SWPAH:
+  case AArch64::SWPAW:
+  case AArch64::SWPAX:
+  case AArch64::SWPALB:
+  case AArch64::SWPALH:
+  case AArch64::SWPALW:
+  case AArch64::SWPALX:
+  case AArch64::SWPLB:
+  case AArch64::SWPLH:
+  case AArch64::SWPLW:
+  case AArch64::SWPLX:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    break;
+  // LD(ADD|CLR|EOR|SET)(A)?(L)?(B|H|W|X)
+  // op Rs, Rt, [Xn]
+  // if Rt == xzr, LD* aliases to ST*
+  case AArch64::LDADDB:
+  case AArch64::LDADDH:
+  case AArch64::LDADDX:
+  case AArch64::LDADDW:
+  case AArch64::LDADDAB:
+  case AArch64::LDADDAH:
+  case AArch64::LDADDAW:
+  case AArch64::LDADDAX:
+  case AArch64::LDADDALB:
+  case AArch64::LDADDALH:
+  case AArch64::LDADDALW:
+  case AArch64::LDADDALX:
+  case AArch64::LDADDLB:
+  case AArch64::LDADDLH:
+  case AArch64::LDADDLX:
+  case AArch64::LDADDLW:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    break;
+  case AArch64::LDCLRB:
+  case AArch64::LDCLRH:
+  case AArch64::LDCLRX:
+  case AArch64::LDCLRW:
+  case AArch64::LDCLRAB:
+  case AArch64::LDCLRAH:
+  case AArch64::LDCLRAW:
+  case AArch64::LDCLRAX:
+  case AArch64::LDCLRALB:
+  case AArch64::LDCLRALH:
+  case AArch64::LDCLRALW:
+  case AArch64::LDCLRALX:
+  case AArch64::LDCLRLB:
+  case AArch64::LDCLRLH:
+  case AArch64::LDCLRLX:
+  case AArch64::LDCLRLW:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    break;
+  case AArch64::LDEORB:
+  case AArch64::LDEORH:
+  case AArch64::LDEORX:
+  case AArch64::LDEORW:
+  case AArch64::LDEORAB:
+  case AArch64::LDEORAH:
+  case AArch64::LDEORAW:
+  case AArch64::LDEORAX:
+  case AArch64::LDEORALB:
+  case AArch64::LDEORALH:
+  case AArch64::LDEORALW:
+  case AArch64::LDEORALX:
+  case AArch64::LDEORLB:
+  case AArch64::LDEORLH:
+  case AArch64::LDEORLX:
+  case AArch64::LDEORLW:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    break;
+  case AArch64::LDSETB:
+  case AArch64::LDSETH:
+  case AArch64::LDSETX:
+  case AArch64::LDSETW:
+  case AArch64::LDSETAB:
+  case AArch64::LDSETAH:
+  case AArch64::LDSETAW:
+  case AArch64::LDSETAX:
+  case AArch64::LDSETALB:
+  case AArch64::LDSETALH:
+  case AArch64::LDSETALW:
+  case AArch64::LDSETALX:
+  case AArch64::LDSETLB:
+  case AArch64::LDSETLH:
+  case AArch64::LDSETLX:
+  case AArch64::LDSETLW:
+    DestRegIdx = 1;
+    BaseRegIdx = 2;
+    break;
+  default:
+    return std::nullopt;
+  }
+  return MemInstInfo{DestRegIdx, BaseRegIdx, OffsetIdx, IsPrePost, IsPair};
+}
diff --git a/llvm/test/MC/AArch64/LFI/lse.s b/llvm/test/MC/AArch64/LFI/lse.s
new file mode 100644
index 0000000000000..bda6c45c7a850
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/lse.s
@@ -0,0 +1,15 @@
+// RUN: llvm-mc -filetype asm -triple aarch64_lfi %s | FileCheck %s
+
+.arch_extension lse
+
+ldadd x0, x1, [x2]
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldadd x0, x1, [x28]
+
+swpal w0, w0, [x1]
+// CHECK:      add x28, x27, w1, uxtw
+// CHECK-NEXT: swpal w0, w0, [x28]
+
+caspal w0, w1, w2, w3, [x4]
+// CHECK:      add x28, x27, w4, uxtw
+// CHECK-NEXT: caspal w0, w1, w2, w3, [x28]
diff --git a/llvm/test/MC/AArch64/LFI/mem.s b/llvm/test/MC/AArch64/LFI/mem.s
index 469b068a786f2..6d87d3b29ba8f 100644
--- a/llvm/test/MC/AArch64/LFI/mem.s
+++ b/llvm/test/MC/AArch64/LFI/mem.s
@@ -146,3 +146,36 @@ st2 { v1.8b, v2.8b }, [x14], #16
 st2 { v1.8b, v2.8b }, [x14]
 // CHECK:      add x28, x27, w14, uxtw
 // CHECK-NEXT: st2 { v1.8b, v2.8b }, [x28]
+
+ld1 { v0.s }[1], [x8]
+// CHECK:      add x28, x27, w8, uxtw
+// CHECK-NEXT: ld1 { v0.s }[1], [x28]
+
+ld1r { v3.2d }, [x9]
+// CHECK:      add x28, x27, w9, uxtw
+// CHECK-NEXT: ld1r { v3.2d }, [x28]
+
+ld1 { v0.s }[1], [x8], x10
+// CHECK:      add x28, x27, w8, uxtw
+// CHECK-NEXT: ld1 { v0.s }[1], [x28]
+// CHECK-NEXT: add x8, x8, x10
+
+ldaxr x0, [x2]
+// CHECK:      add x28, x27, w2, uxtw
+// CHECK-NEXT: ldaxr x0, [x28]
+
+stlxr w15, w17, [x1]
+// CHECK:      add x28, x27, w1, uxtw
+// CHECK-NEXT: stlxr w15, w17, [x28]
+
+ldr w4, [sp, w3, uxtw #2]
+// CHECK:      add x26, sp, w3, uxtw #2
+// CHECK-NEXT: ldr w4, [x27, w26, uxtw]
+
+stxrb w11, w10, [x8]
+// CHECK:      add x28, x27, w8, uxtw
+// CHECK-NEXT: stxrb w11, w10, [x28]
+
+ldr x0, [x0, :got_lo12:x]
+// CHECK:      add x28, x27, w0, uxtw
+// CHECK-NEXT: ldr x0, [x28, :got_lo12:x]

From bdbb526b6bf0c65395604045389bd32d973575eb Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 11 Sep 2025 11:15:01 -0700
Subject: [PATCH 24/41] [LFI] [X86] Add x86-64 LFI target

---
 llvm/include/llvm/TargetParser/Triple.h | 9 ++++++++-
 llvm/lib/MC/MCLFI.cpp                   | 4 ++++
 llvm/lib/Target/X86/X86AsmPrinter.cpp   | 4 ++++
 llvm/lib/TargetParser/Triple.cpp        | 8 ++++++++
 4 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index 7806b780fc5f1..fa04dbbf3eec3 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -167,6 +167,8 @@ class Triple {
     SPIRVSubArch_v15,
     SPIRVSubArch_v16,
 
+    X8664SubArch_lfi,
+
     // DXIL sub-arch corresponds to its version.
     DXILSubArch_v1_0,
     DXILSubArch_v1_1,
@@ -679,7 +681,7 @@ class Triple {
   }
 
   bool isLFI() const {
-    return isAArch64LFI();
+    return isAArch64LFI() || isX8664LFI();
   }
 
   /// Checks if we're targeting the AArch64 LFI subarch.
@@ -688,6 +690,11 @@ class Triple {
            getSubArch() == Triple::AArch64SubArch_lfi;
   }
 
+  bool isX8664LFI() const {
+    return getArch() == Triple::x86_64 &&
+           getSubArch() == Triple::X8664SubArch_lfi;
+  }
+
   bool isWindowsCoreCLREnvironment() const {
     return isOSWindows() && getEnvironment() == Triple::CoreCLR;
   }
diff --git a/llvm/lib/MC/MCLFI.cpp b/llvm/lib/MC/MCLFI.cpp
index 34c187c67c3c7..0ce806496faf2 100644
--- a/llvm/lib/MC/MCLFI.cpp
+++ b/llvm/lib/MC/MCLFI.cpp
@@ -33,6 +33,10 @@ void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
     NoteName = ".note.LFI.ABI.aarch64";
     NoteArch = "aarch64";
     break;
+  case Triple::x86_64:
+    NoteName = ".note.LFI.ABI.x86_64";
+    NoteArch = "x86_64";
+    break;
   default:
     report_fatal_error("Unsupported architecture for LFI");
   }
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index f01e47b41cf5e..141c3af015c45 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -38,6 +38,7 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstBuilder.h"
+#include "llvm/MC/MCLFI.h"
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionMachO.h"
@@ -928,6 +929,9 @@ void X86AsmPrinter::emitStartOfAsmFile(Module &M) {
   bool is16 = TT.getEnvironment() == Triple::CODE16;
   if (M.getModuleInlineAsm().empty() && is16)
     OutStreamer->emitAssemblerFlag(MCAF_Code16);
+
+  if (TT.isLFI())
+    initializeLFIMCStreamer(*OutStreamer.get(), OutContext, TT);
 }
 
 static void
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index b5d5889363801..4aeb1bf625112 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -91,6 +91,10 @@ StringRef Triple::getArchTypeName(ArchType Kind) {
 
 StringRef Triple::getArchName(ArchType Kind, SubArchType SubArch) {
   switch (Kind) {
+  case Triple::x86_64:
+    if (SubArch == X8664SubArch_lfi)
+      return "x86_64_lfi";
+    break;
   case Triple::mips:
     if (SubArch == MipsSubArch_r6)
       return "mipsisa32r6";
@@ -563,6 +567,7 @@ static Triple::ArchType parseArch(StringRef ArchName) {
           // FIXME: Do we need to support these?
           .Cases("i786", "i886", "i986", Triple::x86)
           .Cases("amd64", "x86_64", "x86_64h", Triple::x86_64)
+          .Cases("amd64_lfi", "x86_64_lfi", Triple::x86_64)
           .Cases("powerpc", "powerpcspe", "ppc", "ppc32", Triple::ppc)
           .Cases("powerpcle", "ppcle", "ppc32le", Triple::ppcle)
           .Cases("powerpc64", "ppu", "ppc64", Triple::ppc64)
@@ -803,6 +808,9 @@ static Triple::SubArchType parseSubArch(StringRef SubArchName) {
   if (SubArchName == "aarch64_lfi")
     return Triple::AArch64SubArch_lfi;
 
+  if (SubArchName == "x86_64_lfi")
+    return Triple::X8664SubArch_lfi;
+
   if (SubArchName.starts_with("spirv"))
     return StringSwitch<Triple::SubArchType>(SubArchName)
         .EndsWith("v1.0", Triple::SPIRVSubArch_v10)

From 1cf1f41191328ae16f6a4f7e88141457826517c7 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 11 Sep 2025 11:53:39 -0700
Subject: [PATCH 25/41] [LFI] [X86] Reserve registers and align blocks

---
 clang/lib/Basic/Targets/X86.cpp           |  4 ++
 llvm/lib/Target/X86/CMakeLists.txt        |  1 +
 llvm/lib/Target/X86/X86.h                 |  3 ++
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp   |  4 ++
 llvm/lib/Target/X86/X86LFIRewritePass.cpp | 61 +++++++++++++++++++++++
 llvm/lib/Target/X86/X86RegisterInfo.cpp   |  7 +++
 llvm/lib/Target/X86/X86Subtarget.h        |  2 +
 llvm/lib/Target/X86/X86TargetMachine.cpp  |  3 ++
 8 files changed, 85 insertions(+)
 create mode 100644 llvm/lib/Target/X86/X86LFIRewritePass.cpp

diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 40ad8fd9a0967..10c5b9cd6543a 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -557,6 +557,10 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     DefineStd(Builder, "i386", Opts);
   }
 
+  if (getTriple().isX8664LFI()) {
+    Builder.defineMacro("__LFI__");
+  }
+
   Builder.defineMacro("__SEG_GS");
   Builder.defineMacro("__SEG_FS");
   Builder.defineMacro("__seg_gs", "__attribute__((address_space(256)))");
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index 9553a8619feb5..ccdf45b65a621 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -63,6 +63,7 @@ set(sources
   X86InstrFoldTables.cpp
   X86InstrInfo.cpp
   X86CompressEVEX.cpp
+  X86LFIRewritePass.cpp
   X86LoadValueInjectionLoadHardening.cpp
   X86LoadValueInjectionRetHardening.cpp
   X86MCInstLower.cpp
diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h
index 48a3fe1934a96..d3fd0315d18ae 100644
--- a/llvm/lib/Target/X86/X86.h
+++ b/llvm/lib/Target/X86/X86.h
@@ -42,6 +42,9 @@ FunctionPass *createCleanupLocalDynamicTLSPass();
 /// physical instructions.
 FunctionPass *createX86FloatingPointStackifierPass();
 
+// Creates a pass to modify code generation for LFI rules.
+FunctionPass *createX86LFIRewritePass();
+
 /// This pass inserts AVX vzeroupper instructions before each call to avoid
 /// transition penalty between functions encoded with AVX and SSE.
 FunctionPass *createX86IssueVZeroUpperPass();
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 84bcdae520885..3a60a668fff56 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -182,6 +182,10 @@ namespace {
       IndirectTlsSegRefs = MF.getFunction().hasFnAttribute(
                              "indirect-tls-seg-refs");
 
+      // Always enable IndirectTlsSegRefs for LFI.
+      if (Subtarget->isLFI())
+        IndirectTlsSegRefs = true;
+
       // OptFor[Min]Size are used in pattern predicates that isel is matching.
       OptForMinSize = MF.getFunction().hasMinSize();
       assert((!OptForMinSize || MF.getFunction().hasOptSize()) &&
diff --git a/llvm/lib/Target/X86/X86LFIRewritePass.cpp b/llvm/lib/Target/X86/X86LFIRewritePass.cpp
new file mode 100644
index 0000000000000..09bcb0eff1306
--- /dev/null
+++ b/llvm/lib/Target/X86/X86LFIRewritePass.cpp
@@ -0,0 +1,61 @@
+//=== X86LFIRewritePAss.cpp - Modify instructions for LFI ---------*- C++ -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#define DEBUG_TYPE "lfi-rewrite-pass"
+
+#include "X86.h"
+#include "X86InstrInfo.h"
+#include "X86Subtarget.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+namespace {
+  class X86LFIRewritePass : public MachineFunctionPass {
+  public:
+    static char ID;
+    X86LFIRewritePass() : MachineFunctionPass(ID) {}
+
+    virtual bool runOnMachineFunction(MachineFunction &Fn) override;
+
+    virtual StringRef getPassName() const override {
+      return "LFI Rewrites";
+    }
+
+  private:
+    const TargetInstrInfo *TII;
+    const X86Subtarget *Subtarget;
+  };
+
+  char X86LFIRewritePass::ID = 0;
+} // namespace
+
+bool X86LFIRewritePass::runOnMachineFunction(MachineFunction &MF) {
+  bool Modified = false;
+
+  TII = MF.getSubtarget().getInstrInfo();
+  Subtarget = &MF.getSubtarget<X86Subtarget>();
+  assert(Subtarget->isLFI() && "Unexpected target in LFIRewritePass!");
+
+  MF.setAlignment(llvm::Align(32));
+
+  for (MachineBasicBlock &MBB : MF) {
+    // TODO: consider only setting this if MBB.hasAddressTaken() is true.
+    MBB.setAlignment(llvm::Align(32));
+  }
+  return Modified;
+}
+
+/// createX86LFIRewritePassPass - returns an instance of the pass.
+namespace llvm {
+  FunctionPass* createX86LFIRewritePass() {
+    return new X86LFIRewritePass();
+  }
+} // namespace llvm
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 4faf8bca4f9e0..0d33b2ac2af73 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -557,6 +557,13 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   // Set the SIMD floating point control register as reserved.
   Reserved.set(X86::MXCSR);
 
+  if (MF.getSubtarget<X86Subtarget>().isLFI()) {
+    for (const MCPhysReg &SubReg : subregs_inclusive(X86::R11)) // scratch
+      Reserved.set(SubReg);
+    for (const MCPhysReg &SubReg : subregs_inclusive(X86::R14)) // base
+      Reserved.set(SubReg);
+  }
+
   // Set the stack-pointer register and its aliases as reserved.
   for (const MCPhysReg &SubReg : subregs_inclusive(X86::RSP))
     Reserved.set(SubReg);
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index 722076ca88c9c..db7376ba56612 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -179,6 +179,8 @@ class X86Subtarget final : public X86GenSubtargetInfo {
     return Is64Bit && (!TargetTriple.isX32() && !TargetTriple.isOSNaCl());
   }
 
+  bool isLFI() const { return TargetTriple.isX8664LFI(); }
+
   PICStyles::Style getPICStyle() const { return PICStyle; }
   void setPICStyle(PICStyles::Style Style)  { PICStyle = Style; }
 
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 20dfdd27b33df..4a8785b0f2b96 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -611,6 +611,9 @@ void X86PassConfig::addPreEmitPass() {
   addPass(createX86DiscriminateMemOpsPass());
   addPass(createX86InsertPrefetchPass());
   addPass(createX86InsertX87waitPass());
+
+  if (Triple(TM->getTargetTriple()).isLFI())
+    addPass(createX86LFIRewritePass());
 }
 
 void X86PassConfig::addPreEmitPass2() {

From 3112dfc4938ead105bf0b8c3625a636d885fecce Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 11 Sep 2025 12:28:21 -0700
Subject: [PATCH 26/41] [LFI] [X86] Add stub X86MCLFIExpander

---
 .../Target/X86/MCTargetDesc/CMakeLists.txt    |   1 +
 .../X86/MCTargetDesc/X86MCLFIExpander.cpp     | 108 ++++++++++++++++++
 .../X86/MCTargetDesc/X86MCLFIExpander.h       |  92 +++++++++++++++
 .../X86/MCTargetDesc/X86MCTargetDesc.cpp      |  12 ++
 4 files changed, 213 insertions(+)
 create mode 100644 llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
 create mode 100644 llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h

diff --git a/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt
index f2e7d43fc17f6..dd03aa612374d 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt
@@ -9,6 +9,7 @@ add_llvm_component_library(LLVMX86Desc
   X86MCTargetDesc.cpp
   X86MCAsmInfo.cpp
   X86MCCodeEmitter.cpp
+  X86MCLFIExpander.cpp
   X86MachObjectWriter.cpp
   X86MnemonicTables.cpp
   X86ELFObjectWriter.cpp
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
new file mode 100644
index 0000000000000..76a1476410292
--- /dev/null
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
@@ -0,0 +1,108 @@
+//===- X86MCLFIExpander.cpp - X86 LFI Expander --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the X86MCLFIExpander class, the X86 specific
+// subclass of MCLFIExpander.
+//
+// This file was written by the Native Client authors.
+//
+//===----------------------------------------------------------------------===//
+#include "X86MCLFIExpander.h"
+#include "X86BaseInfo.h"
+
+#include "X86MCTargetDesc.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrDesc.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "saigo"
+
+static const int BundleSize = 32;
+
+bool X86::X86MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
+  return false;
+}
+
+void X86::X86MCLFIExpander::expandDirectCall(const MCInst &Inst,
+                                              MCStreamer &Out,
+                                              const MCSubtargetInfo &STI) {
+}
+
+void X86::X86MCLFIExpander::emitIndirectJumpReg(MCRegister Reg, MCStreamer &Out,
+                                                 const MCSubtargetInfo &STI) {
+
+}
+
+void X86::X86MCLFIExpander::emitIndirectCallReg(MCRegister Reg, MCStreamer &Out,
+                                                 const MCSubtargetInfo &STI) {
+}
+
+void X86::X86MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
+                                                  MCStreamer &Out,
+                                                  const MCSubtargetInfo &STI) {
+}
+
+void X86::X86MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
+                                          const MCSubtargetInfo &STI) {
+}
+
+// Expands any operations that load to or store from memory, but do
+// not explicitly modify the stack or base pointer.
+void X86::X86MCLFIExpander::expandLoadStore(const MCInst &Inst,
+                                             MCStreamer &Out,
+                                             const MCSubtargetInfo &STI,
+                                             bool EmitPrefixes) {
+}
+
+void X86::X86MCLFIExpander::expandStringOperation(
+    const MCInst &Inst,
+    MCStreamer &Out,
+    const MCSubtargetInfo &STI,
+    bool EmitPrefixes) {
+}
+
+void X86::X86MCLFIExpander::expandExplicitStackManipulation(
+    MCRegister StackReg, const MCInst &Inst, MCStreamer &Out,
+    const MCSubtargetInfo &STI, bool EmitPrefixes) {
+}
+
+void X86::X86MCLFIExpander::expandStackRegPush(const MCInst &Inst,
+                                                MCStreamer &Out,
+                                                const MCSubtargetInfo &STI) {
+}
+
+void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst,
+                                          MCStreamer &Out,
+                                          const MCSubtargetInfo &STI,
+                                          bool EmitPrefixes) {
+  printf("TODO: doExpandInst\n");
+  Out.emitInstruction(Inst, STI);
+}
+
+bool X86::X86MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
+                                        const MCSubtargetInfo &STI) {
+  if (Guard)
+    return false;
+  Guard = true;
+
+  doExpandInst(Inst, Out, STI, true);
+  invalidateScratchRegs(Inst);
+
+  Guard = false;
+  return true;
+}
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
new file mode 100644
index 0000000000000..ba8f21b0acef8
--- /dev/null
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
@@ -0,0 +1,92 @@
+//===- X86MCLFIExpander.h - X86 LFI Expander -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// This file was written by the Native Client authors.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the X86MCLFIExpander class, the X86 specific
+// subclass of MCLFIExpander.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_MC_X86MCNACLEXPANDER_H
+#define LLVM_MC_X86MCNACLEXPANDER_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MC/MCLFIExpander.h"
+#include "llvm/MC/MCRegisterInfo.h"
+
+namespace llvm {
+class MCContext;
+class MCStreamer;
+class MCSubtargetInfo;
+
+namespace X86 {
+class X86MCLFIExpander : public MCLFIExpander {
+public:
+  X86MCLFIExpander(MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
+                    std::unique_ptr<MCInstrInfo> &&II)
+      : MCLFIExpander(Ctx, std::move(RI), std::move(II)) {}
+
+  bool expandInst(const MCInst &Inst, MCStreamer &Out,
+                  const MCSubtargetInfo &STI) override;
+protected:
+  bool isValidScratchRegister(MCRegister Reg) const override;
+
+private:
+  bool Guard = false; // recursion guard
+  SmallVector<MCInst, 4> Prefixes;
+
+  void emitPrefixes(MCStreamer &Out, const MCSubtargetInfo &STI);
+
+  void expandDirectCall(const MCInst &Inst, MCStreamer &Out,
+                        const MCSubtargetInfo &STI);
+
+  void expandIndirectBranch(const MCInst &Inst, MCStreamer &Out,
+                            const MCSubtargetInfo &STI);
+
+  void expandReturn(const MCInst &Inst, MCStreamer &Out,
+                    const MCSubtargetInfo &STI);
+
+  void expandLoadStore(const MCInst &Inst, MCStreamer &Out,
+                       const MCSubtargetInfo &STI, bool EmitPrefixes);
+
+  void expandStringOperation(const MCInst &Inst, MCStreamer &Out,
+           const MCSubtargetInfo &STI, bool EmitPrefixes);
+
+  void doExpandInst(const MCInst &Inst, MCStreamer &Out,
+                    const MCSubtargetInfo &STI, bool EmitPrefixes);
+
+  void expandExplicitStackManipulation(MCRegister StackReg, const MCInst &Inst,
+                                       MCStreamer &Out,
+                                       const MCSubtargetInfo &STI,
+                                       bool EmitPrefixes);
+
+  void expandStackRegPush(const MCInst &Inst,
+                          MCStreamer &Out,
+                          const MCSubtargetInfo &STI);
+
+  void emitSandboxMemOp(MCInst &Inst, int MemIdx, MCRegister ScratchReg,
+                        MCStreamer &Out, const MCSubtargetInfo &STI);
+
+  bool emitSandboxMemOps(MCInst &Inst, MCRegister ScratchReg, MCStreamer &Out,
+                         const MCSubtargetInfo &STI, bool EmitInstructions);
+
+  void emitInstruction(const MCInst &Inst, MCStreamer &Out,
+                       const MCSubtargetInfo &STI, bool EmitPrefixes);
+
+  void emitSandboxBranchReg(MCRegister Reg, MCStreamer &Out,
+                            const MCSubtargetInfo &STI);
+  void emitIndirectJumpReg(MCRegister Reg, MCStreamer &Out,
+                           const MCSubtargetInfo &STI);
+  void emitIndirectCallReg(MCRegister Reg, MCStreamer &Out,
+                           const MCSubtargetInfo &STI);
+};
+}
+}
+#endif
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
index 1c4d68d5448d6..f649bdf09b714 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
@@ -16,6 +16,7 @@
 #include "X86BaseInfo.h"
 #include "X86IntelInstPrinter.h"
 #include "X86MCAsmInfo.h"
+#include "X86MCLFIExpander.h"
 #include "X86TargetStreamer.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
@@ -708,6 +709,14 @@ static MCInstrAnalysis *createX86MCInstrAnalysis(const MCInstrInfo *Info) {
   return new X86_MC::X86MCInstrAnalysis(Info);
 }
 
+static MCLFIExpander *createX86_64MCLFIExpander(MCStreamer &S, std::unique_ptr<MCRegisterInfo> &&RegInfo, std::unique_ptr<MCInstrInfo> &&InstInfo) {
+  auto *Exp =
+      new X86::X86MCLFIExpander(S.getContext(), std::move(RegInfo),
+                                std::move(InstInfo));
+  S.setLFIExpander(Exp);
+  return Exp;
+}
+
 // Force static initialization.
 extern "C" LLVM_C_ABI void LLVMInitializeX86TargetMC() {
   for (Target *T : {&getTheX86_32Target(), &getTheX86_64Target()}) {
@@ -755,6 +764,9 @@ extern "C" LLVM_C_ABI void LLVMInitializeX86TargetMC() {
                                        createX86_32AsmBackend);
   TargetRegistry::RegisterMCAsmBackend(getTheX86_64Target(),
                                        createX86_64AsmBackend);
+
+  TargetRegistry::RegisterMCLFIExpander(getTheX86_64Target(),
+                                        createX86_64MCLFIExpander);
 }
 
 MCRegister llvm::getX86SubSuperRegister(MCRegister Reg, unsigned Size,

From 18751876c26101520d6b178c21c01c3ccf437110 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 11 Sep 2025 13:00:07 -0700
Subject: [PATCH 27/41] [LFI] [X86] Enable 32-byte bundling

---
 llvm/lib/MC/MCLFI.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/llvm/lib/MC/MCLFI.cpp b/llvm/lib/MC/MCLFI.cpp
index 0ce806496faf2..c3cfdc3e21b14 100644
--- a/llvm/lib/MC/MCLFI.cpp
+++ b/llvm/lib/MC/MCLFI.cpp
@@ -28,6 +28,7 @@ void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
   assert(TheTriple.isLFI());
   const char *NoteName;
   const char *NoteArch;
+  Align BundleAlign = Align(1);
   switch (TheTriple.getArch()) {
   case Triple::aarch64:
     NoteName = ".note.LFI.ABI.aarch64";
@@ -36,6 +37,7 @@ void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
   case Triple::x86_64:
     NoteName = ".note.LFI.ABI.x86_64";
     NoteArch = "x86_64";
+    BundleAlign = Align(32);
     break;
   default:
     report_fatal_error("Unsupported architecture for LFI");
@@ -53,6 +55,9 @@ void initializeLFIMCStreamer(MCStreamer &Streamer, MCContext &Ctx,
           TheTarget->createMCRegInfo(TheTriple.getTriple())),
       std::unique_ptr<MCInstrInfo>(TheTarget->createMCInstrInfo()));
 
+  if (BundleAlign != Align(1))
+    Streamer.emitBundleAlignMode(BundleAlign);
+
   // Emit an ELF Note section in its own COMDAT group which identifies LFI
   // object files.
   MCSectionELF *Note = Ctx.getELFSection(NoteName, ELF::SHT_NOTE,

From 81fedaff43a55193f52960727d7767153e663f8f Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 11 Sep 2025 13:37:22 -0700
Subject: [PATCH 28/41] [LFI] [X86] Add rewrites for control flow

---
 .../X86/MCTargetDesc/X86MCLFIExpander.cpp     | 182 +++++++++++++++++-
 llvm/test/MC/X86/LFI/branch.s                 |  36 ++++
 llvm/test/MC/X86/LFI/return.s                 |   9 +
 3 files changed, 224 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/MC/X86/LFI/branch.s
 create mode 100644 llvm/test/MC/X86/LFI/return.s

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
index 76a1476410292..bfa2d86582e53 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
@@ -30,10 +30,35 @@
 
 using namespace llvm;
 
-#define DEBUG_TYPE "saigo"
+#define DEBUG_TYPE "lfi"
 
 static const int BundleSize = 32;
 
+static const MCRegister LFIBaseReg = X86::R14;
+
+static MCRegister getReg64(MCRegister Reg) {
+  switch (Reg) {
+  default:
+    return getX86SubSuperRegister(Reg, 64, false);
+  case X86::IP:
+  case X86::EIP:
+  case X86::RIP:
+    return X86::RIP;
+  }
+}
+
+static MCRegister getReg32(MCRegister Reg) {
+  switch (Reg) {
+  default:
+    return getX86SubSuperRegister(Reg, 32, false);
+  case X86::IP:
+  case X86::EIP:
+    return X86::EIP;
+  case X86::RIP:
+    llvm_unreachable("Trying to demote %rip");
+  }
+}
+
 bool X86::X86MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
   return false;
 }
@@ -41,24 +66,128 @@ bool X86::X86MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
 void X86::X86MCLFIExpander::expandDirectCall(const MCInst &Inst,
                                               MCStreamer &Out,
                                               const MCSubtargetInfo &STI) {
+  Out.emitInstruction(Inst, STI);
+  Out.emitCodeAlignment(Align(BundleSize), &STI);
+}
+
+void X86::X86MCLFIExpander::emitSandboxBranchReg(MCRegister Reg, MCStreamer &Out,
+                                                  const MCSubtargetInfo &STI) {
+
+  MCInst AndInst;
+  AndInst.setOpcode(X86::AND32ri8);
+
+  MCOperand Target32 = MCOperand::createReg(getReg32(Reg));
+  AndInst.addOperand(Target32);
+  AndInst.addOperand(Target32);
+  AndInst.addOperand(MCOperand::createImm(-BundleSize));
+  Out.emitInstruction(AndInst, STI);
+
+  MCInst Add;
+  Add.setOpcode(X86::ADD64rr);
+
+  MCOperand Target64 = MCOperand::createReg(getReg64(Reg));
+  Add.addOperand(Target64);
+  Add.addOperand(Target64);
+  Add.addOperand(MCOperand::createReg(LFIBaseReg));
+  Out.emitInstruction(Add, STI);
 }
 
 void X86::X86MCLFIExpander::emitIndirectJumpReg(MCRegister Reg, MCStreamer &Out,
                                                  const MCSubtargetInfo &STI) {
+  Out.emitBundleLock(false);
+  emitSandboxBranchReg(Reg, Out, STI);
 
+  MCInst Jmp;
+  Jmp.setOpcode(X86::JMP64r);
+  MCOperand Target = MCOperand::createReg(getReg64(Reg));
+  Jmp.addOperand(Target);
+
+  Out.emitInstruction(Jmp, STI);
+  Out.emitBundleUnlock();
 }
 
 void X86::X86MCLFIExpander::emitIndirectCallReg(MCRegister Reg, MCStreamer &Out,
                                                  const MCSubtargetInfo &STI) {
+  MCOperand Target = MCOperand::createReg(getReg64(Reg));
+
+  Out.emitBundleLock(false);
+  emitSandboxBranchReg(Reg, Out, STI);
+  MCInst Call;
+  Call.setOpcode(X86::CALL64r);
+  Call.addOperand(Target);
+  Out.emitInstruction(Call, STI);
+  Out.emitBundleUnlock();
+  Out.emitCodeAlignment(Align(BundleSize), &STI);
 }
 
 void X86::X86MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
                                                   MCStreamer &Out,
                                                   const MCSubtargetInfo &STI) {
+  MCRegister Target;
+  if (mayLoad(Inst)) {
+    // indirect jmp/call through memory
+    MCInst Mov;
+    Mov.setOpcode(X86::MOV64rm);
+
+    Target = X86::R11;
+
+    Mov.addOperand(MCOperand::createReg(getReg64(Target)));
+    Mov.addOperand(Inst.getOperand(0));
+    Mov.addOperand(Inst.getOperand(1));
+    Mov.addOperand(Inst.getOperand(2));
+    Mov.addOperand(Inst.getOperand(3));
+    Mov.addOperand(Inst.getOperand(4));
+    doExpandInst(Mov, Out, STI, false);
+  } else {
+    Target = Inst.getOperand(0).getReg();
+  }
+
+  if (isCall(Inst))
+    emitIndirectCallReg(Target, Out, STI);
+  else
+    emitIndirectJumpReg(Target, Out, STI);
+}
+
+static bool isValidReturnRegister(const MCRegister& Reg) {
+  return Reg == X86::EAX ||
+         Reg == X86::RAX ||
+         Reg == X86::AL ||
+         Reg == X86::AX ||
+         Reg == X86::XMM0;
 }
 
 void X86::X86MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
                                           const MCSubtargetInfo &STI) {
+  MCRegister ScratchReg = X86::R11;
+
+  MCInst Pop;
+  Pop.setOpcode(X86::POP64r);
+  Pop.addOperand(MCOperand::createReg(ScratchReg));
+  Out.emitInstruction(Pop, STI);
+
+  if (Inst.getNumOperands() > 0) {
+    if (Inst.getOpcode() == X86::RETI32 || Inst.getOpcode() == X86::RETI64) {
+      MCOperand StackPointer = MCOperand::createReg(X86::RSP);
+
+      MCInst Add;
+      Add.setOpcode(X86::ADD64ri32);
+      Add.addOperand(StackPointer);
+      Add.addOperand(StackPointer);
+      Add.addOperand(Inst.getOperand(0));
+
+      doExpandInst(Add, Out, STI, false);
+    } else if (Inst.getOpcode() == X86::RET32 ||
+               Inst.getOpcode() == X86::RET64) {
+      LLVM_DEBUG(Inst.dump());
+      assert(Inst.getOperand(0).isReg());
+      assert(isValidReturnRegister(Inst.getOperand(0).getReg()));
+    } else {
+      LLVM_DEBUG(Inst.dump());
+      Error(Inst, "Unexpected return instruction.");
+    }
+  }
+
+  emitIndirectJumpReg(ScratchReg, Out, STI);
 }
 
 // Expands any operations that load to or store from memory, but do
@@ -86,12 +215,59 @@ void X86::X86MCLFIExpander::expandStackRegPush(const MCInst &Inst,
                                                 const MCSubtargetInfo &STI) {
 }
 
+// Returns true if Inst is an X86 prefix
+static bool isPrefix(const MCInst &Inst) {
+  switch (Inst.getOpcode()) {
+  case X86::LOCK_PREFIX:
+  case X86::REP_PREFIX:
+  case X86::REPNE_PREFIX:
+  case X86::REX64_PREFIX:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static bool isDirectCall(const MCInst &Inst) {
+  switch (Inst.getOpcode()) {
+    case X86::CALLpcrel32:
+    case X86::CALL64pcrel32:
+      return true;
+    default:
+      return false;
+  }
+}
+
+// Emit prefixes + instruction if EmitPrefixes argument is true.
+// Otherwise, emit the bare instruction.
+void X86::X86MCLFIExpander::emitInstruction(const MCInst &Inst,
+                                             MCStreamer &Out,
+                                             const MCSubtargetInfo &STI,
+                                             bool EmitPrefixes) {
+  if (EmitPrefixes) {
+    for (const MCInst &Prefix : Prefixes)
+      Out.emitInstruction(Prefix, STI);
+    Prefixes.clear();
+  }
+  Out.emitInstruction(Inst, STI);
+}
+
 void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst,
                                           MCStreamer &Out,
                                           const MCSubtargetInfo &STI,
                                           bool EmitPrefixes) {
-  printf("TODO: doExpandInst\n");
-  Out.emitInstruction(Inst, STI);
+  if (isPrefix(Inst)) {
+    return Prefixes.push_back(Inst);
+  }
+  if (isDirectCall(Inst)) {
+    expandDirectCall(Inst, Out, STI);
+  } else if (isIndirectBranch(Inst) || isCall(Inst)) {
+    expandIndirectBranch(Inst, Out, STI);
+  } else if (isReturn(Inst)) {
+    expandReturn(Inst, Out, STI);
+  } else {
+    emitInstruction(Inst, Out, STI, EmitPrefixes);
+  }
 }
 
 bool X86::X86MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
diff --git a/llvm/test/MC/X86/LFI/branch.s b/llvm/test/MC/X86/LFI/branch.s
new file mode 100644
index 0000000000000..d7a983acff888
--- /dev/null
+++ b/llvm/test/MC/X86/LFI/branch.s
@@ -0,0 +1,36 @@
+// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s
+
+callq foo
+// CHECK:      callq foo
+// CHECK-NEXT: .p2align 5
+
+callq *%rax
+// CHECK:      .bundle_lock
+// CHECK-NEXT: andl $-32, %eax
+// CHECK-NEXT: addq %r14, %rax
+// CHECK-NEXT: callq *%rax
+// CHECK-NEXT: .bundle_unlock
+
+callq *(%rax)
+// CHECK:      movq (%rax), %r11
+// CHECK-NEXT: .bundle_lock
+// CHECK-NEXT: andl $-32, %r11d
+// CHECK-NEXT: addq %r14, %r11
+// CHECK-NEXT: callq *%r11
+// CHECK-NEXT: .bundle_unlock
+// CHECK-NEXT: .p2align 5
+
+jmpq *%rax
+// CHECK:      .bundle_lock
+// CHECK-NEXT: andl $-32, %eax
+// CHECK-NEXT: addq %r14, %rax
+// CHECK-NEXT: jmpq *%rax
+// CHECK-NEXT: .bundle_unlock
+
+jmpq *(%rax)
+// CHECK:      movq (%rax), %r11
+// CHECK-NEXT: .bundle_lock
+// CHECK-NEXT: andl $-32, %r11d
+// CHECK-NEXT: addq %r14, %r11
+// CHECK-NEXT: jmpq *%r11
+// CHECK-NEXT: .bundle_unlock
diff --git a/llvm/test/MC/X86/LFI/return.s b/llvm/test/MC/X86/LFI/return.s
new file mode 100644
index 0000000000000..1920f1cf489fe
--- /dev/null
+++ b/llvm/test/MC/X86/LFI/return.s
@@ -0,0 +1,9 @@
+// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s
+
+ret
+// CHECK:      popq %r11
+// CHECK-NEXT: .bundle_lock
+// CHECK-NEXT: andl $-32, %r11d
+// CHECK-NEXT: addq %r14, %r11
+// CHECK-NEXT: jmpq *%r11
+// CHECK-NEXT: .bundle_unlock

From 1d6f6016053ddd92822e45356a3822c30610bb5b Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 11 Sep 2025 15:53:55 -0700
Subject: [PATCH 29/41] [LFI] [X86] Add string instruction rewrites

---
 .../X86/MCTargetDesc/X86MCLFIExpander.cpp     | 110 ++++++++++++++++--
 .../X86/MCTargetDesc/X86MCLFIExpander.h       |   4 -
 llvm/test/MC/X86/LFI/string.s                 |  17 +++
 3 files changed, 120 insertions(+), 11 deletions(-)
 create mode 100644 llvm/test/MC/X86/LFI/string.s

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
index bfa2d86582e53..8675840c9ac48 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
@@ -9,7 +9,7 @@
 // This file implements the X86MCLFIExpander class, the X86 specific
 // subclass of MCLFIExpander.
 //
-// This file was written by the Native Client authors.
+// This file was written by the Native Client authors, modified for LFI.
 //
 //===----------------------------------------------------------------------===//
 #include "X86MCLFIExpander.h"
@@ -59,6 +59,25 @@ static MCRegister getReg32(MCRegister Reg) {
   }
 }
 
+static bool isStringOperation(const MCInst &Inst) {
+  switch (Inst.getOpcode()) {
+  case X86::CMPSB:
+  case X86::CMPSW:
+  case X86::CMPSL:
+  case X86::CMPSQ:
+  case X86::MOVSB:
+  case X86::MOVSW:
+  case X86::MOVSL:
+  case X86::MOVSQ:
+  case X86::STOSB:
+  case X86::STOSW:
+  case X86::STOSL:
+  case X86::STOSQ:
+    return true;
+  }
+  return false;
+}
+
 bool X86::X86MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
   return false;
 }
@@ -196,6 +215,35 @@ void X86::X86MCLFIExpander::expandLoadStore(const MCInst &Inst,
                                              MCStreamer &Out,
                                              const MCSubtargetInfo &STI,
                                              bool EmitPrefixes) {
+  emitInstruction(Inst, Out, STI, EmitPrefixes);
+}
+
+// Emits movl Reg32, Reg32
+// Used as a helper in various places.
+static void clearHighBits(const MCOperand &Reg, MCStreamer &Out,
+                          const MCSubtargetInfo &STI) {
+  MCInst Mov;
+  Mov.setOpcode(X86::MOV32rr);
+  MCOperand Op = MCOperand::createReg(getReg32(Reg.getReg()));
+
+  Mov.addOperand(Op);
+  Mov.addOperand(Op);
+  Out.emitInstruction(Mov, STI);
+}
+
+static void fixupStringOpReg(const MCOperand &Op, MCStreamer &Out,
+                             const MCSubtargetInfo &STI) {
+  clearHighBits(Op, Out, STI);
+
+  MCInst Lea;
+  Lea.setOpcode(X86::LEA64r);
+  Lea.addOperand(MCOperand::createReg(getReg64(Op.getReg())));
+  Lea.addOperand(MCOperand::createReg(LFIBaseReg));
+  Lea.addOperand(MCOperand::createImm(1));
+  Lea.addOperand(MCOperand::createReg(getReg64(Op.getReg())));
+  Lea.addOperand(MCOperand::createImm(0));
+  Lea.addOperand(MCOperand::createReg(0));
+  Out.emitInstruction(Lea, STI);
 }
 
 void X86::X86MCLFIExpander::expandStringOperation(
@@ -203,6 +251,28 @@ void X86::X86MCLFIExpander::expandStringOperation(
     MCStreamer &Out,
     const MCSubtargetInfo &STI,
     bool EmitPrefixes) {
+  Out.emitBundleLock(false);
+  switch (Inst.getOpcode()) {
+  case X86::CMPSB:
+  case X86::CMPSW:
+  case X86::CMPSL:
+  case X86::CMPSQ:
+  case X86::MOVSB:
+  case X86::MOVSW:
+  case X86::MOVSL:
+  case X86::MOVSQ:
+    fixupStringOpReg(Inst.getOperand(1), Out, STI);
+    fixupStringOpReg(Inst.getOperand(0), Out, STI);
+    break;
+  case X86::STOSB:
+  case X86::STOSW:
+  case X86::STOSL:
+  case X86::STOSQ:
+    fixupStringOpReg(Inst.getOperand(0), Out, STI);
+    break;
+  }
+  emitInstruction(Inst, Out, STI, EmitPrefixes);
+  Out.emitBundleUnlock();
 }
 
 void X86::X86MCLFIExpander::expandExplicitStackManipulation(
@@ -210,11 +280,6 @@ void X86::X86MCLFIExpander::expandExplicitStackManipulation(
     const MCSubtargetInfo &STI, bool EmitPrefixes) {
 }
 
-void X86::X86MCLFIExpander::expandStackRegPush(const MCInst &Inst,
-                                                MCStreamer &Out,
-                                                const MCSubtargetInfo &STI) {
-}
-
 // Returns true if Inst is an X86 prefix
 static bool isPrefix(const MCInst &Inst) {
   switch (Inst.getOpcode()) {
@@ -252,6 +317,30 @@ void X86::X86MCLFIExpander::emitInstruction(const MCInst &Inst,
   Out.emitInstruction(Inst, STI);
 }
 
+// returns the stack register that is used in xchg instruction
+static MCRegister xchgStackReg(const MCInst &Inst) {
+  MCRegister Reg1 = 0, Reg2 = 0;
+  switch (Inst.getOpcode()) {
+  case X86::XCHG64ar:
+  case X86::XCHG64rm:
+    Reg1 = Inst.getOperand(0).getReg();
+    break;
+  case X86::XCHG64rr:
+    Reg1 = Inst.getOperand(0).getReg();
+    Reg2 = Inst.getOperand(2).getReg();
+    break;
+  default:
+    return 0;
+  }
+  if (Reg1 == X86::RSP)
+    return Reg1;
+
+  if (Reg2 == X86::RSP)
+    return Reg2;
+
+  return 0;
+}
+
 void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst,
                                           MCStreamer &Out,
                                           const MCSubtargetInfo &STI,
@@ -265,8 +354,15 @@ void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst,
     expandIndirectBranch(Inst, Out, STI);
   } else if (isReturn(Inst)) {
     expandReturn(Inst, Out, STI);
+  } else if (isStringOperation(Inst)) {
+    expandStringOperation(Inst, Out, STI, EmitPrefixes);
+  } else if (explicitlyModifiesRegister(Inst, X86::RSP)) {
+    expandExplicitStackManipulation(X86::RSP, Inst, Out, STI, EmitPrefixes);
+  } else if (MCRegister StackReg = xchgStackReg(Inst)) {
+    // The previous case doesn't catch xchg so special case it.
+    expandExplicitStackManipulation(X86::RSP, Inst, Out, STI, EmitPrefixes);
   } else {
-    emitInstruction(Inst, Out, STI, EmitPrefixes);
+    expandLoadStore(Inst, Out, STI, EmitPrefixes);
   }
 }
 
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
index ba8f21b0acef8..2f26ce4fe5a9d 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
@@ -67,10 +67,6 @@ class X86MCLFIExpander : public MCLFIExpander {
                                        const MCSubtargetInfo &STI,
                                        bool EmitPrefixes);
 
-  void expandStackRegPush(const MCInst &Inst,
-                          MCStreamer &Out,
-                          const MCSubtargetInfo &STI);
-
   void emitSandboxMemOp(MCInst &Inst, int MemIdx, MCRegister ScratchReg,
                         MCStreamer &Out, const MCSubtargetInfo &STI);
 
diff --git a/llvm/test/MC/X86/LFI/string.s b/llvm/test/MC/X86/LFI/string.s
new file mode 100644
index 0000000000000..beef15ef61a3e
--- /dev/null
+++ b/llvm/test/MC/X86/LFI/string.s
@@ -0,0 +1,17 @@
+// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s
+
+rep stosq
+// CHECK:      .bundle_lock
+// CHECK-NEXT: movl %edi, %edi
+// CHECK-NEXT: leaq (%r14,%rdi), %rdi
+// CHECK-NEXT: rep stosq %rax, %es:(%rdi)
+// CHECK-NEXT: .bundle_unlock
+
+rep movsq
+// CHECK:      .bundle_lock
+// CHECK-NEXT: movl %esi, %esi
+// CHECK-NEXT: leaq (%r14,%rsi), %rsi
+// CHECK-NEXT: movl %edi, %edi
+// CHECK-NEXT: leaq (%r14,%rdi), %rdi
+// CHECK-NEXT: rep movsq (%rsi), %es:(%rdi)
+// CHECK-NEXT: .bundle_unlock

From 7a8c8896e28e953659d8c6ac4c80639c16d8fd4d Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 11 Sep 2025 16:09:02 -0700
Subject: [PATCH 30/41] [LFI] [X86] Add rewrites for stack modifications

---
 .../X86/MCTargetDesc/X86MCLFIExpander.cpp     | 680 ++++++++++++++++++
 llvm/test/MC/X86/LFI/stack.s                  |  19 +
 2 files changed, 699 insertions(+)
 create mode 100644 llvm/test/MC/X86/LFI/stack.s

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
index 8675840c9ac48..8debbdcdc98f4 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
@@ -275,9 +275,164 @@ void X86::X86MCLFIExpander::expandStringOperation(
   Out.emitBundleUnlock();
 }
 
+static unsigned demoteOpcode(unsigned Reg);
+
+static bool isAbsoluteReg(MCRegister Reg) {
+  Reg = getReg64(Reg); // Normalize to 64 bits
+  return (Reg == LFIBaseReg || Reg == X86::RSP || Reg == X86::RIP);
+}
+
+static void demoteInst(MCInst &Inst, const MCInstrInfo &InstInfo) {
+  unsigned NewOpc = demoteOpcode(Inst.getOpcode());
+  Inst.setOpcode(NewOpc);
+  // demote all general purpose 64 bit registers to 32 bits
+  const llvm::ArrayRef<llvm::MCOperandInfo> OpInfo = InstInfo.get(Inst.getOpcode()).operands();
+  for (int i = 0, e = Inst.getNumOperands(); i < e; ++i) {
+    if (OpInfo[i].OperandType == MCOI::OPERAND_REGISTER) {
+      assert(Inst.getOperand(i).isReg());
+      MCRegister Reg = Inst.getOperand(i).getReg();
+      if (getReg64(Reg) == Reg) {
+        Inst.getOperand(i).setReg(getReg32(Reg));
+      }
+    }
+  }
+}
+
+// Emits the sandboxing operations necessary, and modifies the memory
+// operand specified by MemIdx.
+// Used as a helper function for emitSandboxMemOps.
+void X86::X86MCLFIExpander::emitSandboxMemOp(MCInst &Inst, int MemIdx,
+                                             MCRegister ScratchReg,
+                                             MCStreamer &Out,
+                                             const MCSubtargetInfo &STI) {
+}
+
+// Returns true if sandboxing the memory operand specified at Idx of
+// Inst will emit any auxillary instructions.
+// Used in emitSandboxMemOps as a helper.
+static bool willEmitSandboxInsts(const MCInst &Inst, int Idx) {
+  const MCOperand &Base = Inst.getOperand(Idx);
+  const MCOperand &Scale = Inst.getOperand(Idx + 1);
+  const MCOperand &Index = Inst.getOperand(Idx + 2);
+
+  if (isAbsoluteReg(Base.getReg()) && Index.getReg() == 0) {
+    return false;
+  } else if (Base.getReg() == 0 && isAbsoluteReg(Index.getReg()) &&
+             Scale.getImm() == 1) {
+    return false;
+  }
+
+  return true;
+}
+
+// Emits the instructions that are used to sandbox the memory operands.
+// Modifies memory operands of Inst in place, but does NOT EMIT Inst.
+// If any instructions are emitted, it will precede them with a .bundle_lock
+// Returns true if any instructions were emitted, otherwise false.
+// Note that this method can modify Inst, but still return false if no
+// auxiliary sandboxing instructions were emitted.
+// If EmitInstructions is set to false, this will not emit anything and return
+// whether it would emit any auxiliary instructions.
+bool X86::X86MCLFIExpander::emitSandboxMemOps(MCInst &Inst,
+                                               MCRegister ScratchReg,
+                                               MCStreamer &Out,
+                                               const MCSubtargetInfo &STI,
+                                               bool EmitInstructions) {
+
+  const llvm::ArrayRef<llvm::MCOperandInfo> OpInfo = InstInfo->get(Inst.getOpcode()).operands();
+
+  bool anyInstsEmitted = false;
+
+  for (int i = 0, e = Inst.getNumOperands(); i < e; ++i) {
+    if (OpInfo[i].OperandType == MCOI::OPERAND_MEMORY) {
+      if (!anyInstsEmitted && willEmitSandboxInsts(Inst, i)) {
+        if (!EmitInstructions)
+          return true;
+
+        Out.emitBundleLock(false);
+        anyInstsEmitted = true;
+      }
+      emitSandboxMemOp(Inst, i, ScratchReg, Out, STI);
+      i += 4;
+    }
+  }
+
+  return anyInstsEmitted;
+}
+
+static void emitStackFixup(MCRegister StackReg, MCStreamer &Out,
+                           const MCSubtargetInfo &STI) {
+  MCInst Lea;
+  Lea.setOpcode(X86::LEA64r);
+  Lea.addOperand(MCOperand::createReg(StackReg));
+  Lea.addOperand(MCOperand::createReg(StackReg));
+  Lea.addOperand(MCOperand::createImm(1));
+  Lea.addOperand(MCOperand::createReg(LFIBaseReg));
+  Lea.addOperand(MCOperand::createImm(0));
+  Lea.addOperand(MCOperand::createReg(0));
+  Out.emitInstruction(Lea, STI);
+}
+
 void X86::X86MCLFIExpander::expandExplicitStackManipulation(
     MCRegister StackReg, const MCInst &Inst, MCStreamer &Out,
     const MCSubtargetInfo &STI, bool EmitPrefixes) {
+  if (Inst.getOpcode() == X86::POP64r) {
+    // Transform
+    // pop   %rsp
+    // into
+    // pop %r11
+    // .bundle_lock
+    // movl %r11d, %esp
+    // add %r15, %rsp
+    // .bundle_unlock
+    // where %r11 is the scratch register, and %rsp the stack register.
+
+    MCInst PopR11Inst;
+    PopR11Inst.setOpcode(X86::POP64r);
+    PopR11Inst.addOperand(MCOperand::createReg(X86::R11));
+    Out.emitInstruction(PopR11Inst, STI);
+
+    Out.emitBundleLock(false);
+
+    MCInst MovR11ToESP;
+    MovR11ToESP.setOpcode(X86::MOV32rr);
+    MovR11ToESP.addOperand(MCOperand::createReg(getReg32(StackReg)));
+    MovR11ToESP.addOperand(MCOperand::createReg(X86::R11D));
+    Out.emitInstruction(MovR11ToESP, STI);
+
+    MCInst AddR15Inst;
+    AddR15Inst.setOpcode(X86::ADD64rr);
+    AddR15Inst.addOperand(MCOperand::createReg(StackReg));
+    AddR15Inst.addOperand(MCOperand::createReg(StackReg));
+    AddR15Inst.addOperand(MCOperand::createReg(X86::R15));
+    Out.emitInstruction(AddR15Inst, STI);
+
+    return Out.emitBundleUnlock();
+  }
+
+  MCInst SandboxedInst(Inst);
+  demoteInst(SandboxedInst, *InstInfo);
+
+  MCRegister ScratchReg;
+  if (numScratchRegs() > 0)
+    ScratchReg = getScratchReg(0);
+  else
+    ScratchReg = 0;
+
+  bool MemSandboxed = emitSandboxMemOps(SandboxedInst,
+                                        ScratchReg,
+                                        Out,
+                                        STI,
+                                        /*emitInstructions=*/true);
+
+  Out.emitBundleLock(false); // for stack fixup
+
+  emitInstruction(SandboxedInst, Out, STI, EmitPrefixes);
+  if (MemSandboxed)
+    Out.emitBundleUnlock(); // for memory reference
+  emitStackFixup(StackReg, Out, STI);
+
+  Out.emitBundleUnlock(); // for stack fixup
 }
 
 // Returns true if Inst is an X86 prefix
@@ -378,3 +533,528 @@ bool X86::X86MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
   Guard = false;
   return true;
 }
+
+static unsigned demoteOpcode(unsigned Opcode) {
+  switch (Opcode) {
+  case X86::ADC64rr:
+    return X86::ADC32rr;
+  case X86::ADC64ri8:
+    return X86::ADC32ri8;
+  case X86::ADC64ri32:
+    return X86::ADC32ri;
+  case X86::ADC64rm:
+    return X86::ADC32rm;
+  case X86::ADCX64rr:
+    return X86::ADCX32rr;
+  case X86::ADCX64rm:
+    return X86::ADCX32rm;
+  case X86::ADD64rr:
+    return X86::ADD32rr;
+  case X86::ADD64ri8:
+    return X86::ADD32ri8;
+  case X86::ADD64ri32:
+    return X86::ADD32ri;
+  case X86::ADD64rm:
+    return X86::ADD32rm;
+  case X86::ADOX64rr:
+    return X86::ADOX32rr;
+  case X86::ADOX64rm:
+    return X86::ADOX32rm;
+  case X86::ANDN64rr:
+    return X86::ANDN32rr;
+  case X86::ANDN64rm:
+    return X86::ANDN32rm;
+  case X86::AND64rr:
+    return X86::AND32rr;
+  case X86::AND64ri8:
+    return X86::AND32ri8;
+  case X86::AND64ri32:
+    return X86::AND32ri;
+  case X86::AND64rm:
+    return X86::AND32rm;
+  case X86::BEXTRI64ri:
+    return X86::BEXTRI32ri;
+  case X86::BEXTRI64mi:
+    return X86::BEXTRI32mi;
+  case X86::BEXTR64rr:
+    return X86::BEXTR32rr;
+  case X86::BEXTR64rm:
+    return X86::BEXTR32rm;
+  case X86::BLCFILL64rr:
+    return X86::BLCFILL32rr;
+  case X86::BLCFILL64rm:
+    return X86::BLCFILL32rm;
+  case X86::BLCI64rr:
+    return X86::BLCI32rr;
+  case X86::BLCI64rm:
+    return X86::BLCI32rm;
+  case X86::BLCIC64rr:
+    return X86::BLCIC32rr;
+  case X86::BLCIC64rm:
+    return X86::BLCIC32rm;
+  case X86::BLCMSK64rr:
+    return X86::BLCMSK32rr;
+  case X86::BLCMSK64rm:
+    return X86::BLCMSK32rm;
+  case X86::BLCS64rr:
+    return X86::BLCS32rr;
+  case X86::BLCS64rm:
+    return X86::BLCS32rm;
+  case X86::BLSFILL64rr:
+    return X86::BLSFILL32rr;
+  case X86::BLSFILL64rm:
+    return X86::BLSFILL32rm;
+  case X86::BLSIC64rr:
+    return X86::BLSIC32rr;
+  case X86::BLSIC64rm:
+    return X86::BLSIC32rm;
+  case X86::BLSI64rr:
+    return X86::BLSI32rr;
+  case X86::BLSI64rm:
+    return X86::BLSI32rm;
+  case X86::BLSMSK64rr:
+    return X86::BLSMSK32rr;
+  case X86::BLSMSK64rm:
+    return X86::BLSMSK32rm;
+  case X86::BLSR64rr:
+    return X86::BLSR32rr;
+  case X86::BLSR64rm:
+    return X86::BLSR32rm;
+  case X86::BSF64rr:
+    return X86::BSF32rr;
+  case X86::BSF64rm:
+    return X86::BSF32rm;
+  case X86::BSR64rr:
+    return X86::BSR32rr;
+  case X86::BSR64rm:
+    return X86::BSR32rm;
+  case X86::BSWAP64r:
+    return X86::BSWAP32r;
+  case X86::BTC64rr:
+    return X86::BTC32rr;
+  case X86::BTC64ri8:
+    return X86::BTC32ri8;
+  case X86::BT64rr:
+    return X86::BT32rr;
+  case X86::BT64ri8:
+    return X86::BT32ri8;
+  case X86::BTR64rr:
+    return X86::BTR32rr;
+  case X86::BTR64ri8:
+    return X86::BTR32ri8;
+  case X86::BTS64rr:
+    return X86::BTS32rr;
+  case X86::BTS64ri8:
+    return X86::BTS32ri8;
+  case X86::BZHI64rr:
+    return X86::BZHI32rr;
+  case X86::BZHI64rm:
+    return X86::BZHI32rm;
+  case X86::CALL64r:
+    return X86::CALL32r;
+  case X86::XOR64rr:
+    return X86::XOR32rr;
+  // NaCl 2020 note: there used to be a lot of cmov opcodes, now there are only
+  // two left since they were unified. The condition (above, equal, ...) used
+  // to be part of the opcode, now it is encoded  differently as per
+  // https://github.com/llvm/llvm-project/commit/e0bfeb5f24979416144c16e8b99204f5f163b889
+  // If there are errors, maybe the reason could be that here it is not sufficient
+  // to just lower the opcode, but the operation needs to be constructed
+  // in a more sophisticated way.
+  case X86::CMOV64rr:
+    return X86::CMOV32rr;
+  case X86::CMOV64rm:
+    return X86::CMOV32rm;
+  case X86::CMP64rr:
+    return X86::CMP32rr;
+  case X86::CMP64ri8:
+    return X86::CMP32ri8;
+  case X86::CMP64ri32:
+    return X86::CMP32ri;
+  case X86::CMP64rm:
+    return X86::CMP32rm;
+  case X86::CMPXCHG64rr:
+    return X86::CMPXCHG32rr;
+  case X86::CRC32r64r8:
+    return X86::CRC32r32r8;
+  case X86::CRC32r64m8:
+    return X86::CRC32r64m8;
+  case X86::CRC32r64r64:
+    return X86::CRC32r32r32;
+  case X86::CRC32r64m64:
+    return X86::CRC32r32m32;
+  case X86::CVTSD2SI64rr_Int:
+    return X86::CVTSD2SIrr_Int;
+  case X86::CVTSD2SI64rm_Int:
+    return X86::CVTSD2SIrm_Int;
+  case X86::CVTSS2SI64rr_Int:
+    return X86::CVTSS2SIrr_Int;
+  case X86::CVTSS2SI64rm_Int:
+    return X86::CVTSS2SIrm_Int;
+  case X86::CVTTSD2SI64rr:
+    return X86::CVTTSD2SIrr;
+  case X86::CVTTSD2SI64rm:
+    return X86::CVTTSD2SIrm;
+  case X86::CVTTSS2SI64rr:
+    return X86::CVTTSS2SIrr;
+  case X86::CVTTSS2SI64rm:
+    return X86::CVTTSS2SIrm;
+  case X86::DEC64r:
+    return X86::DEC32r;
+  case X86::DIV64r:
+    return X86::DIV32r;
+  case X86::IDIV64r:
+    return X86::IDIV32r;
+  case X86::IMUL64r:
+    return X86::IMUL32r;
+  case X86::IMUL64rr:
+    return X86::IMUL32rr;
+  case X86::IMUL64rri8:
+    return X86::IMUL32rri8;
+  case X86::IMUL64rri32:
+    return X86::IMUL32rri;
+  case X86::IMUL64rm:
+    return X86::IMUL32rm;
+  case X86::IMUL64rmi8:
+    return X86::IMUL32rmi8;
+  case X86::IMUL64rmi32:
+    return X86::IMUL32rmi;
+  case X86::INC64r:
+    return X86::INC32r;
+  case X86::INVEPT64:
+    return X86::INVEPT32;
+  case X86::INVPCID64:
+    return X86::INVPCID32;
+  case X86::INVVPID64:
+    return X86::INVVPID32;
+  case X86::JMP64r:
+    return X86::JMP32r;
+  case X86::KMOVQrk:
+    return X86::KMOVQrk;
+  case X86::LAR64rr:
+    return X86::LAR32rr;
+  case X86::LAR64rm:
+    return X86::LAR32rm;
+  case X86::LEA64r:
+    return X86::LEA32r;
+  case X86::LFS64rm:
+    return X86::LFS32rm;
+  case X86::LGS64rm:
+    return X86::LGS32rm;
+  case X86::LSL64rr:
+    return X86::LSL32rr;
+  case X86::LSL64rm:
+    return X86::LSL32rm;
+  case X86::LSS64rm:
+    return X86::LSS32rm;
+  case X86::LZCNT64rr:
+    return X86::LZCNT32rr;
+  case X86::LZCNT64rm:
+    return X86::LZCNT32rm;
+  case X86::MOV64ri:
+    return X86::MOV32ri;
+  case X86::MOVBE64rm:
+    return X86::MOVBE32rm;
+  case X86::MOV64rr:
+    return X86::MOV32rr;
+  case X86::MMX_MOVD64from64rr:
+    return X86::MMX_MOVD64grr;
+  case X86::MOVPQIto64rr:
+    return X86::MOVPDI2DIrr;
+  case X86::MOV64rs:
+    return X86::MOV32rs;
+  case X86::MOV64rd:
+    return X86::MOV32rd;
+  case X86::MOV64rc:
+    return X86::MOV32rc;
+  case X86::MOV64ri32:
+    return X86::MOV32ri;
+  case X86::MOV64rm:
+    return X86::MOV32rm;
+  case X86::MOVSX64rr8:
+    return X86::MOVSX32rr8;
+  case X86::MOVSX64rm8:
+    return X86::MOVSX32rm8;
+  case X86::MOVSX64rr32:
+    return X86::MOV32rr;
+  case X86::MOVSX64rm32:
+    return X86::MOV32rm;
+  case X86::MOVSX64rr16:
+    return X86::MOVSX32rr16;
+  case X86::MOVSX64rm16:
+    return X86::MOVSX32rm16;
+  case X86::MOVZX64rr8:
+    return X86::MOVZX32rr8;
+  case X86::MOVZX64rm8:
+    return X86::MOVZX32rm8;
+  case X86::MOVZX64rr16:
+    return X86::MOVZX32rr16;
+  case X86::MOVZX64rm16:
+    return X86::MOVZX32rm16;
+  case X86::MUL64r:
+    return X86::MUL32r;
+  case X86::MULX64rr:
+    return X86::MULX32rr;
+  case X86::MULX64rm:
+    return X86::MULX32rm;
+  case X86::NEG64r:
+    return X86::NEG32r;
+  case X86::NOT64r:
+    return X86::NOT32r;
+  case X86::OR64rr:
+    return X86::OR32rr;
+  case X86::OR64ri8:
+    return X86::OR32ri8;
+  case X86::OR64ri32:
+    return X86::OR32ri;
+  case X86::OR64rm:
+    return X86::OR32rm;
+  case X86::PDEP64rr:
+    return X86::PDEP32rr;
+  case X86::PDEP64rm:
+    return X86::PDEP32rm;
+  case X86::PEXT64rr:
+    return X86::PEXT32rr;
+  case X86::PEXT64rm:
+    return X86::PEXT32rm;
+  case X86::PEXTRQrri:
+    return X86::PEXTRQrri;
+  case X86::POPCNT64rr:
+    return X86::POPCNT32rr;
+  case X86::POPCNT64rm:
+    return X86::POPCNT32rm;
+  case X86::POP64r:
+    return X86::POP32r;
+  case X86::POP64rmr:
+    return X86::POP32rmr;
+  // TODO POP64rmm?
+  case X86::PUSH64r:
+    return X86::PUSH32r;
+  case X86::PUSH64rmr:
+    return X86::PUSH32rmr;
+  case X86::RCL64r1:
+    return X86::RCL32r1;
+  case X86::RCL64rCL:
+    return X86::RCL32rCL;
+  case X86::RCL64ri:
+    return X86::RCL32ri;
+  case X86::RCR64r1:
+    return X86::RCR32r1;
+  case X86::RCR64rCL:
+    return X86::RCR32rCL;
+  case X86::RCR64ri:
+    return X86::RCR32ri;
+  case X86::RDFSBASE64:
+    return X86::RDFSBASE;
+  case X86::RDGSBASE64:
+    return X86::RDGSBASE;
+  case X86::RDRAND64r:
+    return X86::RDRAND32r;
+  case X86::RDSEED64r:
+    return X86::RDSEED32r;
+  case X86::ROL64r1:
+    return X86::ROL32r1;
+  case X86::ROL64rCL:
+    return X86::ROL32rCL;
+  case X86::ROL64ri:
+    return X86::ROL32ri;
+  case X86::ROR64r1:
+    return X86::ROR32r1;
+  case X86::ROR64rCL:
+    return X86::ROR32rCL;
+  case X86::ROR64ri:
+    return X86::ROR32ri;
+  case X86::RORX64ri:
+    return X86::RORX32ri;
+  case X86::RORX64mi:
+    return X86::RORX64mi;
+  case X86::SAR64r1:
+    return X86::SAR32r1;
+  case X86::SAR64rCL:
+    return X86::SAR32rCL;
+  case X86::SAR64ri:
+    return X86::SAR32ri;
+  case X86::SARX64rr:
+    return X86::SARX32rr;
+  case X86::SARX64rm:
+    return X86::SARX32rm;
+  case X86::SBB64rr:
+    return X86::SBB32rr;
+  case X86::SBB64ri8:
+    return X86::SBB32ri8;
+  case X86::SBB64ri32:
+    return X86::SBB32ri;
+  case X86::SBB64rm:
+    return X86::SBB32rm;
+  case X86::SHLD64rrCL:
+    return X86::SHLD32rrCL;
+  case X86::SHLD64rri8:
+    return X86::SHLD32rri8;
+  case X86::SHL64r1:
+    return X86::SHL32r1;
+  case X86::SHL64rCL:
+    return X86::SHL32rCL;
+  case X86::SHL64ri:
+    return X86::SHL32ri;
+  case X86::SHLX64rr:
+    return X86::SHLX32rr;
+  case X86::SHLX64rm:
+    return X86::SHLX32rm;
+  case X86::SHRD64rrCL:
+    return X86::SHRD32rrCL;
+  case X86::SHRD64rri8:
+    return X86::SHRD32rri8;
+  case X86::SHR64r1:
+    return X86::SHR32r1;
+  case X86::SHR64rCL:
+    return X86::SHR32rCL;
+  case X86::SHR64ri:
+    return X86::SHR32ri;
+  case X86::SHRX64rr:
+    return X86::SHRX32rr;
+  case X86::SHRX64rm:
+    return X86::SHRX32rm;
+  case X86::SLDT64r:
+    return X86::SLDT32r;
+  case X86::SMSW64r:
+    return X86::SMSW32r;
+  case X86::STR64r:
+    return X86::STR32r;
+  case X86::SUB64rr:
+    return X86::SUB32rr;
+  case X86::SUB64ri8:
+    return X86::SUB32ri8;
+  case X86::SUB64ri32:
+    return X86::SUB32ri;
+  case X86::SUB64rm:
+    return X86::SUB32rm;
+  case X86::T1MSKC64rr:
+    return X86::T1MSKC32rr;
+  case X86::T1MSKC64rm:
+    return X86::T1MSKC32rm;
+  case X86::TEST64rr:
+    return X86::TEST32rr;
+  case X86::TEST64ri32:
+    return X86::TEST32ri;
+  case X86::TEST64mr:
+    return X86::TEST32mr;
+  case X86::TZCNT64rr:
+    return X86::TZCNT32rr;
+  case X86::TZCNT64rm:
+    return X86::TZCNT32rm;
+  case X86::TZMSK64rr:
+    return X86::TZMSK32rr;
+  case X86::TZMSK64rm:
+    return X86::TZMSK32rm;
+  /*
+  Some more Candidates for this are:
+  // Candidates start
+  case X86::VCVTSI2SSZrr:
+  case X86::VCVTSI2SSZrm:
+  case X86::Int_VCVTSI2SSZrr:
+  case X86::Int_VCVTSI2SSZrm:
+  case X86::VCVTSI2SSZrr_Int:
+  case X86::VCVTSI2SSZrm_Int:
+  case X86::VCVTSI642SSZrr:
+  case X86::VCVTSI642SSZrm:
+  case X86::Int_VCVTSI2SS64Zrr:
+  case X86::Int_VCVTSI2SS64Zrm:
+  case X86::VCVTSI642SSZrr_Int:
+  case X86::VCVTSI642SSZrm_Int:
+  case X86::VCVTSI2SDZrr:
+  case X86::VCVTSI2SDZrm:
+  case X86::Int_VCVTSI2SDZrr:
+  case X86::Int_VCVTSI2SDZrm:
+  case X86::VCVTSI2SDZrr_Int:
+  case X86::VCVTSI2SDZrm_Int:
+  case X86::VCVTSI642SDZrr:
+  case X86::VCVTSI642SDZrm:
+  case X86::Int_VCVTSI2SD64Zrr:
+  case X86::Int_VCVTSI2SD64Zrm:
+  case X86::VCVTSI642SDZrr_Int:
+  case X86::VCVTSI642SDZrm_Int:
+  Maybe everything that is listed in hasUndefRegUpdate() in X86InstrInfo.cpp
+  // Candidates end
+  */
+  case X86::VCVTSD2SI64rr_Int:
+    return X86::VCVTSD2SIrr_Int;
+  case X86::VCVTSD2SI64Zrr_Int:
+    return X86::VCVTSD2SIZrr_Int;
+  case X86::VCVTSD2SI64Zrm_Int:
+    return X86::VCVTSD2SIZrm_Int;
+  case X86::VCVTSD2SI64rm_Int:
+    return X86::VCVTSD2SIrm_Int;
+  case X86::VCVTSD2USI64Zrr_Int:
+    return X86::VCVTSD2USIZrr_Int;
+  case X86::VCVTSD2USI64Zrm_Int:
+    return X86::VCVTSD2USIZrm_Int;
+  case X86::VCVTSS2SI64rr_Int:
+    return X86::VCVTSS2SIrr_Int;
+  case X86::VCVTSS2SI64Zrr_Int:
+    return X86::VCVTSS2SIZrr_Int;
+  case X86::VCVTSS2SI64Zrm_Int:
+    return X86::VCVTSS2SIZrm_Int;
+  case X86::VCVTSS2SI64rm_Int:
+    return X86::VCVTSS2SIrm_Int;
+  case X86::VCVTSS2USI64Zrr_Int:
+    return X86::VCVTSS2USIZrr_Int;
+  case X86::VCVTSS2USI64Zrm_Int:
+    return X86::VCVTSS2USIZrm_Int;
+  case X86::VCVTTSD2SI64rr:
+    return X86::VCVTTSD2SIrr;
+  case X86::VCVTTSD2SI64Zrr:
+    return X86::VCVTTSD2SIZrr;
+  case X86::VCVTTSD2SI64Zrm:
+    return X86::VCVTTSD2SIZrm;
+  case X86::VCVTTSD2SI64rm:
+    return X86::VCVTTSD2SIrm;
+  case X86::VCVTTSD2USI64Zrr:
+    return X86::VCVTTSD2USIZrr;
+  case X86::VCVTTSD2USI64Zrm:
+    return X86::VCVTTSD2USIZrm;
+  case X86::VCVTTSS2SI64rr:
+    return X86::VCVTTSS2SIrr;
+  case X86::VCVTTSS2SI64Zrr:
+    return X86::VCVTTSS2SIZrr;
+  case X86::VCVTTSS2SI64Zrm:
+    return X86::VCVTTSS2SIZrm;
+  case X86::VCVTTSS2SI64rm:
+    return X86::VCVTTSS2SIrm;
+  case X86::VCVTTSS2USI64Zrr:
+    return X86::VCVTTSS2USIZrr;
+  case X86::VCVTTSS2USI64Zrm:
+    return X86::VCVTTSS2USIZrm;
+  case X86::VMOVPQIto64rr:
+    return X86::VMOVPDI2DIrr;
+  case X86::VMOVPQIto64Zrr:
+    return X86::VMOVPDI2DIZrr;
+  case X86::VMREAD64rr:
+    return X86::VMREAD32rr;
+  case X86::VMWRITE64rr:
+    return X86::VMWRITE32rr;
+  case X86::VMWRITE64rm:
+    return X86::VMWRITE32rm;
+  case X86::VPEXTRQrri:
+    return X86::VPEXTRQrri;
+  case X86::WRFSBASE64:
+    return X86::WRFSBASE;
+  case X86::WRGSBASE64:
+    return X86::WRGSBASE;
+  case X86::XADD64rr:
+    return X86::XADD32rr;
+  case X86::XCHG64ar:
+    return X86::XCHG32ar;
+  case X86::XCHG64rr:
+    return X86::XCHG32rr;
+  case X86::XCHG64rm:
+    return X86::XCHG32rm;
+  case X86::XOR64ri8:
+    return X86::XOR32ri8;
+  case X86::XOR64ri32:
+    return X86::XOR32ri;
+  case X86::XOR64rm:
+    return X86::XOR32rm;
+  default:
+    return Opcode;
+  }
+}
diff --git a/llvm/test/MC/X86/LFI/stack.s b/llvm/test/MC/X86/LFI/stack.s
new file mode 100644
index 0000000000000..98d5f05b1a7b7
--- /dev/null
+++ b/llvm/test/MC/X86/LFI/stack.s
@@ -0,0 +1,19 @@
+// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s
+
+movq %rdi, %rsp
+// CHECK:      .bundle_lock
+// CHECK-NEXT: movl %edi, %esp
+// CHECK-NEXT: leaq (%rsp,%r14), %rsp
+// CHECK-NEXT: .bundle_unlock
+
+addq %rax, %rsp
+// CHECK:      .bundle_lock
+// CHECK-NEXT: addl %eax, %esp
+// CHECK-NEXT: leaq (%rsp,%r14), %rsp
+// CHECK-NEXT: .bundle_unlock
+
+addq $8, %rsp
+// CHECK:      .bundle_lock
+// CHECK-NEXT: addl $8, %esp
+// CHECK-NEXT: leaq (%rsp,%r14), %rsp
+// CHECK-NEXT: .bundle_unlock

From e7f6de0a57fde8450ef8509c2a42507f5febcb9c Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 11 Sep 2025 17:26:27 -0700
Subject: [PATCH 31/41] [LFI] [X86] Rewrites for system calls

---
 llvm/include/llvm/MC/MCLFIExpander.h          |  2 +-
 .../X86/MCTargetDesc/X86MCLFIExpander.cpp     | 63 +++++++++++++++++--
 .../X86/MCTargetDesc/X86MCLFIExpander.h       | 12 ++++
 llvm/test/MC/X86/LFI/sys.s                    |  8 +++
 4 files changed, 80 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/MC/X86/LFI/sys.s

diff --git a/llvm/include/llvm/MC/MCLFIExpander.h b/llvm/include/llvm/MC/MCLFIExpander.h
index db0b254785ac2..d3861c6455793 100644
--- a/llvm/include/llvm/MC/MCLFIExpander.h
+++ b/llvm/include/llvm/MC/MCLFIExpander.h
@@ -30,10 +30,10 @@ class MCStreamer;
 class MCLFIExpander {
 private:
   SmallVector<MCRegister, 2> ScratchRegs;
-  MCContext &Ctx;
   bool Enabled = true;
 
 protected:
+  MCContext &Ctx;
   std::unique_ptr<MCInstrInfo> InstInfo;
   std::unique_ptr<MCRegisterInfo> RegInfo;
   void invalidateScratchRegs(const MCInst &Inst);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
index 8debbdcdc98f4..5cd2013750e59 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
@@ -496,14 +496,69 @@ static MCRegister xchgStackReg(const MCInst &Inst) {
   return 0;
 }
 
+static bool isSyscall(const MCInst &Inst) {
+  return Inst.getOpcode() == X86::SYSCALL;
+}
+
+void X86::X86MCLFIExpander::emitLFICall(LFICallType CallType,
+                                        MCStreamer &Out,
+                                        const MCSubtargetInfo &STI) {
+  Out.emitBundleLock(false);
+
+  MCSymbol *Symbol = Ctx.createTempSymbol();
+
+  MCInst Lea;
+  Lea.setOpcode(X86::LEA64r);
+  Lea.addOperand(MCOperand::createReg(X86::R11));
+  Lea.addOperand(MCOperand::createReg(X86::RIP));
+  Lea.addOperand(MCOperand::createImm(1));
+  Lea.addOperand(MCOperand::createReg(X86::NoRegister));
+  Lea.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(Symbol, Ctx)));
+  Lea.addOperand(MCOperand::createReg(0));
+  Out.emitInstruction(Lea, STI);
+
+  unsigned Offset;
+  switch (CallType) {
+  case LFISyscall:
+    Offset = 0;
+    break;
+  case LFITLSRead:
+    Offset = 8;
+    break;
+  case LFITLSWrite:
+    Offset = 16;
+    break;
+  }
+
+  MCInst Jmp;
+  Jmp.setOpcode(X86::JMP64m);
+  Jmp.addOperand(MCOperand::createReg(LFIBaseReg));
+  Jmp.addOperand(MCOperand::createImm(1));
+  Jmp.addOperand(MCOperand::createReg(X86::NoRegister));
+  Jmp.addOperand(MCOperand::createImm(Offset));
+  Jmp.addOperand(MCOperand::createReg(0));
+  Out.emitInstruction(Jmp, STI);
+
+  Out.emitLabel(Symbol);
+
+  Out.emitBundleUnlock();
+}
+
+void X86::X86MCLFIExpander::expandSyscall(const MCInst &Inst, MCStreamer &Out,
+                                          const MCSubtargetInfo &STI) {
+  emitLFICall(LFISyscall, Out, STI);
+}
+
 void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst,
-                                          MCStreamer &Out,
-                                          const MCSubtargetInfo &STI,
-                                          bool EmitPrefixes) {
+                                         MCStreamer &Out,
+                                         const MCSubtargetInfo &STI,
+                                         bool EmitPrefixes) {
   if (isPrefix(Inst)) {
     return Prefixes.push_back(Inst);
   }
-  if (isDirectCall(Inst)) {
+  if (isSyscall(Inst)) {
+    expandSyscall(Inst, Out, STI);
+  } else if (isDirectCall(Inst)) {
     expandDirectCall(Inst, Out, STI);
   } else if (isIndirectBranch(Inst) || isCall(Inst)) {
     expandIndirectBranch(Inst, Out, STI);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
index 2f26ce4fe5a9d..4e91375d91b8d 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
@@ -53,6 +53,9 @@ class X86MCLFIExpander : public MCLFIExpander {
   void expandReturn(const MCInst &Inst, MCStreamer &Out,
                     const MCSubtargetInfo &STI);
 
+  void expandSyscall(const MCInst &Inst, MCStreamer &Out,
+                     const MCSubtargetInfo &STI);
+
   void expandLoadStore(const MCInst &Inst, MCStreamer &Out,
                        const MCSubtargetInfo &STI, bool EmitPrefixes);
 
@@ -82,6 +85,15 @@ class X86MCLFIExpander : public MCLFIExpander {
                            const MCSubtargetInfo &STI);
   void emitIndirectCallReg(MCRegister Reg, MCStreamer &Out,
                            const MCSubtargetInfo &STI);
+
+  enum LFICallType {
+    LFISyscall,
+    LFITLSRead,
+    LFITLSWrite,
+  };
+
+  void emitLFICall(LFICallType CallType, MCStreamer &Out,
+                   const MCSubtargetInfo &STI);
 };
 }
 }
diff --git a/llvm/test/MC/X86/LFI/sys.s b/llvm/test/MC/X86/LFI/sys.s
new file mode 100644
index 0000000000000..b4c9468c29f7b
--- /dev/null
+++ b/llvm/test/MC/X86/LFI/sys.s
@@ -0,0 +1,8 @@
+// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s
+
+syscall
+// CHECK:      .bundle_lock
+// CHECK-NEXT: leaq .Ltmp0(%rip), %r11
+// CHECK-NEXT: jmpq *(%r14)
+// CHECK-NEXT: .Ltmp0:
+// CHECK-NEXT: .bundle_unlock

From 5a02be8fd841b188d1cecc7b3d125e80b7f2ab8f Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 11 Sep 2025 21:02:59 -0700
Subject: [PATCH 32/41] [LFI] [X86] Rewrites for TLS reads

---
 .../X86/MCTargetDesc/X86MCLFIExpander.cpp     | 43 +++++++++++++++++++
 .../X86/MCTargetDesc/X86MCLFIExpander.h       |  3 ++
 llvm/test/MC/X86/LFI/sys.s                    | 16 +++++++
 3 files changed, 62 insertions(+)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
index 5cd2013750e59..362544057f6e9 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
@@ -500,6 +500,15 @@ static bool isSyscall(const MCInst &Inst) {
   return Inst.getOpcode() == X86::SYSCALL;
 }
 
+static bool isTLSRead(const MCInst &Inst) {
+  return Inst.getOpcode() == X86::MOV64rm &&
+    Inst.getOperand(1).getReg() == X86::NoRegister &&
+    Inst.getOperand(2).getImm() == 1 &&
+    Inst.getOperand(3).getReg() == X86::NoRegister &&
+    Inst.getOperand(4).getImm() == 0 &&
+    Inst.getOperand(5).getReg() == X86::FS;
+}
+
 void X86::X86MCLFIExpander::emitLFICall(LFICallType CallType,
                                         MCStreamer &Out,
                                         const MCSubtargetInfo &STI) {
@@ -549,6 +558,29 @@ void X86::X86MCLFIExpander::expandSyscall(const MCInst &Inst, MCStreamer &Out,
   emitLFICall(LFISyscall, Out, STI);
 }
 
+void X86::X86MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer &Out,
+                                          const MCSubtargetInfo &STI) {
+  if (Inst.getOperand(0).getReg() != X86::RAX) {
+    MCInst Mov;
+    Mov.setOpcode(X86::MOV64rr);
+    Mov.addOperand(Inst.getOperand(0));
+    Mov.addOperand(MCOperand::createReg(X86::RAX));
+    Out.emitInstruction(Mov, STI);
+  }
+
+  emitLFICall(LFITLSRead, Out, STI);
+
+  if (Inst.getOperand(0).getReg() != X86::RAX) {
+    MCInst Xchg;
+    Xchg.setOpcode(X86::XCHG64rr);
+    Xchg.addOperand(Inst.getOperand(0));
+    Xchg.addOperand(MCOperand::createReg(X86::RAX));
+    Xchg.addOperand(Inst.getOperand(0));
+    Xchg.addOperand(MCOperand::createReg(X86::RAX));
+    Out.emitInstruction(Xchg, STI);
+  }
+}
+
 void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst,
                                          MCStreamer &Out,
                                          const MCSubtargetInfo &STI,
@@ -558,6 +590,8 @@ void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst,
   }
   if (isSyscall(Inst)) {
     expandSyscall(Inst, Out, STI);
+  } else if (isTLSRead(Inst)) {
+    expandTLSRead(Inst, Out, STI);
   } else if (isDirectCall(Inst)) {
     expandDirectCall(Inst, Out, STI);
   } else if (isIndirectBranch(Inst) || isCall(Inst)) {
@@ -572,6 +606,15 @@ void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst,
     // The previous case doesn't catch xchg so special case it.
     expandExplicitStackManipulation(X86::RSP, Inst, Out, STI, EmitPrefixes);
   } else {
+    for (int i = 0, e = Inst.getNumOperands(); i < e; ++i) {
+      if (Inst.getOperand(i).isReg()) {
+        MCRegister Reg = Inst.getOperand(i).getReg();
+        if (Reg == X86::FS || Reg == X86::GS)
+          return Out.getContext().reportError(
+              Inst.getLoc(), "invalid use of %fs or %gs segment register");
+      }
+    }
+
     expandLoadStore(Inst, Out, STI, EmitPrefixes);
   }
 }
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
index 4e91375d91b8d..1e114375c5434 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
@@ -56,6 +56,9 @@ class X86MCLFIExpander : public MCLFIExpander {
   void expandSyscall(const MCInst &Inst, MCStreamer &Out,
                      const MCSubtargetInfo &STI);
 
+  void expandTLSRead(const MCInst &Inst, MCStreamer &Out,
+                     const MCSubtargetInfo &STI);
+
   void expandLoadStore(const MCInst &Inst, MCStreamer &Out,
                        const MCSubtargetInfo &STI, bool EmitPrefixes);
 
diff --git a/llvm/test/MC/X86/LFI/sys.s b/llvm/test/MC/X86/LFI/sys.s
index b4c9468c29f7b..4cd495196cb9f 100644
--- a/llvm/test/MC/X86/LFI/sys.s
+++ b/llvm/test/MC/X86/LFI/sys.s
@@ -6,3 +6,19 @@ syscall
 // CHECK-NEXT: jmpq *(%r14)
 // CHECK-NEXT: .Ltmp0:
 // CHECK-NEXT: .bundle_unlock
+
+movq %fs:0, %rax
+// CHECK:      .bundle_lock
+// CHECK-NEXT: leaq .Ltmp1(%rip), %r11
+// CHECK-NEXT: jmpq *8(%r14)
+// CHECK-NEXT: .Ltmp1:
+// CHECK-NEXT: .bundle_unlock
+
+movq %fs:0, %rdi
+// CHECK:      movq %rax, %rdi
+// CHECK-NEXT: .bundle_lock
+// CHECK-NEXT: leaq .Ltmp2(%rip), %r11
+// CHECK-NEXT: jmpq *8(%r14)
+// CHECK-NEXT: .Ltmp2:
+// CHECK-NEXT: .bundle_unlock
+// CHECK-NEXT: xchgq %rax, %rdi

From b18dda4883f6cb031fb99b6028e9ebcbabf24b24 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 11 Sep 2025 21:07:07 -0700
Subject: [PATCH 33/41] [LFI] [X86] Autoformat

---
 .../X86/MCTargetDesc/X86MCLFIExpander.cpp     | 109 ++++++++----------
 .../X86/MCTargetDesc/X86MCLFIExpander.h       |   9 +-
 llvm/lib/Target/X86/X86LFIRewritePass.cpp     |  28 ++---
 3 files changed, 67 insertions(+), 79 deletions(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
index 362544057f6e9..4c605cc176f26 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
@@ -21,8 +21,8 @@
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCLFIExpander.h"
-#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/CommandLine.h"
@@ -83,14 +83,15 @@ bool X86::X86MCLFIExpander::isValidScratchRegister(MCRegister Reg) const {
 }
 
 void X86::X86MCLFIExpander::expandDirectCall(const MCInst &Inst,
-                                              MCStreamer &Out,
-                                              const MCSubtargetInfo &STI) {
+                                             MCStreamer &Out,
+                                             const MCSubtargetInfo &STI) {
   Out.emitInstruction(Inst, STI);
   Out.emitCodeAlignment(Align(BundleSize), &STI);
 }
 
-void X86::X86MCLFIExpander::emitSandboxBranchReg(MCRegister Reg, MCStreamer &Out,
-                                                  const MCSubtargetInfo &STI) {
+void X86::X86MCLFIExpander::emitSandboxBranchReg(MCRegister Reg,
+                                                 MCStreamer &Out,
+                                                 const MCSubtargetInfo &STI) {
 
   MCInst AndInst;
   AndInst.setOpcode(X86::AND32ri8);
@@ -112,7 +113,7 @@ void X86::X86MCLFIExpander::emitSandboxBranchReg(MCRegister Reg, MCStreamer &Out
 }
 
 void X86::X86MCLFIExpander::emitIndirectJumpReg(MCRegister Reg, MCStreamer &Out,
-                                                 const MCSubtargetInfo &STI) {
+                                                const MCSubtargetInfo &STI) {
   Out.emitBundleLock(false);
   emitSandboxBranchReg(Reg, Out, STI);
 
@@ -126,7 +127,7 @@ void X86::X86MCLFIExpander::emitIndirectJumpReg(MCRegister Reg, MCStreamer &Out,
 }
 
 void X86::X86MCLFIExpander::emitIndirectCallReg(MCRegister Reg, MCStreamer &Out,
-                                                 const MCSubtargetInfo &STI) {
+                                                const MCSubtargetInfo &STI) {
   MCOperand Target = MCOperand::createReg(getReg64(Reg));
 
   Out.emitBundleLock(false);
@@ -140,8 +141,8 @@ void X86::X86MCLFIExpander::emitIndirectCallReg(MCRegister Reg, MCStreamer &Out,
 }
 
 void X86::X86MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
-                                                  MCStreamer &Out,
-                                                  const MCSubtargetInfo &STI) {
+                                                 MCStreamer &Out,
+                                                 const MCSubtargetInfo &STI) {
   MCRegister Target;
   if (mayLoad(Inst)) {
     // indirect jmp/call through memory
@@ -167,16 +168,13 @@ void X86::X86MCLFIExpander::expandIndirectBranch(const MCInst &Inst,
     emitIndirectJumpReg(Target, Out, STI);
 }
 
-static bool isValidReturnRegister(const MCRegister& Reg) {
-  return Reg == X86::EAX ||
-         Reg == X86::RAX ||
-         Reg == X86::AL ||
-         Reg == X86::AX ||
-         Reg == X86::XMM0;
+static bool isValidReturnRegister(const MCRegister &Reg) {
+  return Reg == X86::EAX || Reg == X86::RAX || Reg == X86::AL ||
+         Reg == X86::AX || Reg == X86::XMM0;
 }
 
 void X86::X86MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
-                                          const MCSubtargetInfo &STI) {
+                                         const MCSubtargetInfo &STI) {
   MCRegister ScratchReg = X86::R11;
 
   MCInst Pop;
@@ -211,10 +209,9 @@ void X86::X86MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
 
 // Expands any operations that load to or store from memory, but do
 // not explicitly modify the stack or base pointer.
-void X86::X86MCLFIExpander::expandLoadStore(const MCInst &Inst,
-                                             MCStreamer &Out,
-                                             const MCSubtargetInfo &STI,
-                                             bool EmitPrefixes) {
+void X86::X86MCLFIExpander::expandLoadStore(const MCInst &Inst, MCStreamer &Out,
+                                            const MCSubtargetInfo &STI,
+                                            bool EmitPrefixes) {
   emitInstruction(Inst, Out, STI, EmitPrefixes);
 }
 
@@ -246,11 +243,10 @@ static void fixupStringOpReg(const MCOperand &Op, MCStreamer &Out,
   Out.emitInstruction(Lea, STI);
 }
 
-void X86::X86MCLFIExpander::expandStringOperation(
-    const MCInst &Inst,
-    MCStreamer &Out,
-    const MCSubtargetInfo &STI,
-    bool EmitPrefixes) {
+void X86::X86MCLFIExpander::expandStringOperation(const MCInst &Inst,
+                                                  MCStreamer &Out,
+                                                  const MCSubtargetInfo &STI,
+                                                  bool EmitPrefixes) {
   Out.emitBundleLock(false);
   switch (Inst.getOpcode()) {
   case X86::CMPSB:
@@ -286,7 +282,8 @@ static void demoteInst(MCInst &Inst, const MCInstrInfo &InstInfo) {
   unsigned NewOpc = demoteOpcode(Inst.getOpcode());
   Inst.setOpcode(NewOpc);
   // demote all general purpose 64 bit registers to 32 bits
-  const llvm::ArrayRef<llvm::MCOperandInfo> OpInfo = InstInfo.get(Inst.getOpcode()).operands();
+  const llvm::ArrayRef<llvm::MCOperandInfo> OpInfo =
+      InstInfo.get(Inst.getOpcode()).operands();
   for (int i = 0, e = Inst.getNumOperands(); i < e; ++i) {
     if (OpInfo[i].OperandType == MCOI::OPERAND_REGISTER) {
       assert(Inst.getOperand(i).isReg());
@@ -304,8 +301,7 @@ static void demoteInst(MCInst &Inst, const MCInstrInfo &InstInfo) {
 void X86::X86MCLFIExpander::emitSandboxMemOp(MCInst &Inst, int MemIdx,
                                              MCRegister ScratchReg,
                                              MCStreamer &Out,
-                                             const MCSubtargetInfo &STI) {
-}
+                                             const MCSubtargetInfo &STI) {}
 
 // Returns true if sandboxing the memory operand specified at Idx of
 // Inst will emit any auxillary instructions.
@@ -334,12 +330,13 @@ static bool willEmitSandboxInsts(const MCInst &Inst, int Idx) {
 // If EmitInstructions is set to false, this will not emit anything and return
 // whether it would emit any auxiliary instructions.
 bool X86::X86MCLFIExpander::emitSandboxMemOps(MCInst &Inst,
-                                               MCRegister ScratchReg,
-                                               MCStreamer &Out,
-                                               const MCSubtargetInfo &STI,
-                                               bool EmitInstructions) {
+                                              MCRegister ScratchReg,
+                                              MCStreamer &Out,
+                                              const MCSubtargetInfo &STI,
+                                              bool EmitInstructions) {
 
-  const llvm::ArrayRef<llvm::MCOperandInfo> OpInfo = InstInfo->get(Inst.getOpcode()).operands();
+  const llvm::ArrayRef<llvm::MCOperandInfo> OpInfo =
+      InstInfo->get(Inst.getOpcode()).operands();
 
   bool anyInstsEmitted = false;
 
@@ -419,10 +416,7 @@ void X86::X86MCLFIExpander::expandExplicitStackManipulation(
   else
     ScratchReg = 0;
 
-  bool MemSandboxed = emitSandboxMemOps(SandboxedInst,
-                                        ScratchReg,
-                                        Out,
-                                        STI,
+  bool MemSandboxed = emitSandboxMemOps(SandboxedInst, ScratchReg, Out, STI,
                                         /*emitInstructions=*/true);
 
   Out.emitBundleLock(false); // for stack fixup
@@ -450,20 +444,19 @@ static bool isPrefix(const MCInst &Inst) {
 
 static bool isDirectCall(const MCInst &Inst) {
   switch (Inst.getOpcode()) {
-    case X86::CALLpcrel32:
-    case X86::CALL64pcrel32:
-      return true;
-    default:
-      return false;
+  case X86::CALLpcrel32:
+  case X86::CALL64pcrel32:
+    return true;
+  default:
+    return false;
   }
 }
 
 // Emit prefixes + instruction if EmitPrefixes argument is true.
 // Otherwise, emit the bare instruction.
-void X86::X86MCLFIExpander::emitInstruction(const MCInst &Inst,
-                                             MCStreamer &Out,
-                                             const MCSubtargetInfo &STI,
-                                             bool EmitPrefixes) {
+void X86::X86MCLFIExpander::emitInstruction(const MCInst &Inst, MCStreamer &Out,
+                                            const MCSubtargetInfo &STI,
+                                            bool EmitPrefixes) {
   if (EmitPrefixes) {
     for (const MCInst &Prefix : Prefixes)
       Out.emitInstruction(Prefix, STI);
@@ -502,15 +495,14 @@ static bool isSyscall(const MCInst &Inst) {
 
 static bool isTLSRead(const MCInst &Inst) {
   return Inst.getOpcode() == X86::MOV64rm &&
-    Inst.getOperand(1).getReg() == X86::NoRegister &&
-    Inst.getOperand(2).getImm() == 1 &&
-    Inst.getOperand(3).getReg() == X86::NoRegister &&
-    Inst.getOperand(4).getImm() == 0 &&
-    Inst.getOperand(5).getReg() == X86::FS;
+         Inst.getOperand(1).getReg() == X86::NoRegister &&
+         Inst.getOperand(2).getImm() == 1 &&
+         Inst.getOperand(3).getReg() == X86::NoRegister &&
+         Inst.getOperand(4).getImm() == 0 &&
+         Inst.getOperand(5).getReg() == X86::FS;
 }
 
-void X86::X86MCLFIExpander::emitLFICall(LFICallType CallType,
-                                        MCStreamer &Out,
+void X86::X86MCLFIExpander::emitLFICall(LFICallType CallType, MCStreamer &Out,
                                         const MCSubtargetInfo &STI) {
   Out.emitBundleLock(false);
 
@@ -581,8 +573,7 @@ void X86::X86MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer &Out,
   }
 }
 
-void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst,
-                                         MCStreamer &Out,
+void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
                                          const MCSubtargetInfo &STI,
                                          bool EmitPrefixes) {
   if (isPrefix(Inst)) {
@@ -620,7 +611,7 @@ void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst,
 }
 
 bool X86::X86MCLFIExpander::expandInst(const MCInst &Inst, MCStreamer &Out,
-                                        const MCSubtargetInfo &STI) {
+                                       const MCSubtargetInfo &STI) {
   if (Guard)
     return false;
   Guard = true;
@@ -756,9 +747,9 @@ static unsigned demoteOpcode(unsigned Opcode) {
   // two left since they were unified. The condition (above, equal, ...) used
   // to be part of the opcode, now it is encoded  differently as per
   // https://github.com/llvm/llvm-project/commit/e0bfeb5f24979416144c16e8b99204f5f163b889
-  // If there are errors, maybe the reason could be that here it is not sufficient
-  // to just lower the opcode, but the operation needs to be constructed
-  // in a more sophisticated way.
+  // If there are errors, maybe the reason could be that here it is not
+  // sufficient to just lower the opcode, but the operation needs to be
+  // constructed in a more sophisticated way.
   case X86::CMOV64rr:
     return X86::CMOV32rr;
   case X86::CMOV64rm:
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
index 1e114375c5434..47d52e648504e 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
@@ -30,11 +30,12 @@ namespace X86 {
 class X86MCLFIExpander : public MCLFIExpander {
 public:
   X86MCLFIExpander(MCContext &Ctx, std::unique_ptr<MCRegisterInfo> &&RI,
-                    std::unique_ptr<MCInstrInfo> &&II)
+                   std::unique_ptr<MCInstrInfo> &&II)
       : MCLFIExpander(Ctx, std::move(RI), std::move(II)) {}
 
   bool expandInst(const MCInst &Inst, MCStreamer &Out,
                   const MCSubtargetInfo &STI) override;
+
 protected:
   bool isValidScratchRegister(MCRegister Reg) const override;
 
@@ -63,7 +64,7 @@ class X86MCLFIExpander : public MCLFIExpander {
                        const MCSubtargetInfo &STI, bool EmitPrefixes);
 
   void expandStringOperation(const MCInst &Inst, MCStreamer &Out,
-           const MCSubtargetInfo &STI, bool EmitPrefixes);
+                             const MCSubtargetInfo &STI, bool EmitPrefixes);
 
   void doExpandInst(const MCInst &Inst, MCStreamer &Out,
                     const MCSubtargetInfo &STI, bool EmitPrefixes);
@@ -98,6 +99,6 @@ class X86MCLFIExpander : public MCLFIExpander {
   void emitLFICall(LFICallType CallType, MCStreamer &Out,
                    const MCSubtargetInfo &STI);
 };
-}
-}
+} // namespace X86
+} // namespace llvm
 #endif
diff --git a/llvm/lib/Target/X86/X86LFIRewritePass.cpp b/llvm/lib/Target/X86/X86LFIRewritePass.cpp
index 09bcb0eff1306..bc1f3040fbbbc 100644
--- a/llvm/lib/Target/X86/X86LFIRewritePass.cpp
+++ b/llvm/lib/Target/X86/X86LFIRewritePass.cpp
@@ -18,23 +18,21 @@
 using namespace llvm;
 
 namespace {
-  class X86LFIRewritePass : public MachineFunctionPass {
-  public:
-    static char ID;
-    X86LFIRewritePass() : MachineFunctionPass(ID) {}
+class X86LFIRewritePass : public MachineFunctionPass {
+public:
+  static char ID;
+  X86LFIRewritePass() : MachineFunctionPass(ID) {}
 
-    virtual bool runOnMachineFunction(MachineFunction &Fn) override;
+  virtual bool runOnMachineFunction(MachineFunction &Fn) override;
 
-    virtual StringRef getPassName() const override {
-      return "LFI Rewrites";
-    }
+  virtual StringRef getPassName() const override { return "LFI Rewrites"; }
 
-  private:
-    const TargetInstrInfo *TII;
-    const X86Subtarget *Subtarget;
-  };
+private:
+  const TargetInstrInfo *TII;
+  const X86Subtarget *Subtarget;
+};
 
-  char X86LFIRewritePass::ID = 0;
+char X86LFIRewritePass::ID = 0;
 } // namespace
 
 bool X86LFIRewritePass::runOnMachineFunction(MachineFunction &MF) {
@@ -55,7 +53,5 @@ bool X86LFIRewritePass::runOnMachineFunction(MachineFunction &MF) {
 
 /// createX86LFIRewritePassPass - returns an instance of the pass.
 namespace llvm {
-  FunctionPass* createX86LFIRewritePass() {
-    return new X86LFIRewritePass();
-  }
+FunctionPass *createX86LFIRewritePass() { return new X86LFIRewritePass(); }
 } // namespace llvm

From 526830150be6a2a159d2ac7db71041db33b82051 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Fri, 12 Sep 2025 11:30:01 -0700
Subject: [PATCH 34/41] [LFI] [X86] Rewrites for loads/stores

---
 .../X86/MCTargetDesc/X86MCLFIExpander.cpp     | 228 +++++++++++++++++-
 llvm/lib/Target/X86/X86.td                    |  16 ++
 llvm/test/MC/X86/LFI/branch.s                 |   4 +-
 llvm/test/MC/X86/LFI/mem.s                    |  10 +
 4 files changed, 245 insertions(+), 13 deletions(-)
 create mode 100644 llvm/test/MC/X86/LFI/mem.s

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
index 4c605cc176f26..8d634ea155828 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
@@ -24,6 +24,7 @@
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -35,6 +36,16 @@ using namespace llvm;
 static const int BundleSize = 32;
 
 static const MCRegister LFIBaseReg = X86::R14;
+static const MCRegister LFIBaseSeg = X86::GS;
+
+static bool hasFeature(const FeatureBitset Feature,
+                       const MCSubtargetInfo &STI) {
+  return (STI.getFeatureBits() & Feature) == Feature;
+}
+
+static bool hasSegue(const MCSubtargetInfo &STI) {
+  return !hasFeature(FeatureBitset({X86::FeatureLFINoSegue}), STI);
+}
 
 static MCRegister getReg64(MCRegister Reg) {
   switch (Reg) {
@@ -173,6 +184,10 @@ static bool isValidReturnRegister(const MCRegister &Reg) {
          Reg == X86::AX || Reg == X86::XMM0;
 }
 
+static bool isHighReg(MCRegister Reg) {
+  return Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH;
+}
+
 void X86::X86MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
                                          const MCSubtargetInfo &STI) {
   MCRegister ScratchReg = X86::R11;
@@ -212,7 +227,88 @@ void X86::X86MCLFIExpander::expandReturn(const MCInst &Inst, MCStreamer &Out,
 void X86::X86MCLFIExpander::expandLoadStore(const MCInst &Inst, MCStreamer &Out,
                                             const MCSubtargetInfo &STI,
                                             bool EmitPrefixes) {
-  emitInstruction(Inst, Out, STI, EmitPrefixes);
+  assert(!explicitlyModifiesRegister(Inst, X86::RSP));
+
+  // Optimize if we are doing a mov into a register
+  bool ElideScratchReg = false;
+  switch (Inst.getOpcode()) {
+  case X86::MOV64rm:
+  case X86::MOV32rm:
+  case X86::MOV16rm:
+  case X86::MOV8rm:
+    ElideScratchReg = true;
+    break;
+  default:
+    break;
+  }
+
+  MCInst SandboxedInst(Inst);
+
+  MCRegister ScratchReg;
+  if (ElideScratchReg)
+    ScratchReg = Inst.getOperand(0).getReg();
+  else if (numScratchRegs() > 0)
+    ScratchReg = getScratchReg(0);
+  else
+    ScratchReg = X86::R11D;
+
+  // Determine if the instruction requires sandboxing
+  bool InstNeedsSandboxing = emitSandboxMemOps(SandboxedInst,
+                                               ScratchReg,
+                                               Out,
+                                               STI,
+                                               /*EmitInstructions=*/false);
+
+  // We may want to load/store to a high byte register, which is not possible
+  // in combination with using Base for sandboxing.
+  // Instead, rotate the target register so that the load/store operates on the
+  // low byte instead.
+  MCRegister RotateRegister = X86::NoRegister;
+  if (InstNeedsSandboxing &&
+      (SandboxedInst.getOpcode() == X86::MOV8rm_NOREX ||
+       SandboxedInst.getOpcode() == X86::MOV8rm) &&
+      isHighReg(SandboxedInst.getOperand(0).getReg())) {
+    RotateRegister = SandboxedInst.getOperand(0).getReg();
+    SandboxedInst.setOpcode(X86::MOV8rm);
+    SandboxedInst.getOperand(0).setReg(
+        getX86SubSuperRegister(RotateRegister, 8, /*High=*/false));
+  } else if (InstNeedsSandboxing &&
+             (SandboxedInst.getOpcode() == X86::MOV8mr_NOREX ||
+              SandboxedInst.getOpcode() == X86::MOV8mr) &&
+             isHighReg(SandboxedInst.getOperand(5).getReg())) {
+    RotateRegister = SandboxedInst.getOperand(5).getReg();
+    SandboxedInst.setOpcode(X86::MOV8mr);
+    SandboxedInst.getOperand(5).setReg(
+        getX86SubSuperRegister(RotateRegister, 8, /*High=*/false));
+  }
+
+  if (RotateRegister != X86::NoRegister) {
+    MCInst RotateHtoLInst;
+    RotateHtoLInst.setOpcode(X86::ROR64ri);
+    RotateHtoLInst.addOperand(MCOperand::createReg(getReg64(RotateRegister)));
+    RotateHtoLInst.addOperand(MCOperand::createReg(getReg64(RotateRegister)));
+    RotateHtoLInst.addOperand(MCOperand::createImm(8));
+    Out.emitInstruction(RotateHtoLInst, STI);
+  }
+
+  bool BundleLock = emitSandboxMemOps(SandboxedInst,
+                                      ScratchReg,
+                                      Out,
+                                      STI,
+                                      /*EmitInstructions=*/true);
+  emitInstruction(SandboxedInst, Out, STI, EmitPrefixes);
+  if (BundleLock)
+    Out.emitBundleUnlock();
+
+  if (RotateRegister != X86::NoRegister) {
+    // Rotate the register back.
+    MCInst RotateLtoHInst;
+    RotateLtoHInst.setOpcode(X86::ROL64ri);
+    RotateLtoHInst.addOperand(MCOperand::createReg(getReg64(RotateRegister)));
+    RotateLtoHInst.addOperand(MCOperand::createReg(getReg64(RotateRegister)));
+    RotateLtoHInst.addOperand(MCOperand::createImm(8));
+    Out.emitInstruction(RotateLtoHInst, STI);
+  }
 }
 
 // Emits movl Reg32, Reg32
@@ -301,7 +397,115 @@ static void demoteInst(MCInst &Inst, const MCInstrInfo &InstInfo) {
 void X86::X86MCLFIExpander::emitSandboxMemOp(MCInst &Inst, int MemIdx,
                                              MCRegister ScratchReg,
                                              MCStreamer &Out,
-                                             const MCSubtargetInfo &STI) {}
+                                             const MCSubtargetInfo &STI) {
+  MCOperand &Base = Inst.getOperand(MemIdx);
+  MCOperand &Scale = Inst.getOperand(MemIdx + 1);
+  MCOperand &Index = Inst.getOperand(MemIdx + 2);
+  MCOperand &Offset = Inst.getOperand(MemIdx + 3);
+  MCOperand &Segment = Inst.getOperand(MemIdx + 4);
+
+  // In the cases below, we want to promote any registers in the
+  // memory operand to 64 bits.
+  if (isAbsoluteReg(Base.getReg()) && Index.getReg() == 0) {
+    Base.setReg(getReg64(Base.getReg()));
+    return;
+  }
+
+  if (Base.getReg() == 0 && isAbsoluteReg(Index.getReg()) &&
+             Scale.getImm() == 1) {
+    Base.setReg(getReg64(Index.getReg()));
+    Index.setReg(0);
+    return;
+  }
+
+  if (Index.getReg() == 0 && Base.getReg() == 0) {
+    Base.setReg(LFIBaseReg);
+    return;
+  }
+
+  if (hasSegue(STI) && Segment.getReg() == 0) {
+    Segment.setReg(LFIBaseSeg);
+    Base.setReg(getReg32(Base.getReg()));
+    Index.setReg(getReg32(Index.getReg()));
+    return;
+  }
+
+  MCRegister ScratchReg32 = 0;
+  if (ScratchReg != 0) {
+    ScratchReg32 = getReg32(ScratchReg);
+  } else {
+    Error(Inst,
+        "Not enough scratch registers when emitting sandboxed memory operation."
+    );
+  }
+
+  if (isAbsoluteReg(Base.getReg()) && !isAbsoluteReg(Index.getReg()) &&
+      Offset.isImm() && Offset.getImm() == 0) {
+    // Validation requires that for memory access, a 64-bit register is used,
+    // with its upper 32 bits clobbered by a 32-bit move just before.
+    // Move Index to 32 bit ScratchReg, and use ScratchReg32 instead of Index
+    // memory access.
+    MCInst MovIdxToScratchReg;
+    MovIdxToScratchReg.setOpcode(X86::MOV32rr);
+    MovIdxToScratchReg.addOperand(MCOperand::createReg(ScratchReg32));
+    MovIdxToScratchReg.addOperand(
+        MCOperand::createReg(getReg32(Index.getReg())));
+    Out.emitInstruction(MovIdxToScratchReg, STI);
+
+    Base.setReg(getReg64(Base.getReg()));
+    Index.setReg(getReg64(ScratchReg32));
+    return;
+  }
+
+  if (Index.getReg() == 0 && Base.getReg() != 0 && Offset.isImm() &&
+      Offset.getImm() == 0) {
+    // Validation requires that for memory access, a 64-bit register is used,
+    // with its upper 32 bits clobbered by a 32-bit move just before.
+    // Move Base to 32 bit ScratchReg, and use ScratchReg32 instead of Base for
+    // memory access.
+    MCInst MovBaseToScratchReg;
+    MovBaseToScratchReg.setOpcode(X86::MOV32rr);
+    MovBaseToScratchReg.addOperand(MCOperand::createReg(ScratchReg32));
+    MovBaseToScratchReg.addOperand(
+        MCOperand::createReg(getReg32(Base.getReg())));
+    Out.emitInstruction(MovBaseToScratchReg, STI);
+
+    Index.setReg(getReg64(ScratchReg32));
+    Base.setReg(LFIBaseReg);
+    return;
+  }
+
+  MCRegister ScratchReg64 = getReg64(ScratchReg32);
+  MCRegister BaseReg64 = getReg64(Base.getReg());
+  MCRegister IndexReg64 = getReg64(Index.getReg());
+
+  MCInst Lea;
+  Lea.setOpcode(X86::LEA64_32r);
+  Lea.addOperand(MCOperand::createReg(ScratchReg32));
+  Lea.addOperand(MCOperand::createReg(BaseReg64));
+  Lea.addOperand(Scale);
+  Lea.addOperand(MCOperand::createReg(IndexReg64));
+  Lea.addOperand(Offset);
+  Lea.addOperand(Segment);
+
+  // Specical case if there is no base or scale
+  if (Base.getReg() == 0 && Scale.getImm() == 1) {
+    Lea.getOperand(1).setReg(IndexReg64); // Base
+    Lea.getOperand(3).setReg(0);          // Index
+  }
+
+  Out.emitInstruction(Lea, STI);
+
+  Base.setReg(LFIBaseReg);
+  Scale.setImm(1);
+  Index.setReg(ScratchReg64);
+  if (Offset.isImm()) {
+    Offset.setImm(0);
+  } else {
+    Inst.erase(Inst.begin() + MemIdx + 3);
+    Inst.insert(Inst.begin() + MemIdx + 3, MCOperand::createImm(0));
+  }
+}
 
 // Returns true if sandboxing the memory operand specified at Idx of
 // Inst will emit any auxillary instructions.
@@ -346,8 +550,10 @@ bool X86::X86MCLFIExpander::emitSandboxMemOps(MCInst &Inst,
         if (!EmitInstructions)
           return true;
 
-        Out.emitBundleLock(false);
-        anyInstsEmitted = true;
+        if (!hasSegue(STI)) {
+          Out.emitBundleLock(false);
+          anyInstsEmitted = true;
+        }
       }
       emitSandboxMemOp(Inst, i, ScratchReg, Out, STI);
       i += 4;
@@ -380,7 +586,7 @@ void X86::X86MCLFIExpander::expandExplicitStackManipulation(
     // pop %r11
     // .bundle_lock
     // movl %r11d, %esp
-    // add %r15, %rsp
+    // add %r14, %rsp
     // .bundle_unlock
     // where %r11 is the scratch register, and %rsp the stack register.
 
@@ -397,12 +603,12 @@ void X86::X86MCLFIExpander::expandExplicitStackManipulation(
     MovR11ToESP.addOperand(MCOperand::createReg(X86::R11D));
     Out.emitInstruction(MovR11ToESP, STI);
 
-    MCInst AddR15Inst;
-    AddR15Inst.setOpcode(X86::ADD64rr);
-    AddR15Inst.addOperand(MCOperand::createReg(StackReg));
-    AddR15Inst.addOperand(MCOperand::createReg(StackReg));
-    AddR15Inst.addOperand(MCOperand::createReg(X86::R15));
-    Out.emitInstruction(AddR15Inst, STI);
+    MCInst AddBaseInst;
+    AddBaseInst.setOpcode(X86::ADD64rr);
+    AddBaseInst.addOperand(MCOperand::createReg(StackReg));
+    AddBaseInst.addOperand(MCOperand::createReg(StackReg));
+    AddBaseInst.addOperand(MCOperand::createReg(LFIBaseReg));
+    Out.emitInstruction(AddBaseInst, STI);
 
     return Out.emitBundleUnlock();
   }
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 577428cad6d61..e285eaac4ce18 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -772,6 +772,22 @@ def TuningUseGLMDivSqrtCosts
 def TuningBranchHint: SubtargetFeature<"branch-hint", "HasBranchHint", "true",
                                         "Target has branch hint feature">;
 
+//===----------------------------------------------------------------------===//
+// LFI Subtarget Features
+//===----------------------------------------------------------------------===//
+
+def FeatureLFIStores : SubtargetFeature<"lfi-stores", "LFIStores", "true",
+                                        "Isolate only store+jump instructions">;
+
+def FeatureLFIJumps : SubtargetFeature<"lfi-jumps", "LFIJumps", "true",
+                                       "Isolate only jump instructions">;
+
+def FeatureLFITLSReg : SubtargetFeature<"lfi-tls-reg", "LFITLSReg", "true",
+                                        "Reserve a register for thread-local LFI data">;
+
+def FeatureLFINoSegue : SubtargetFeature<"lfi-no-segue", "LFINoSegue", "true",
+                                         "Do not use segmentation register for LFI">;
+
 //===----------------------------------------------------------------------===//
 // X86 CPU Families
 // TODO: Remove these - use general tuning features to determine codegen.
diff --git a/llvm/test/MC/X86/LFI/branch.s b/llvm/test/MC/X86/LFI/branch.s
index d7a983acff888..0f039b43f7cf7 100644
--- a/llvm/test/MC/X86/LFI/branch.s
+++ b/llvm/test/MC/X86/LFI/branch.s
@@ -12,7 +12,7 @@ callq *%rax
 // CHECK-NEXT: .bundle_unlock
 
 callq *(%rax)
-// CHECK:      movq (%rax), %r11
+// CHECK:      movq %gs:(%eax), %r11
 // CHECK-NEXT: .bundle_lock
 // CHECK-NEXT: andl $-32, %r11d
 // CHECK-NEXT: addq %r14, %r11
@@ -28,7 +28,7 @@ jmpq *%rax
 // CHECK-NEXT: .bundle_unlock
 
 jmpq *(%rax)
-// CHECK:      movq (%rax), %r11
+// CHECK:      movq %gs:(%eax), %r11
 // CHECK-NEXT: .bundle_lock
 // CHECK-NEXT: andl $-32, %r11d
 // CHECK-NEXT: addq %r14, %r11
diff --git a/llvm/test/MC/X86/LFI/mem.s b/llvm/test/MC/X86/LFI/mem.s
new file mode 100644
index 0000000000000..b4ed56dfd4c0c
--- /dev/null
+++ b/llvm/test/MC/X86/LFI/mem.s
@@ -0,0 +1,10 @@
+// RUN: llvm-mc -filetype asm -triple x86_64_lfi %s | FileCheck %s
+
+movq (%rax), %rdi
+// CHECK: movq %gs:(%eax), %rdi
+
+movq %rcx, (%rax)
+// CHECK: movq %rcx, %gs:(%eax)
+
+movq %rcx, (%rax, %rdi)
+// CHECK: movq %rcx, %gs:(%eax,%edi)

From 9cb11c5a5a185bea23838ddf88ae04985fa06992 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Fri, 12 Sep 2025 12:49:22 -0700
Subject: [PATCH 35/41] [LFI] [X86] Fixes for TLS rewrites

---
 .../X86/MCTargetDesc/X86MCLFIExpander.cpp     | 48 ++++++++++++++++++-
 .../X86/MCTargetDesc/X86MCLFIExpander.h       |  3 ++
 2 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
index 8d634ea155828..6709e9b3e35a9 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
@@ -391,6 +391,48 @@ static void demoteInst(MCInst &Inst, const MCInstrInfo &InstInfo) {
   }
 }
 
+void X86::X86MCLFIExpander::prepareSandboxMemOp(MCInst &Inst, int MemIdx,
+                                             MCRegister ScratchReg,
+                                             MCStreamer &Out,
+                                             const MCSubtargetInfo &STI) {
+  MCOperand &Base = Inst.getOperand(MemIdx);
+  MCOperand &Index = Inst.getOperand(MemIdx + 2);
+  MCOperand &Segment = Inst.getOperand(MemIdx + 4);
+
+  if (Segment.getReg() == X86::FS) {
+    MCInst Push;
+    Push.setOpcode(X86::PUSH64r);
+    Push.addOperand(MCOperand::createReg(X86::RAX));
+    Out.emitInstruction(Push, STI);
+    MCInst Read;
+    Read.setOpcode(X86::MOV64rm);
+    Read.addOperand(MCOperand::createReg(X86::RAX));
+    Read.addOperand(Inst.getOperand(MemIdx));
+    Read.addOperand(Inst.getOperand(MemIdx + 1));
+    Read.addOperand(Inst.getOperand(MemIdx + 2));
+    Read.addOperand(Inst.getOperand(MemIdx + 3));
+    Read.addOperand(Inst.getOperand(MemIdx + 4));
+    expandTLSRead(Read, Out, STI);
+    MCInst Mov;
+    Mov.setOpcode(X86::MOV64rr);
+    Mov.addOperand(MCOperand::createReg(ScratchReg));
+    Mov.addOperand(MCOperand::createReg(X86::RAX));
+    Out.emitInstruction(Mov, STI);
+    MCInst Pop;
+    Pop.setOpcode(X86::POP64r);
+    Pop.addOperand(MCOperand::createReg(X86::RAX));
+    Out.emitInstruction(Pop, STI);
+    if (Base.getReg() == 0)
+      Base.setReg(ScratchReg);
+    else if (Index.getReg() == 0)
+      Index.setReg(ScratchReg);
+    else
+      return Out.getContext().reportError(
+          Inst.getLoc(), "invalid use of %fs segment register");
+    Segment.setReg(0);
+  }
+}
+
 // Emits the sandboxing operations necessary, and modifies the memory
 // operand specified by MemIdx.
 // Used as a helper function for emitSandboxMemOps.
@@ -546,6 +588,8 @@ bool X86::X86MCLFIExpander::emitSandboxMemOps(MCInst &Inst,
 
   for (int i = 0, e = Inst.getNumOperands(); i < e; ++i) {
     if (OpInfo[i].OperandType == MCOI::OPERAND_MEMORY) {
+      prepareSandboxMemOp(Inst, i, ScratchReg, Out, STI);
+
       if (!anyInstsEmitted && willEmitSandboxInsts(Inst, i)) {
         if (!EmitInstructions)
           return true;
@@ -806,9 +850,9 @@ void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
     for (int i = 0, e = Inst.getNumOperands(); i < e; ++i) {
       if (Inst.getOperand(i).isReg()) {
         MCRegister Reg = Inst.getOperand(i).getReg();
-        if (Reg == X86::FS || Reg == X86::GS)
+        if (Reg == X86::GS)
           return Out.getContext().reportError(
-              Inst.getLoc(), "invalid use of %fs or %gs segment register");
+              Inst.getLoc(), "invalid use of %gs segment register");
       }
     }
 
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
index 47d52e648504e..5bde669aa3d8e 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.h
@@ -74,6 +74,9 @@ class X86MCLFIExpander : public MCLFIExpander {
                                        const MCSubtargetInfo &STI,
                                        bool EmitPrefixes);
 
+  void prepareSandboxMemOp(MCInst &Inst, int MemIdx, MCRegister ScratchReg,
+                           MCStreamer &Out, const MCSubtargetInfo &STI);
+
   void emitSandboxMemOp(MCInst &Inst, int MemIdx, MCRegister ScratchReg,
                         MCStreamer &Out, const MCSubtargetInfo &STI);
 

From 935badf98006898b7facec7fd992a789578efe1b Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Mon, 15 Sep 2025 13:46:34 -0700
Subject: [PATCH 36/41] [LFI] [X86] Disable TCO with more than 6 args

---
 llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
index 3f27166080d5a..72b59e192d5ac 100644
--- a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
@@ -77,6 +77,7 @@
 #include "llvm/Pass.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 using namespace llvm;
@@ -256,6 +257,11 @@ static bool markTails(Function &F, OptimizationRemarkEmitter *ORE) {
                           {LLVMContext::OB_clang_arc_attachedcall,
                            LLVMContext::OB_ptrauth, LLVMContext::OB_kcfi});
 
+      // LFI: the CI->getNumOperands() >= 6 is needed because we reserve %r11
+      // on x86-64. See https://issuetracker.google.com/issues/42403689?pli=1
+      if (llvm::Triple(CI->getModule()->getTargetTriple()).isX8664LFI() && CI->getNumOperands() >= 6)
+        IsNoTail = true;
+
       if (!IsNoTail && CI->doesNotAccessMemory()) {
         // A call to a readnone function whose arguments are all things computed
         // outside this function can be marked tail. Even if you stored the

From 8f007949943cd687b4c503ad1d5cdf7545a5b62c Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Mon, 15 Sep 2025 14:03:34 -0700
Subject: [PATCH 37/41] [LFI] [X86] Updates for building runtime libraries

---
 compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake | 2 +-
 compiler-rt/cmake/builtin-config-ix.cmake            | 2 +-
 compiler-rt/lib/builtins/CMakeLists.txt              | 2 ++
 compiler-rt/lib/hwasan/hwasan_interceptors.cpp       | 2 ++
 compiler-rt/lib/orc/sysv_reenter.x86-64.S            | 2 ++
 compiler-rt/lib/xray/xray_trampoline_x86_64.S        | 2 ++
 libunwind/src/UnwindRegistersRestore.S               | 2 ++
 lld/ELF/Writer.cpp                                   | 2 +-
 8 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
index f19b0b7961a9e..2e97293156dbe 100644
--- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
+++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
@@ -2,7 +2,7 @@ set(ARM64 aarch64 aarch64_lfi)
 set(ARM32 arm armhf)
 set(HEXAGON hexagon)
 set(X86 i386)
-set(X86_64 x86_64)
+set(X86_64 x86_64 x86_64_lfi)
 set(LOONGARCH64 loongarch64)
 set(MIPS32 mips mipsel)
 set(MIPS64 mips64 mips64el)
diff --git a/compiler-rt/cmake/builtin-config-ix.cmake b/compiler-rt/cmake/builtin-config-ix.cmake
index 8e043ba24c985..d3b9ed487db61 100644
--- a/compiler-rt/cmake/builtin-config-ix.cmake
+++ b/compiler-rt/cmake/builtin-config-ix.cmake
@@ -63,7 +63,7 @@ set(ARM32 arm armhf armv4t armv5te armv6 armv6m armv7m armv7em armv7 armv7s armv
 set(AVR avr)
 set(HEXAGON hexagon)
 set(X86 i386)
-set(X86_64 x86_64)
+set(X86_64 x86_64 x86_64_lfi)
 set(LOONGARCH64 loongarch64)
 set(MIPS32 mips mipsel)
 set(MIPS64 mips64 mips64el)
diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index d059277e6f006..a69e7e35848ca 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -639,6 +639,8 @@ endif()
 
 set(amdgcn_SOURCES ${GENERIC_SOURCES})
 
+set(x86_64_lfi_SOURCES ${x86_64_SOURCES})
+
 set(armv4t_SOURCES ${arm_min_SOURCES})
 set(armv5te_SOURCES ${arm_min_SOURCES})
 set(armv6_SOURCES ${arm_min_SOURCES})
diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
index fbe9f04db3669..6b10b9ed246ae 100644
--- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
@@ -409,7 +409,9 @@ InternalLongjmp(__hw_register_buf env, int retval) {
       "mov (1*8)(%0),%%rbp;"
       "mov (2*8)(%0),%%r12;"
       "mov (3*8)(%0),%%r13;"
+#ifndef __LFI__
       "mov (4*8)(%0),%%r14;"
+#endif
       "mov (5*8)(%0),%%r15;"
       "mov (6*8)(%0),%%rsp;"
       "mov (7*8)(%0),%%rdx;"
diff --git a/compiler-rt/lib/orc/sysv_reenter.x86-64.S b/compiler-rt/lib/orc/sysv_reenter.x86-64.S
index 0a36280f1d1f8..a59ee0fd701e6 100644
--- a/compiler-rt/lib/orc/sysv_reenter.x86-64.S
+++ b/compiler-rt/lib/orc/sysv_reenter.x86-64.S
@@ -61,7 +61,9 @@ __orc_rt_sysv_reenter:
         movq            %rax,   8(%rbp)
         fxrstor64       (%rsp)
         movq            -112(%rbp),     %r15
+#ifndef __LFI__
         movq            -104(%rbp),     %r14
+#endif
         movq            -96(%rbp),      %r13
         movq            -88(%rbp),      %r12
         movq            -80(%rbp),      %r11
diff --git a/compiler-rt/lib/xray/xray_trampoline_x86_64.S b/compiler-rt/lib/xray/xray_trampoline_x86_64.S
index 0f480547b52cc..93a12e1340fa2 100644
--- a/compiler-rt/lib/xray/xray_trampoline_x86_64.S
+++ b/compiler-rt/lib/xray/xray_trampoline_x86_64.S
@@ -91,7 +91,9 @@
 	movq  32(%rsp), %r11
 	movq  24(%rsp), %r12
 	movq  16(%rsp), %r13
+#ifndef __LFI__
 	movq  8(%rsp), %r14
+#endif
 	movq  0(%rsp), %r15
 	addq	$240, %rsp
 	CFI_ADJUST_CFA_OFFSET(-240)
diff --git a/libunwind/src/UnwindRegistersRestore.S b/libunwind/src/UnwindRegistersRestore.S
index bc3fff38a13dc..12507c792998a 100644
--- a/libunwind/src/UnwindRegistersRestore.S
+++ b/libunwind/src/UnwindRegistersRestore.S
@@ -106,7 +106,9 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_x86_64_jumpto)
   movq   88(%rdi), %r11
   movq   96(%rdi), %r12
   movq  104(%rdi), %r13
+#ifndef __LFI__
   movq  112(%rdi), %r14
+#endif
   movq  120(%rdi), %r15
   # skip rflags
   # skip cs
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 0db922b07aabf..39848de8755a4 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -2901,7 +2901,7 @@ template <class ELFT> void Writer<ELFT>::writeTrapInstr() {
       if (p->p_type == PT_LOAD && (p->p_flags & PF_X))
         fillTrap(
             ctx.target->trapInstr,
-            ctx.bufferStart + alignDown(p->firstSec->offset + p->p_filesz, 4),
+            ctx.bufferStart + p->firstSec->offset,
             ctx.bufferStart + alignToPowerOf2(p->firstSec->offset + p->p_filesz,
                                               ctx.arg.maxPageSize));
 

From aa32defa2a2d3ecd37e32deaf882dd0a3f945609 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 18 Sep 2025 13:34:40 -0700
Subject: [PATCH 38/41] [LFI] [X86] Disable TCO only for vararg 6-arg functions

---
 llvm/lib/Target/X86/X86ISelLoweringCall.cpp             | 5 +++++
 llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp | 5 -----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 6835c7e336a5c..d5181466edfc9 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -2820,6 +2820,11 @@ bool X86TargetLowering::IsEligibleForTailCallOptimization(
     if (IsCalleeWin64 || IsCallerWin64)
       return false;
 
+    // Do not optimize vararg calls with 6 arguments for LFI because LFI
+    // reserves %r11.
+    if (Subtarget.isLFI() && ArgLocs.size() > 5)
+      return false;
+
     for (const auto &VA : ArgLocs)
       if (!VA.isRegLoc())
         return false;
diff --git a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
index 72b59e192d5ac..f78a88832db15 100644
--- a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
@@ -257,11 +257,6 @@ static bool markTails(Function &F, OptimizationRemarkEmitter *ORE) {
                           {LLVMContext::OB_clang_arc_attachedcall,
                            LLVMContext::OB_ptrauth, LLVMContext::OB_kcfi});
 
-      // LFI: the CI->getNumOperands() >= 6 is needed because we reserve %r11
-      // on x86-64. See https://issuetracker.google.com/issues/42403689?pli=1
-      if (llvm::Triple(CI->getModule()->getTargetTriple()).isX8664LFI() && CI->getNumOperands() >= 6)
-        IsNoTail = true;
-
       if (!IsNoTail && CI->doesNotAccessMemory()) {
         // A call to a readnone function whose arguments are all things computed
         // outside this function can be marked tail. Even if you stored the

From 7b3766014122de3500f81fe32705e6fd7c3c6bc4 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Mon, 22 Sep 2025 16:40:37 -0700
Subject: [PATCH 39/41] [LFI] [X86] Warn or error for reserved register
 modification

---
 .../X86/MCTargetDesc/X86MCLFIExpander.cpp     | 30 ++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
index 6709e9b3e35a9..470460125cedc 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
@@ -33,9 +33,14 @@ using namespace llvm;
 
 #define DEBUG_TYPE "lfi"
 
+static cl::opt<bool> X86LFIErrorReserved(
+    "x86-lfi-error-reserved", cl::Hidden, cl::init(false),
+    cl::desc("Produce errors for uses of LFI reserved registers"));
+
 static const int BundleSize = 32;
 
 static const MCRegister LFIBaseReg = X86::R14;
+static const MCRegister LFIScratchReg = X86::R11;
 static const MCRegister LFIBaseSeg = X86::GS;
 
 static bool hasFeature(const FeatureBitset Feature,
@@ -823,13 +828,36 @@ void X86::X86MCLFIExpander::expandTLSRead(const MCInst &Inst, MCStreamer &Out,
   }
 }
 
+static MCInst replaceReg(const MCInst &Inst, MCRegister Dest, MCRegister Src) {
+  MCInst New;
+  New.setOpcode(Inst.getOpcode());
+  New.setLoc(Inst.getLoc());
+  for (unsigned I = 0; I < Inst.getNumOperands(); ++I) {
+    const MCOperand &Op = Inst.getOperand(I);
+    if (Op.isReg() && Op.getReg() == Src) {
+      New.addOperand(MCOperand::createReg(Dest));
+    } else {
+      New.addOperand(Op);
+    }
+  }
+  return New;
+}
+
 void X86::X86MCLFIExpander::doExpandInst(const MCInst &Inst, MCStreamer &Out,
                                          const MCSubtargetInfo &STI,
                                          bool EmitPrefixes) {
   if (isPrefix(Inst)) {
     return Prefixes.push_back(Inst);
   }
-  if (isSyscall(Inst)) {
+  if (explicitlyModifiesRegister(Inst, LFIBaseReg)) {
+    if (X86LFIErrorReserved)
+      return Out.getContext().reportError(
+          Inst.getLoc(), "illegal modification of reserved LFI register");
+    Out.getContext().reportWarning(
+        Inst.getLoc(), "deleting modification of reserved LFI register");
+    MCInst New = replaceReg(Inst, LFIScratchReg, LFIBaseReg);
+    return doExpandInst(New, Out, STI, EmitPrefixes);
+  } else if (isSyscall(Inst)) {
     expandSyscall(Inst, Out, STI);
   } else if (isTLSRead(Inst)) {
     expandTLSRead(Inst, Out, STI);

From a09000fba4c115a714d5f785e07ade02cb9241e3 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 25 Sep 2025 12:57:58 -0700
Subject: [PATCH 40/41] [LFI] [X86] Add support for +lfi-stores and +lfi-jumps

---
 .../X86/MCTargetDesc/X86MCLFIExpander.cpp     | 25 ++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
index 470460125cedc..43a19f2f192b9 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCLFIExpander.cpp
@@ -52,6 +52,14 @@ static bool hasSegue(const MCSubtargetInfo &STI) {
   return !hasFeature(FeatureBitset({X86::FeatureLFINoSegue}), STI);
 }
 
+static bool hasLFIStores(const MCSubtargetInfo &STI) {
+  return hasFeature(FeatureBitset({X86::FeatureLFIStores}), STI);
+}
+
+static bool hasLFIJumps(const MCSubtargetInfo &STI) {
+  return hasFeature(FeatureBitset({X86::FeatureLFIJumps}), STI);
+}
+
 static MCRegister getReg64(MCRegister Reg) {
   switch (Reg) {
   default:
@@ -358,14 +366,17 @@ void X86::X86MCLFIExpander::expandStringOperation(const MCInst &Inst,
   case X86::MOVSW:
   case X86::MOVSL:
   case X86::MOVSQ:
-    fixupStringOpReg(Inst.getOperand(1), Out, STI);
-    fixupStringOpReg(Inst.getOperand(0), Out, STI);
+    if (!hasLFIJumps(STI) && !hasLFIStores(STI))
+      fixupStringOpReg(Inst.getOperand(1), Out, STI);
+    if (!hasLFIJumps(STI))
+      fixupStringOpReg(Inst.getOperand(0), Out, STI);
     break;
   case X86::STOSB:
   case X86::STOSW:
   case X86::STOSL:
   case X86::STOSQ:
-    fixupStringOpReg(Inst.getOperand(0), Out, STI);
+    if (!hasLFIJumps(STI))
+      fixupStringOpReg(Inst.getOperand(0), Out, STI);
     break;
   }
   emitInstruction(Inst, Out, STI, EmitPrefixes);
@@ -445,6 +456,11 @@ void X86::X86MCLFIExpander::emitSandboxMemOp(MCInst &Inst, int MemIdx,
                                              MCRegister ScratchReg,
                                              MCStreamer &Out,
                                              const MCSubtargetInfo &STI) {
+  if (hasLFIJumps(STI))
+    return;
+  if (hasLFIStores(STI) && !mayStore(Inst))
+    return;
+
   MCOperand &Base = Inst.getOperand(MemIdx);
   MCOperand &Scale = Inst.getOperand(MemIdx + 1);
   MCOperand &Index = Inst.getOperand(MemIdx + 2);
@@ -628,6 +644,9 @@ static void emitStackFixup(MCRegister StackReg, MCStreamer &Out,
 void X86::X86MCLFIExpander::expandExplicitStackManipulation(
     MCRegister StackReg, const MCInst &Inst, MCStreamer &Out,
     const MCSubtargetInfo &STI, bool EmitPrefixes) {
+  if (hasLFIJumps(STI))
+    return emitInstruction(Inst, Out, STI, EmitPrefixes);
+
   if (Inst.getOpcode() == X86::POP64r) {
     // Transform
     // pop   %rsp

From 3458f34028ad019aafe7aacbdd527d4d6e59bdb7 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia@gmail.com>
Date: Thu, 25 Sep 2025 12:59:55 -0700
Subject: [PATCH 41/41] [LFI] [AArch64] Don't expand stack mod for +lfi-jumps

---
 llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
index ebf11f7f3e8e8..1fa6590640c32 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIExpander.cpp
@@ -213,6 +213,9 @@ void AArch64::AArch64MCLFIExpander::expandStackModification(
     return Out.emitInstruction(Inst, STI);
   }
 
+  if (hasFeature(FeatureBitset({AArch64::FeatureLFIJumps}), STI))
+    return Out.emitInstruction(Inst, STI);
+
   MCInst ModInst;
   MCRegister Scratch = getScratch();
   assert(Inst.getOperand(0).isReg() &&